diff --git a/.gitignore b/.gitignore index bfc1237c7..d6aa06ad1 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,10 @@ docs/plans/ !.env.example # Per-Bot egress proxies. Carries credentials in the URL, like .env does. egress.env -node_modules/ +# No trailing slash, deliberately. With one, this pattern matches a DIRECTORY only, and a symlink +# named node_modules is not a directory — so a sandbox that links its dependencies at the repo root +# leaves three links git will happily commit. That happened. +node_modules **/dist/ app/src/lib/generated/application-config.ts .logs/ diff --git a/app/serve.ts b/app/serve.ts index 85fc6d2fc..c13ebc287 100644 --- a/app/serve.ts +++ b/app/serve.ts @@ -74,14 +74,33 @@ export function upstreamWebSocketHeaders(requestHeaders: Headers): Headers { return headers; } +/** + * Bun's WebSocket client: the DOM one, plus the two things Bun adds and the browser does not. + * + * `bun-types` hands the global `WebSocket` over to `lib.dom` whenever DOM is in `lib`, and this + * project has DOM in `lib` for the browser code that is the rest of `app/`. What is left describes + * the BROWSER's client, which has no `terminate()` and takes subprotocols where Bun takes options. + * This file only ever runs under Bun and uses both. + * + * Stated as an extension of the DOM type rather than by reaching for `Bun.WebSocket`, because that + * interface is written for projects WITHOUT DOM in `lib` and degrades to `{}` here — its events + * come back untyped, which costs more than the two members it would buy. Same socket either way; + * this is a type-level correction, not a different client. + */ +export type BunWebSocket = WebSocket & { terminate(): void }; +export const BunWebSocket = globalThis.WebSocket as unknown as { + new (url: string | URL, options?: { headers?: HeadersInit }): BunWebSocket; + readonly OPEN: number; +}; + type WebSocketBridge = { - upstream: WebSocket; + upstream: BunWebSocket; attach: (downstream: ServerWebSocket) => void; dispose: () => void; }; /** Own the upstream before awaiting its handshake, including any immediate welcome frames. */ -function prepareWebSocketBridge(upstream: WebSocket, signal: AbortSignal) { +function prepareWebSocketBridge(upstream: BunWebSocket, signal: AbortSignal) { upstream.binaryType = "arraybuffer"; let downstream: ServerWebSocket | undefined; const pending: (string | ArrayBuffer)[] = []; @@ -185,7 +204,7 @@ if (import.meta.main) { }, message(ws, message) { const { upstream } = ws.data; - if (upstream.readyState === WebSocket.OPEN) { + if (upstream.readyState === BunWebSocket.OPEN) { upstream.send(message); } else { ws.close(1011, "Upstream connection closed"); @@ -202,13 +221,13 @@ if (import.meta.main) { if (isApiCall(url.pathname)) { const target = SERVER + url.pathname + url.search; if (request.headers.get("upgrade")?.toLowerCase() === "websocket") { - const upstream = new WebSocket(target.replace(/^http/, "ws"), { + const upstream = new BunWebSocket(target.replace(/^http/, "ws"), { headers: upstreamWebSocketHeaders(request.headers), }); const bridge = prepareWebSocketBridge(upstream, request.signal); if ( !(await bridge.opened) || - upstream.readyState !== WebSocket.OPEN + upstream.readyState !== BunWebSocket.OPEN ) { bridge.data.dispose(); return new Response("Could not connect to the upstream WebSocket", { diff --git a/app/src/components/channels/channel-chat.tsx b/app/src/components/channels/channel-chat.tsx index f31a53db1..2fc3c6610 100644 --- a/app/src/components/channels/channel-chat.tsx +++ b/app/src/components/channels/channel-chat.tsx @@ -1,11 +1,13 @@ import type { Message } from "@ag-ui/core"; import { + type Attachment, UseAgentUpdate, useAgent, useCopilotKit, } from "@copilotkit/react-core/v2"; import { useMutation, useQuery } from "@tanstack/react-query"; import { useCallback, useEffect, useRef, useState } from "react"; +import { attachmentModality } from "@/components/channels/chat-messages"; import { toAgentOptions } from "@/components/channels/composer"; import { ConversationView } from "@/components/channels/conversation-view"; import { @@ -14,6 +16,7 @@ import { transcriptMessages, } from "@/components/channels/transcript-messages"; import { agentListQueryOptions } from "@/lib/agents/queries"; +import { attachmentUrl } from "@/lib/channels/attachments"; import { recordChannelActivityMutationOptions, setChannelBusy, @@ -126,6 +129,95 @@ function mergeStoredMessages(local: Message[], stored: Message[]): Message[] { ]; } +/** + * The uploaded id and filename an `Attachment` carries once it is `ready`, read from the + * `metadata` the composer's `onUpload` stamped on it — see `composer/attachments.ts`. Not the + * SDK's own `attachment.id`, which is a client-side handle for the upload placeholder rather than + * the id this deployment stored the file under. + */ +function uploadedAttachment(attachment: Attachment): { + attachmentId: string; + filename?: string; +} { + const metadata = attachment.metadata as + | { attachmentId?: unknown; filename?: unknown } + | undefined; + const attachmentId = metadata?.attachmentId; + if (typeof attachmentId !== "string") { + throw new Error("Attachment is missing its uploaded id."); + } + const filename = + typeof metadata?.filename === "string" ? metadata.filename : undefined; + return filename ? { attachmentId, filename } : { attachmentId }; +} + +/** + * One attachment, turned into the part shape a stored message carries. + * + * THE MODALITY COMES FROM THE BYTES, NOT FROM `attachment.type`, AND THIS IS THE ONLY PLACE IT CAN. + * + * `attachment.type` is the browser's claim, fixed before the upload and never reconciled with what + * the file turned out to be. The server stopped trusting it — `resolvePart` decides an attachment's + * modality with `classifyAttachment` on its own sniffed `mimeType` — but that correction lives on + * the server and never comes back here. What this function writes IS the stored message, so a + * `document` written here is what every later render of that message reads, for ever: a screenshot + * whose part the browser mislabelled drew a grey file card over the picture, and the transcript's + * document probe then paid a whole-file read per render for the privilege. + * + * Narrowed through the source union rather than read straight off, for the reason `parkedTiles` in + * `chat-transcript.tsx` narrows the same field: a `data` source's `mimeType` is `file.type`, the + * very claim being refused, and only a `url` source has been past the server. `attachmentModality` + * falls back to the declared type when there is no corroborated one, so an attachment that somehow + * arrives unuploaded is written exactly as it used to be. + * + * The comment this replaces said only "image" and "document" reach here because the composer's + * upload config accepts no other kind of file. That reason is no longer true — the config's + * `accept` is now the wildcard, and it is `screenPickedFiles` that holds the line. The conclusion still + * holds; the justification had rotted, which is why the modality is now derived rather than cast. + */ +function toAttachmentPart(attachment: Attachment) { + const { attachmentId, filename } = uploadedAttachment(attachment); + const { source } = attachment; + const mimeType = + source.type === "url" && source.mimeType ? source.mimeType : undefined; + return { + type: attachmentModality(attachment.type, mimeType), + source: { type: "url" as const, value: attachmentUrl(attachmentId) }, + metadata: filename ? { attachmentId, filename } : { attachmentId }, + }; +} + +/** + * A plain string when there is nothing attached, exactly as every message in every channel has + * always been sent — never a single-element array wrapping the same text, which every existing + * reader would take a different path for no gain. With attachments, the text goes first as its + * own part and is left out entirely when empty, since an empty text part is noise the model has + * to read past. + * + * Exported for the test that pins this wire format. Reaching it through `deliver`/`say` would mean + * standing up `useAgent`'s runtime, the thread join and the ready/join gates around it just to + * observe a pure string-in-object-out mapping — none of that machinery bears on what this function + * decides, so a narrow export is the honest way to test the contract without restructuring the + * module around a test. + */ +export function toMessageContent( + trimmed: string, + attachments: readonly Attachment[], +) { + if (attachments.length === 0) return trimmed; + const refs = attachments.map(toAttachmentPart); + return trimmed ? [{ type: "text" as const, text: trimmed }, ...refs] : refs; +} + +/** What the roster's "last thing said" reads when a message carried no caption. */ +function describeAttachments(attachments: readonly Attachment[]): string { + if (attachments.length === 1) { + const { filename } = uploadedAttachment(attachments[0]); + return filename ? `Sent ${filename}` : "Sent an attachment"; + } + return `Sent ${attachments.length} attachments`; +} + /** * One channel's conversation with one coworker. * @@ -408,6 +500,36 @@ export function ChannelChat({ // Run failures arrive as events and are reported only for turns started in this mount. const [runError, setRunError] = useState(null); const awaitingReply = useRef(false); + /** + * WHY THIS TURN ENDED WITHOUT AN ANSWER, KEPT WHERE `deliver` CAN STILL SEE IT — because the one + * thing that knows is a subscriber, and the one thing that has to act on it is an `await`. + * + * `copilotkit.runAgent` DOES NOT REJECT ON A FAILED RUN. `CopilotKitCore.runAgent` catches + * everything the agent throws, reports it through `emitError` as `AGENT_RUN_FAILED`, and returns + * `{ result: undefined, newMessages: [] }` — a value indistinguishable from a run that finished + * with nothing to say. So a gateway 503, a stream that dies, a model that refuses the request: + * every one of them arrived here as a resolved promise, and `say` reported success for a turn + * that never reached the server. + * + * WHAT THAT COST, WHICH IS THE REASON THIS EXISTS. `say` resolving is what every caller reads as + * "it went". The composer clears the box and gives up the chips it was riding; the queue empties + * into a draft nothing retries; `conversation-view.tsx` never runs either of the failure paths it + * has written for exactly this. The person is left with the failed turn in the transcript and a + * notice under it, the words unretryable, and the files behind them staged rows that nothing on + * any screen points at any more. The notice is honest and everything under it was not. + * + * READ OFF THE SAME `fail` THE NOTICE IS, and deliberately not from a second subscription of its + * own. `fail` already answers the one question a separate subscriber would get wrong: a turn the + * PERSON stopped also reaches `onRunFailed`, with an abort, and `onStop` clears `awaitingReply` + * before it — so Stop is not a failure here and nothing restores a draft somebody chose to end. + * + * ONE SLOT FOR ONE TURN AT A TIME, the same assumption `awaitingReply` beside it already makes. + * Two overlapping turns — a component button pressed during a composer send — would have the + * second clear the first's reason, which reports the earlier turn as successful. That is the + * pre-existing shape of `awaitingReply`, not a new one, and narrowing it means giving a run a + * handle that `copilotkit.runAgent` does not hand back. + */ + const turnFailure = useRef(null); const assistantMessagesBeforeRun = useRef>(new Set()); /* @@ -466,7 +588,11 @@ export function ChannelChat({ * Everything `say` does once it has something worth sending, split out so the counter it is * wrapped in covers every way out of here, a throw included. */ - const deliver = async (trimmed: string, skillInstructions: string[]) => { + const deliver = async ( + trimmed: string, + skillInstructions: string[], + attachments: Attachment[], + ) => { // Wait briefly for the runtime agent instance before adding the message. if (!isReadyRef.current) { await Promise.race([ @@ -490,6 +616,7 @@ export function ChannelChat({ const target = agentRef.current; setRunError(null); + turnFailure.current = null; assistantMessagesBeforeRun.current = new Set( target.messages .filter((message) => message.role === "assistant") @@ -518,11 +645,11 @@ export function ChannelChat({ } target.addMessage({ - content: trimmed, + content: toMessageContent(trimmed, attachments), id: newId(), role: "user", }); - report(trimmed, null); + report(trimmed || describeAttachments(attachments), null); // Providers reject later turns if prior tool calls have no result; repair before sending. const repaired = repairUnansweredToolCalls(target.messages); @@ -536,6 +663,30 @@ export function ChannelChat({ } finally { setRunsInFlight((count) => count - 1); } + + /* + * A TURN THAT DID NOT HAPPEN FAILS THE SEND, which is the only way anything upstream can tell. + * See `turnFailure` for why the resolved promise above says nothing about that. + * + * AFTER the `finally`, not inside the `try`: the run is over either way, so the counter that + * draws the Stop button must come down before this throws. Throwing from inside would leave + * `runsInFlight` high for a run that has already ended. + * + * WHAT THE THROW REACHES, so it is clear this is a message and not a crash. The composer's + * `catch` puts the words and the chips back; `conversation-view.tsx` puts a drained queue back + * as retryable entries carrying their files. Nothing here reports the failure — `runError` was + * already set from the same `fail` that set this, and the transcript already draws it — so this + * adds a retry, not a second sentence. + * + * THE MESSAGE STAYS ON SCREEN. `deliver` added it above and nothing takes it away: it is what + * the failed turn WAS, it is what the notice under it is about, and removing it would delete a + * partial answer that a mid-stream failure had already produced. The restored draft beside it + * is the retry, the same way a failed composer send has always put its words back while the + * transcript kept the turn. + */ + if (turnFailure.current !== null) { + throw new Error(turnFailure.current); + } }; /** @@ -546,9 +697,16 @@ export function ChannelChat({ * keeping here rather than in the view: the view sees only the turns it started itself, and a * queue that drains on the wrong one of those posts a correction into the middle of an answer. */ - const say = async (text: string, skillInstructions: string[] = []) => { + const say = async ( + text: string, + skillInstructions: string[] = [], + attachments: Attachment[] = [], + ) => { const trimmed = text.trim(); - if (!trimmed) return; + // A pasted screenshot with no caption is still a message to send: `canSendDraft` already + // unlocks the button for exactly this case, so refusing it here would leave the button + // enabled and inert. + if (!trimmed && attachments.length === 0) return; turnsRef.current += 1; setTurnsInFlight(turnsRef.current); @@ -556,7 +714,7 @@ export function ChannelChat({ void setChannelBusy({ channelId: channel.id, busy: true }); } try { - await deliver(trimmed, skillInstructions); + await deliver(trimmed, skillInstructions, attachments); } finally { turnsRef.current -= 1; setTurnsInFlight(turnsRef.current); @@ -572,6 +730,9 @@ export function ChannelChat({ const fail = (message: string) => { if (!awaitingReply.current) return; awaitingReply.current = false; + // Both halves of one fact: the sentence the transcript shows, and the reason `deliver` throws + // so the draft behind the turn is restored rather than counted as sent. See `turnFailure`. + turnFailure.current = message; setRunError(message); }; const subscription = agent.subscribe?.({ @@ -604,9 +765,16 @@ export function ChannelChat({ /** * Component buttons speak as user turns without forcing every transcript card to re-render. + * + * The rejection is swallowed HERE rather than left to the void, and that is not a style choice: + * `say` throws on a failed turn now (see `turnFailure`), and a voided promise with nothing on the + * end of it is an unhandled rejection — in this repository's test runner, a failure attributed to + * whichever test happened to be running when it surfaced. There is nothing to restore for this + * caller either way: the words came from a button inside a rendered card, not from a box somebody + * is still holding, and the failed turn is already reported by `runError` under the transcript. */ const askFromComponent = useCallback((text: string) => { - void sayRef.current(text); + void sayRef.current(text).catch(() => undefined); }, []); /** @@ -618,9 +786,12 @@ export function ChannelChat({ if (!pending) return; seedRef.current = null; - void sayRef.current( - typeof pending.content === "string" ? pending.content : "", - ); + // Swallowed for the reason `askFromComponent` above records: `say` throws on a failed turn, and + // the seed has no box to go back into — it was typed on a screen that has already navigated + // away. The transcript keeps the seeded message and the notice under it says what happened. + void sayRef + .current(typeof pending.content === "string" ? pending.content : "") + .catch(() => undefined); // Keep `seed` in state; transcriptMessages gives it up once the agent holds a user turn. }, []); @@ -629,6 +800,7 @@ export function ChannelChat({ ?v=2` became `"?v=2"` — which is a wrong value sitting in a + * field named for an id, and the field is typed `string` so nothing downstream has any reason to + * doubt it. Only `sameAttachmentRow` compares it today, and it compares two values built the same + * wrong way, which is exactly why this went unnoticed and why it is worth cutting properly now + * rather than when the first reader builds a url back out of it. + * + * A fallback at all, rather than an empty string, because the id IS in the url for every url this + * projection accepts: `SentAttachmentTile` refuses anything that does not start with + * `attachmentUrl("")`, so the last segment of a servable url is the id the route will be asked for. + */ +function attachmentIdFromUrl(url: string): string { + const [path = ""] = url.split(/[?#]/); + return path.split("/").at(-1) ?? ""; +} + +/** + * The two fields a tile reads off a part's `metadata`, checked rather than cast. + * + * THE LAST UNCHECKED READ IN THIS FILE, and it was unchecked for the least good reason: it does not + * throw, so nothing ever pointed at it. `isReadablePart` and `isReadableToolCall` above were both + * written after a TypeError took the channel view down; `metadata` fails quietly instead. `?.` + * covers a null and a non-object yields `undefined` for both keys, so a bad value does not crash — + * it just arrives. `metadata: { attachmentId: 42 }` put a number in `SentAttachment.attachmentId`, + * which is DECLARED `string` and compared for identity by `sameAttachmentRow`, and a number + * `filename` passed the truthiness spread below into `title={filename}` and an `alt` template. + * + * The source is the same unvalidated live-run array everything else here guards against: a stored + * message is parsed against a schema on its way out of the database, a live one is whatever the run + * put in the array. The types are not load-bearing here, which is this file's whole thesis. + * + * An EMPTY string is refused alongside a non-string, for both fields. An `attachmentId` of `""` + * names nothing and would beat the url fallback that does; a `filename` of `""` draws a tile with a + * blank name where "Untitled file" is the honest answer. + */ +function readAttachmentMetadata(part: { type: string }): { + attachmentId?: string; + filename?: string; +} { + const raw = (part as { metadata?: unknown }).metadata; + if (typeof raw !== "object" || raw === null) return {}; + const { attachmentId, filename } = raw as { + attachmentId?: unknown; + filename?: unknown; + }; + + return { + ...(typeof attachmentId === "string" && attachmentId + ? { attachmentId } + : {}), + ...(typeof filename === "string" && filename ? { filename } : {}), + }; +} + +/** + * The url a part points at and the type the SERVER gave the bytes behind it, or null when the part + * does not point at a url at all. + * + * A `data` source is the ordinary reason for null — `copilot.ts` swaps the bytes in as the run is + * built, so one can exist on a live turn — and a source that is missing, or carries no `value`, is + * the malformed reason. Both answer the same question the same way: there is nothing to draw. + * + * `mimeType` IS READ OFF THE SOURCE AND NOT OFF `metadata` BECAUSE IT IS A REAL FIELD OF THE + * SOURCE. `AttachmentSource` in `shared/attachments.ts` declares it optional on the url member, and + * AG-UI's own `InputContentUrlSourceSchema` has it as `z.string().optional()` — so unlike a sibling + * key hung off `metadata`, which that file's comment notes would be silently stripped, it survives + * the `RunAgentInputSchema.parse` every stored message is put through on its way back through a + * run. It is a declared field, not one invented here. + * + * Returned together with the url rather than through a second reader, because both come off ONE + * value that has to be proved an object first. Two readers meant validating `source` twice and left + * it possible for a caller to take the url from a source whose type it never checked. + * + * OPTIONAL, AND NARROWED THE WAY `readAttachmentMetadata` NARROWS ITS TWO. The same unvalidated + * live-run array feeds this, so a non-string is refused — and so is `""`, which matters more than + * it looks: `classifyAttachment("")` answers `"unsupported"`, so an empty string taken as an answer + * would draw a file card over a screenshot on the strength of a field that says nothing. + */ +function readUrlSource(part: { type: string }): { + url: string; + mimeType?: string; +} | null { + const source = (part as { source?: unknown }).source; + if (typeof source !== "object" || source === null) return null; + const { type, value, mimeType } = source as { + type?: unknown; + value?: unknown; + mimeType?: unknown; + }; + /* + * `""` IS THE "CARRIES NO `value`" CASE THIS DOC ALREADY PROMISED TO REFUSE, and `typeof` alone + * let it through. What came out was a row rather than a skip: `attachmentIdFromUrl("")` answers + * `""`, so `attachmentId` — a field named for an id, declared `string`, compared for identity by + * `sameAttachmentRow` — held a value naming nothing, and `SentAttachmentTile` refuses any url + * that does not start with `attachmentUrl("")`, so the reader was shown "This attachment is + * unavailable." in the name of a file the server had never been asked about and had not lost. + * That is the accusation the whole absent-file path is written to avoid, arrived at from a part + * that simply had nothing in it. + * + * Narrowed the way every sibling in this file narrows: `readText`, both fields of + * `readAttachmentMetadata`, and `mimeType` two lines below all refuse `""` as well as a + * non-string, because an empty string is a value in good standing to `typeof` and an answer to + * nobody. + */ + if (type !== "url" || typeof value !== "string" || value === "") return null; + return { + url: value, + ...(typeof mimeType === "string" && mimeType ? { mimeType } : {}), + }; +} + +/** + * WHAT A FILE IS DRAWN AS, DECIDED FROM THE BYTES WHEN THE BYTES HAVE BEEN READ, AND FROM THE + * BROWSER'S GUESS ONLY WHEN THEY HAVE NOT. + * + * A staged attachment's `type` is a guess made before the file was uploaded and never revisited. + * Verified against the installed SDK rather than assumed: `useAttachments.processFiles` sets + * `type: getModalityFromMimeType(file.type)` on the placeholder, and that function maps everything + * that is not `image/`, `audio/` or `video/` to `"document"`; when `onUpload` answers, the merge is + * `{ ...att, source, status: "ready", thumbnail, metadata }`, which replaces the SOURCE and never + * the `type`. So the stale guess and the server's answer sit side by side on the same object. + * + * THE DISAGREEMENT IS ENGINEERED, NOT EXOTIC. `composer/picked-files.ts` deliberately passes a + * claim that names no format — `application/octet-stream`, or `""` — so that the server is the one + * that decides, from `sniffMimeType` over the actual bytes. A PNG dragged out of an editor is + * therefore an attachment whose `type` says `document` and whose source says `image/png`. + * + * IT IS NOT ONLY THE WRONG PICTURE, AND THE LOUD HALF IS THE ONE THAT MATTERS. A text file claimed + * as an image draws an `` the browser cannot decode, `onError` fires, and the tile asserts in + * the file's own name that it is unavailable — telling somebody a file was deleted while it sits + * there intact. That is the reason this function exists. + * + * The other direction used to be argued here as a cost and no longer is. `SentAttachmentTile` does + * send a HEAD probe for a document and none for an image, and this paragraph claimed the probe + * "costs the server the whole file out of Postgres", so that a mislabelled screenshot bought "a + * megabyte read on every render of the transcript". The attachment route has since grown a HEAD + * branch that selects `sizeBytes` and never the bytes, so what a mislabelled screenshot buys is one + * cheap metadata round trip nobody needed. Still waste, not worth a sentence in this size — kept + * only to say that the sentence it replaces is wrong, since two review rounds read it as current. + * + * `classifyAttachment` RATHER THAN `mimeType.startsWith("image/")`, and this is the load-bearing + * choice. It is the same function `server/src/channels/attachment-parts.ts` gates on, so a picture + * is drawn to the person exactly when a picture was put in front of the Bot; the two sides cannot + * drift because they ask one question. It also settles HEIC correctly — an image by media type that + * no `` here can render, whose honest tile is the card naming the file. + * + * THE FALLBACK IS THE DECLARED TYPE, NOT `"document"`. `mimeType` is optional on both schemas, so a + * part without one is well-formed rather than malformed, and today that is EVERY sent message: see + * the note at the `modality` field in `toVisibleChatItems`. Collapsing them to file cards would + * break every stored transcript that renders correctly now, which is a far larger population than + * the mislabelled files this exists to fix. + * + * Exported because the QUEUE asks the same question of a parked message (`parkedTiles` in + * `chat-transcript.tsx`) and a second copy of this rule is a second thing to get wrong. Those tiles + * are meant to be the ones the turn will draw once it runs, so two rules that disagreed would show + * as a tile changing shape at the moment of sending. + */ +export function attachmentModality( + declaredType: string, + mimeType?: string, +): "image" | "document" { + if (mimeType === undefined) { + return declaredType === "image" ? "image" : "document"; + } + return classifyAttachment(mimeType) === "image" ? "image" : "document"; +} + +/** + * A tool call with the three fields the transcript reads off it, checked rather than trusted. + * + * The same caution as `isReadablePart`, on the branch beside it: `toolCall.function.name` is read + * three fields deep off whatever a live run put in the array, and the row it builds is keyed on the + * id. Nothing between the run and here checks any of it. + */ +function isReadableToolCall(toolCall: unknown): toolCall is ToolCall { + if (typeof toolCall !== "object" || toolCall === null) return false; + const { id, function: called } = toolCall as { + id?: unknown; + function?: unknown; + }; + if (typeof id !== "string") return false; + if (typeof called !== "object" || called === null) return false; + return typeof (called as { name?: unknown }).name === "string"; +} + /** A tool result, as it arrives, its own message, pointing back at the call it answers. */ type ToolResultMessage = { role: "tool"; toolCallId: string; content?: string }; +/** + * A MESSAGE THAT IS AN OBJECT AT ALL AND CARRIES AN ID, which is a lower bar than any guard above + * and was the one nobody had checked. + * + * Every other guard in this file defends the INSIDE of a message — a bad part, a bad tool call, a + * `content` that is not what it claims. This defends the message itself, and it has to, because a + * hole in the array throws EARLIER than any of them: `isToolResult` reads `.role` in the + * results-gathering pass that runs before the projection begins, so `[null]` cost the whole array + * rather than the one hole in it, and every careful skip below was bypassed on the way past. + * + * Same stakes the rest of the file was written for, and the same answer: `toVisibleChatItems` runs + * inside `ChatTranscript`'s render, so the TypeError escapes into React and unmounts the channel + * view. One malformed turn anywhere in a history and the conversation is a blank screen. + * + * Reachable for the reason `isReadablePart` gives: a stored message is parsed against a schema on + * its way out of the database, but a LIVE one is whatever the run put in the array the agent hands + * back, and nothing between that array and this function checks it. + * + * IT DOES NOT CHECK THE ID, AND `isDrawableMessage` BELOW IS WHERE THAT LIVES. Both passes need + * the object check; only one of them keys anything on an id. + */ +function isReadableMessage(message: unknown): message is Readonly { + return typeof message === "object" && message !== null; +} + +/** + * A MESSAGE THIS PROJECTION CAN KEY A ROW ON, which is the check `message.id` never had. + * + * IT IS THE FIELD THIS FILE WAS KEYED ON HARDEST WHILE TRUSTING IT MOST. `isReadableToolCall` below + * has checked `toolCall.id` since the day it was written, with the reason in its own comment — "the + * row it builds is keyed on the id". `message.id` is keyed on harder and was checked nowhere: it + * becomes `VisibleChatItem.id`, which is DECLARED `string`, and downstream that one value is the + * React key, `MessageScrollerItem`'s `messageId`, the turn-grouping key `anchorRowIds` and `turnOf` + * cut apart at the last colon, and the memo key `createFirstPaintDelays` hangs an entrance delay + * off. + * + * IT SURVIVED BECAUSE IT DOES NOT THROW — the same reason `readAttachmentMetadata` gives for its + * own fields having gone unchecked, and the claim there that `metadata` was "the last unchecked + * read in this file" was simply wrong; this was. A hole gets a text row `key={undefined}`, so React + * warns and reconciles those rows by POSITION, and `data-message-id` is omitted so the scroller + * never registers the row and it can never be a scroll anchor. The attachments row is quieter and + * worse: `` `${message.id}:attachments` `` stringifies the hole, so every id-less turn in a history + * collides on the literal `"undefined:attachments"` — one render key, one registration, one delay, + * and two people's files drawn as one row. + * + * SEPARATE FROM `isReadableMessage` RATHER THAN FOLDED INTO IT, and the difference is which pass + * runs it. The results-gathering pass above reads only `role`, `toolCallId` and `content` off a + * tool result and keys the map on the TOOL CALL's id, which `isReadableToolCall` already checks; it + * never keys anything on `message.id`. Refusing an id-less tool result there would drop the entry + * from `results`, and the tool line it answers would then render as still-in-flight and shimmer for + * ever — trading a real defect for a quieter one. So the object check guards both passes and the id + * check guards only the pass that needs it. + * + * SKIPPING THE MESSAGE IS THE ANSWER RATHER THAN SYNTHESISING AN ID, because an id we invented + * would be stable only within one render: `toVisibleChatItems` runs again on every chunk of a + * streaming answer, so a counter or a `crypto.randomUUID()` would hand React a different key for + * the same row on every frame and remount the message — entrance animation, scroll registration and + * all — several times a second. A row nobody can key is a row this projection cannot draw, and + * dropping it costs that row alone, which is the caution every other guard in this file takes. + * + * `""` IS REFUSED ALONGSIDE A NON-STRING, for the reason `readAttachmentMetadata` refuses an empty + * `attachmentId`: it is a string, so a bare `typeof` admits it, and it names nothing. Two turns + * carrying it collide on `":attachments"` exactly as two holes collide on `"undefined:attachments"`. + */ +function isDrawableMessage(message: Readonly): boolean { + const { id } = message as { id?: unknown }; + return typeof id === "string" && id !== ""; +} + function isToolResult( message: Readonly, ): message is Readonly & ToolResultMessage { @@ -47,13 +368,30 @@ export function toVisibleChatItems( // Gather results first so calls render with their current completion state in the same pass. const results = new Map(); for (const message of messages) { + // Both passes over this array are guarded, not just this one: a guard on only the first would + // move the throw into the flatMap below rather than remove it. + if (!isReadableMessage(message)) continue; if (isToolResult(message)) results.set(message.toolCallId, message.content); } return messages.flatMap((message): VisibleChatItem[] => { + if (!isReadableMessage(message)) return []; + // Every row built below is keyed on `message.id`, including the one that keys on it by string + // interpolation. See `isDrawableMessage` for why an unkeyable row is dropped rather than given + // an id of our own. + if (!isDrawableMessage(message)) return []; + if (message.role === "assistant") { const items: VisibleChatItem[] = []; - if (message.content) { + /* + * `typeof`, NOT TRUTHINESS, and for the same reason the user branch below checks its own + * content: a live turn is whatever the run produced, not whatever the type says. `[]` and + * `{}` are both truthy, so both used to be pushed on as the `text` of a text item and handed + * to the markdown renderer, which reads a string and throws on anything else — one flatMap + * away from the throw the user branch was already defended against, and with the same cost: + * the transcript renders this, so the exception unmounts the channel. + */ + if (typeof message.content === "string" && message.content) { items.push({ kind: "text", id: message.id, @@ -61,7 +399,13 @@ export function toVisibleChatItems( text: message.content, }); } - for (const toolCall of message.toolCalls ?? []) { + // A `toolCalls` that is not a list is not iterable, and `for...of` reports that by throwing. + for (const toolCall of Array.isArray(message.toolCalls) + ? message.toolCalls + : []) { + // Read three fields deep off something nothing has validated: a hole in the array, or a + // call with no `function`, threw before a single row could be drawn. + if (!isReadableToolCall(toolCall)) continue; /* * The call that draws an interface is not a row of its own; the interface is. * @@ -104,14 +448,118 @@ export function toVisibleChatItems( ) { return []; } - const text = - typeof message.content === "string" - ? message.content - : message.content - .filter((part) => part.type === "text") - .map((part) => part.text) - .join("\n"); - - return text ? [{ kind: "text", id: message.id, role: "user", text }] : []; + + if (typeof message.content === "string") { + const text = message.content; + return text ? [{ kind: "text", id: message.id, role: "user", text }] : []; + } + + const text = message.content + .map((part) => (isReadablePart(part) ? readText(part) : null)) + .filter((part) => part !== null) + .join("\n"); + + /* + * ATTACHMENTS FIRST, THEN THE CAPTION, which is the order they are read in and the order every + * chat that carries files puts them in: the picture is what the sentence is about, so a + * question that arrives above its own screenshot asks about something the reader has not seen + * yet. This is the reverse of the parts' order inside the message — a composer sends the text + * part first — and deliberately so; nothing downstream depends on matching the wire order, and + * `anchorRowIds` picks whichever of these comes first, so a turn with a picture now anchors on + * the picture. + */ + const items: VisibleChatItem[] = []; + + /* + * One row for all of them, with the caption appended after it (and absent rather than empty + * when there is none). Only a `url` source ever reaches the browser this way — + * `copilot.ts` swaps in the `data` source later, building the run — but a part is skipped + * rather than thrown on if one ever did arrive here, and so is a part that is not a part at + * all: see `isReadablePart`, and the whole-message caution above it that it extends. + */ + const attachments: SentAttachment[] = []; + message.content.forEach((part, index) => { + if (!isReadablePart(part)) return; + /* + * THE SOURCE IS THE GATE AND `part.type` IS NOT, WHICH IS THE RULE THE SERVER ALREADY + * APPLIES TO THE SAME PART. + * + * This used to require `part.type` to be `image` or `document`, and + * `attachmentIdFor` in `server/src/channels/attachment-parts.ts` deliberately does the + * opposite — its comment names gating on those two as "the alternative and is worse". AG-UI's + * part union is `text | image | audio | video | document | binary`, and a client writing its + * own message content can send any of the six naming one of our urls. The server therefore + * RESOLVES such a part: it inlines the bytes into the run and stamps `attachedAt`, so the + * sweeper spares the row. This file dropped it. The file went to the model, stayed on the + * shelf for ever, and was drawn to the person who sent it nowhere at all — the one shape of + * bug where the two sides disagreeing is invisible from either side alone. + * + * `readUrlSource` below is now the whole gate, and it is the same question the server asks: + * is there a `url` source with something in it. A `text` part is unaffected, and that is not + * luck — a well-formed one carries no `source` key, including all three the server + * substitutes for an attachment it could not send (`unavailableNote`, `notIncludedNote` and + * `unreadableNote` each write `{ type: "text", text }` and nothing else), so they all still + * fall to `readText` and become the caption. + * + * WHAT THE PART TYPE IS STILL GOOD FOR IS THE MODALITY, AND ONLY AS A FALLBACK. It is handed + * to `attachmentModality` below, which prefers the type the SERVER sniffed off the bytes + * whenever the part carries one. So an `audio` part gets the file card rather than an `` + * that cannot decode — the honest tile for a kind this app has no viewer for — and a declared + * type is never given the chance to claim a picture the bytes do not support. + */ + const source = readUrlSource(part); + if (source === null) return; + const { url } = source; + + const metadata = readAttachmentMetadata(part); + const attachmentId = metadata.attachmentId ?? attachmentIdFromUrl(url); + + attachments.push({ + // The PART's index, not the attachment's, so the id survives a caption being added or + // removed above it and stays unique when one turn carries the same file twice. Counted + // over the whole array, malformed parts included: skipping them in the count would + // renumber every file after one and change the render key of a row already on screen. + id: `${message.id}:${index}`, + attachmentId, + url, + // Already narrowed to a non-empty string, or absent: see `readAttachmentMetadata`. + ...(metadata.filename ? { filename: metadata.filename } : {}), + /* + * `source.mimeType` IS ALWAYS ABSENT HERE, AND THAT IS NO LONGER THE PROBLEM IT WAS. + * + * A sent message's part is built by `toAttachmentPart` in `channel-chat.tsx`, which writes + * `source: { type: "url", value }` and carries no `mimeType` — so this call always falls + * through to `part.type`. What changed is what `part.type` MEANS: that function now derives + * it with this same `attachmentModality`, off the `mimeType` the server sniffed from the + * bytes, instead of casting the browser's claim. The stored `type` is the corroborated + * answer, so falling back to it is right rather than merely tolerable. + * + * The call keeps both arguments anyway. It costs nothing, it is the same rule the parked + * tiles apply, and if a stored part ever does carry a `mimeType` — see the paragraph below, + * which is the obvious way to fix history — this reads it without another change here. + * + * WHAT IS STILL WRONG IS HISTORY, AND ONLY HISTORY. A thread stored before that change kept + * whatever the browser claimed, and no amount of client work will correct a row that is + * already written. Putting it right means whoever serves a stored thread to the browser + * supplying the type the attachment row already holds — a server change, written down here + * rather than quietly half-done. + */ + modality: attachmentModality(part.type, source.mimeType), + }); + }); + + if (attachments.length > 0) { + items.push({ + kind: "attachments", + // `turnOf` reads the message id back off this by cutting at the last colon, which is why + // the suffix is a word rather than something that could be mistaken for one. + id: `${message.id}:attachments`, + attachments, + }); + } + + if (text) items.push({ kind: "text", id: message.id, role: "user", text }); + + return items; }); } diff --git a/app/src/components/channels/chat-transcript.tsx b/app/src/components/channels/chat-transcript.tsx index 80b7a20cb..6aed60500 100644 --- a/app/src/components/channels/chat-transcript.tsx +++ b/app/src/components/channels/chat-transcript.tsx @@ -1,13 +1,34 @@ import type { ActivityMessage, Message } from "@ag-ui/core"; +import type { Attachment } from "@copilotkit/react-core/v2"; import { useRenderActivityMessage, useRenderToolCall, } from "@copilotkit/react-core/v2"; -import { IconBox, IconClock } from "@tabler/icons-react"; +import { + IconAlertTriangle, + IconBox, + IconClock, + IconFile, + IconX, +} from "@tabler/icons-react"; import { motion, useReducedMotion } from "motion/react"; -import { memo, useEffect, useLayoutEffect, useMemo, useRef } from "react"; +import { + memo, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from "react"; import { Streamdown } from "streamdown"; import { Bubble, BubbleContent } from "@/components/ui/bubble"; +import { + Dialog, + DialogClose, + DialogContent, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; import { MessageContent, MessageFooter, @@ -23,12 +44,19 @@ import { useMessageScroller, } from "@/components/ui/message-scroller"; import { Skeleton } from "@/components/ui/skeleton"; +import { attachmentUrl } from "@/lib/channels/attachments"; import { readFiring } from "@/lib/channels/routine-firing"; import { markdownComponents } from "@/lib/markdown"; import { EASE_OUT, ENTRANCE_SECONDS } from "@/lib/motion"; import { readToolName } from "@/lib/plugins/tool-name"; import { asText, forDisplay, REFUSAL_MARKER } from "@/lib/plugins/tool-result"; -import { toVisibleChatItems } from "./chat-messages"; +import { cn } from "@/lib/utils"; +import { + attachmentModality, + type SentAttachment, + toVisibleChatItems, + type VisibleChatItem, +} from "./chat-messages"; import type { QueuedMessage } from "./composer"; import { ToolRenderBoundary } from "./tool-boundary"; import { ToolLine } from "./tool-line"; @@ -167,6 +195,83 @@ function Stopped({ reason }: { reason: string }) { ); } +/** + * A file staged on the composer, described the way a sent one is. + * + * So that a parked message draws the SAME tiles the turn will draw once it runs, rather than a + * second rendering of an attachment that has to be kept in step with this one. The composer's + * `onUpload` has already put the row on the server and handed back its url — `canSendDraft` refuses + * to send, and therefore to park, anything still uploading — so there is always something to point + * at by the time one of these reaches here. + * + * Anything that is not a picture becomes a document tile, which is what the sent row does with the + * modalities it has no preview for: a card naming the file is the honest drawing of "this came + * along", and the alternative is a broken thumbnail. + * + * WHICH ONES ARE PICTURES IS ASKED OF THE SERVER'S ANSWER, NOT OF `attachment.type`, and this is + * the one surface in the browser where that answer is actually in hand. `attachment.type` is the + * SDK's `getModalityFromMimeType(file.type)` from before the upload, never revisited when the + * upload replies; `attachment.source.mimeType` is what OUR `onUpload` put there, and that is + * `body.mimeType` — the type the server earned from `sniffMimeType` over the bytes + * (`composer/attachments.ts`). A PNG the browser called `text/plain` has `type: "document"` and + * `source.mimeType: "image/png"`, and this used to draw a grey card over it. + * + * A PARKED MESSAGE HOLDS THE LIVE `Attachment`, WHICH IS WHY THIS CAN BE PUT RIGHT AND THE SENT ROW + * CANNOT. `QueuedMessage.attachments` is `Attachment[]` — the staged object itself, source and all + * — whereas a sent turn has been through `toAttachmentPart` (`channel-chat.tsx`), which rebuilds + * the source as `{ type: "url", value }` and drops the `mimeType` on the floor. So the two tiles + * genuinely can disagree for as long as that line stands: a mislabelled picture draws correctly + * here and reverts to a file card the moment the turn runs. + * + * That flip is a real cost and it is still the right way round. It is the symptom of the missing + * line rather than a reason to keep this tile wrong on purpose, and the alternative — throwing away + * an answer we hold so that both surfaces are wrong together — is the kind of consistency that + * hides the defect instead of paying it down. `attachmentModality` is shared with the sent path + * precisely so that fixing the source there needs no second change here. + */ +function parkedTiles(attachments: readonly Attachment[]): SentAttachment[] { + return attachments.map((attachment) => { + const attachmentId = (attachment.metadata as { attachmentId?: unknown }) + ?.attachmentId; + /* + * Narrowed rather than read straight through, for the reason the sent path narrows the same + * field: `Attachment["source"]` is a union whose `data` member REQUIRES `mimeType` and whose + * `url` member does not, and a `data` source here carries `file.type` — the browser's claim, + * which is the very thing this is refusing to trust. Only a url source has been near the + * server. A staged attachment still uploading is exactly that case (`{ type: "data", value: "", + * mimeType: file.type }`), and while `canSendDraft` refuses to park one, this costs nothing and + * means the guard does not depend on that staying true. + */ + const { source } = attachment; + const mimeType = + source.type === "url" && source.mimeType ? source.mimeType : undefined; + + return { + id: attachment.id, + attachmentId: + typeof attachmentId === "string" ? attachmentId : attachment.id, + url: source.value, + ...(attachment.filename ? { filename: attachment.filename } : {}), + modality: attachmentModality(attachment.type, mimeType), + }; + }); +} + +/** What a parked message is called, for somebody who cannot see it: its words, or else its files. */ +function describeParked( + text: string, + files: readonly SentAttachment[], +): string { + if (text) { + return text; + } + const named = files + .map((file) => file.filename) + .filter((filename) => filename !== undefined); + + return named.length > 0 ? named.join(", ") : "attachment"; +} + /** * Something the person said while the Bot was working, waiting its turn. * @@ -181,21 +286,42 @@ function Stopped({ reason }: { reason: string }) { * hover over the words it is offering to delete. */ function Queued({ + attachments, text, onRemove, }: { + attachments: readonly Attachment[]; text: string; onRemove?: (() => void) | undefined; }) { + /* + * THE FILES COME WITH IT, and until they did they were on NO SURFACE IN THE APP AT ALL. Parking + * consumes the draft, so the composer's strip empties in the same beat this line appears; drawing + * only `text` meant somebody who attached a screenshot mid-turn watched it vanish from the + * composer and never show up anywhere else. Same tiles as a sent turn, in the same order — + * pictures above the words — because this IS their message, just not yet run. + */ + const files = parkedTiles(attachments); + return ( - - - {/* Shown exactly as typed, for the same reason a sent message is. */} - {text} - - + {files.length > 0 ? ( + + ) : null} + {/* + * NO WORDS, NO BUBBLE. A screenshot pasted mid-turn with nothing typed is the ordinary way + * this gets used, and it drew an empty muted bubble above the footer — which reads as a + * message sent by mistake rather than as a file waiting its turn. + */} + {text ? ( + + + {/* Shown exactly as typed, for the same reason a sent message is. */} + {text} + + + ) : null} {/* * `status` rather than `alert`, matching the thinking line: a person who has just chosen @@ -209,8 +335,12 @@ function Queued({ * called "Remove" in a row, and somebody reading by name alone is told what they can * do and nothing about which one it would happen to. The visible word stays short * because the bubble it sits under is the answer for everybody who can see it. + * + * With no sentence to name it by, the FILES are what it deletes — see + * `describeParked`. An attachment-only message named the label after an empty string + * and read as "Remove queued message:", which is the same nothing three times over. */ - aria-label={`Remove queued message: ${text}`} + aria-label={`Remove queued message: ${describeParked(text, files)}`} className="ml-2 underline underline-offset-2 hover:text-foreground focus-visible:outline-none focus-visible:ring-3 focus-visible:ring-ring/50" onClick={onRemove} type="button" @@ -511,6 +641,517 @@ const TranscriptMessage = memo(function TranscriptMessage({ ); }); +/** + * The one shape an attachment url is allowed to have, taken from the helper that builds it rather + * than written out again, so the check cannot drift from the route that serves the file. + */ +const ATTACHMENT_URL_PREFIX = attachmentUrl(""); + +/** + * THE FILES A PERSON ATTACHED TO ONE TURN, drawn as a row where their own message would be. + * + * ALWAYS THE PERSON'S OWN, so it aligns end like their bubble does — `toVisibleChatItems` only ever + * produces this kind from a user turn's content, and there is no assistant equivalent to confuse it + * with. `justify-end` is what actually puts a short row against the right edge; `align="end"` gets + * the row there, not the tiles inside it. + * + * SQUARE, AND ALL THE SAME SIZE. Three photos of different shapes drawn at their own aspect ratios + * make a ragged row whose only signal is which camera took what, and a single tall screenshot drawn + * at its own aspect pushed the rest of the turn off the screen. A grid of equal tiles says "three + * files" at a glance, which is the thing worth saying here; the picture itself is one click away. + */ +type AttachmentRowProps = { + attachments: readonly SentAttachment[]; + delay: number; +}; + +/** + * Whether two renders of this row are the same row, compared BY VALUE because identity says no + * every time. + * + * `toVisibleChatItems` runs on every render of the transcript — deliberately, and the comment on + * that call says why: the agent hands back the same array and mutates it, so a `useMemo` over it + * never invalidates and a reply never appears. The price is that `attachments` is a fresh array of + * fresh objects on every chunk of a streaming answer, and `memo`'s default `Object.is` on two + * different arrays is false however identical they are. So this memo missed EVERY time, and every + * tile in a channel's history — each one carrying an image `Dialog` — re-rendered on every token + * of an answer being typed further down. The memoised message rows above it were paying for this + * one's misses. + * + * Field by field, because the fields are what the tiles draw: a row whose files have the same ids, + * urls, names and kinds in the same order draws exactly the same pixels. + * + * Exported so it can be checked without mounting anything, the same reason `isPersonSentMessage` + * is. + */ +export function sameAttachmentRow( + previous: AttachmentRowProps, + next: AttachmentRowProps, +): boolean { + if (previous.delay !== next.delay) { + return false; + } + if (previous.attachments.length !== next.attachments.length) { + return false; + } + + return previous.attachments.every((attachment, index) => { + const other = next.attachments[index]; + return ( + other !== undefined && + attachment.id === other.id && + attachment.attachmentId === other.attachmentId && + attachment.url === other.url && + attachment.filename === other.filename && + attachment.modality === other.modality + ); + }); +} + +export const TranscriptAttachments = memo(function TranscriptAttachments({ + attachments, + delay, +}: AttachmentRowProps) { + return ( + + + + + + + + ); +}, sameAttachmentRow); + +/** + * The tiles themselves, as one row. + * + * Its own component because the QUEUE draws this row too, faded, for a message parked mid-turn — + * and two copies of a list of tiles is two places for the alignment below to be got right. + * + * `self-end` because `align="end"` does not reach this far on its own: both callers put this + * inside a `flex w-full flex-col` — `Arriving` for a sent turn, `MessageContent` for a parked one — + * so a block child stretches to the transcript's full width and its contents draw hard against the + * LEFT edge, under the person's own right-aligned bubble, reading as though the Bot had sent them. + * `Bubble` escapes this because it carries its own `group-data-[align=end]/message:self-end`. + */ +function AttachmentTiles({ + attachments, + className, +}: { + attachments: readonly SentAttachment[]; + className?: string; +}) { + return ( +
    + {attachments.map((attachment) => ( +
  • + +
    • + ))} +
+ ); +} + +/** + * One tile in that row, and all three shapes are the same size on purpose. + * + * A document used to draw as a `ToolLine`, which was the right call while an attachment was a row + * of its own: that component is one line for one thing a Bot did, and a filename beside a label is + * exactly that shape. It is the wrong thing inside a ROW. `ToolLine` has no width of its own, so + * two documents and a photo came out as a pair of bare sentences stretched across the empty half of + * the line with the picture stranded at the end — the same tiles, laid out as if they were prose. + * + * So a document gets a card the height of a thumbnail instead. What the row is saying is "these + * files came with this message", and it can only say it if every tile in it reads as a file. + * + * A BROKEN-IMAGE GLYPH SAYS NOTHING TO THE READER — the browser's placeholder tells them a box + * failed to load, not that a file is gone. `onError` catches that and swaps it for a sentence in the + * file's own name, in the same destructive vocabulary `Stopped` already uses for "the thing that was + * supposed to be here isn't." It borrows the LOOK and not the urgency: see the missing tile below + * for why the same absence is a note here and an alert there. + */ +/** + * The probes currently outstanding, so that two tiles asking the same question ask it once. + * + * ONE TURN CAN CARRY THE SAME FILE TWICE — `toVisibleChatItems` keys tiles on the PART index + * precisely so that it can — and a message parked mid-turn draws its files a second time beside the + * sent row while the queue holds it. Each of those tiles mounts its own effect, and each one used + * to send its own request for an answer that is the same by construction. + * + * WHAT THAT SHARING IS WORTH, CORRECTED. This comment used to say a duplicate probe was "a + * duplicate megabyte read, not a duplicate status line", because at the time a HEAD reached a + * handler whose single statement selected `attachments.bytes`. It does not any more: the route + * grew a branch that answers a HEAD from `name, mimeType, sizeBytes` and never touches the bytes. + * A duplicate probe is now exactly the duplicate status line this once said it was not. + * + * IT STILL EARNS ITS KEEP, FOR A REASON THAT DOES NOT DEPEND ON THE OLD COST. Nothing else in the + * stack will coalesce these: the route serves `private, no-cache`, so the browser's HTTP cache is + * required to revalidate every probe rather than answer one from the other, and two tiles for one + * file are two components with two effects and no knowledge of each other. Without this map the + * same question goes over the wire once per tile, every time the transcript mounts. It is a dozen + * lines to ask it once, and cheap-per-answer is not the same as free-per-answer. + * + * IN FLIGHT ONLY, AND DELIBERATELY NOT A CACHE OF THE ANSWERS — and THIS is the half the old cost + * model was never holding up. Holding onto "this file is still there" across remounts is exactly + * the lie this hook exists to stop: a file deleted while the reader has the app open would go on + * drawing as an intact card for as long as the tab lived, which is the "worse of the two lies" the + * comment below names. That was the argument then and it is the whole argument now. Holding onto + * "this file is GONE" is sound, deletion being terminal, but it buys nothing worth the branch. + */ +const probesInFlight = new Map>(); + +/** + * Asks the route whether this file is missing, and answers the SAME question only once at a time. + * + * Resolves true for 404 and false for every other answer; a fetch that never arrives rejects, and + * the caller declines to draw a conclusion from it. + */ +function probeDocument(url: string): Promise { + const existing = probesInFlight.get(url); + if (existing) return existing; + + const probe = fetch(url, { method: "HEAD", credentials: "include" }) + .then((response) => response.status === 404) + .finally(() => { + // Cleared however it settled, so the next mount asks again rather than inheriting an answer + // that has had time to stop being true. + probesInFlight.delete(url); + }); + + probesInFlight.set(url, probe); + return probe; +} + +/** + * Whether the row behind a document has been deleted, asked of the server that serves it. + * + * A DOCUMENT HAS NO OTHER WAY TO FIND OUT. The picture beside it learns its file is gone by + * fetching it: the `` requests the url, the route answers 404, and `onError` fires. A document + * tile is a filename and a label — it requests nothing, so no event about the file can ever reach + * it — and `failedToLoad` was the only thing feeding the absent branch. A deleted document + * therefore kept drawing as an intact card naming a file the server was answering 404 for, which + * is the worse of the two lies: a broken picture at least looks broken. + * + * `HEAD` because the question is whether the row exists and the status line is the whole answer, so + * the reader is not made to download a PDF to learn it is still there. AND IT IS NOW CHEAP ON THE + * SERVER TOO, WHICH IT ONCE WAS NOT AND WHICH THIS COMMENT WENT ON ASSERTING AFTER IT STOPPED BEING + * TRUE. The old paragraph was right about the mechanism — Hono does answer a HEAD by dispatching + * the GET handler in full and dropping the body at the last step — and it concluded that the server + * therefore still read the whole file out of Postgres, and that a route which could answer "is it + * there" without the bytes "is not this file's to write". Somebody wrote it. The attachment route + * now branches on the method INSIDE the handler Hono actually dispatches, selects `sizeBytes` + * instead of `bytes`, and sets `Content-Length` by hand; the access join, the statuses and the + * revalidation are the GET's exactly, so a probe still learns nothing a fetch would not tell you. + * + * That is recorded here rather than quietly deleted because this file spent two review rounds being + * read as evidence that the cheap probe did not exist. A comment describing a path as broken, about + * a path that works, costs more than no comment at all. + * + * A REQUEST THAT NEVER ARRIVED IS NOT A DELETED FILE, so a rejected fetch — offline, a dropped + * connection, a proxy in the way — leaves the card alone rather than accusing the server of having + * lost somebody's file. + * + * AND NEITHER IS A REQUEST THE SERVER REFUSED, which is the same rule and was the bug. This asked + * `!response.ok`, which is every status outside 200-299, when exactly one of them means what the + * tile then says. The route collapses "no such row", "channel deleted" and "not a channel of + * yours" into 404 precisely so that probing ids learns nothing — that is the one status that means + * "there is no file here for you". Everything else is a fact about the request: + * + * - 401 is a session that expired while the channel sat open, and it turned EVERY document tile + * in the transcript into a red card asserting, in each file's own name, that somebody's files + * had been deleted. Nothing had been; they need to sign in again. + * - 500 is a bad moment on the server, and it stuck: the deps below are stable, so nothing asks + * again and the accusation stands until the component remounts. + * - 304 is the strongest proof of PRESENCE this route can give — the row was found AND the + * membership join passed — and `Response.ok` is false for it. + * + * That is the same lie this hook exists to stop, pointing the other way, and it is the louder one: + * drawn in the destructive vocabulary, naming the file. So absence is claimed on 404 alone, which + * also puts a document back in step with the picture beside it — an `` handed a 401 shows a + * broken image, not a sentence swearing the file was deleted. + */ +function useDocumentIsGone(url: string, ask: boolean): boolean { + const [gone, setGone] = useState(false); + + useEffect(() => { + if (!ask) { + return; + } + // A tile reused at the same position for a different file starts the question over rather than + // inheriting the previous file's answer. + setGone(false); + + let cancelled = false; + void probeDocument(url) + .then((missing) => { + if (!cancelled && missing) { + setGone(true); + } + }) + .catch(() => { + // Deliberately nothing: see above. Not knowing is not the same as knowing it is gone. + }); + + return () => { + cancelled = true; + }; + }, [ask, url]); + + return gone; +} + +function SentAttachmentTile({ attachment }: { attachment: SentAttachment }) { + const [failedToLoad, setFailedToLoad] = useState(false); + const { filename, modality, url } = attachment; + /* + * An off-site url is treated as a missing file, whatever kind of file it says it is. + * + * `toVisibleChatItems` passes on whatever a `url` source carried, and nothing between here and + * the wire narrows it. A sent message only ever carries the relative form — `shared/attachments.ts` + * says the url "is never fetched by a provider", and `copilot.ts` swaps the source for the bytes + * as the run is built — so an absolute url arriving here did not come from this app's composer, + * and putting it in `src` would have the reader's browser fetch a third party the instant the + * transcript drew, announcing to whoever owns it that this person opened this channel. + */ + const servable = url.startsWith(ATTACHMENT_URL_PREFIX); + /* + * ONLY OURS IS EVER ASKED ABOUT, and that is the same rule as the line above rather than a second + * one: a probe is a request like any other, so asking a third party whether a file is still there + * announces the reader exactly as fetching it would. An off-site url is already unavailable + * without anybody being asked. + */ + const gone = useDocumentIsGone(url, servable && modality === "document"); + const unavailable = failedToLoad || gone || !servable; + + /* + * ABSENCE IS DECIDED BEFORE MODALITY IS, and that ordering is the whole point of this block + * sitting above the document card rather than below it. A document whose url is not ours to + * serve, or whose row `useDocumentIsGone` found deleted, drew as an intact card naming a file + * that is not there. A missing picture at least looked missing; a missing document looked + * present. Whether the file can be shown at all is the first question either kind asks. + */ + if (unavailable) { + return ( + /* + * `note`, NOT `alert`, AND THE DIFFERENCE IS WHO IS INTERRUPTED. An alert is an assertive + * live region: it cuts across whatever a screen reader is saying the moment it appears. That + * is right for `Stopped`, which reports something that just happened in answer to what + * somebody did, and wrong for every one of these — a transcript with three deleted files + * fired three interruptions on mount, before the reader had heard a word of the conversation, + * to report absences that predate their opening the channel. + * + * `note` is not a live region at all, so nothing is announced over anything; it still marks + * the tile as a thing to stop on, and the sentence inside it — unchanged, in the file's own + * name — is what says the file is gone when the reader reaches it. + */ +
+ +

{unavailableSentence(filename)}

+
+ ); + } + + if (modality === "document") { + return ( +
+ +
+ {/* `title` because the tile is fixed-width and a long name is cut, not wrapped. */} +

+ {filename ?? "Untitled file"} +

+

Attachment

+
+
+ ); + } + + return ( + + {filename setFailedToLoad(true)} + src={url} + /> + + ); +} + +/** + * What an absent file is called, in one place, because two surfaces now say it. + * + * The tile said it and the lightbox behind the tile said nothing at all. Sharing the wording rather + * than writing it twice is what keeps them from drifting into two different accounts of the same + * absence — the tile naming the file and the dialog saying something vaguer, or worse, later. + * + * A file with no name still gets a sentence rather than a blank: the reader clicked on something, + * and "this attachment" is the honest way to refer to a thing whose name we never had. + */ +function unavailableSentence(filename?: string): string { + return filename + ? `${filename} is unavailable.` + : "This attachment is unavailable."; +} + +/** + * The full-size picture inside the lightbox, and what it draws when the file will not load. + * + * IT NEEDED A FAILURE STATE AND HAD NONE. The tile in front of it has had one from the start — an + * `onError` swapping the broken-image box for a sentence, with a comment above it about why the + * browser's own placeholder "says nothing to the reader" — and this ``, the one drawn at full + * size against a dark backdrop with the reader's whole attention on it, had no `onError` at all. A + * file deleted between the tile painting and the reader clicking it opened a dialog containing + * exactly the placeholder the tile path exists to avoid. + * + * THE TILE CANNOT COVER THIS ONE. Its own `` has already loaded by the time there is anything + * to click, and a loaded image does not fire `error` again because the file behind it went away; the + * request that finds out is this one. Nor can the probe beside it: `useDocumentIsGone` deliberately + * never asks about a picture, because a picture's own load is supposed to be the answer — and this + * is the load it meant. + * + * IT STAYS OPEN AND SAYS SO, rather than closing itself. The reader opened this deliberately, and a + * dialog that vanishes on its own leaves them looking at the transcript with no idea what happened + * and their focus wherever the close put it. The sentence in place answers the question they + * actually asked. Escape and the close button still work, and the state resets when the popup + * unmounts, so reopening genuinely tries again rather than remembering a failure. + * + * ITS OWN COMPONENT, AND EXPORTED, SO THE FAILURE CAN BE TESTED AT ALL: Base UI portals this popup + * and under happy-dom the portal never mounts — checked, and recorded in the test named "a thumbnail + * is a crop" — so there is no way to reach this `` through the trigger. Same reason + * `sameAttachmentRow` is exported: the behaviour is worth pinning and the thing it lives inside + * cannot be mounted here. + */ +export function LightboxPicture({ + filename, + url, +}: { + filename?: string; + url: string; +}) { + const [failedToLoad, setFailedToLoad] = useState(false); + + if (failedToLoad) { + return ( + /* + * `note` rather than `alert`, for the reason the tile's own missing card gives at length: an + * assertive live region cuts across whatever a screen reader is saying, and this is an answer + * to something the reader just did rather than an emergency. Light-on-dark because it is + * drawn against the lightbox's own backdrop, where `text-destructive` is unreadable. + */ +
+ +

{unavailableSentence(filename)}

+
+ ); + } + + return ( + {filename setFailedToLoad(true)} + src={url} + /> + ); +} + +/** + * The square, opened. + * + * The tile is a crop — that is the price of a tidy row — so there has to be a way to see the whole + * picture, and it used to be `target="_blank"`. A new tab is a worse answer than it looks: it drops + * the reader out of the conversation they were reading, the browser shows the raw file against its + * own chrome with no way back but the back button, and on a phone it is a context switch away from + * the channel entirely. A dialog closes on Escape and puts them back exactly where they were. + * + * Built on the app's `Dialog` so focus trapping, scroll locking and Escape behave the way they do + * everywhere else, but with its card stripped off: `max-w-none border-0 bg-transparent p-0 + * shadow-none` leaves the picture as the only lit thing against a dark backdrop. + * + * THE CLOSE BUTTON IS FIXED TO THE VIEWPORT, not to the popup, which is why `showCloseButton` is + * off and this draws its own. Pinned to the popup it would sit on the picture — invisible over a + * pale one, and moving with every image's shape. + */ +function AttachmentLightbox({ + children, + filename, + url, +}: { + children: React.ReactNode; + filename?: string; + url: string; +}) { + const label = filename ?? "Attachment"; + /* + * Controlled only so that clicking the dark space around the picture closes it, the way every + * lightbox a reader has used does. The popup covers the viewport (see below), so it — not the + * backdrop underneath — is what receives that click, and Base UI's own dismiss never fires. + */ + const [open, setOpen] = useState(false); + + return ( + + } + > + {children} + + { + if (event.target === event.currentTarget) setOpen(false); + }} + overlayClassName="bg-black/80 supports-backdrop-filter:backdrop-blur-sm" + showCloseButton={false} + > + {/* Named for a screen reader; the picture carries the same name in its alt text. */} + {label} + + + } + > + + + + + ); +} + /** * What a failed activity is called on screen. * @@ -680,6 +1321,70 @@ export function isPersonSentMessage( return role === "user" && readFiring(text) === null; } +/** + * The turn a row belongs to. + * + * The attachments row is identified as `${messageId}:attachments` by `toVisibleChatItems`, so that + * it and the caption beneath it can be told apart while still belonging to one turn. The id of the + * message they were sent in is the part before the last colon. + */ +function turnOf(item: VisibleChatItem): string { + if (item.kind !== "attachments") { + return item.id; + } + const separator = item.id.lastIndexOf(":"); + return separator === -1 ? item.id : item.id.slice(0, separator); +} + +/** + * Which rows the scroller may lift to the top of the viewport: THE FIRST ROW OF EACH TURN THE + * PERSON SENT, and only that one. + * + * The scroller reads `data-scroll-anchor` off the rows appended in one go, takes the first it finds + * and scrolls it to the top with a peek of the answer above it — but finding MORE THAN ONE among + * that same batch it gives up on the ambiguity and jumps to the end instead. One turn is very often + * several rows: a caption and its screenshot, or three files pasted together, all arrive at once. + * So "every row a person sent" is not the rule; "the first row of every turn a person sent" is, and + * the difference between them is the whole anchoring behaviour of an ordinary captioned message. + * + * A Bot's prose is never an anchor — the anchor exists to hold the QUESTION at the top while the + * answer streams in underneath it — and neither is a routine firing, which arrived wearing + * `role: "user"` without anybody having typed anything. + * + * Order is `toVisibleChatItems`' order, which puts a turn's attachments before its caption, so a + * turn carrying a file is anchored on the first file and a plain one on its text. That is the right + * end to hold: the picture is the top of what the person sent, and anchoring on the caption + * underneath it would scroll the picture off the top of the pane. + */ +function anchorRowIds(items: readonly VisibleChatItem[]): Set { + const anchors = new Set(); + const claimed = new Set(); + + for (const item of items) { + const sent = + item.kind === "attachments" || + (item.kind === "text" && isPersonSentMessage(item.role, item.text)); + if (!sent) continue; + + const turn = turnOf(item); + if (claimed.has(turn)) continue; + claimed.add(turn); + anchors.add(item.id); + } + + return anchors; +} + +/** + * The only way to actually enforce exhaustiveness over `VisibleChatItem`: the + * `item` parameter is typed `never`, so a branch that reaches here with a + * member of the union still unhandled fails to compile, rather than a + * trailing `: null` that accepts anything and renders nothing for it. + */ +function assertNever(_item: never): null { + return null; +} + const SEND_SCROLL_MS = 700; function useSmoothSendScroll( @@ -747,16 +1452,35 @@ export function ChatTranscript({ */ const lastItem = items.at(-1); const waitingOnFirstToken = - busy && lastItem?.kind === "text" && lastItem.role === "user"; + busy && + ((lastItem?.kind === "text" && lastItem.role === "user") || + /* + * A screenshot pasted with no caption is still a person sending something, and they are + * watching the same spot under it for the Bot's answer as they would under a typed question. + * `toVisibleChatItems` only ever produces an `attachment` item from a user turn, so seeing one + * last means the person went last — without this an attachment-only turn silently swallowed + * the Thinking indicator. + */ + lastItem?.kind === "attachments"); const viewportRef = useRef(null); const newestUserMessageId = items.findLast( (item) => - item.kind === "text" && isPersonSentMessage(item.role, item.text), + (item.kind === "text" && isPersonSentMessage(item.role, item.text)) || + /* + * Same reasoning as `waitingOnFirstToken` above: an attachment is always the person's own, + * so a turn that is only a file still counts as them sending something. This id decides + * nothing but whether the viewport scrolls smoothly for a beat; where it lands is + * `anchoredTurns` below. + */ + item.kind === "attachments", )?.id ?? null; useSmoothSendScroll(viewportRef, newestUserMessageId); + /* One rule for both kinds of row — see `anchorRowIds`, which is where it is written down. */ + const anchorRows = anchorRowIds(items); + /* * One decider per mounted transcript, so opening a different channel starts the cascade over and * a message never inherits a delay from a conversation it was not in. @@ -790,6 +1514,54 @@ export function ChatTranscript({ className="mx-auto w-full max-w-2xl px-4 py-6" spacerClassName="order-2" > + {/* + * DRAWN LAST, WRITTEN FIRST, AND THE SCROLLER IS WHY. Three reviewers have now arrived + * at this block, worked the mechanism out from the library's bundle, and reached the + * same place; it had no comment for any of them to read. This is that comment. + * + * `MessageScrollerContent` is `flex flex-col`, so `order` is live: the spacer takes + * `order-2`, these children take `order-1`, and the transcript rows below keep the + * default `0`. Visual order is therefore items, then anything parked or in flight, then + * the spacer — while DOM order puts this block first. + * + * IT CANNOT SIMPLY BE MOVED DOWN. The scroller finds a newly appended row POSITIONALLY. + * On every content change it takes `Array.from(content.children)` minus the spacer, + * compares the length against the previous length, and when it grew scans FROM THE OLD + * LENGTH FORWARD for the next `data-scroll-anchor="true"` (`je(a, T)` in + * `@shadcn/react/dist/message-scroller`). A new row is only found when it lands at the + * very end of that list. Put this block after `items.map` and every appended row lands + * one slot short of the end, the scan finds this div instead, returns null, and the + * caller falls through to its follow-the-bottom branch — so a new turn stops aligning + * to the top of the viewport with a peek of the previous one, silently, with no test + * failing. + * + * `display: contents` IS LOAD-BEARING FOR THE SAME REASON, and not a layout trick. It + * promotes these children to flex items of the column so they can carry `order`, while + * the div itself stays a single, always-present entry in `content.children` — one + * stable slot the row count can be offset by. Wrapping `items.map` the same way would + * be the natural symmetry and is fatal: the rows would leave `content.children` + * entirely and the scroller would see a transcript of two elements, neither carrying a + * `data-message-id`, so nothing would register, be tracked as visible, or anchor. + * + * WHAT THIS COSTS, STATED RATHER THAN LEFT TO BE REDISCOVERED. `order` moves paint and + * not the DOM, so it moves neither focus order nor the reading order of the enclosing + * `role="log"`. A keyboard user tabbing in reaches the "Remove queued message: …" + * button of every parked message before any control in the conversation, though those + * lines are drawn at the very bottom (WCAG 2.4.3); a screen reader reading the log + * linearly hears the parked messages, and `Stopped`, ahead of the conversation they + * follow on screen (WCAG 1.3.2). + * + * THAT IS A KNOWN, UNPAID DEBT AND NOT AN OVERSIGHT. The fixes available from inside + * this file were each tried on paper and each breaks something worse: `aria-owns` needs + * a generated id per row and re-sequences only the accessibility tree, leaving tab + * order inverted; positive `tabIndex` hijacks the tab sequence of the whole page; + * hoisting the queue out of `MessageScrollerContent` into a sibling region — the + * cleanest END STATE, since a parked message is genuinely not a log entry — puts it + * outside a column that is `min-h-full`, so it lands below the fold on a short + * transcript, outside the `gap-6` rhythm, and outside the spacer's height arithmetic. + * Paying it properly means the scroller identifying new rows by identity rather than by + * position, which is the library's to change and is worth asking for. + */}
{stopped ? ( @@ -798,6 +1570,7 @@ export function ChatTranscript({ ) : null} {queued.map((message) => ( - ) : ( + ) : item.kind === "text" ? ( + ) : item.kind === "attachments" ? ( + + + + ) : ( + // Every member of the union is handled above. `assertNever` types `item` as + // `never` here, so a future addition to `VisibleChatItem` fails to typecheck at + // this call instead of silently falling into this branch and rendering nothing. + assertNever(item) ), )} diff --git a/app/src/components/channels/composer/attachment-strip.tsx b/app/src/components/channels/composer/attachment-strip.tsx new file mode 100644 index 000000000..eadbe3efb --- /dev/null +++ b/app/src/components/channels/composer/attachment-strip.tsx @@ -0,0 +1,214 @@ +import { IconFile, IconX } from "@tabler/icons-react"; +import { motion, useReducedMotion } from "motion/react"; + +import { Skeleton } from "@/components/ui/skeleton"; +import { EASE_OUT, ENTRANCE_SECONDS } from "@/lib/motion"; +import { Collapse } from "./collapse"; + +/** + * WHAT IS ON THE COMPOSER, DRAWN AS ITS OWN ROW ACROSS THE TOP. + * + * `PromptArea` will draw this strip itself, given `images` and `files`, and it did until this + * existed. Three things were wrong with that, and all three come from the same fact: the strip is + * inside the editor's column, and on the compact composer that column is the MIDDLE of a row whose + * first child is the attach button. + * + * So the strip started 42px in from the composer's left edge, hanging off nothing, instead of + * lining up with the frame the way every reference composer draws it. Its thumbnails are a + * hardcoded `h-16 w-16` with no prop to say otherwise. And its padding is a hardcoded `pb-2`, + * which this file previously reached in and overrode by class name. + * + * Each of those could be forced from outside with a selector into somebody else's DOM, and for a + * while one of them was. Three of them stacked is a component we have forked in CSS without saying + * so, and it breaks silently on the next `prompt-area` release. Fifty lines of our own markup is + * the cheaper of the two. + * + * Both composer branches use this, so there is one strip in the app rather than a compact one and + * a full-size one that drift. + */ + +/** + * `url` is empty while the upload is in flight — the SDK has no bytes to point at yet — which is + * why `loading` is not merely cosmetic here: an `` re-requests the whole page in some + * browsers, and React says so in the console. + */ +export type StagedImage = { + id: string; + url: string; + alt?: string; + loading: boolean; +}; + +export type StagedFile = { + id: string; + name: string; + size?: number; + loading: boolean; +}; + +export function AttachmentStrip({ + files, + images, + onRemove, +}: { + files: readonly StagedFile[]; + images: readonly StagedImage[]; + onRemove: (id: string) => void; +}) { + const occupied = images.length > 0 || files.length > 0; + + return ( + /* + * `items-start` rather than a stretch, so a wrapped second line of thumbnails sits under the + * first rather than growing to match the tallest thing on its own line. + * + * The list stays mounted while empty, which is what gives `Collapse` something to measure: a + * box with nothing in it cannot report the height it is meant to animate to. + * + * `aria-hidden` is the price of that, and it is not optional. Height 0 under `overflow-hidden` + * is a VISUAL state and nothing more — it does not take an element out of the accessibility + * tree the way `display: none` does. Left as it was, a composer carrying nothing still offered + * a screen reader a labelled list to walk into and announce as "Attachments, list, 0 items". + * Empty and unheard is the state we want; empty is only half of it. + * + * Hiding a subtree that contains something focusable is its own defect, and this cannot commit + * it: `occupied` is false exactly when both lists are empty, so there is nothing inside to + * reach when the attribute is on. + */ + +
    + {images.map((image) => ( + + {image.loading || image.url === "" ? ( + /* + * The placeholder stands in for the picture, so it carries the picture's role and + * the picture's name — otherwise the tile is a shape and nothing else, and the only + * thing a screen reader finds in it is a button offering to remove something + * unnamed. The file tile has said "Uploading…" in plain text since it was written. + * + * A name, not an announcement. `role="status"` would make each of these a live + * region, and a paste of eight images would interrupt eight times to say so. + */ + + ) : ( + {image.alt + )} + onRemove(image.id)} + /> + + ))} + {files.map((file) => ( + /* + * The same height as a thumbnail, so a message carrying one of each reads as one row of + * attachments rather than as two things that happened to land next to each other. + */ + + +
    +

    + {file.name} +

    + {file.loading ? ( +

    Uploading…

    + ) : file.size === undefined ? null : ( +

    + {formatBytes(file.size)} +

    + )} +
    + onRemove(file.id)} /> +     + ))} +
+ + ); +} + +/** + * One tile, arriving. + * + * Entrance only. It leaves the instant it is removed, because that is what the person asked for and + * because an element held on screen by an animation is an element a test cannot prove is gone — see + * the note on the collapse above. + * + * `layout` is what keeps the removal from reading as a glitch anyway: the tiles to the right of the + * one that went slide into its place over the same 200ms rather than jumping on a single frame. It + * is also why `scale` is motion's own prop here, where the rest of this app writes full transform + * strings for hardware acceleration — a layout animation composes its own transform, and a literal + * `transform` in `animate` would be overwritten by it mid-flight. + */ +function Staged({ + children, + className, +}: { + children: React.ReactNode; + className?: string; +}) { + const shouldReduceMotion = useReducedMotion(); + + return ( + + {children} + + ); +} + +/** + * The label names the file, because a composer carrying four attachments otherwise offers a screen + * reader four buttons all called "Remove". + * + * Inside the tile rather than hanging off its corner: the strip animates its own height inside an + * `overflow-hidden` box, and anything outside the tile is clipped while that runs. + */ +function RemoveButton({ + name, + onRemove, +}: { + name: string; + onRemove: () => void; +}) { + return ( + + ); +} + +/** + * Whole kilobytes below a megabyte and one decimal above it. A staged attachment is capped at 8MB + * (`shared/attachments.ts`), so this never has to reach gigabytes, and "0.1MB" for a 100KB text + * file tells the reader less than "98KB" does. + */ +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes}B`; + if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)}KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)}MB`; +} diff --git a/app/src/components/channels/composer/attachments.ts b/app/src/components/channels/composer/attachments.ts new file mode 100644 index 000000000..81b933acb --- /dev/null +++ b/app/src/components/channels/composer/attachments.ts @@ -0,0 +1,278 @@ +import type { AttachmentsConfig } from "@copilotkit/react-core/v2"; +import { + ACCEPTED_IMAGE_MIME, + ACCEPTED_TEXT_MIME, + attachmentUrl, + MAX_IMAGE_BYTES, +} from "@/lib/channels/attachments"; + +/** + * Uploads a picked file to this deployment and hands the composer a link to it, never the bytes. + * + * Neither `AttachmentUploadResult` nor `AttachmentUploadError` is exported by anything installed + * here: `@copilotkit/shared` declares both, but it is not a dependency of `app/` — the only + * installed CopilotKit package in this workspace is `react-core`, and its `v2` entry point does + * not re-export these two. They are derived instead from `AttachmentsConfig["onUpload"]` / + * `["onUploadFailed"]`, the one signature that IS public, so they can never drift from what the + * SDK actually calls. + */ +type AttachmentUploadResult = Awaited< + ReturnType> +>; +type AttachmentUploadError = Parameters< + NonNullable +>[0]; + +/** The shape `POST /api/channels/:channelId/attachments` sends back on success. */ +type UploadedAttachment = { + id: string; + name: string; + mimeType: string; +}; + +/** + * The SDK's `onUpload`, scoped to one channel. + * + * `fetch` directly rather than `client()` from `@/lib/client`: `client()` JSON-stringifies its + * body and sets a JSON content type, neither of which can carry a multipart upload. + * + * Supplying this is what keeps the bytes out of browser state: with `onUpload` set, the SDK never + * calls its own `readFileAsBase64` and never retains the `File` it was handed, so no copy of the + * bytes lives on past this call. Returning a `url` source rather than a `data` one is the other + * half of that: it is what puts a small reference in the sent message instead of megabytes of + * base64 riding along in it. + */ +export function uploadToChannel( + channelId: string, + uploadGroup: string, +): (file: File) => Promise { + return async (file: File): Promise => { + const formData = new FormData(); + formData.append("file", file); + /* + * WHICH COMPOSER STAGED IT, so the server's cap counts the set this composer can see. + * + * The cap is per message. The composer counts what is on its own screen; without this field the + * server counted every unsent row this person had in this channel, which is not the same set + * the moment anything leaves a row behind — a closed tab, a stopped run, a removed queued + * message. The client would then accept a pick the server refused with a 409 naming files that + * were on nobody's screen, and eight such orphans locked uploads in that channel until the + * sweeper's window expired. Sending the group is what makes the two sides count the same rows. + */ + formData.append("uploadGroup", uploadGroup); + + const response = await fetch(`/api/channels/${channelId}/attachments`, { + method: "POST", + credentials: "include", + body: formData, + }); + + /* + * READ ONCE, FOR EITHER OUTCOME, AND NEVER TRUSTED. + * + * The refusal path has been careful since a body that was not JSON put "Failed to parse JSON" + * on screen as the reason a file was rejected. The success path was not: it went straight to + * `as UploadedAttachment`, which is a cast and not a check, so the same malformed body threw a + * raw `SyntaxError` that the SDK reported as the refusal — and a body that parsed but carried + * no `id` was worse, because it succeeded: `attachmentUrl(undefined)` became a chip pointing at + * `/api/attachments/undefined`, Send unlocked, and the message went out carrying a link to no + * attachment. Both halves of one response are held to the same standard here. + */ + const body: unknown = await response.json().catch(() => undefined); + // Named after the file rather than left empty, for every way this can go wrong. It is the only + // sentence anybody gets: the composer keeps the string verbatim and shows it as the reason. + const couldNotUpload = new Error(`Could not upload "${file.name}".`); + + if (!response.ok) { + // The server phrases refusals in the product's voice — naming the mime type or the limit + // that was actually hit — so its sentence is preferred whenever it has really sent one. + throw refusalIn(body) ?? couldNotUpload; + } + + if (!isUploadedAttachment(body)) { + throw couldNotUpload; + } + + return { + type: "url", + value: attachmentUrl(body.id), + mimeType: body.mimeType, + metadata: { attachmentId: body.id, filename: body.name }, + }; + }; +} + +/** + * The server's own sentence, when it has actually sent one. + * + * `??` was doing this job and could not: it steps in only for `null` and `undefined`, so + * `{"error":""}` — or a whitespace-only one, or a number — was preferred over the fallback and + * reached the strip as a file refused with NO REASON BESIDE IT. A refusal with nothing to read is + * indistinguishable from a bug, which is the one thing this whole path exists to avoid. + */ +function refusalIn(body: unknown): Error | undefined { + if (typeof body !== "object" || body === null) { + return undefined; + } + const { error } = body as { error?: unknown }; + if (typeof error !== "string" || error.trim().length === 0) { + return undefined; + } + return new Error(error); +} + +/** + * Every field this function is about to hand the SDK, present and a string. + * + * `id` is checked for content and not merely for type, because it is the one that becomes a URL: + * an empty id makes `/api/attachments/`, which is a different endpoint entirely rather than a + * broken one. + */ +function isUploadedAttachment(body: unknown): body is UploadedAttachment { + if (typeof body !== "object" || body === null) { + return false; + } + const { id, name, mimeType } = body as Record; + return ( + typeof id === "string" && + id.length > 0 && + typeof name === "string" && + typeof mimeType === "string" + ); +} + +/** + * WHAT THE `+` BUTTON'S FILE DIALOG OFFERS — A HINT, AND THE ONLY PLACE `accept` IS STILL SPENT. + * + * This is NOT a gate. A file dialog's `accept` greys files out; it refuses nothing, every desktop + * browser offers a way past it, and drag and paste never see it at all. So it may be as narrow as + * is useful, where the SDK's `accept` (see `attachmentsConfigFor`) may not be narrow at all. + * + * THE EXTENSIONS ARE HERE FOR THE SAME REASON THE `unnamed` BRANCH EXISTS IN `picked-files.ts`. A + * MIME-only list is matched against what the browser CLAIMS a file is, and the whole point of that + * branch is that for a `.txt` dragged out of an editor — or anything with an extension the + * platform has no mapping for — the browser claims `application/octet-stream` or nothing. Listing + * the media types alone therefore greys out, in the dialog, exactly the files this composer went + * to some trouble to accept everywhere else. `matchesAcceptFilter` and every browser's dialog both + * read a leading-dot entry as a filename suffix, so the two doors now agree. + */ +export const FILE_PICKER_ACCEPT = [ + ...ACCEPTED_IMAGE_MIME, + ...ACCEPTED_TEXT_MIME, + ".png", + ".jpg", + ".jpeg", + ".gif", + ".webp", + ".txt", + ".md", + ".csv", + ".json", +].join(","); + +/** + * `AttachmentsConfig` for one channel's composer. + * + * `accept` IS THE WILDCARD, AND THAT IS THE WHOLE OF THE TYPE POLICY MOVING TO ONE PLACE RATHER + * THAN BEING ABANDONED. + * + * It used to be `[...ACCEPTED_IMAGE_MIME, ...ACCEPTED_TEXT_MIME].join(",")`, and that was a second + * gate standing behind `screenPickedFiles` — a stricter one, applying a different rule, phrased for + * a machine. `useAttachments.processFiles` runs it with `matchesAcceptFilter`, whose non-wildcard, + * non-extension branch is a bare `file.type === filter`. `screenPickedFiles` deliberately passes a + * claim that names no format (`application/octet-stream`, `""` — see the long note at + * `picked-files.ts`) so the server can sniff the bytes; none of those strings equals any of the + * eight, so the SDK refused every one of them before a request was ever made, with + * `File "notes.txt" is not accepted. Supported types: image/png,…`. That branch was dead in the + * running app, the drift it exists to close was still open, and the sentence somebody read was the + * one `stageFiles` promises a screened file can never produce. + * + * Widening the list instead was the other option and is not enough: `""` cannot be written as an + * accept entry at all, and `namesNoFormat` also passes anything not shaped like a MIME type, which + * is an open set. There is no string that means what the screen means. + * + * WHAT THIS COSTS, STATED PLAINLY: nothing, because the SDK's filter was never reachable except + * through us. `stageFiles` is the only caller of `processFiles` — the composer supplies its own + * drop handler and its own `onChange`, so the hook's `handleDrop` and `handleFileUpload` are never + * used — and the hook's own `document` paste listener is inert here, because it is scoped by a + * `containerRef` this composer deliberately never hands it (see `containerRef` in `composer.tsx`). + * So every file that reaches `processFiles` has already been through `screenPickedFiles`, which + * enforces kind, BOTH size ceilings and the per-message cap — all of which the SDK's filter does + * not — and phrases its refusals for a person. The narrow list is kept where it is honest about + * being a hint: `FILE_PICKER_ACCEPT`, above, on the file dialog. + * + * `maxSize` is `MAX_IMAGE_BYTES`, the larger of the two ceilings: the SDK config has only one + * number for every kind of file. The tighter `MAX_FILE_BYTES` limit for text attachments is + * enforced by our own pre-check ahead of upload, and again by the server on arrival — which is + * the authority either way, so letting a text file past this one number costs nothing. It is left + * in place, unlike `accept`, because it can only ever agree with the screen: the loosest thing + * `screenPickedFiles` passes is a file of exactly `MAX_IMAGE_BYTES`, so this backstop cannot fire + * on a file we accepted. + */ +export function attachmentsConfigFor( + channelId: string, + uploadGroup: string, + onUploadFailed: (error: AttachmentUploadError) => void, + /** + * EVERY SERVER ROW THIS COMPOSER CREATES, REPORTED THE MOMENT IT EXISTS. + * + * `onUpload`'s answer normally reaches the composer the long way round, by the SDK writing it + * onto the placeholder it minted — and that write is a no-op if the placeholder is gone, which is + * what happens when somebody removes a chip whose upload is still in flight. The row is on the + * server by then and the only handle to it was in that discarded answer, so the count the server + * enforces and the count the screen shows drift apart by one, and the next pick is refused with a + * 409 naming a file nobody can see. That is verbatim the failure `uploadGroup` was added to end. + * + * So the id is reported here as well, where it cannot be dropped, and `composer.tsx` reconciles + * what it was told against what is actually on the strip. It is a separate channel rather than a + * `wasCancelled(placeholderId)` predicate on the way in — the shape that suggests itself — for a + * blunt reason: `onUpload` is handed a `File` and nothing else. The SDK mints the placeholder id + * itself and never tells us which one this call belongs to, so there is no id here to be asked + * about. Reporting the row and reconciling afterwards needs no such correlation. + */ + onUploaded: (attachmentId: string) => void, +): AttachmentsConfig { + const upload = uploadToChannel(channelId, uploadGroup); + return { + enabled: true, + accept: "*/*", + maxSize: MAX_IMAGE_BYTES, + onUpload: async (file) => { + const uploaded = await upload(file); + const rowId = stagedRowId(uploaded); + /* + * `uploadToChannel` has already refused any answer without a non-empty `id` + * (`isUploadedAttachment`), so this is narrowing rather than a real branch — `metadata` is + * declared `unknown` by the SDK and has to be re-read as something. It is not silently + * skipped if it ever does come back empty: the row would be one this composer never learned + * about, which is the sweeper's to collect, and pretending otherwise by passing `undefined` + * down would put `DELETE /api/attachments/undefined` on the wire. + */ + if (rowId !== undefined) { + onUploaded(rowId); + } + return uploaded; + }, + onUploadFailed, + }; +} + +/** + * The server row an SDK attachment stands for, if it stands for one. + * + * `metadata` is `unknown` on the SDK's `Attachment` — it is whatever `onUpload` chose to return — + * so every reader has to narrow it, and this is the one place that knows what we put there. Both + * the composer's reconciler and this file's `onUpload` wrapper read it through here rather than + * each spelling out the same cast, because the two must never disagree about what counts as a row. + * + * `undefined` for an attachment still uploading, which has no row yet, and for one the SDK built + * itself without our `onUpload` — neither has anything on the server to give back. + */ +export function stagedRowId( + carrier: { metadata?: unknown } | undefined, +): string | undefined { + const metadata = carrier?.metadata as { attachmentId?: unknown } | undefined; + return typeof metadata?.attachmentId === "string" + ? metadata.attachmentId + : undefined; +} diff --git a/app/src/components/channels/composer/collapse.tsx b/app/src/components/channels/composer/collapse.tsx new file mode 100644 index 000000000..ebddc86fa --- /dev/null +++ b/app/src/components/channels/composer/collapse.tsx @@ -0,0 +1,142 @@ +import { motion, useReducedMotion } from "motion/react"; +import { useEffect, useLayoutEffect, useRef, useState } from "react"; + +import { EASE_OUT, ENTRANCE_SECONDS } from "@/lib/motion"; + +/** + * A box that opens and closes by growing and shrinking, used for the two things above the compact + * composer's text: the attachment strip and the list of refusals. + * + * WHY THIS EXISTS AT ALL. Both of those change the composer's height by far more than a line, and + * both did it on a single frame — paste a screenshot and the box jumped 86px, taking whatever + * somebody was mid-sentence in with it. Growing into the space instead is what makes a picture read + * as having been added to the message rather than as the message having been replaced. + * + * WHY IT IS NOT `AnimatePresence`, WHICH IS THE OBVIOUS SPELLING. It was written that way first and + * cost eight tests. An exiting element stays MOUNTED until its animation reports finished, and + * under happy-dom that report never comes: every assertion that a removed attachment is gone hung + * until it timed out. `MotionGlobalConfig.skipAnimations` is motion's own documented hook for + * exactly this and did not fix it either — it moved the wait from never to about four seconds, + * against a suite that runs in three. A flourish that makes the behaviour underneath it + * unverifiable is not worth having. + * + * WHY IT MEASURES INSTEAD OF ANIMATING TO `auto`. That was the second attempt, and it looked right + * in one direction only. Motion resolves `auto` by measuring at the moment the animation starts, so + * closing — where the content has already left the DOM — measured the EMPTY box and animated 0 to + * 0. The height snapped 141px to 66px in four milliseconds and then eased the last eleven, which is + * a jump wearing an animation's clothes. Measuring the content ourselves gives the close a real + * number to leave from. + * + * WHAT A CALLER OWES THIS BOX. Closed here means height 0 under `overflow-hidden`, and that is a + * VISUAL state only — it is not `display: none`, so everything inside a closed box is still in the + * accessibility tree, still labelled, still reachable by a screen reader's own navigation. So + * whatever goes in here has to deal with its own absence: `RejectedFiles` unmounts its alert on + * the way closed, `AttachmentStrip` keeps its list mounted to be measured and `aria-hidden`s it + * while it is empty. Both are recorded where they are done. + * + * That is deliberately not enforced from in here. This box cannot know whether a caller has left + * something focusable inside it, and `aria-hidden` over a focusable element is its own defect — + * worse than the one it would be papering over. The caller knows; this does not. + */ +export function Collapse({ + children, + className, + open, +}: { + children: React.ReactNode; + /** Classes for the content, not the animating box — that one owns its own overflow. */ + className?: string; + open: boolean; +}) { + const shouldReduceMotion = useReducedMotion(); + const content = useRef(null); + const [openHeight, setOpenHeight] = useState(0); + + /** + * The last height the content had while it was OPEN, and the guard is the whole point. + * + * A caller may unmount its content on the way closed — `RejectedFiles` does, because a + * `role="alert"` left in the tree is still an alert to a screen reader. The observer fires for + * that too, and taking the measurement would overwrite the height we are about to animate FROM + * with the zero we are animating TO. Read through a ref rather than the closure so this sees the + * commit that closed it rather than the render that installed the observer. + */ + const isOpen = useRef(open); + + /** + * WHY THE ASSIGNMENT IS HERE AND NOT IN THE BODY OF THE RENDER, WHERE IT USED TO BE. + * + * `isOpen.current = open` written during render records what React was CONSIDERING, and React is + * free to render a component and then throw the work away — that is what every interrupted or + * suspended transition does. A guard fed by a render that never committed answers about a box + * that never changed: it turns away the observer while the content is still open and visibly + * resizing, and the box goes on animating to a height its content no longer has. + * + * A layout effect only runs on a COMMIT, which is the state the DOM is actually in, and it runs + * synchronously before the browser lays out — so it is in place before any `ResizeObserver` + * callback for that same commit can be delivered. That ordering is the reason this is not a + * plain `useEffect`: passive effects can be flushed after paint, and the observer would get + * there first. + */ + useLayoutEffect(() => { + isOpen.current = open; + + /** + * AND THE FIRST OPEN IS MEASURED HERE, NOT LEFT TO THE OBSERVER. + * + * `openHeight` starts at 0 and the observer is the only other thing that moves it, but the + * observer's first callback arrives while the box is still closed — where the guard above + * correctly refuses it. So the first time `open` went true there was still no height to go to, + * and `animate` ran 0 to 0. The real number turned up on the frame after, once the observer + * had fired again and its `setState` had landed, and what that reads as is not an animation: + * the composer sits still for a frame and then jumps. On the first attachment of every + * session, which is the one moment this component exists to smooth. + * + * Measuring on the commit that opens gives that first animation a real destination. It is + * every open rather than only the first because what goes in this box is a different size each + * time, and a mount-only measurement would send the second attachment to the height of the + * first. + */ + const element = content.current; + if (open && element) setOpenHeight(element.offsetHeight); + }, [open]); + + /** + * And afterwards: the content is not a fixed size once it is open. A thumbnail finishes loading, + * a filename wraps to a second line, the window narrows — each of those changes the height the + * box should be holding without changing `open`, and only the observer sees them. + * + * It is not asked for an opening measurement here. The layout effect above has already taken one + * for this same commit, and it ran first; a browser delivers an initial callback on `observe()` + * in any case. + */ + useEffect(() => { + const element = content.current; + if (!element) return; + const measure = () => { + if (isOpen.current) setOpenHeight(element.offsetHeight); + }; + const observer = new ResizeObserver(measure); + observer.observe(element); + return () => observer.disconnect(); + }, []); + + return ( + +
+ {children} +
+ + ); +} diff --git a/app/src/components/channels/composer/composer.tsx b/app/src/components/channels/composer/composer.tsx index f642ccfb2..9a86a34fd 100644 --- a/app/src/components/channels/composer/composer.tsx +++ b/app/src/components/channels/composer/composer.tsx @@ -1,11 +1,23 @@ +import { + type Attachment, + type AttachmentsConfig, + useAttachments, +} from "@copilotkit/react-core/v2"; import { IconArrowUp, IconPlayerStopFilled, IconPlus, } from "@tabler/icons-react"; import { PromptArea, type PromptAreaHandle } from "prompt-area"; -import type { Segment } from "prompt-area/helpers"; import { + isSegmentsEmpty, + mergeAdjacentTextSegments, + type Segment, + text, +} from "prompt-area/helpers"; +import { + type ChangeEvent, + type DragEvent, type FormEvent, useCallback, useEffect, @@ -13,18 +25,89 @@ import { useRef, useState, } from "react"; +import { attachmentModality } from "@/components/channels/chat-messages"; +import { + attachmentUrl, + classifyAttachment, + MAX_ATTACHMENTS_PER_MESSAGE, + mediaTypeOf, + shouldClaimPaste, +} from "@/lib/channels/attachments"; +import { newId } from "@/lib/new-id"; import { cn } from "@/lib/utils"; import { Button } from "../../ui/button"; +import { + attachmentsConfigFor, + FILE_PICKER_ACCEPT, + stagedRowId, +} from "./attachments"; import { applyCommandChips, + canSendDraft, type CommandOption, type ComposerDraft, enforceSingleAgent, toDraft, } from "./draft"; +import { screenPickedFiles } from "./picked-files"; +import { AttachmentStrip } from "./attachment-strip"; +import { type RejectedFile, RejectedFiles } from "./rejected-files"; import { PLACEHOLDER_COMMANDS } from "./sources"; import { type AgentOption, buildTriggers } from "./triggers"; +/** + * The SDK's upload failure, derived rather than imported for the reason `attachments.ts` records + * against the same derivation: `@copilotkit/shared` declares `AttachmentUploadError` and is not a + * dependency of `app/`, so the only public spelling of this shape is the config it is passed to. + */ +type UploadFailure = Parameters< + NonNullable +>[0]; + +/** + * WHY A STAGED ATTACHMENT LEFT THE COMPOSER WITHOUT BEING SENT. + * + * Two different things push files down this channel and they want different words. The composer + * used to build ONE sentence for both — "dropped when queued messages were merged into one: a + * message can carry at most 8" — which was true of the first and false of the second: nothing is + * merged when somebody removes a parked message, and no cap is hit. Telling them about a limit they + * never reached is worse than saying nothing, because it sends them looking for a limit to work + * around. + * + * `reduceQueue` reports the same `Attachment[]` either way, so the cause cannot be recovered from + * the files. It is read off the queue ACTION instead, by `conversation-view.tsx`, which is the one + * place that has both halves in hand. + */ +export type DroppedAttachmentCause = + /** Several parked messages became one turn, and the joined message overran the per-message cap. */ + | "merged-over-cap" + /** A parked message was taken back out of the queue, and it was carrying these. */ + | "queued-message-removed"; + +export type DroppedAttachments = { + cause: DroppedAttachmentCause; + attachments: readonly Attachment[]; +}; + +/** + * WHAT A STAGED ATTACHMENT ACTUALLY IS, rather than what the browser called it when it was picked + * up. See the `images` memo for the whole account, and `attachmentModality` in `chat-messages.ts` + * for the rule itself — the send path and the parked tiles ask this same question the same way, and + * a second copy of the rule here would be a second thing to get wrong. + * + * A one-line function rather than an inline expression because BOTH halves of the strip call it and + * they are complements: `images` keeps what this calls an image and `files` keeps everything else, + * so the two must never be able to drift into disagreeing about one attachment. + */ +function stagedModality(attachment: Attachment) { + const { source } = attachment; + // Only a `url` source has been past the server, which sniffed the bytes. A `data` source's + // `mimeType` is `file.type`: the same claim, not a second opinion on it. + const corroborated = + source.type === "url" && source.mimeType ? source.mimeType : undefined; + return attachmentModality(attachment.type, corroborated); +} + const MAX_HEIGHT_PX = 220; /** * Tracks the compact `text-sm` line box so PromptArea stays vertically centered in one row. @@ -87,6 +170,18 @@ export type ComposerProps = { * caret, because that one they asked for. */ autoFocus?: boolean; + /** + * The channel a picked file is uploaded to, and the one switch for the whole attachment feature. + * + * WITH NO `channelId` THIS COMPOSER BEHAVES EXACTLY AS IT DID BEFORE ATTACHMENTS EXISTED, which + * is what keeps `/channel/new`, the home screen and the onboarding poster working untouched. No + * config reaches `useAttachments`, so it installs no paste listener at all; the form takes no + * drag handlers, no file input is rendered, and the `+` button goes on saying it has nothing to + * offer. That is the right answer rather than a degraded one for a screen with nowhere to put an + * upload: the compose screen creates its channel on send, so a file staged there would belong to + * a channel that does not exist yet. + */ + channelId?: string; /** * There is a run on the wire for Stop to reach. * @@ -101,8 +196,117 @@ export type ComposerProps = { */ stoppable?: boolean; initialValue?: string; + /** + * Files a caller's queue left behind, and WHY, reported so they read as a refusal rather than + * vanishing. + * + * Two causes reach here and they are not interchangeable — see `DroppedAttachments` above. + * Joining several parked messages into one drained turn can overflow + * `MAX_ATTACHMENTS_PER_MESSAGE` in a way no per-draft check ever saw coming (see `reduceQueue`'s + * `droppedAttachments` in `queue.ts`), and removing a parked message takes whatever it was + * carrying with it. Either way those files are still staged server-side, so saying nothing is not + * neutral: it is a person finding out from a 409 on their next upload, with nothing connecting + * that refusal to the files that silently disappeared. + * + * A FRESH OBJECT EACH TIME SOMETHING IS DROPPED is what this relies on — see the effect below. + * The caller builds one per event, which it has to do anyway to say which cause it was. + */ + droppedAttachments?: DroppedAttachments; + /** + * HOW MANY ATTACHMENTS ARE SITTING IN THE CALLER'S QUEUE, BECAUSE THE SERVER IS STILL COUNTING + * THEM AND THIS COMPOSER CANNOT SEE THEM. + * + * Parking a message takes its chips off the strip — the queue owns them now — but nothing about + * the rows behind them changes: `attachedAt` is written only when the message is really sent, so + * they stay `attached_at IS NULL` in this composer's `uploadGroup` for the whole life of the + * in-flight turn, and the server's upload cap counts exactly that set. The strip is empty, + * `screenPickedFiles` is told `alreadyStaged: 0`, the pick is accepted, the upload round-trips, + * and the server answers 409 for a file the client had already said yes to. Park eight files + * during a turn — or three messages carrying three each — and the next one is refused. + * + * A per-draft cap that ignores the queue cannot agree with a per-group cap that does not, so the + * one number the client screens against has to include both halves. + * + * NOT THE CLOSED-TAB LOCKOUT, and worth saying so because the sentence is the same: parked files + * are drawn in the transcript, come back if the message is taken out of the queue, and their rows + * are stamped the moment the turn drains. The cost is a refusal that should have been instant and + * in our words arriving after a round trip in the server's. + * + * The count rather than the attachments, because that is all the screen needs and the queue's + * shape is the caller's business. `conversation-view.tsx` owns the queue and is the only thing + * that can sum this; a caller with no queue leaves it out and nothing changes. + */ + queuedAttachmentCount?: number; }; +/** + * A draft holding more files than one message may carry. + * + * A function rather than a comparison written twice, because the two readers must never disagree: + * the button that refuses the press, and `submitDraft`, which refuses the Enter key that never + * looks at the button. See `tooManyStaged` for how a strip gets over the cap at all. + */ +function overCap(draft: ComposerDraft): boolean { + return draft.attachments.length > MAX_ATTACHMENTS_PER_MESSAGE; +} + +/** What the person is told when Send is held shut by the cap, and what to do about it. */ +function tooManyStagedReason(count: number): string { + return ( + `${count} attachments are staged and a message can carry at most ` + + `${MAX_ATTACHMENTS_PER_MESSAGE}. Remove ` + + `${count - MAX_ATTACHMENTS_PER_MESSAGE} to send the rest.` + ); +} + +/** + * One sentence per cause, and they say different things because different things happened. + * + * A merge names the cap, because that is the limit the person can work around — send fewer at once. + * A removal names the removal and nothing else: there is no limit to explain, and inventing one + * would send somebody hunting for a rule they never hit. + */ +const DROPPED_REASON: Record = { + "merged-over-cap": + "dropped when queued messages were merged into one: a " + + `message can carry at most ${MAX_ATTACHMENTS_PER_MESSAGE} ` + + "attachments.", + "queued-message-removed": + "removed along with the queued message it was attached to.", +}; + +/** + * WHAT A FILE DROPPED ON A COMPOSER THAT CANNOT TAKE IT IS TOLD, AND WHY IT IS TOLD ANYTHING. + * + * Catching the drop (see `refuseDragOver`/`refuseDrop` below) is what stops the browser from + * navigating the app away, and that alone would be a fix. It would also be a file that vanished: + * the person aimed a screenshot at the one box on the screen that takes screenshots, let go, and + * got nothing back — no chip, no error, no cursor change that outlasts the gesture. That is the + * exact failure `rejected` exists to end for every other door into this composer, and a drop is + * not a lesser door than the `+` button. + * + * TWO SENTENCES, BECAUSE THE TWO REFUSALS ARE NOT THE SAME REFUSAL. One is a "not yet" with a next + * step the person can take immediately; the other is a "not here, ever". Sharing one string + * between them would either promise a send that will never work or hide the send that will. + * + * NEITHER NAMES A CAUSE THIS COMPONENT CANNOT SEE. `disabled` is a plain boolean prop — + * `channel-chat.tsx` passes `!channel.active` and its own notice says the coworker was deleted, + * but that is the CALLER'S knowledge, and a composer that repeated it would be guessing on behalf + * of every future caller that disables it for some other reason. So this says only what is true of + * all of them: nothing can be sent from here, so nothing can be attached to it either. + * + * The filename is not repeated inside the reason — `RejectedFiles` renders `: ` — + * matching `DROPPED_REASON` above rather than `picked-files.ts`, whose strings quote the name a + * second time. + */ +const UNACCEPTED_DROP_REASON = { + "cannot-send": + "was not attached: this conversation can no longer take messages.", + "no-conversation": + "was not attached: there is no conversation here yet. Send this " + + "message first, then attach to the one it opens.", +} as const; + export function Composer({ className, editorClassName, @@ -115,8 +319,12 @@ export function Composer({ disabled = false, pending = false, autoFocus = false, + channelId, stoppable, initialValue, + droppedAttachments, + // Nothing parked is the right answer for every caller without a queue, which is most of them. + queuedAttachmentCount = 0, }: ComposerProps) { const [value, setValue] = useState( initialValue ? [{ type: "text", text: initialValue }] : [], @@ -124,17 +332,712 @@ export function Composer({ const [isSubmitting, setIsSubmitting] = useState(false); const submitInFlight = useRef(false); const promptAreaRef = useRef(null); + /** + * The composer's outer element, and the reason the hook's own `containerRef` is left on the + * floor. + * + * `useAttachments` scopes its `document` paste listener with exactly this test — is the event's + * target inside the element holding my ref — so an unheld ref is a listener that returns on + * every paste. That is the point. This composer deliberately does its own paste handling (see + * the listener below for why), and while ours claimed every paste carrying a file the SDK's + * never got a look in. It does now, on the pastes we decline, and it does not decline them: a + * spreadsheet cell was staged TWICE, once by us and once by it, and a Word paste that we + * correctly let through as text had its picture attached anyway. Holding the ref ourselves is + * what makes this composer the only thing in the app that decides what a paste means. + */ + const containerRef = useRef(null); /** A send has completed and the caret is owed back, as soon as the editor will take it. */ const wantsFocus = useRef(false); /** `autoFocus` has been honoured once, and is not owed again for the life of this composer. */ const claimedAutoFocus = useRef(false); + /** + * EVERY REFUSAL THIS COMPOSER HAS EVER MADE, BECAUSE NOTHING ELSE REMEMBERS THEM. + * + * An `Attachment` has no error state and the hook has no retry: a failed upload removes the + * placeholder from the strip and calls `onUploadFailed` exactly once, so a refusal that is not + * kept here is a file that vanished from the composer with nothing said about it. Our own + * pre-check adds to the same list, so the two sources of "no" read as one list of reasons. + */ + const [rejected, setRejected] = useState([]); + + const dismissRejections = useCallback(() => setRejected([]), []); + + /** + * `newId()` rather than `crypto.randomUUID()`, for the reason `uploadGroup` below sets out at + * length: the function is ABSENT outside a secure context, so on a deployment served over plain + * `http://
` this throws rather than returning something worse. + * + * The throw would land in the worst possible place. This is the SDK's `onUploadFailed`, called + * from inside `processFiles` — so the report of a failed upload would itself fail, the file + * would leave the strip the way a failed upload always does, and the sentence explaining it + * would never arrive. Every upload refusal in the app comes through here. + */ + const recordRejection = useCallback((failure: UploadFailure) => { + setRejected((current) => [ + ...current, + { + id: newId(), + name: failure.file.name, + reason: failure.message, + }, + ]); + }, []); + + /** + * THIS COMPOSER'S UPLOAD GROUP: one key, minted once, sent with every upload it makes. + * + * The per-message cap has two enforcers, and before this they counted different sets. The client + * counts what is on its own screen; the server counted every unsent row this person had in this + * channel. They agree only while nothing has been left behind — and a closed tab, a stopped run + * or a removed queued message leaves exactly that. The client would then accept a pick the server + * refused with a 409 naming files nobody could see, and eight orphans locked uploads in that + * channel until the sweeper's 24-hour window expired. Grouping the rows by the composer that + * staged them is what makes both sides count the same set. + * + * NOT A DRAFT ID. Nothing about the message is saved anywhere by this, and nothing survives a + * reload: a remount mints a new key and the rows the old one staged become somebody else's + * problem, which is to say the sweeper's. + * + * `newId()` rather than `crypto.randomUUID()`, which does not exist outside a secure context — + * on a deployment reached at plain `http://
` the call is not merely wrong, it throws, + * and the throw here would abort the whole screening pass rather than fail one upload. See + * `lib/new-id.ts`. + * + * Lazy `useState` rather than `useRef(newId())`, whose argument would be evaluated on every + * render to be thrown away — cheap, but it would also mean the id nobody keeps is minted + * hundreds of times per composer. + */ + const [uploadGroup] = useState(newId); + + /** + * EVERY SERVER ROW THIS COMPOSER HAS CREATED AND NOT YET ACCOUNTED FOR. + * + * A row leaves this set exactly one of two ways: a message takes ownership of it + * (`releaseStagedRows`, on send and on queue), or the reconciler below finds it has no chip left + * and gives it back to the server. Anything still in here is a row whose fate is undecided. + * + * A ref rather than state: nothing on screen is drawn from it, and a set that re-rendered the + * composer on every upload would invalidate all four attachment memos for no visible reason. + */ + const uploadedRows = useRef>(new Set()); + + const recordUpload = useCallback((attachmentId: string) => { + uploadedRows.current.add(attachmentId); + }, []); + + /** + * These rows belong to a message now, not to this composer, so it must stop offering to delete + * them. Called before the chips are taken off, on both paths where a draft carries its + * attachments away — the send that became a message, and the parked message the queue now owns. + * + * Getting this wrong in the other direction is the expensive one: leave a row in the set and the + * reconciler will notice the chip has gone and DELETE an attachment that has just been sent, or + * that a queued message is still waiting to send. + */ + const releaseStagedRows = useCallback((released: readonly Attachment[]) => { + for (const attachment of released) { + const rowId = stagedRowId(attachment); + if (rowId !== undefined) { + uploadedRows.current.delete(rowId); + } + } + }, []); + + const attachmentsConfig = useMemo( + () => + channelId + ? attachmentsConfigFor( + channelId, + uploadGroup, + recordRejection, + recordUpload, + ) + : undefined, + [channelId, uploadGroup, recordRejection, recordUpload], + ); + + /** + * The queue just left these behind, and they need the same rejection treatment a pick-time + * refusal gets — see `droppedAttachments` above for why staying quiet is not an option. + * + * Keyed on the object itself rather than on something derived from it, because the caller is + * expected to hand over a fresh one only when a new drop actually happened; an empty one on + * mount, or between drops, runs this once for nothing and then not again until the next real + * drop. + */ + useEffect(() => { + if (!droppedAttachments || droppedAttachments.attachments.length === 0) { + return; + } + const reason = DROPPED_REASON[droppedAttachments.cause]; + setRejected((current) => [ + ...current, + ...droppedAttachments.attachments.map((attachment) => ({ + // `newId()` again, and this is the least survivable of the three sites: an effect body + // that throws propagates out of React's commit, so on a deployment served over plain + // `http://` a queue drop would answer by tearing down the tree that was about to explain + // it — a blank conversation instead of a list of the files it just lost. + id: newId(), + name: attachment.filename ?? "Attachment", + reason, + })), + ]); + }, [droppedAttachments]); + + /** + * Called unconditionally, with a config only when there is a channel to upload to. The hook + * reads no React context — it is state, refs and one `document` paste listener that it does not + * install while disabled — so an undefined config is a hook that does nothing, not a hook that + * throws. + */ + const { + attachments, + enabled: attachmentsEnabled, + fileInputRef, + processFiles, + handleDragOver, + handleDragLeave, + removeAttachment, + } = useAttachments({ config: attachmentsConfig }); + + /** + * THERE IS A CHANNEL TO UPLOAD TO **AND** THIS CONVERSATION CAN STILL TAKE A MESSAGE. + * + * `disabled` used to gate sending and nothing else, so every one of the three doors — the drop, + * the paste, and the `+` button with its input — went on accepting files for a channel whose + * coworker has been deleted. Each one becomes a row staged server-side that counts against that + * channel's cap and waits for the sweeper, attached to a message the screen has already said can + * never be sent, with nothing connecting the two. + * + * Every door reads this rather than `attachmentsEnabled`, and `stageFiles` re-asks it besides: + * `onImagePaste` is PromptArea's call and not ours, so the choke point has to answer for itself. + */ + const canAttach = attachmentsEnabled && !disabled; + + /** + * THE ATTACHMENTS A SEND IN FLIGHT IS ALREADY CARRYING, HIDDEN FOR THE LENGTH OF THE RUN. + * + * `onSubmit` does not resolve until the whole run does, so between the press and the answer a + * screenshot was on screen twice — once in the transcript and once still in the strip — and + * `canSendDraft` unlocks on attachments alone, so the text box being empty did not disable + * anything: one more press queued the identical attachment ids into a second message. That + * duplicate is what this exists to close. + * + * THE OTHER WAY TO CLOSE IT, AND WHY NOT IT. Calling `consumeAttachments()` optimistically + * alongside `setValue([])` is three lines and loses the strip whenever a send fails. This file + * will not make that trade: a failed send already puts the words back, and files that vanished + * from the composer at the same moment would be exactly the thing it never does quietly — the + * whole `RejectedFiles` apparatus above exists because a file that disappears with nothing said + * about it is the worst outcome here, and these ones are still staged server-side, so nothing on + * screen would connect them to the 409 they later cause. So the send takes a snapshot of the ids + * instead, they are hidden while it runs, and the catch hands them straight back with the words. + * + * State rather than a ref, because `images`, `files` and `draft` are memos keyed on the + * attachments: a ref would not invalidate them, and the chips would sit there for the whole run, + * which is the bug itself. + */ + const [sending, setSending] = useState([]); + + /** + * What is on the composer right now: everything staged, less whatever a send in flight already + * took. Every reader goes through this — the per-message cap, both halves of the strip, and the + * draft both buttons are enabled from — so there is no path left on which an attachment that has + * already been sent can be counted, drawn, or sent again. + */ + const staged = useMemo( + () => attachments.filter((attachment) => !sending.includes(attachment.id)), + [attachments, sending], + ); + + /** + * Take a chip off the strip. That is all this does, and the row it stood for is deliberately + * somebody else's problem — `reconcileStagedRows` below. + * + * IT USED TO DELETE THE ROW ITSELF, AND COULD ONLY EVER DO HALF THE JOB. It read the id off the + * attachment, so it worked for a `ready` chip and did nothing at all for one whose upload was + * still in flight — there is no id on that one yet. The comment said the in-flight case was the + * sweeper's and treated it as harmless. It is not: `uploadGroup` is minted once per composer and + * lives for the whole mount, so the row that lands a moment later is counted by the server + * against THIS composer, while the chip it belonged to is gone from the strip. The screen counts + * seven, the server counts eight, and the next pick comes back 409 naming a file nobody can see — + * verbatim the failure `uploadGroup` was introduced to remove, reached through a Remove button + * instead of through a closed tab. + * + * Splitting the answer across two functions is what left the gap, so there is one function now, + * and it is the one that can see both halves. + */ + const discardAttachment = useCallback( + // Unconditional: the person asked for the chip to be gone, and nothing here is allowed to make + // that wait on a round trip. + (id: string) => removeAttachment(id), + [removeAttachment], + ); + + /** + * GIVE BACK EVERY ROW THAT NO LONGER HAS A CHIP — THE ONE PLACE THAT DECIDES THIS. + * + * A row this composer uploaded is either on the strip, or owned by a message, or nobody's. The + * third is the leak, and it has two sources that used to be handled differently and one of them + * not at all: a `ready` chip removed by hand, and a chip removed while its upload was still in + * the air. Reconciling what was uploaded against what is drawn catches both without having to + * know which happened. + * + * WHY IT CANNOT BE DONE AT THE MOMENT OF THE REMOVAL, WHICH IS THE OBVIOUS PLACE. `onUpload` is + * handed a `File` and nothing else; the SDK mints the placeholder id itself and never says which + * placeholder a given call belongs to. So at the moment somebody removes an uploading chip there + * is no id to record, and when the upload lands there is no way to ask whether the chip it + * belonged to is the one that went. Absence is the only signal that survives that, and it is a + * complete one. + * + * WAITING FOR NOTHING TO BE UPLOADING IS THE WHOLE OF THE SAFETY. Between `onUpload` returning an + * id and the SDK writing it onto its placeholder there is a window in which the row is known and + * not yet drawn, and deleting there would destroy a live attachment. While that window is open + * the placeholder is still `uploading` in committed state, so this gate closes exactly over it. + * The gate is read off `attachments` rather than off the `staging` ref next to `stagedCount` + * deliberately: a ref is not a render, so a ref-gated pass could skip the last commit and never + * be woken again — a stale count self-corrects on the next commit, an un-deleted row does not. + * + * `attachments` and not `staged`: a send in flight only HIDES its attachments, and they are still + * the composer's until the send becomes a message. + */ + useEffect(() => { + if (uploadedRows.current.size === 0) { + return; + } + if (attachments.some((attachment) => attachment.status === "uploading")) { + return; + } + const drawn = new Set(); + for (const attachment of attachments) { + const rowId = stagedRowId(attachment); + if (rowId !== undefined) { + drawn.add(rowId); + } + } + for (const rowId of uploadedRows.current) { + if (drawn.has(rowId)) { + // Still on the strip. Left in the set on purpose: it may yet be removed, and this is the + // only thing watching for that. + continue; + } + uploadedRows.current.delete(rowId); + discardStagedAttachment(rowId); + } + }, [attachments]); + + /** + * How many attachments the next screen must count, including any this tick has already accepted. + * + * `staged.length` is a number captured at render, and `stageFiles` is async, so two gestures in + * one tick — two drops in a row, or a drop landing on a paste — both read the same stale count + * and both accept a full cap's worth. Five files and five more went out as ten uploads against a + * cap of eight, and the server refused the surplus: a 409 the person had been given no chance to + * avoid, which is the exact failure this whole change exists to remove. + * + * A ref, because the correction has to be visible to the second caller in the SAME tick, and no + * state update is. + */ + const stagedCount = useRef(0); + + /** + * How many accepted files are between `stageFiles` and their placeholders. + * + * This exists only to say when `stagedCount` may be resynced, and it is not optional: the hook + * adds placeholders ONE AT A TIME, so mid-flight `staged.length` is a number that is still + * climbing. Resyncing from it there would hand the reservation straight back — the first version + * of this did, and the second gesture screened against 1 instead of 5. + */ + const staging = useRef(0); + + /* + * Every commit, and cheap: whenever nothing is in flight, the hook's own count is the truth, and + * that is how a removal, a send taking its attachments off, or a file the SDK refused after we + * accepted it all get back into the number. No dependency array on purpose — the condition that + * matters is a ref, which no dependency list can watch. + */ + useEffect(() => { + if (staging.current === 0) { + stagedCount.current = staged.length; + } + }); + + /** + * The one way a file gets from a person's hands into the upload queue. + * + * SCREENING HAPPENS HERE, BEFORE `processFiles`, AND THAT ORDER IS THE WHOLE POINT. The SDK + * enforces `accept` and `maxSize` itself and phrases its refusals for a machine — `File + * "logo.svg" is not accepted. Supported types: image/png,…` — where `screenPickedFiles` writes a + * sentence naming the file and the limit it hit. Anything this refuses never reaches the SDK, so + * a refused file produces exactly one reason, in our words. + */ + const stageFiles = useCallback( + async (picked: readonly File[]) => { + // The last gate, and the only one `onImagePaste` passes: PromptArea decides on its own when + // to hand an image over, so a door we do not own still arrives here. + if (disabled) { + return; + } + const screened = screenPickedFiles(picked, { + // What is on the composer, not what the hook is holding: an attachment riding on a send + // in flight belongs to that message's count, not to the one being built now. + // + // WHICH IS TRUE OF A SEND THAT LANDS AND NOT OF ONE THAT FAILS, and this number cannot + // tell them apart at the moment it is read — the send is still out. A failure hands the + // riding files back onto a strip that has filled up behind them, and the draft is then + // over the cap however carefully this counted. `tooManyStaged` is what catches that, + // because it asks about the strip as it actually is rather than predicting it. + // + // PLUS WHAT IS PARKED, which is on nobody's strip and is still counted by the server — + // see `queuedAttachmentCount`. Without the second half the client accepts a pick the + // server then refuses with a 409 the person was given no chance to avoid. + alreadyStaged: stagedCount.current + queuedAttachmentCount, + }); + if (screened.rejected.length > 0) { + setRejected((current) => [...current, ...screened.rejected]); + } + if (screened.accepted.length === 0) { + return; + } + // Both bumps happen BEFORE the await, so a second gesture in this same tick screens against + // a number that already includes these. + stagedCount.current += screened.accepted.length; + staging.current += screened.accepted.length; + try { + await processFiles(screened.accepted.map(withMediaTypeOnly)); + } finally { + staging.current -= screened.accepted.length; + } + }, + [disabled, processFiles, queuedAttachmentCount], + ); + + /** + * The image half of a paste that our own listener let through. + * + * `PromptArea` has its own rule for a clipboard carrying BOTH text and an image: if the + * `text/html` is Microsoft Office markup it inserts the TEXT and ignores the image, because a + * copied Excel cell or Word block is text that happens to ship a picture of itself. Anything else + * it treats as an image, calls this, and RETURNS — having already called `preventDefault` and + * inserted nothing. + * + * WHICH SOURCES LAND WHERE, SAID PLAINLY, BECAUSE THE SECOND GROUP IS NOT COVERED. Word and Excel + * write Office markup, so their text is typed. Google Sheets and Numbers do not, so they take the + * second branch: this prop attaches their image and THEIR TEXT IS NOT TYPED. Having the prop is + * still strictly better than not having it — without it that same paste calls nothing at all, so + * there is no text AND no attachment, which is the failure our own listener was written to avoid + * — but it does not make the spreadsheet case right, and this does not claim it does. See + * `shouldClaimPaste` in `shared/attachments.ts` for the rule and for why it is being kept. + * + * Routing it into `stageFiles` means the file arrives by the same door as the picker and the + * drop, so it is screened, capped and refused in the same words. + * + * EVERY BRANCH BELOW HAS TO PASS IT, WHICH IS THE WHOLE OF THIS PROP'S CORRECTNESS. It went on + * the compact one alone for a while, and the full-size branch — the shape the home screen draws + * — swallowed exactly this paste in silence. `composer-paste.test.tsx` renders that branch for + * no other reason. + */ + const stagePastedImage = useCallback( + (file: File) => { + void stageFiles([file]); + }, + [stageFiles], + ); + + /** + * THE THIRD DOOR, SHUT — AND SHUT AHEAD OF THE SDK'S OWN. + * + * `useAttachments` installs a bubble-phase `document` paste listener that pre-filters the + * clipboard with the same exact `file.type === filter` comparison `withMediaTypeOnly` exists to + * survive, and then calls `processFiles` directly. A pasted `text/plain;charset=utf-8` file — + * which is what a browser reports for some clipboard entries — fails that comparison, and the + * listener then returns having uploaded nothing, said nothing and called no `onUploadFailed`. + * Silent. Paste also skipped the per-message cap and `MAX_FILE_BYTES`, which our screen enforces + * and the SDK's does not. + * + * So this one listens in the CAPTURE phase, which runs before the hook's, and `stopPropagation` + * on a paste we claim means the hook's listener never runs at all. What we claim goes through + * `stageFiles`, the same path the `+` button and the drop already take, so screening, the cap and + * the wording of a refusal are identical whichever way the file arrived. + * + * `shouldClaimPaste` decides whether the paste is ours: text wins whenever the clipboard carries + * any, and a file is ours only when there is none. Copying a cell from a spreadsheet puts BOTH an + * image and text on the clipboard, and claiming that paste attached a screenshot of the cell and + * typed nothing. + * + * DECLINING IS NOT THE SAME AS THE TEXT BEING TYPED. What a declined paste does next is + * PromptArea's rule, and it types the text only when the `text/html` is Microsoft Office markup. + * So Word and Excel are handled; Google Sheets and Numbers are not — their image is attached + * through `stagePastedImage` below and their text is lost. That is a known defect being kept for + * now, and it is written out in full on `shouldClaimPaste` in `shared/attachments.ts`. + */ + useEffect(() => { + if (!canAttach) { + return; + } + const handlePaste = (event: ClipboardEvent) => { + // The hook's own containment gate, kept exactly: several composers can be mounted at once, + // and a paste into a search box is nobody's attachment. + const target = event.target as Node | null; + if (!target || !containerRef.current?.contains(target)) { + return; + } + const clipboard = event.clipboardData; + if (!clipboard) { + return; + } + const files = Array.from(clipboard.items) + .filter((item) => item.kind === "file") + .map((item) => item.getAsFile()) + .filter((file): file is File => file !== null); + const claimed = shouldClaimPaste({ + kinds: files.map((file) => classifyAttachment(file.type)), + plainText: clipboard.getData("text/plain"), + }); + if (!claimed) { + return; + } + event.preventDefault(); + event.stopPropagation(); + void stageFiles(files); + }; + document.addEventListener("paste", handlePaste, true); + return () => document.removeEventListener("paste", handlePaste, true); + }, [canAttach, stageFiles]); + + const handleDrop = useCallback( + (event: DragEvent) => { + event.preventDefault(); + /* + * The hook's `dragOver` flag, put back down. + * + * NOTHING DRAWS IT TODAY, AND THIS IS STILL NOT DEAD CODE. `useAttachments` keeps a + * `dragOver` boolean that its own chat components render a highlight from; this composer + * does not destructure it and no border or overlay anywhere in `app/src` changes while a + * file hovers. The comment that used to sit here said a missed drag-leave "would leave the + * composer looking like a file is still hovering over it", which was describing a highlight + * that does not exist — a reader chasing that sentence finds nothing. + * + * The call stays because the flag is the hook's, not ours: leaving it stuck at `true` after + * a drop would hand a trap to whoever wires the highlight up, and they would have no reason + * to suspect the drop path. It costs one function call per drop. + */ + handleDragLeave(event); + void stageFiles(Array.from(event.dataTransfer.files)); + }, + [handleDragLeave, stageFiles], + ); + + /** + * THE DROP THIS COMPOSER CANNOT ACCEPT, CAUGHT ANYWAY — BECAUSE "NOT A DROP TARGET" IS NOT THE + * SAME THING AS "IGNORES DROPS", AND THE DIFFERENCE IS THE WHOLE APP. + * + * An element becomes a drop target only when something calls `preventDefault` on its `dragover`. + * With no handler at all — which is what `dropZone` used to spread whenever `canAttach` was + * false — the composer is not a target, `drop` never fires on it, and the browser performs ITS + * default for a file dropped on a document: it NAVIGATES THE TOP-LEVEL DOCUMENT TO THAT FILE. + * + * The comment this replaces called those "drag handlers it would only ignore". Ignoring is the + * one thing that does not happen. The single-page app unloads and everything held in memory goes + * with it: the sentence being typed on `/channel/new`, and on a `disabled` channel the entire + * parked queue — whose own teardown effect in `conversation-view.tsx` never gets to run, because + * the document is being REPLACED rather than unmounted, so the rows behind those parked messages + * stay orphaned until the 24-hour sweep. The person sees their raw PNG on a blank page and + * presses Back. + * + * ALWAYS INSTALLED; ONLY THE ACCEPTANCE IS CONDITIONAL. The states that reach here are the three + * the `channelId` docblock names — `/channel/new`, the home screen, the onboarding poster — plus + * every channel whose composer is `disabled`. + * + * THE APP-WIDE GUARD NOW EXISTS, AND THIS IS STILL NOT REDUNDANT. `useUnclaimedDropGuard` in + * `routes/__root.tsx` refuses any drop nobody claimed, which covers the transcript, the sidebar, + * the page margin and the onboarding poster's `pointer-events-none` composer — every surface a + * leaf could never reach. It deliberately stands down the moment an event is already + * `defaultPrevented`, because that is the browser's own signal that something in the tree owns + * the drop. So the two do not fight, and they do different jobs: the root one can only refuse + * silently, since it has no idea what the person was aiming at. This one KNOWS, and names the + * file and the reason in `RejectedFiles`. Deleting it would turn a sentence into silence on the + * one surface people actually aim files at. + */ + const refuseDragOver = useCallback((event: DragEvent) => { + event.preventDefault(); + /* + * `dropEffect = "none"` SO THE CURSOR TELLS THE TRUTH BEFORE THE PERSON LETS GO. + * + * `preventDefault` alone leaves the effect at its default, which draws the same copy-badge + * cursor the working composer draws — so the guard would advertise an acceptance it is about + * to refuse, and the refusal would then read as a bug rather than as an answer. "none" draws + * the no-entry cursor and changes nothing else: the element is still a drop target, `drop` + * still fires on it, and the browser still never gets the file. + */ + if (event.dataTransfer) { + event.dataTransfer.dropEffect = "none"; + } + }, []); + + const refuseDrop = useCallback( + (event: DragEvent) => { + // FIRST, AND UNCONDITIONALLY. Everything below this line is the explanation; this line is + // the fix, and no branch of the explanation may be able to skip it. + event.preventDefault(); + // `?.` and `??` because a drag event can reach a handler with no `dataTransfer` — happy-dom + // builds one only when a test supplies it — and a throw here would be a page saved and a + // refusal never written down. + const files = Array.from(event.dataTransfer?.files ?? []); + if (files.length === 0) { + /* + * A dragged link, or a selection of text, or a drag from another app carrying no file. + * The default is still refused above — a dropped URL navigates just as a dropped file does + * — but there is nothing to name and nothing was attempted, so a refusal written here + * would be one invented for a gesture nobody made. + */ + return; + } + /* + * CHOSEN BY WHAT IS DEAD, NOT BY WHAT IS MISSING. + * + * This asked `attachmentsEnabled` — is there a channel — and got one of the four states + * wrong. A composer that is BOTH `disabled` and channel-less was told "there is no + * conversation here yet. Send this message first, then attach to the one it opens", while + * the Send button beside the sentence is shut and pressing it does nothing. Advice that + * cannot be followed is worse than none: it sends somebody to a control that is already + * refusing them, and they conclude the app is broken rather than that this conversation is + * over. + * + * `disabled` is the question actually being answered — CAN THIS PERSON DO THE THING THE + * SENTENCE WILL TELL THEM TO DO. It is also the only one of the two that can be true while + * the other is: `canAttach` is `attachmentsEnabled && !disabled`, so reaching here means at + * least one is against us, and `disabled` is the one that makes the next step impossible. + * The three states that were already right are unchanged: a live channel-less composer still + * gets the "not yet" sentence, and a disabled channel still gets "can no longer take + * messages". + */ + const reason = + UNACCEPTED_DROP_REASON[disabled ? "cannot-send" : "no-conversation"]; + /* + * `setRejected` directly rather than through `recordRejection`, which takes the SDK's + * `UploadFailure` shape — a `file` plus a `message` — and would mean fabricating an upload + * failure for a file no upload was ever attempted for. One line per file, matching every + * other refusal on this composer: dropping two files on a dead channel is two files that did + * not arrive, and folding them into one line loses which. + */ + setRejected((current) => [ + ...current, + ...files.map((file) => ({ id: newId(), name: file.name, reason })), + ]); + }, + [disabled], + ); + + const handlePickedFiles = useCallback( + (event: ChangeEvent) => { + const picked = Array.from(event.target.files ?? []); + // Emptied so that picking the same file again still fires a change event: a file refused by + // the screen above is a file somebody may well pick a second time by mistake, and an input + // still holding it would silently do nothing. + event.target.value = ""; + void stageFiles(picked); + }, + [stageFiles], + ); + + /** + * PromptArea already draws an attachment strip, so these map onto its props rather than adding a + * second one. An upload in flight has no preview to show — its placeholder source is an empty + * string until `onUpload` answers — which is what `loading` covers. + * + * SPLIT ON WHAT THE FILE IS, NOT ON WHAT THE BROWSER CALLED IT — the same correction the send + * path (`toAttachmentPart` in `channel-chat.tsx`) and the parked tiles (`parkedTiles` in + * `chat-transcript.tsx`) already made, through the same shared `attachmentModality`. + * + * `attachment.type` is the SDK's modality, and the SDK derives it from `file.type`: the + * browser's guess from a file extension, made before anything read a byte. A screenshot dragged + * out of an app that names it `application/octet-stream` was drawn here as a grey file card and + * then turned into a thumbnail the instant it was parked or sent — the same file, three + * surfaces, two answers, and the tile visibly changing shape at the moment of sending. The other + * direction is worse: a text file the browser calls `image/png` was handed to an `` that + * can never render it, which is a broken-image icon where a filename should be. + * + * ONLY A `url` SOURCE'S `mimeType` IS EVIDENCE, which is why the narrowing below is not + * defensive noise. A `data` source's `mimeType` is `file.type` — the very claim being refused, + * wearing the name of an answer — and only a `url` source has been past the server, which sniffs + * the bytes. `attachmentModality` falls back to the declared type when there is nothing + * corroborated, so an attachment still uploading is drawn exactly as it used to be and settles + * onto the truth when the server answers. That settle is the one visible cost: a mislabelled + * screenshot spends its upload as a file card. The alternative is holding every tile back until + * the answer, which would make every ordinary attachment feel slower to spare a rare one a + * flicker. + */ + const images = useMemo( + () => + staged + .filter((attachment) => stagedModality(attachment) === "image") + .map((attachment) => ({ + id: attachment.id, + url: attachment.source.value, + alt: attachment.filename, + loading: attachment.status === "uploading", + })), + [staged], + ); + + /** + * NO `type` HERE, BECAUSE NOTHING HAS EVER READ ONE. This carried + * `type: attachment.source.mimeType`, `StagedFile` (`attachment-strip.tsx`) declares no such + * field, and `AttachmentStrip` draws the same `IconFile` for every file whatever its type. It + * survived because the array is a variable rather than a fresh object literal at the JSX site, so + * TypeScript's excess-property check — the thing that would have caught it — never ran. + * + * Deleted rather than adopted. Adding `type` to `StagedFile` and drawing a per-format label off + * it is a real improvement and a deliberately separate one: it is a design change to the tile, + * not the removal of a line that pretends to feed something. + * + * The complement of `images` above, and it has to be read as one: whatever is not an image is a + * card, so both halves must ask the same question of the same field or an attachment lands in + * both strips or in neither. + */ + const files = useMemo( + () => + staged + .filter((attachment) => stagedModality(attachment) !== "image") + .map((attachment) => ({ + id: attachment.id, + name: attachment.filename ?? "Attachment", + size: attachment.size, + loading: attachment.status === "uploading", + })), + [staged], + ); const isBusy = pending || isSubmitting; const triggers = useMemo( () => buildTriggers({ agents, commands }), [agents, commands], ); - const draft = useMemo(() => toDraft(value), [value]); + const draft = useMemo(() => toDraft(value, staged), [staged, value]); + + /** + * MORE FILES ON THIS STRIP THAN ONE MESSAGE MAY CARRY — a state the screening is supposed to make + * unreachable, and does not. + * + * HOW THE STRIP GETS HERE. `sending` hides an outgoing message's attachments from `staged`, and + * `stagedCount` resyncs off `staged`, so while a send is out the client-side count reads zero and + * a second full batch is accepted behind the first. That is deliberate and right for a send that + * lands: those files belong to the message that went, not to the one being built now. It is only + * wrong when the send FAILS, because the `finally` then hands the first batch back — onto a strip + * that has since filled up — and one draft is holding two messages' worth of files. + * + * WHY THE CAP AND NOT `canSendDraft`. That function asks about upload status and emptiness, which + * are facts about each attachment; this is a fact about the draft as a whole, and it belongs + * beside the other thing the composer knows and the draft model does not — see `stageFiles`, + * which screens against the same number. + * + * NOTHING IS DROPPED TO RESOLVE IT, and that is the whole reason this is a gate rather than a + * slice. Trimming the strip back to the cap would delete files somebody picked, on a path they + * did not ask for, with the rows behind them released behind their back — which is verbatim the + * failure every other release in this file exists to avoid. So every chip stays, every one is + * removable by hand, and Send says no with a sentence naming the limit until it is. + */ + const tooManyStaged = overCap(draft); const handleChange = useCallback( (next: Segment[]) => { @@ -160,8 +1063,14 @@ export function Composer({ */ const submitDraft = useCallback( async (segments: Segment[]) => { - const submitted = toDraft(segments); - if (submitted.isEmpty || disabled) { + // `canSendDraft` rather than `isEmpty`: a screenshot with no words is a message, and an + // upload still in flight is not one yet, whatever is typed alongside it. + // + // `overCap` is asked here as well as at the button, and not only there: Enter reaches this + // function through prompt-area's own `onSubmit`, which has never looked at `canSend`. A gate + // drawn on the button alone would refuse the press and accept the keystroke. + const submitted = toDraft(segments, staged); + if (!canSendDraft(submitted) || overCap(submitted) || disabled) { return; } @@ -182,6 +1091,21 @@ export function Composer({ return; } setValue([]); + dismissRejections(); + // Taken off the composer as the message is parked, so the strip empties with the words + // rather than leaving the files looking like they are still to be sent. Only these — + // `consumeAttachments()` takes every `ready` attachment, including ones a send already + // in flight is riding, and that send's own `finally` still needs them to hand back if it + // fails. `submitted.attachments` is `staged`, which already excludes those. + // + // Released BEFORE the chips go, and this order is not cosmetic: the parked message owns + // these rows now and will send them later, so the reconciler must be told before it sees + // the chips disappear — otherwise it reads a queued message's attachments as abandoned and + // deletes the rows out from under it. + releaseStagedRows(submitted.attachments); + for (const attachment of submitted.attachments) { + removeAttachment(attachment.id); + } onQueue(submitted); return; } @@ -194,12 +1118,77 @@ export function Composer({ setIsSubmitting(true); // Clear optimistically; restore if the send fails before becoming a message. setValue([]); + /* + * The refusals go with them, and do NOT come back if the send fails. They are about files + * that never made it onto this message, so the words being restored has nothing to do with + * them; putting them back would be an old complaint reappearing next to a new attempt. + */ + dismissRejections(); + // The strip is cleared optimistically too, but by hiding rather than by consuming: these + // ids leave the composer now, so nothing on screen can send them a second time, and the + // `finally` below is what decides whether they come back. + const riding = submitted.attachments.map((attachment) => attachment.id); + setSending(riding); try { await onSubmit(submitted); - } catch (error) { - setValue(segments); - throw error; + // Only once the send has become a message, and only the ones that rode on it — + // `consumeAttachments()` would also swallow anything attached while the run was in + // flight, which belongs to the next message and has never been sent. + // + // Released first, for the reason the queue branch above gives: the message carries these + // rows now, and a reconciler that saw the chips go without being told would delete + // attachments that have just been sent. + releaseStagedRows(submitted.attachments); + for (const id of riding) { + removeAttachment(id); + } + } catch { + /* + * THE WORDS COME BACK IN FRONT OF WHATEVER ARRIVED WHILE THE SEND WAS OUT, NOT OVER IT. + * + * The editor is never disabled mid-turn, and that is deliberate — it is how a correction + * gets typed at a Bot that is already working. So by the time a send fails the box may well + * hold something newer than the message that failed. `setValue(segments)` wrote straight + * over it: the failed message came back and the sentence typed after it was gone, with + * nothing said about either. Both are somebody's words, so neither may be dropped; the + * restored ones go ahead of the newer ones and the person edits the join. + */ + setValue((current) => { + // An attachment-only message has no words to restore, and putting an empty segment list + // back would clear a box somebody has since typed into. + if (isSegmentsEmpty(segments)) { + return current; + } + if (isSegmentsEmpty(current)) { + return segments; + } + return mergeAdjacentTextSegments([ + ...segments, + text(" "), + ...current, + ]); + }); + /* + * NOT RETHROWN, AND THE RETHROW THIS REPLACES REACHED NOTHING. + * + * `submitDraft` has exactly two callers: `handleFormSubmit`, which does + * `void submitDraft(value)`, and PromptArea's `onSubmit`, which calls it and ignores the + * promise. Neither awaits and neither catches, so every failed send became an unhandled + * rejection — and in this repository's test runner, a failure attributed to whichever test + * happened to be running when it surfaced, which is why two composer suites carry notes + * saying a failed send could not be driven from them at all. + * + * Nothing is hidden by stopping. The caller is the half that knows what went wrong and it + * already reports it: `conversation-view.tsx` shows a failed turn through its own notice, + * and its drain path catches this same rejection under a comment saying the composer's + * throw exists only so the composer can put the words back. Putting the words back is what + * this catch does, so there is nothing left for the throw to carry. + */ } finally { + // Handed back to the composer either way: on success they are already gone from the + // hook's own state, and on failure this is what puts the chips back beside the restored + // words rather than leaving somebody to re-pick files that are still staged server-side. + setSending([]); submitInFlight.current = false; setIsSubmitting(false); // Asked for here, performed in the effect below, which runs after the commit that clears @@ -207,7 +1196,16 @@ export function Composer({ wantsFocus.current = true; } }, - [disabled, isBusy, onQueue, onSubmit], + [ + disabled, + dismissRejections, + isBusy, + onQueue, + onSubmit, + releaseStagedRows, + removeAttachment, + staged, + ], ); /** @@ -247,9 +1245,18 @@ export function Composer({ * and the button it wants is Stop. */ const canQueue = Boolean(onQueue) && isBusy && !disabled; - /** Something is typed, mid-turn, with a queue to put it in. */ - const parking = canQueue && !draft.isEmpty; - const canSend = !disabled && !draft.isEmpty && (!isBusy || canQueue); + /** + * There is a message here to send or to park. + * + * `canSendDraft` rather than a local "is anything typed": it is the half of this answer that + * knows about attachments — an image on its own is a message, and one still uploading is not + * one yet. Both buttons read it, so Send and Queue can never disagree about whether there is + * anything to do. + */ + const sendable = canSendDraft(draft) && !tooManyStaged; + /** Something to send, mid-turn, with a queue to put it in. */ + const parking = canQueue && sendable; + const canSend = !disabled && sendable && (!isBusy || canQueue); /** * Stop is available only once there is a run for it to reach, and it gives way to Send the moment * there is something typed to park. @@ -271,88 +1278,234 @@ export function Composer({ */ const sendLabel = parking ? "Queue message" : "Send message"; + /** + * EVERY COMPOSER IS A DROP TARGET. What changes with `canAttach` is what happens to the file, + * never whether the browser gets to keep it — see `refuseDragOver` above for what an + * uninstalled `dragover` handler actually does to this app. + * + * SPREAD ON THE CONTAINER, NOT ON THE FORM, AND THE DIFFERENCE IS A REAL GAP THAT WAS OPEN. The + * refusal strip renders ABOVE the form — `RejectedFiles` is a sibling, moved there so the reasons + * sit next to the drop that caused them rather than below the box — so with the handlers on the + * form, a file let go over a refusal was let go over nothing. The most likely second drop in the + * whole app lands exactly there: somebody drops two files, reads why the first was refused, and + * aims the retry at the sentence they are reading. + * + * The container is also the element `containerRef` is on, which is the boundary our own paste + * listener already uses to decide whether a paste is ours. One element now answers both + * questions, so "inside this composer" means the same thing for a file that arrives by clipboard + * and for one that arrives by hand. + * + * No `onDragLeave` on the refusing branch, and that is not an omission: `handleDragLeave` only + * puts the hook's `dragOver` flag back down, and the branch that never raises it has nothing to + * put down. (Nothing renders that flag today — see `handleDrop` above, which says why the call + * is kept anyway.) + */ + const dropZone = canAttach + ? { + onDragLeave: handleDragLeave, + onDragOver: handleDragOver, + onDrop: handleDrop, + } + : { + onDragOver: refuseDragOver, + onDrop: refuseDrop, + }; + + /** + * Rendered only with a channel behind it: without one there is nothing to upload to, and a file + * dialog that leads nowhere is worse than no button. + * + * `FILE_PICKER_ACCEPT` RATHER THAN `attachmentsConfig?.accept`, WHICH THESE TWO NO LONGER SHARE. + * The config's `accept` is a GATE the SDK enforces on every file we hand it, and it is a wildcard + * now for the reason set out on `attachmentsConfigFor` — a second, stricter, machine-worded gate + * behind `screenPickedFiles` refused the very files that screen exists to pass. A dialog filter is + * a HINT: it greys files out, refuses nothing, and drag and paste never meet it. Spending one + * string on both meant the honest widening of the gate would have widened the hint to "any file", + * which helps nobody. Two names, two jobs. + */ + const filePicker = attachmentsEnabled ? ( + + ) : null; + + /** + * WHY SEND IS SHUT WITH A FULL STRIP IN FRONT OF IT — drawn wherever the refusals are, because it + * is the same kind of sentence and answers the same question. + * + * NOT DISMISSABLE, unlike `RejectedFiles`. A refusal is news about a file that is already gone, + * so it is read once and cleared; this is a live description of the strip, and it goes away by + * removing a chip rather than by being acknowledged. A dismiss would leave a dead Send button + * with nothing on screen explaining it. + * + * `role="status"` and not `alert`: nothing happened at this instant — the person did not just do + * something refusable — and interrupting a screen reader mid-sentence to describe a button's + * state is not what an alert is for. + */ + const tooManyStagedNotice = tooManyStaged ? ( +

+ {tooManyStagedReason(draft.attachments.length)} +

+ ) : null; + if (compact) { return ( -
- - + + {tooManyStagedNotice} + - {canStop ? ( - - ) : ( - - )} - + onSubmit={handleFormSubmit} + > + {filePicker} + + {/* + * `self-end` ON THE THREE CONTROLS, AND IT COSTS NOTHING ON ONE LINE. The row is + * `items-center`, which is right while everything in it is a single line high. A long + * message grows the editor to `COMPACT_MAX_HEIGHT_PX`, and centred buttons then float to + * the middle of that block rather than sitting on the line the person is typing. Empty, + * the row is exactly a button tall, so centred and bottom are the same pixel. + */} +
+ {attachmentsEnabled ? ( + + ) : ( + + )} + + {canStop ? ( + + ) : ( + + )} +
+ +
); } return ( -
+ // The same `containerRef`, on the wrapper this branch already had: the paste listener is scoped + // to whatever holds the ref, so a branch without it can be pasted into and nothing happens. +
+ + {tooManyStagedNotice}
- {}} type="file" /> + {filePicker}
+
-
+ {attachmentsEnabled ? ( + + ) : ( +
+ )}
{canStop ? ( @@ -401,3 +1568,53 @@ export function Composer({
); } + +/** + * GIVE BACK THE STAGED ROW A REMOVED CHIP STOOD FOR — THE ONLY CALLER THIS ENDPOINT HAS. + * + * `removeAttachment` only filters client state, so until this existed, attach-then-change-your-mind + * left the row staged with `attachedAt IS NULL` for good. The upload handler counts exactly those + * rows per person per channel and refuses the ninth with "You already have 8 attachments waiting to + * send in this channel." — naming files that are on nobody's screen, with no way back. + * + * BEST-EFFORT, AND DELIBERATELY UNINSPECTED. The chip is gone from the strip before this is called, + * because that is what the person asked for; making that wait on a round trip, or reporting what + * comes back, would put an error in front of somebody for a request they never made. There are two + * such answers — 404 for a non-uploader, 409 for an attachment already sent — and neither is + * actionable. `cull-staged-attachments.ts` is the backstop for whatever this misses. + */ +function discardStagedAttachment(attachmentId: string): void { + void fetch(attachmentUrl(attachmentId), { + method: "DELETE", + credentials: "include", + }).catch(() => undefined); +} + +/** + * THE SECOND DOOR, SHUT. + * + * `processFiles` re-checks `accept` itself with an exact, CASE-SENSITIVE `file.type === filter` + * comparison, while `classifyAttachment` — the rule the server also applies — normalises first. + * Left alone, the two gates disagree, and a text file this composer accepted would be refused a + * second time in the SDK's own machine wording. + * + * BOTH HALVES OF THE NORMALISATION, WHICH IS WHY THIS GOES THROUGH `mediaTypeOf` RATHER THAN DOING + * ITS OWN SPLIT. It used to drop the parameter and leave the case, which closed the + * `text/plain;charset=utf-8` half of the gap and left the other half open: a type differing from + * the accept list only in case came back from the split unchanged, so this function decided nothing + * needed doing and handed the SDK a file it was about to refuse. That case cannot be reproduced + * with `new File(...)`, which lower-cases `type` per the Blob spec — but this function's whole + * purpose is the files this app did not build. + * + * The new `File` is a handle onto the same bytes, not a copy of them. + */ +function withMediaTypeOnly(file: File): File { + const mediaType = mediaTypeOf(file.type); + if (mediaType === file.type) { + return file; + } + return new File([file], file.name, { + lastModified: file.lastModified, + type: mediaType, + }); +} diff --git a/app/src/components/channels/composer/draft.test.ts b/app/src/components/channels/composer/draft.test.ts index 73c714af0..74bf9f9a7 100644 --- a/app/src/components/channels/composer/draft.test.ts +++ b/app/src/components/channels/composer/draft.test.ts @@ -1,7 +1,9 @@ +import type { Attachment } from "@copilotkit/react-core/v2"; import { describe, expect, test } from "bun:test"; import { chip, type Segment, text } from "prompt-area/helpers"; import { applyCommandChips, + canSendDraft, type CommandOption, enforceSingleAgent, toDraft, @@ -15,6 +17,15 @@ function command(id: string, name: string) { return chip({ trigger: "/", value: id, displayText: name }); } +function attachment(status: Attachment["status"]): Attachment { + return { + id: status, + type: "image", + source: { type: "url", value: "https://example.com/a.png" }, + status, + }; +} + describe("toDraft", () => { test("flattens chips back into the plain text sent to the runtime", () => { const draft = toDraft([ @@ -45,6 +56,32 @@ describe("toDraft", () => { expect(toDraft([text(" ")]).isEmpty).toBe(true); expect(toDraft([]).isEmpty).toBe(true); }); + + test("defaults attachments to empty when called with one argument", () => { + expect(toDraft([text("hello")]).attachments).toEqual([]); + }); +}); + +describe("canSendDraft", () => { + test("holds while an attachment is still uploading", () => { + const draft = toDraft([text("hi")], [attachment("uploading")]); + expect(canSendDraft(draft)).toBe(false); + }); + + test("releases once every attachment is ready", () => { + const draft = toDraft([text("hi")], [attachment("ready")]); + expect(canSendDraft(draft)).toBe(true); + }); + + test("allows a ready attachment alone, with no text typed", () => { + const draft = toDraft([], [attachment("ready")]); + expect(canSendDraft(draft)).toBe(true); + }); + + test("blocks an empty draft with no attachments", () => { + const draft = toDraft([]); + expect(canSendDraft(draft)).toBe(false); + }); }); describe("enforceSingleAgent", () => { diff --git a/app/src/components/channels/composer/draft.ts b/app/src/components/channels/composer/draft.ts index bfd4a2ef9..4fb8ca5e3 100644 --- a/app/src/components/channels/composer/draft.ts +++ b/app/src/components/channels/composer/draft.ts @@ -1,3 +1,4 @@ +import type { Attachment } from "@copilotkit/react-core/v2"; import { getChipsByTrigger, isSegmentsEmpty, @@ -27,9 +28,13 @@ export type ComposerDraft = { /** Commands that survive into the sent message, in the order they were typed. */ commandIds: string[]; isEmpty: boolean; + attachments: Attachment[]; }; -export function toDraft(segments: Segment[]): ComposerDraft { +export function toDraft( + segments: Segment[], + attachments: Attachment[] = [], +): ComposerDraft { const agentChips = getChipsByTrigger(segments, AGENT_TRIGGER); const commandChips = getChipsByTrigger(segments, COMMAND_TRIGGER); @@ -38,9 +43,26 @@ export function toDraft(segments: Segment[]): ComposerDraft { agentId: agentChips.at(-1)?.value ?? null, commandIds: commandChips.map((chip) => chip.value), isEmpty: isSegmentsEmpty(segments), + attachments, }; } +/** + * A pasted screenshot with no text is the whole feature: `isEmpty` only answers "is there text", + * so an attachment alone must be enough to unlock Send rather than riding on top of it. An + * upload still in flight holds the gate either way, since sending would race the file that has + * not finished becoming a source yet. + */ +export function canSendDraft(draft: ComposerDraft): boolean { + if ( + draft.attachments.some((attachment) => attachment.status === "uploading") + ) { + return false; + } + + return draft.attachments.length > 0 || !draft.isEmpty; +} + /** Collapse multiple agent mentions to the most recent one while preserving identity on no-op. */ export function enforceSingleAgent(segments: Segment[]): Segment[] { const agentChipCount = getChipsByTrigger(segments, AGENT_TRIGGER).length; diff --git a/app/src/components/channels/composer/index.ts b/app/src/components/channels/composer/index.ts index 875477f38..aee99ca77 100644 --- a/app/src/components/channels/composer/index.ts +++ b/app/src/components/channels/composer/index.ts @@ -1,6 +1,12 @@ -export { Composer, type ComposerProps } from "./composer"; +export { + Composer, + type ComposerProps, + type DroppedAttachmentCause, + type DroppedAttachments, +} from "./composer"; export { AGENT_TRIGGER, + canSendDraft, COMMAND_TRIGGER, type CommandKind, type CommandOption, diff --git a/app/src/components/channels/composer/picked-files.test.ts b/app/src/components/channels/composer/picked-files.test.ts new file mode 100644 index 000000000..b9f2b0866 --- /dev/null +++ b/app/src/components/channels/composer/picked-files.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, test } from "bun:test"; +import { + MAX_ATTACHMENTS_PER_MESSAGE, + MAX_FILE_BYTES, + MAX_IMAGE_BYTES, +} from "@/lib/channels/attachments"; +import { screenPickedFiles } from "./picked-files"; + +function file(name: string, type: string, size = 3): File { + return new File(["x".repeat(size)], name, { type }); +} + +describe("screenPickedFiles", () => { + test("accepts a PNG under the ceiling", () => { + const png = file("photo.png", "image/png"); + const result = screenPickedFiles([png], { alreadyStaged: 0 }); + + expect(result.accepted).toEqual([png]); + expect(result.rejected).toEqual([]); + }); + + test("rejects an SVG with a reason naming the script risk", () => { + const svg = file("logo.svg", "image/svg+xml"); + const result = screenPickedFiles([svg], { alreadyStaged: 0 }); + + expect(result.accepted).toEqual([]); + expect(result.rejected).toHaveLength(1); + expect(result.rejected[0].reason).toMatch(/SVG/); + expect(result.rejected[0].name).toBe("logo.svg"); + }); + + /** + * The wording used to compare `file.type` raw while `classifyAttachment` — and the server, and + * the SDK's accept check — all normalise first. An SVG off a clipboard arrives as + * `image/svg+xml;charset=utf-8`, so it was still refused, but with the generic "an image format + * that is not supported" instead of the one sentence that says WHY this one in particular: + * an SVG can carry script. The person is left thinking their editor exported the wrong format. + */ + test("keeps the script-risk wording for an SVG whose type carries a charset", () => { + const svg = file("logo.svg", "image/svg+xml;charset=utf-8"); + const result = screenPickedFiles([svg], { alreadyStaged: 0 }); + + expect(result.accepted).toEqual([]); + expect(result.rejected[0].reason).toMatch(/SVG/); + }); + + test("keeps the script-risk wording for an SVG whose type differs only in case", () => { + const svg = file("logo.svg", "image/png"); + Object.defineProperty(svg, "type", { value: "IMAGE/SVG+XML" }); + const result = screenPickedFiles([svg], { alreadyStaged: 0 }); + + expect(result.accepted).toEqual([]); + expect(result.rejected[0].reason).toMatch(/SVG/); + }); + + test("rejects an oversized text file with a reason naming the size problem", () => { + const big = file("notes.md", "text/plain", MAX_FILE_BYTES + 1); + const result = screenPickedFiles([big], { alreadyStaged: 0 }); + + expect(result.accepted).toEqual([]); + expect(result.rejected[0].reason).toMatch(/too large/); + }); + + test("rejects an oversized image", () => { + const big = file("huge.png", "image/png", MAX_IMAGE_BYTES + 1); + const result = screenPickedFiles([big], { alreadyStaged: 0 }); + + expect(result.accepted).toEqual([]); + expect(result.rejected[0].reason).toMatch(/too large/); + }); + + /** + * THE CLIENT MUST NOT REFUSE WHAT THE SERVER ACCEPTS, AND IT DID. + * + * A browser reports `""` or `application/octet-stream` for a file it has no mapping for — a + * `.txt` dragged out of an editor, anything with an unfamiliar extension. `sniffMimeType` + * discards exactly those claims and reads the bytes instead, so the server takes such a file and + * stores it as `text/plain`. Screening on the claim alone refused it here first, and the person + * was told their plain text file "is not a file type this chat accepts" by the half of the + * system that had not looked at it. + */ + test("lets a file the browser could not name through for the server to sniff", () => { + const unnamed = file("notes.txt", ""); + const generic = file("notes.txt", "application/octet-stream"); + + const result = screenPickedFiles([unnamed, generic], { alreadyStaged: 0 }); + + expect(result.accepted).toEqual([unnamed, generic]); + expect(result.rejected).toEqual([]); + }); + + /** + * The loose ceiling, not the tight one, for the same reason: the client does not yet know which + * of the two limits applies, and guessing the tight one would put back a refusal the server would + * not have made. The server has the bytes and applies the right limit on arrival. + */ + test("still holds a file the browser could not name to the larger ceiling", () => { + const huge = file("notes.txt", "", MAX_IMAGE_BYTES + 1); + const result = screenPickedFiles([huge], { alreadyStaged: 0 }); + + expect(result.accepted).toEqual([]); + expect(result.rejected[0].reason).toMatch(/too large|larger than/); + }); + + test("a claim that names a format it does not accept is still refused here", () => { + // The fall-through is only for claims that name NOTHING. `application/zip` names a format, and + // `sniffMimeType` hands that name straight back for the server to refuse by name, so refusing + // it at pick time is the two halves agreeing rather than disagreeing. + const zip = file("archive.zip", "application/zip"); + const result = screenPickedFiles([zip], { alreadyStaged: 0 }); + + expect(result.accepted).toEqual([]); + expect(result.rejected).toHaveLength(1); + }); + + test("rejects an unsupported file type with a reason distinct from the SVG one", () => { + const zip = file("archive.zip", "application/zip"); + const result = screenPickedFiles([zip], { alreadyStaged: 0 }); + + expect(result.accepted).toEqual([]); + expect(result.rejected[0].reason).not.toMatch(/SVG/); + }); + + test("reports one rejection per bad file, each naming its own file", () => { + const svg = file("a.svg", "image/svg+xml"); + const zip = file("b.zip", "application/zip"); + const result = screenPickedFiles([svg, zip], { alreadyStaged: 0 }); + + expect(result.rejected).toHaveLength(2); + expect(result.rejected[0].name).toBe("a.svg"); + expect(result.rejected[1].name).toBe("b.zip"); + }); + + test("counts the cap after kind and size, against already-staged files", () => { + const first = file("one.png", "image/png"); + const second = file("two.png", "image/png"); + const result = screenPickedFiles([first, second], { + alreadyStaged: MAX_ATTACHMENTS_PER_MESSAGE - 1, + }); + + expect(result.accepted).toEqual([first]); + expect(result.rejected).toHaveLength(1); + expect(result.rejected[0].name).toBe("two.png"); + expect(result.rejected[0].reason).toMatch( + new RegExp(String(MAX_ATTACHMENTS_PER_MESSAGE)), + ); + }); + + /** + * THE ORDERING THE DOCSTRING NAMES, WHICH THE TEST ABOVE CANNOT SEE. + * + * "Counts the cap after kind and size" was pinned with two acceptable PNGs, so no file in it ever + * failed kind or size AND the cap — the two orderings produce identical output for that input, and + * hoisting the cap check to the top of the loop left the whole suite green. + * + * The observable difference is WHICH SENTENCE the person is given, and it only appears for a file + * that would fail both. A full composer and one SVG: judged in the documented order it is refused + * for being an SVG, which tells somebody what to do about it. Judged cap-first it is refused for a + * limit that had nothing to do with why it was never going to be accepted — and re-sending with + * fewer files would not help, because the SVG is still an SVG. + * + * The other half of the docstring — that a refused file must never eat a slot — is true under + * either ordering, because the cap counts `accepted.length`. That is exactly why the reason half + * is the one that needs a test. + */ + test("a file that fails kind is refused for its kind, not for the cap", () => { + const svg = file("logo.svg", "image/svg+xml"); + const result = screenPickedFiles([svg], { + alreadyStaged: MAX_ATTACHMENTS_PER_MESSAGE, + }); + + expect(result.accepted).toEqual([]); + expect(result.rejected).toHaveLength(1); + expect(result.rejected[0].reason).toMatch(/SVG/); + expect(result.rejected[0].reason).not.toMatch(/at most/); + }); + + /** The same, for the size ceiling: an oversized image is too large, not one file too many. */ + test("a file that fails size is refused for its size, not for the cap", () => { + const huge = file("huge.png", "image/png", MAX_IMAGE_BYTES + 1); + const result = screenPickedFiles([huge], { + alreadyStaged: MAX_ATTACHMENTS_PER_MESSAGE, + }); + + expect(result.accepted).toEqual([]); + expect(result.rejected).toHaveLength(1); + expect(result.rejected[0].reason).toMatch(/too large for an image/); + expect(result.rejected[0].reason).not.toMatch(/at most/); + }); + + test("keeps accepted files in input order when good and bad are interleaved", () => { + const good1 = file("good1.png", "image/png"); + const bad = file("bad.svg", "image/svg+xml"); + const good2 = file("good2.md", "text/plain"); + + const result = screenPickedFiles([good1, bad, good2], { + alreadyStaged: 0, + }); + + expect(result.accepted).toEqual([good1, good2]); + expect(result.rejected).toHaveLength(1); + expect(result.rejected[0].name).toBe("bad.svg"); + }); +}); diff --git a/app/src/components/channels/composer/picked-files.ts b/app/src/components/channels/composer/picked-files.ts new file mode 100644 index 000000000..0a50c1575 --- /dev/null +++ b/app/src/components/channels/composer/picked-files.ts @@ -0,0 +1,153 @@ +import { + classifyAttachment, + MAX_ATTACHMENTS_PER_MESSAGE, + MAX_FILE_BYTES, + MAX_IMAGE_BYTES, + mediaTypeOf, + namesNoFormat, +} from "@/lib/channels/attachments"; +import { newId } from "@/lib/new-id"; +import type { RejectedFile } from "./rejected-files"; + +/** + * Pure boundary between a raw file pick (drag, paste, or the file dialog) and what the composer + * is willing to stage. Nothing here touches state or the network: it only sorts files the caller + * already has in hand into what to keep and what to refuse, and why. + */ + +export type ScreenedFiles = { + accepted: File[]; + rejected: RejectedFile[]; +}; + +/** + * Kind and size are judged before the per-message cap is ever consulted, and the cap only counts + * files that already cleared both. A refused SVG (or an oversized image) must never eat a slot + * that a real, acceptable file could have used — so a file that fails kind or size is rejected + * before it can be charged against `MAX_ATTACHMENTS_PER_MESSAGE`, and the cap check only runs + * against files that made it this far. + */ +export function screenPickedFiles( + files: readonly File[], + options: { alreadyStaged: number }, +): ScreenedFiles { + const accepted: File[] = []; + const rejected: RejectedFile[] = []; + + for (const file of files) { + /* + * The same normalisation `classifyAttachment` runs, because the wording below has to be about + * the same string the kind was decided from. It used to compare `file.type` raw, so an SVG off + * a clipboard (`image/svg+xml;charset=utf-8`) was still refused — but with the generic image + * sentence rather than the one that says why this format in particular is not accepted. The + * server gives the specific reason for the same file, so the two refusals disagreed. + */ + const mediaType = mediaTypeOf(file.type); + const kind = classifyAttachment(file.type); + + if (kind === "unsupported-image") { + rejected.push( + reject( + file, + mediaType === "image/svg+xml" + ? `'${file.name}' is an SVG, which can carry scripts and is not accepted.` + : `'${file.name}' is an image format that is not supported.`, + ), + ); + continue; + } + + /* + * THE BROWSER TOLD US NOTHING, SO THE SERVER GETS TO LOOK — AND ONLY THEN. + * + * A claim that names no format is not a refusal, it is an absence: `sniffMimeType` throws + * exactly these claims away and reads the bytes, so the server accepts the plain text file + * behind an `application/octet-stream` that this screen used to turn away with "not a file type + * this chat accepts" — a sentence written by the half of the system that had not looked at it. + * + * A claim that DOES name a format is still refused here, because there the two halves already + * agree: `sniffMimeType` hands such a name straight back for the server to refuse by name. + * + * The cost, stated because it is real: a genuinely unreadable file the browser could not name + * now takes a round trip to be refused, in the server's words rather than ours. That is the + * right way round. The server has the bytes; this screen has a string somebody else wrote. + */ + const unnamed = kind === "unsupported" && namesNoFormat(mediaType); + + if (kind === "unsupported" && !unnamed) { + rejected.push( + reject(file, `'${file.name}' is not a file type this chat accepts.`), + ); + continue; + } + + if (kind === "image" && file.size > MAX_IMAGE_BYTES) { + rejected.push( + reject( + file, + `'${file.name}' is too large for an image attachment (limit ${formatBytes(MAX_IMAGE_BYTES)}).`, + ), + ); + continue; + } + + if (kind === "text" && file.size > MAX_FILE_BYTES) { + rejected.push( + reject( + file, + `'${file.name}' is too large for a text attachment (limit ${formatBytes(MAX_FILE_BYTES)}).`, + ), + ); + continue; + } + + // The LOOSE ceiling for a file nobody has named yet, because this screen does not know which + // of the two limits applies to it. Guessing the tight one would put back exactly the refusal + // this branch exists to remove; the server applies the right limit once it has the bytes. + if (unnamed && file.size > MAX_IMAGE_BYTES) { + rejected.push( + reject( + file, + `'${file.name}' is larger than the ${formatBytes(MAX_IMAGE_BYTES)} limit for an attachment.`, + ), + ); + continue; + } + + if ( + options.alreadyStaged + accepted.length >= + MAX_ATTACHMENTS_PER_MESSAGE + ) { + rejected.push( + reject( + file, + `'${file.name}' was not added: a message can carry at most ${MAX_ATTACHMENTS_PER_MESSAGE} attachments.`, + ), + ); + continue; + } + + accepted.push(file); + } + + return { accepted, rejected }; +} + +/** + * `newId()` RATHER THAN `crypto.randomUUID()`, AND THE DIFFERENCE HERE IS NOT COSMETIC. + * + * `crypto.randomUUID` exists only in a secure context. On a deployment reached at plain + * `http://
` it is not there at all, so the call does not return a worse id — it THROWS. + * The throw comes out of `screenPickedFiles`, the single door every drag, paste and file dialog + * goes through, and it takes the WHOLE PASS with it: one SVG in a drop of eight and the seven good + * files beside it are never staged either, with no chip, no refusal, and nothing on screen saying + * why the gesture did nothing. The one function whose entire purpose is that a refused file still + * gets a sentence would be the function that swallowed the drop in silence. See `lib/new-id.ts`. + */ +function reject(file: File, reason: string): RejectedFile { + return { id: newId(), name: file.name, reason }; +} + +function formatBytes(bytes: number): string { + return `${Math.round(bytes / (1024 * 1024))}MB`; +} diff --git a/app/src/components/channels/composer/queue.test.ts b/app/src/components/channels/composer/queue.test.ts index 5cd0bb03e..d466deaad 100644 --- a/app/src/components/channels/composer/queue.test.ts +++ b/app/src/components/channels/composer/queue.test.ts @@ -1,9 +1,24 @@ +import type { Attachment } from "@copilotkit/react-core/v2"; import { describe, expect, test } from "bun:test"; +import { MAX_ATTACHMENTS_PER_MESSAGE } from "@/lib/channels/attachments"; import type { ComposerDraft } from "./draft"; import { type QueuedMessage, reduceQueue } from "./queue"; -function draft(text: string, commandIds: string[] = []): ComposerDraft { - return { text, agentId: null, commandIds, isEmpty: false }; +function attachment(id: string): Attachment { + return { + id, + type: "image", + source: { type: "url", value: `https://example.com/${id}.png` }, + status: "ready", + }; +} + +function draft( + text: string, + commandIds: string[] = [], + attachments: Attachment[] = [], +): ComposerDraft { + return { text, agentId: null, commandIds, isEmpty: false, attachments }; } /** Park one message and hand back the queue it produced, which is what every case starts from. */ @@ -12,10 +27,11 @@ function park( id: string, text: string, commandIds: string[] = [], + attachments: Attachment[] = [], ): readonly QueuedMessage[] { return reduceQueue(queue, { busy: true, - draft: draft(text, commandIds), + draft: draft(text, commandIds, attachments), id, type: "submit", }).queue; @@ -62,6 +78,44 @@ describe("submitting", () => { expect(result.run?.commandIds).toEqual(["search", "summarize"]); }); + test("an idle send keeps its own @mention whether or not anything was parked", () => { + // Routing must not depend on a coincidence. The same draft, sent twice: once into an empty + // queue and once into one that happened to hold a leftover, and the coworker it is addressed + // to has to be the same coworker both times. The join used to hardcode `agentId: null`, so the + // second send silently fell back to the channel's default and the mention became decoration. + const addressed: ComposerDraft = { + ...draft("@Knowledge the Q3 file"), + agentId: "knowledge", + }; + + const alone = reduceQueue([], { + busy: false, + draft: addressed, + id: "two", + type: "submit", + }); + const joined = reduceQueue(park([], "one", "no, the other one"), { + busy: false, + draft: addressed, + id: "two", + type: "submit", + }); + + expect(alone.run?.agentId).toBe("knowledge"); + expect(joined.run?.agentId).toBe("knowledge"); + }); + + test("an idle send with no mention still lets the channel pick", () => { + const joined = reduceQueue(park([], "one", "no, the other one"), { + busy: false, + draft: draft("the Q3 file"), + id: "two", + type: "submit", + }); + + expect(joined.run?.agentId).toBeNull(); + }); + test("a send while the Bot is working waits instead of running", () => { const result = reduceQueue([], { busy: true, @@ -72,7 +126,12 @@ describe("submitting", () => { expect(result.run).toBeNull(); expect(result.queue).toEqual([ - { id: "one", text: "no, the other one", commandIds: [] }, + { + id: "one", + text: "no, the other one", + commandIds: [], + attachments: [], + }, ]); }); @@ -139,6 +198,49 @@ describe("settling", () => { ]); }); + test("a parked screenshot with no words does not become a blank line", () => { + // A message can be an attachment and nothing else — `canSendDraft` unlocks Send on attachments + // alone, so a screenshot pasted mid-turn parks with an empty text. Joining that in as a line + // opens the drained turn with a blank one, which reads as an instruction nobody typed. + const shot = attachment("screenshot"); + let queue = park([], "one", "", [], [shot]); + queue = park(queue, "two", "what is wrong with this"); + + const result = reduceQueue(queue, { type: "settle" }); + + expect(result.run?.text).toBe("what is wrong with this"); + // The file still rides along; it is the empty LINE that goes, not the message carrying it. + expect(result.run?.attachments).toEqual([shot]); + }); + + test("an empty message between two typed ones does not split them apart", () => { + let queue = park([], "one", "no, the other one"); + queue = park(queue, "two", "", [], [attachment("screenshot")]); + queue = park(queue, "three", "the Q3 file"); + + expect(reduceQueue(queue, { type: "settle" }).run?.text).toBe( + "no, the other one\nthe Q3 file", + ); + }); + + test("a drain of nothing but attachments has no text and admits it", () => { + // `isEmpty` is a claim about the words, and the drained draft used to assert `false` + // unconditionally. With nothing but a screenshot parked there are no words at all, and the + // one field that answers that question has to say so rather than repeat a constant. + const queue = park([], "one", "", [], [attachment("screenshot")]); + + const result = reduceQueue(queue, { type: "settle" }); + + expect(result.run?.text).toBe(""); + expect(result.run?.isEmpty).toBe(true); + }); + + test("a drain with words still reports itself as non-empty", () => { + const queue = park([], "one", "no, the other one"); + + expect(reduceQueue(queue, { type: "settle" }).run?.isEmpty).toBe(false); + }); + test("draining twice does not resend what has already gone", () => { const queue = park([], "one", "no, the other one"); const drained = reduceQueue(queue, { type: "settle" }); @@ -185,6 +287,67 @@ describe("removing", () => { ); }); + /* + * WHAT THIS BLOCK USED TO NOT SAY, AND WHY IT MATTERED. Every case above is about which words + * survive a removal, and none of them was about the FILES the removed message was carrying. The + * composer hands its staged attachments to the queue and takes them off its own strip in the + * same breath, so a parked message holds the only reference anything has to those rows — and + * `remove` used to drop that reference, leaving the rows staged server-side with `attachedAt + * IS NULL` until the 24-hour sweep, counting against the person's eight-per-channel limit and + * surfacing as a 409 naming files on nobody's screen. `droppedAttachments` is how the removal + * hands them back, and these are the assertions that would have caught it. + */ + test("taking a message back hands its attachments back to be released", () => { + const receipt = attachment("receipt"); + const queue = park([], "one", "here's the file", [], [receipt]); + + const result = reduceQueue(queue, { id: "one", type: "remove" }); + + expect(result.queue).toEqual([]); + expect(result.droppedAttachments).toEqual([receipt]); + }); + + test("only the removed message's attachments come back, not the ones still waiting", () => { + // The survivors are still going to be sent, so handing them over for release would delete the + // rows out from under a turn that has not run yet. + const kept = attachment("kept"); + const dropped = attachment("dropped"); + let queue = park([], "one", "keep this", [], [kept]); + queue = park(queue, "two", "drop this", [], [dropped]); + + const result = reduceQueue(queue, { id: "two", type: "remove" }); + + expect(result.droppedAttachments).toEqual([dropped]); + expect( + reduceQueue(result.queue, { type: "settle" }).run?.attachments, + ).toEqual([kept]); + }); + + test("taking back a message that carried nothing releases nothing", () => { + const queue = park([], "one", "second thoughts"); + + expect( + reduceQueue(queue, { id: "one", type: "remove" }).droppedAttachments, + ).toEqual([]); + }); + + test("a removal that missed releases nothing", () => { + // Nothing left the queue, so nothing may be deleted — a release keyed on a miss would take + // the rows off a message still sitting on screen waiting to run. + const queue = park( + [], + "one", + "here's the file", + [], + [attachment("receipt")], + ); + + expect( + reduceQueue(queue, { id: "elsewhere", type: "remove" }) + .droppedAttachments, + ).toEqual([]); + }); + test("two identical corrections are two entries and only one is taken back", () => { let queue = park([], "one", "no, the other one"); queue = park(queue, "two", "no, the other one"); @@ -192,7 +355,341 @@ describe("removing", () => { const result = reduceQueue(queue, { id: "one", type: "remove" }); expect(result.queue).toEqual([ - { id: "two", text: "no, the other one", commandIds: [] }, + { id: "two", text: "no, the other one", commandIds: [], attachments: [] }, ]); }); }); + +describe("attachments", () => { + test("a queued message's attachments join the drained draft", () => { + const file = attachment("receipt"); + const queue = park([], "one", "here's the file", [], [file]); + + const result = reduceQueue(queue, { type: "settle" }); + + expect(result.run?.attachments).toEqual([file]); + }); + + test("attachments from two queued messages land in queue order", () => { + const first = attachment("first"); + const second = attachment("second"); + let queue = park([], "one", "the first one", [], [first]); + queue = park(queue, "two", "and the second one", [], [second]); + + const result = reduceQueue(queue, { type: "settle" }); + + expect(result.run?.attachments).toEqual([first, second]); + }); + + test("an idle send that empties a queue also merges attachments in order", () => { + const queued = attachment("queued"); + const submitted = attachment("submitted"); + const waiting = park([], "one", "no, the other one", [], [queued]); + + const result = reduceQueue(waiting, { + busy: false, + draft: draft("the Q3 file", [], [submitted]), + id: "two", + type: "submit", + }); + + expect(result.run?.attachments).toEqual([queued, submitted]); + }); + + /* + * THE IDLE-SEND JOIN IS THE SECOND WAY INTO `joinQueued`, AND IT WAS ONLY EVER TESTED FOR ORDER. + * The case above says merged attachments keep their order and stops there — so every rule the + * join applies on the way out was pinned on the `settle` path alone, and an idle send that + * emptied a queue could have overrun the cap, or eaten the overflow silently, with nothing here + * to notice. Both paths produce a draft that has to be sendable, so both get asked. + */ + test("an idle send that empties a queue is capped like any other drain", () => { + // A full load parked, and one more attached to the send that joins it: nine against a cap of + // eight, assembled by a step that never re-asked. + const parked = Array.from( + { length: MAX_ATTACHMENTS_PER_MESSAGE }, + (_, index) => attachment(`parked-${index}`), + ); + const waiting = park([], "one", "the invoices", [], parked); + + const result = reduceQueue(waiting, { + busy: false, + draft: draft("and this one", [], [attachment("live")]), + id: "two", + type: "submit", + }); + + expect(result.run?.attachments).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE); + // The earliest survive, so the live send's own file is the one that goes: it arrived last. + expect(result.run?.attachments).toEqual(parked); + }); + + test("an idle send that empties a queue reports what the cap bumped off", () => { + const live = attachment("live"); + const waiting = park( + [], + "one", + "the invoices", + [], + Array.from({ length: MAX_ATTACHMENTS_PER_MESSAGE }, (_, index) => + attachment(`parked-${index}`), + ), + ); + + const result = reduceQueue(waiting, { + busy: false, + draft: draft("and this one", [], [live]), + id: "two", + type: "submit", + }); + + // Still staged server-side, so a silent slice here is a 409 on this person's next upload with + // no way back to the file that caused it. + expect(result.droppedAttachments).toEqual([live]); + }); + + test("an idle send that empties a queue under the cap reports nothing dropped", () => { + const waiting = park([], "one", "the invoices", [], [attachment("parked")]); + + const result = reduceQueue(waiting, { + busy: false, + draft: draft("and this one", [], [attachment("live")]), + id: "two", + type: "submit", + }); + + expect(result.droppedAttachments).toEqual([]); + }); + + test("a message with no attachments contributes none, leaving text-merging untouched", () => { + let queue = park([], "one", "no, the other one"); + queue = park(queue, "two", "the Q3 file"); + queue = park(queue, "three", "and skip the summary"); + + const result = reduceQueue(queue, { type: "settle" }); + + expect(result.run?.text).toBe( + "no, the other one\nthe Q3 file\nand skip the summary", + ); + expect(result.run?.attachments).toEqual([]); + }); + + test("the drained draft is capped, however many messages fed it", () => { + // The cap is checked as files are staged, one draft at a time, so three parked messages + // carrying a full load each would drain into one draft of three times the limit — a message + // this deployment does not accept, assembled by a step that never re-asked. + const staged = (message: string) => + Array.from({ length: MAX_ATTACHMENTS_PER_MESSAGE }, (_, index) => + attachment(`${message}-${index}`), + ); + let queue = park([], "one", "the invoices", [], staged("one")); + queue = park(queue, "two", "and these", [], staged("two")); + queue = park(queue, "three", "these too", [], staged("three")); + + const result = reduceQueue(queue, { type: "settle" }); + + expect(result.run?.attachments).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE); + // The earliest survive, so what is kept is what the person picked first rather than an + // arbitrary slice of a flattened list. + expect(result.run?.attachments).toEqual(staged("one")); + }); + + test("what the cap bumps off a drained turn is reported, not just dropped", () => { + // Three parked messages of a full load each: eight kept by the cap, sixteen that the cap + // re-check would otherwise erase without a trace. Those sixteen are still staged + // server-side, so losing track of them here is what turns into a confusing 409 on this + // person's next upload. + const staged = (message: string) => + Array.from({ length: MAX_ATTACHMENTS_PER_MESSAGE }, (_, index) => + attachment(`${message}-${index}`), + ); + let queue = park([], "one", "the invoices", [], staged("one")); + queue = park(queue, "two", "and these", [], staged("two")); + queue = park(queue, "three", "these too", [], staged("three")); + + const result = reduceQueue(queue, { type: "settle" }); + + expect(result.run?.attachments).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE); + expect(result.droppedAttachments).toHaveLength( + 2 * MAX_ATTACHMENTS_PER_MESSAGE, + ); + // Queue order, not the reverse: the survivors are message one's files, so the reported + // list is what message two contributed followed by what message three contributed. + expect(result.droppedAttachments).toEqual([ + ...staged("two"), + ...staged("three"), + ]); + }); + + test("a drain inside the cap reports nothing dropped", () => { + const file = attachment("receipt"); + const queue = park([], "one", "here's the file", [], [file]); + + const result = reduceQueue(queue, { type: "settle" }); + + expect(result.droppedAttachments).toEqual([]); + }); +}); + +/** + * WHAT A RUN TAKES OUT OF THE QUEUE, AND WHAT COMES BACK IF THE RUN NEVER BECOMES A MESSAGE. + * + * `droppedAttachments` covers the files a transition refused to carry. These cases are about the + * other half of the same worry: what it DID carry, and whether carrying it was the last anybody + * sees of it. A drained turn is built out of messages the composer let go of as they were parked, + * so a failed drain has to give every one of them back; a live send that joins a non-empty queue is + * built out of both kinds at once, and the composer restores its OWN draft — so re-queueing that + * one as well would send the same words twice. + * + * MESSAGES AND NOT ATTACHMENTS, which is the change these cases pin. The field used to name the + * files alone, and a bag of files with no words around them can only be deleted — which is what + * `conversation-view.tsx` did with them, destroying the rows behind a message the transcript was + * still showing. Handing back the messages is what makes restoring possible at all. + * + * The distinction is only visible here, in the transition. By the time `conversation-view.tsx` has + * a rejected promise in hand, the queue that knew where each message came from is empty. + */ +describe("restoring a failed run", () => { + test("a drain gives back every message it drained, because nothing else was holding them", () => { + const first = attachment("first"); + const second = attachment("second"); + let queue = park([], "one", "the invoices", [], [first]); + queue = park(queue, "two", "and these", [], [second]); + + const result = reduceQueue(queue, { type: "settle" }); + + // The entries themselves, words and all — not a flat list of the two files. Putting these back + // is a queue again, in the order they were typed. + expect(result.restoreIfRunFails).toEqual(queue); + }); + + test("a settle with nothing waiting restores nothing", () => { + const result = reduceQueue([], { type: "settle" }); + + expect(result.restoreIfRunFails).toEqual([]); + }); + + test("an idle send with nothing waiting restores nothing: the composer still holds its own", () => { + // THE CASE THAT MUST STAY EMPTY. The run here IS the draft in the box; a failed send hands the + // words and the chips straight back, so re-queueing them would put a second copy of the + // message on screen and send it again behind the one somebody is editing. + const own = attachment("own"); + const result = reduceQueue([], { + busy: false, + draft: draft("the Q3 file", [], [own]), + id: "one", + type: "submit", + }); + + expect(result.restoreIfRunFails).toEqual([]); + }); + + test("an idle send that empties a queue restores the parked half and not its own", () => { + const parked = attachment("parked"); + const own = attachment("own"); + const waiting = park([], "one", "no, the other one", [], [parked]); + + const result = reduceQueue(waiting, { + busy: false, + draft: draft("the Q3 file", [], [own]), + id: "two", + type: "submit", + }); + + // Both ride out on the same run — the queue was emptied to build it — but only one of them has + // nobody left to give it back. + expect(result.run?.attachments).toEqual([parked, own]); + expect(result.restoreIfRunFails).toEqual(waiting); + }); + + test("a restored message carries only the rows the run actually took", () => { + // The excess is reported through `droppedAttachments` and released there. A restored message + // still pointing at it would be a retry of a file that no longer exists. + const own = attachment("own"); + const parked = Array.from( + { length: MAX_ATTACHMENTS_PER_MESSAGE }, + (_, index) => attachment(`parked-${index}`), + ); + const waiting = park([], "one", "the invoices", [], parked); + + const result = reduceQueue(waiting, { + busy: false, + draft: draft("and this one", [], [own]), + id: "two", + type: "submit", + }); + + expect(result.droppedAttachments).toEqual([own]); + expect(result.restoreIfRunFails).toEqual(waiting); + expect( + result.restoreIfRunFails.flatMap((message) => message.attachments), + ).not.toContain(own); + }); + + test("a message the cap emptied of everything it had is not restored", () => { + // Wordless and skill-less, and every file it was carrying was bumped: there is nothing left to + // send and nothing to draw. Re-queueing it would put a blank row with a Remove button on + // screen for a message that is genuinely gone — and the files it stood for were released as + // `droppedAttachments`, with the composer saying so. + const kept = Array.from( + { length: MAX_ATTACHMENTS_PER_MESSAGE }, + (_, index) => attachment(`kept-${index}`), + ); + const bumped = attachment("bumped"); + let queue = park([], "one", "here are the invoices", [], kept); + queue = park(queue, "two", "", [], [bumped]); + + const result = reduceQueue(queue, { type: "settle" }); + + expect(result.droppedAttachments).toEqual([bumped]); + expect(result.restoreIfRunFails).toHaveLength(1); + expect(result.restoreIfRunFails[0]?.id).toBe("one"); + }); + + test("parking restores nothing: there is no run, and the queue is still holding it", () => { + const file = attachment("receipt"); + const result = reduceQueue([], { + busy: true, + draft: draft("here's the file", [], [file]), + id: "one", + type: "submit", + }); + + expect(result.run).toBeNull(); + expect(result.restoreIfRunFails).toEqual([]); + }); + + test("taking a queued message back restores nothing: those rows go out as dropped instead", () => { + const file = attachment("receipt"); + const queue = park([], "one", "here's the file", [], [file]); + + const result = reduceQueue(queue, { id: "one", type: "remove" }); + + expect(result.droppedAttachments).toEqual([file]); + expect(result.restoreIfRunFails).toEqual([]); + }); + + test("a restore puts messages back at the front, ahead of anything parked since", () => { + // THE ORDER IS THE POINT. The restored message was typed before whatever was parked while the + // failed run was out, and running a correction after the sentence correcting it is the exact + // reordering this queue exists to prevent. + const failed = attachment("failed"); + const later = attachment("later"); + const drained = park([], "one", "use the invoices", [], [failed]); + const since = park([], "two", "actually, hold on", [], [later]); + + const result = reduceQueue(since, { messages: drained, type: "restore" }); + + expect(result.queue.map((message) => message.id)).toEqual(["one", "two"]); + expect(result.run).toBeNull(); + expect(result.droppedAttachments).toEqual([]); + }); + + test("restoring nothing leaves the queue identical, so no render is spent", () => { + const queue = park([], "one", "still waiting", [], []); + + const result = reduceQueue(queue, { messages: [], type: "restore" }); + + expect(result.queue).toBe(queue); + }); +}); diff --git a/app/src/components/channels/composer/queue.ts b/app/src/components/channels/composer/queue.ts index 054fb9fb3..01cc52c26 100644 --- a/app/src/components/channels/composer/queue.ts +++ b/app/src/components/channels/composer/queue.ts @@ -1,3 +1,5 @@ +import type { Attachment } from "@copilotkit/react-core/v2"; +import { MAX_ATTACHMENTS_PER_MESSAGE } from "@/lib/channels/attachments"; import type { ComposerDraft } from "./draft"; /** @@ -29,6 +31,25 @@ import type { ComposerDraft } from "./draft"; * kept from anybody; but a queue is not an outbox and must not be read as one. It is drawn only * while a turn is in flight, so a reload finds no queue and shows none, which is better than a list * of messages quietly promising to run and never running. + * + * A DRAIN THAT DOES NOT LAND PUTS EVERYTHING BACK. Settling is keyed on the turn ending, not on the + * turn succeeding, and the same is true one level down: the send the drain starts can fail, and for + * a while what that cost was the whole queue. The messages were gone — emptied to build the draft — + * and the files under them were deleted outright, while the transcript went on showing the message + * the failed run had already added to it. So a run built out of this queue that never becomes a + * message hands its messages back, whole, through `restoreIfRunFails`; they are parked again, they + * go out with the next turn, and the only thing that deletes their rows is somebody removing one. + * The retry waits for a turn somebody asks for rather than going again on its own — see + * `conversation-view.tsx` — because a queue that re-sent itself would spin against a server that is + * refusing every request. + * + * THE FILES UNDER THOSE WORDS ARE NOT COVERED BY THAT PARAGRAPH, and reading them into it was a + * leak. Words a person watched land on screen can be retyped; a staged attachment is a row on the + * server that the parked entry holds the only reference to, and letting it die with the mount left + * it sitting with `attachedAt IS NULL` until the next day's sweep. So the mount going away releases + * them — see `conversation-view.tsx`'s teardown, which walks the queue on the way out for exactly + * this. A tab CLOSING is still the sweeper's, and honestly so: nothing in a page that is going can + * be relied on to finish a request. */ /** One message waiting for the Bot to finish, in the words the person typed. */ @@ -44,6 +65,11 @@ export type QueuedMessage = { * eventually runs rather than being silently dropped on the way through the queue. */ commandIds: string[]; + /** + * Whatever was staged on the draft when it got parked, so a file somebody attached before the + * Bot was ready still applies when the message eventually runs. + */ + attachments: Attachment[]; }; export type QueueAction = @@ -59,6 +85,19 @@ export type QueueAction = | { type: "submit"; id: string; draft: ComposerDraft; busy: boolean } /** The turn is over, however it ended: finished, failed, or stopped. */ | { type: "settle" } + /** + * A RUN THIS QUEUE EMPTIED ITSELF TO BUILD NEVER BECAME A MESSAGE. Put its messages back. + * + * The caller supplies `restoreIfRunFails` from the transition that emptied the queue — see the + * field for why that list, and not `run.attachments`, is the honest one. They go back at the + * FRONT: they were typed before anything that has parked since, and the whole reason this queue + * exists is that a correction must not be read after the sentence correcting it. + * + * It produces no run of its own. A failed send that immediately re-sent itself would spin against + * a server that is down, so restoring is where this stops and the next turn is what carries them; + * `conversation-view.tsx` holds the drain back until one starts. + */ + | { type: "restore"; messages: readonly QueuedMessage[] } /** Second thoughts, before it has run. */ | { type: "remove"; id: string }; @@ -67,6 +106,67 @@ export type QueueTransition = { queue: readonly QueuedMessage[]; /** A turn to start now, or null when there is nothing to run. */ run: ComposerDraft | null; + /** + * EVERYTHING THIS TRANSITION LET GO OF: rows that were staged server-side and that nothing on + * any screen points at any more, in the order they were parked. Empty on a transition that let + * go of nothing. + * + * Two ways in, and they are the same fact. The cap re-applied in `joinQueued` bumps the excess + * off a drained turn; a `remove` takes back a whole parked message and everything it was + * carrying with it. Either way the composer already dropped its own reference when the message + * was parked, so this list is the last one, and losing it loses the rows. + * + * A file somebody attached and never saw again is the exact failure this feature exists to + * avoid, so neither path can just slice the excess away and say nothing. And the cost of + * staying quiet outlives the draft: the rows stay staged with `attachedAt IS NULL`, counting + * against that person's per-channel limit until the 24-hour sweep, and surface as a confusing + * 409 on their next upload with no way back to the files that caused it. + * + * THE CALLER IS WHAT ACTS ON IT. This is the transition naming them — so a caller can release + * them through `DELETE /api/attachments/:id` and say so on screen — not doing either itself. + * See `conversation-view.tsx`, which is the only caller that produces this from a real queue. + */ + droppedAttachments: readonly Attachment[]; + /** + * OF WHAT `run` IS CARRYING, THE PART NOTHING ELSE IS HOLDING — as the messages it came from, so + * a caller whose run never becomes a message can put them back rather than having to decide what + * to do with a bag of orphaned files. Empty when there is no run, and empty when everything on it + * has somewhere to return to. + * + * IT USED TO BE THE ATTACHMENTS ALONE, AND THAT SHAPE ONLY ALLOWED ONE ANSWER. A list of files + * with no words around them cannot be re-queued — nothing says which message each belonged to, + * what was typed beside it, or which `/` skills it was invoked with — so the only thing a caller + * could do with it was delete the rows, and that is what `conversation-view.tsx` did. Deleting + * them is wrong for the reason the whole area keeps rediscovering: `channel-chat.tsx` adds the + * user message to the transcript BEFORE the run, and nothing removes it when the run fails, so + * the release destroyed the files behind a message that is still on screen. Handing back the + * messages instead makes restoring possible, and restoring is what the caller now does. + * + * `droppedAttachments` is about files this transition REFUSED to carry; this is about what it DID + * carry, named against the possibility that carrying it turns out to have been the last anybody + * sees of it. The two never overlap: an attachment is either kept by the cap or bumped by it — + * which is also why the messages here carry only the SURVIVORS. A restored message pointing at a + * row the cap already gave back would be a retry of a file that no longer exists. + * + * WHY THE QUEUE HAS TO ANSWER THIS AND NOT THE CALLER. A drained turn is built out of messages + * the composer let go of as they were parked, so nothing but this queue ever held them; a live + * send joining a non-empty queue is built out of BOTH — the parked messages, held by nobody now + * that the queue has emptied, and the draft in the box, which the composer puts back beside the + * restored words when the send fails. Restoring the second kind would put a message back in the + * queue whose words and chips are also sitting in the composer, and send it twice. Only the + * transition knows which message came from where, so it is the transition that says. + * + * Which makes the answer per-case rather than "everything on the run": + * - `settle` — every message in it, the whole run came out of the queue. + * - `submit` joining a non-empty queue — the parked ones only; the live draft is the composer's + * to restore. + * - `submit` with nothing waiting — none, the run IS the live draft. + * - `remove`, a park, and a `restore` — none, there is no run. + * + * THE CALLER IS WHAT ACTS ON IT, and only on failure. See `conversation-view.tsx`, where both + * paths that can produce a run answer this list from the same rule. + */ + restoreIfRunFails: readonly QueuedMessage[]; }; /** @@ -93,20 +193,87 @@ export function reduceQueue( */ if (!action.busy) { if (queue.length === 0) { - return { queue, run: action.draft }; + // Nothing stranded: the run IS the draft in the box, and a send that fails hands its + // words and its chips straight back to the composer they came from. + return { + queue, + run: action.draft, + droppedAttachments: [], + restoreIfRunFails: [], + }; } - return { - queue: [], - run: joinQueued([ + /* + * ADDRESSED TO WHOEVER THE LIVE DRAFT IS ADDRESSED TO. This send is going out now, from a + * composer with a caret in it, so its `@mention` is a live routing decision and not a + * leftover — and the branch directly above, the same send with nothing parked behind it, + * honours it. Joining used to hardcode `null` here, which made `@Knowledge` mean one thing + * or the other depending on whether anything happened to be waiting, a coincidence nobody + * typing can see. The parked messages have no say: `QueuedMessage` carries no `agentId` at + * all, for the reason `joinQueued` records. + */ + const joined = joinQueued( + [ ...queue, { id: action.id, text: action.draft.text, commandIds: [...action.draft.commandIds], + attachments: [...action.draft.attachments], }, - ]), + ], + action.draft.agentId, + ); + /* + * THE PARKED HALF OF WHAT THIS RUN IS CARRYING, AND ONLY THAT HALF. The queue is emptied + * here, so nothing holds the parked messages any more; the live draft's words, chips and + * attachments are still the composer's, which puts them back beside each other when the + * send fails. Restoring those as well would queue a second copy of a message somebody can + * already see in their box. + * + * The messages that were waiting BEFORE this send joined them, rather than a slice of what + * went out: the cap keeps the earliest, so the survivors happen to be the parked ones first + * today, and a rule that read that off a slice would quietly go wrong the day the ordering + * does. `carrying` filters by identity against what the join actually kept. + */ + return { + queue: [], + run: joined.draft, + droppedAttachments: joined.dropped, + restoreIfRunFails: carrying(queue, joined.draft.attachments), }; } + /* + * PARKED, WITH ITS FILES — AND THE FILES LEAVE ONE MORE PLACE THAN THE WORDS DO. + * + * The composer clears its own strip as this message is parked, which is what makes the words + * look like they landed. It empties something else at the same time: the number the CLIENT'S + * per-message cap is counted against. `stagedCount` in `composer.tsx` resyncs from the strip + * on every commit, so after a park it reads zero. The SERVER'S cap counts a different set — + * every row this person has staged in this composer's `uploadGroup` with `attachedAt IS NULL` + * — and a parked row is exactly that until the drained turn is sent. The two therefore + * disagree for the length of the turn: pick a ninth file behind eight parked ones and + * `screenPickedFiles` accepts it, the upload goes out, and the server answers 409. + * + * WHICH IS NOT THE FAILURE `uploadGroup` WAS ADDED TO REMOVE, and the difference decides what + * this is worth. That one was a 409 naming rows on NOBODY'S screen — a closed tab's, a + * stopped run's — unreachable by the person holding them, with 24 hours of locked uploads in + * that channel before the sweeper freed the count. These rows are on screen and are theirs to + * act on: `chat-transcript.tsx` draws every parked attachment as a tile under its queued line, + * taking the message back releases them (see `remove` below), and the drain stamps + * `attachedAt` and frees the group. The server's sentence is true when it arrives — there + * really are eight waiting to send. What is wrong is only WHO gets to say no, and how long it + * takes: a refusal that should be instant and in the client's own words costs a round trip and + * arrives in the server's. + * + * IT CANNOT BE CLOSED FROM THIS FILE, and the two ways it looks like it could are both worse. + * Releasing the rows as the message parks would delete files somebody is still waiting to + * send, which is the opposite of what every other release here is for. Re-applying the cap on + * the way IN would bound the queue — it cannot exceed the cap anyway, because the server + * refuses a ninth upload into one group — without changing the one number that decides + * whether a pick is accepted, so the 409 would arrive exactly as before. That number is + * `composer.tsx`'s, built from its own strip, and the queue is not something it can see; the + * fix is one addend on that side, counting what is parked alongside what is staged. + */ return { queue: [ ...queue, @@ -114,29 +281,139 @@ export function reduceQueue( id: action.id, text: action.draft.text, commandIds: [...action.draft.commandIds], + attachments: [...action.draft.attachments], }, ], run: null, + droppedAttachments: [], + restoreIfRunFails: [], + }; + } + + case "restore": { + if (action.messages.length === 0) { + return { + queue, + run: null, + droppedAttachments: [], + restoreIfRunFails: [], + }; + } + return { + // At the front: these were typed before anything that has parked while the failed run was + // out, and the order somebody typed in is the order the Bot has to read. + queue: [...action.messages, ...queue], + run: null, + droppedAttachments: [], + restoreIfRunFails: [], }; } case "settle": { if (queue.length === 0) { - return { queue, run: null }; + return { + queue, + run: null, + droppedAttachments: [], + restoreIfRunFails: [], + }; } - return { queue: [], run: joinQueued(queue) }; + /* + * Nobody in particular. A drain has no live draft behind it — every message in it was parked + * minutes ago into a conversation already pinned to one coworker — so there is no mention + * here to honour and none is invented. + */ + const joined = joinQueued(queue, null); + return { + queue: [], + run: joined.draft, + droppedAttachments: joined.dropped, + /* + * ALL OF THEM. Every message in this drain was parked, which means the composer let go of + * its words and its attachments at the time, and the queue has just emptied itself to build + * this. There is no box for a failed drain to put anything back into — so the queue is the + * box, and a failure puts them back in it. + */ + restoreIfRunFails: carrying(queue, joined.draft.attachments), + }; } case "remove": { - const kept = queue.filter((message) => message.id !== action.id); + const removed = queue.filter((message) => message.id === action.id); + if (removed.length === 0) { + // The same array, not an equal one: a removal that missed must not cost a re-render, and + // it must not release anything either — nothing left the queue, so every row here still + // belongs to a message sitting on screen waiting to run. + return { + queue, + run: null, + droppedAttachments: [], + restoreIfRunFails: [], + }; + } + /* + * WHATEVER IT WAS CARRYING GOES BACK, because this is the last reference to it. The composer + * hands its staged attachments over as the message is parked and calls `removeAttachment` + * on its own strip in the same breath, so from that moment the queue is the only thing + * holding them. Dropping the entry without saying so left the rows staged server-side with + * nothing on any screen pointing at them until the 24-hour sweep — counting against that + * person's per-channel limit the whole time, and surfacing as a 409 naming files they have + * no way back to. The composer's own strip already answers this exact gesture by releasing + * the row behind a removed chip; this is the queue answering it the same way. + */ return { - queue: kept.length === queue.length ? queue : kept, + queue: queue.filter((message) => message.id !== action.id), run: null, + droppedAttachments: removed.flatMap((message) => message.attachments), + restoreIfRunFails: [], }; } } } +/** + * The messages a run is carrying, carrying only the rows it actually took. + * + * ONE RULE FOR BOTH PATHS THAT CAN PRODUCE A RUN, which is the whole reason this is a function. A + * drain and a live send joining a queue answer `restoreIfRunFails` from the same question — of the + * messages the queue gave up, what is restorable — and two call sites each filtering for themselves + * is two places for the cap's survivors and the cap's casualties to be confused with one another. + * + * BY IDENTITY AGAINST WHAT THE JOIN KEPT, not by count or by position. The cap keeps the earliest + * files, so a restored message may have handed over three attachments and get one back; the other + * two were bumped, reported through `droppedAttachments`, and released by the caller as the run was + * built. Restoring those would put a message back pointing at rows that no longer exist. + * + * A MESSAGE LEFT WITH NOTHING AT ALL IS NOT RESTORED. A wordless, skill-less message whose only + * attachments the cap bumped has nothing left to send and nothing to show: re-queueing it would + * draw a blank row with a Remove button and no content, for a message that is genuinely gone. + */ +function carrying( + queue: readonly QueuedMessage[], + kept: readonly Attachment[], +): readonly QueuedMessage[] { + const survivors = new Set(kept); + return queue + .map((message) => ({ + ...message, + attachments: message.attachments.filter((attachment) => + survivors.has(attachment), + ), + })) + .filter( + (message) => + message.text.trim().length > 0 || + message.commandIds.length > 0 || + message.attachments.length > 0, + ); +} + +/** The draft a drain produces, plus whatever the cap re-check would not let it keep. */ +type Joined = { + draft: ComposerDraft; + dropped: Attachment[]; +}; + /** * Everything waiting, as the one turn it is about to become. * @@ -144,21 +421,62 @@ export function reduceQueue( * together into a paragraph invents a sentence nobody wrote; keeping the line breaks keeps them as * lines of a single instruction, which is how a burst of corrections reads out loud anyway. * - * Never empty. The composer refuses an empty draft before it reaches the queue, so a drained turn - * always has something in it to send. + * WORDLESS MESSAGES CONTRIBUTE NO LINE. A message here is not obliged to have any text: a pasted + * screenshot with nothing typed beside it is the whole point of `canSendDraft` unlocking on + * attachments alone, and one parked mid-turn arrives with `text: ""`. Joining that in the way the + * rest are joined opens the drained turn with a blank line, or splits two corrections apart with + * one, and a blank line is an instruction nobody wrote. The message still counts for everything + * else it carries — its files and its skills go in exactly as they would have. + * + * Which means the joined text CAN be empty, and `isEmpty` has to be computed rather than asserted: + * a drain of nothing but screenshots has no words in it, and a field that claims otherwise is a + * field no reader can trust for the case it exists to answer. + * + * `agentId` IS THE CALLER'S TO SUPPLY, and it is the one field of the joined draft that nothing in + * the queue can answer. A `QueuedMessage` does not carry one: a message parked mid-turn lands in a + * conversation already pinned to one coworker for the life of its thread, so there is nothing an + * `@` could change, and the text of the mention stays in the words, where it was typed and where + * it still reads as addressed. A LIVE draft joining the queue on its way out is the other case + * entirely — it is being sent this instant and its mention routes — so which answer applies is + * decided at each call site rather than assumed to be `null` here. */ -function joinQueued(queue: readonly QueuedMessage[]): ComposerDraft { +function joinQueued( + queue: readonly QueuedMessage[], + agentId: string | null, +): Joined { + /* + * A parked message carries whatever the sender had staged when they parked it, in queue + * order. Dropping them here would lose a file somebody attached before the Bot was ready. + * + * THE CAP HAS TO BE RE-APPLIED ON THE WAY OUT. It is checked as files are staged, against one + * draft at a time; joining three parked messages of eight files each would produce a single + * draft of twenty-four, which is a message this deployment does not accept and no later check + * would catch. The earliest files win, the same order-of-arrival rule `screenPickedFiles` + * applies within one draft, so the survivors are the ones the person picked first. That split + * is not free, though: the excess is still staged server-side and nothing that survives this + * function points at it any more, so `dropped` is what lets a caller give the rows back and say + * which files went — instead of the person finding out from a 409 on their next upload with no + * way back to what caused it. + */ + const flattened = queue.flatMap((message) => message.attachments); + const attachments = flattened.slice(0, MAX_ATTACHMENTS_PER_MESSAGE); + const dropped = flattened.slice(MAX_ATTACHMENTS_PER_MESSAGE); + + const text = queue + .map((message) => message.text) + .filter((line) => line.trim().length > 0) + .join("\n"); + return { - text: queue.map((message) => message.text).join("\n"), - /* - * Nothing routes on a mention here. A queued message lands in a conversation already pinned to - * one coworker for the life of its thread, so there is nothing an `@` could change; the text of - * the mention stays in the words, where it was typed and where it still reads as addressed. - */ - agentId: null, - // The same skill queued twice is still one instruction. Sending it twice would put the same - // paragraph in front of the Bot two times and say nothing new by doing it. - commandIds: [...new Set(queue.flatMap((message) => message.commandIds))], - isEmpty: false, + draft: { + text, + agentId, + // The same skill queued twice is still one instruction. Sending it twice would put the + // same paragraph in front of the Bot two times and say nothing new by doing it. + commandIds: [...new Set(queue.flatMap((message) => message.commandIds))], + isEmpty: text.length === 0, + attachments, + }, + dropped, }; } diff --git a/app/src/components/channels/composer/rejected-files.tsx b/app/src/components/channels/composer/rejected-files.tsx new file mode 100644 index 000000000..ee0c1513a --- /dev/null +++ b/app/src/components/channels/composer/rejected-files.tsx @@ -0,0 +1,100 @@ +import { IconX } from "@tabler/icons-react"; + +import { Collapse } from "./collapse"; + +/** + * One line per refusal, not one message for the batch: dropping two bad files onto the composer at + * once must produce two reasons, because folding them into a single string ("2 files were + * rejected") or overwriting one reason with the next reports one refusal for two problems, and the + * person can no longer tell which file failed for which reason. + */ +/** + * `id` exists because `name` cannot be the key: two files sharing a name is the ordinary case here + * (drag one `screenshot.png` from two different folders), not a contrived one, and this component + * exists so that two refusals produce two lines rather than one collapsing into the other. Minting + * that identity is the caller's job — whatever builds this list owes it an `id` per entry. + */ +export type RejectedFile = { id: string; name: string; reason: string }; + +/** + * Purely presentational: the composer decides which files to refuse and why (see + * `shared/attachments.ts` for the limits behind those reasons); this only renders the list it is + * handed. `role="alert"` matches every other error line in this app (standing instructions, channel + * pin errors): a refusal is something the person needs to notice. + */ +export function RejectedFiles({ + onDismiss, + rejected, +}: { + /** + * Required rather than optional, because a list with no way out is the defect this argument + * closes. A refusal is the only thing on this composer with no natural end: an attachment leaves + * when it is sent or removed, typed words leave when they are sent, and a reason for a file that + * never made it in has neither. Sending clears these too (see `submitDraft`), but somebody who + * drops an SVG and then walks away should not have to send a message to be rid of the sentence + * about it. + */ + onDismiss: () => void; + rejected: readonly RejectedFile[]; +}) { + return ( + /* + * Collapsed rather than switched on and off, because this sits directly against the composer: + * its arrival and its dismissal each move the box somebody is typing in, and it is the one + * thing here that appears without being asked for — the worst kind of thing to have jump. + * + * The ALERT unmounts the moment it is dismissed rather than fading, because a `role="alert"` + * left in the tree is still an alert: invisible to the eye, still there to a screen reader, and + * still there to a test asking whether the refusal is gone. `Collapse` is built for exactly + * that — it keeps the height it measured while open, so the empty box still closes over it. + */ + /* + * THE GAP IS `pb-2` ON THE MEASURED BOX, AND IT USED TO BE `mb-2` ON THE ALERT, WHICH IS NOT + * THE SAME THING HERE. + * + * `Collapse` animates to `content.offsetHeight`, and `offsetHeight` is the border-box height: + * padding counts, margins do not. The content wrapper has no border and no padding of its own, + * so the alert's bottom margin collapsed straight out of the number being measured — the box + * settled 8px short and the gap between the refusals and the composer directly under them was + * never drawn. `AttachmentStrip` spends `pb-3` inside its own `Collapse` for exactly this + * reason, and this is the same bargain. + * + * On the wrapper rather than on the alert, unlike the strip's: the alert has a dashed border, + * so padding spent inside it would land within that border instead of under it — a taller + * dashed box rather than a gap below one. + */ + 0}> + {rejected.length > 0 ? ( +
+
+ {rejected.map((file) => ( +

+ {file.name}: {file.reason} +

+ ))} +
+ {/* + * One button for the block, not one per line. These arrive together — a drop of eight + * files refuses several at once — and are read together, so dismissing them one at a + * time is work without a purpose. + */} + +
+ ) : null} + + ); +} diff --git a/app/src/components/channels/conversation-view.tsx b/app/src/components/channels/conversation-view.tsx index dd28b8662..603059be5 100644 --- a/app/src/components/channels/conversation-view.tsx +++ b/app/src/components/channels/conversation-view.tsx @@ -1,4 +1,5 @@ import type { Message } from "@ag-ui/core"; +import type { Attachment } from "@copilotkit/react-core/v2"; import { type ReactNode, useCallback, @@ -12,10 +13,12 @@ import { type CommandOption, Composer, type ComposerDraft, + type DroppedAttachments, type QueueAction, type QueuedMessage, reduceQueue, } from "@/components/channels/composer"; +import { attachmentUrl } from "@/lib/channels/attachments"; import { newId } from "../../lib/new-id"; export function ConversationView({ @@ -24,6 +27,7 @@ export function ConversationView({ notice, agents = [], commands, + channelId, disabled = false, pending = false, autoFocus = false, @@ -43,6 +47,12 @@ export function ConversationView({ * The `/` menu for this Bot's granted skills, supplied by the route that owns grant loading. */ commands?: readonly CommandOption[]; + /** + * The channel this conversation belongs to, forwarded to the composer so it can upload + * attachments to it. Omitted by a caller with no channel yet — `/channel/new` creates one on + * first send — which leaves the composer exactly as it behaved before attachments existed. + */ + channelId?: string; disabled?: boolean; /** * A turn is in flight: the Bot has been asked something and has not come back yet. @@ -96,6 +106,27 @@ export function ConversationView({ const [queued, setQueued] = useState([]); const queuedRef = useRef(queued); + /** + * Files the queue left behind AND the reason it did, forwarded to the composer so it can say so + * rather than leaving somebody to notice on their own that a file they attached is not going to + * be sent. See `reduceQueue`'s `droppedAttachments` for the two ways a row gets here. + * + * Saying so is only half of it, and `releaseStagedAttachment` below is the other half: these rows + * are still staged server-side, and telling somebody about them without giving them back is a + * warning shipped alongside its own cause. + * + * The cause travels with them because `reduceQueue` cannot supply it — it reports the same + * `Attachment[]` whether the files were bumped off a merged draft by the cap or carried out of the + * queue by a message somebody removed, and the composer's sentence for one is false about the + * other. + * + * Only ever set, never appended to: each transition is its own event, and the composer is what + * turns a new object here into new rejection lines, one batch per event, rather than this file + * accumulating a list nobody here has any other use for. + */ + const [droppedAttachments, setDroppedAttachments] = + useState(); + /** * A turn this screen started and has not seen finish. * @@ -125,14 +156,121 @@ export function ConversationView({ * The ref is what the decisions read. React state is a render behind, and both callers below have * to know what is actually queued at the moment they are called rather than at the moment they * were last rendered — one of them is an effect firing on the same commit that emptied the list. + * + * THE WHOLE TRANSITION COMES BACK, NOT JUST THE RUN. It used to hand back `next.run` alone, which + * was every caller's whole interest until a failed run became something either of them had to + * answer for: a run can only be put back if you know which of the messages in it the queue + * contributed, and that is on the transition beside it. Recomputing it out here would mean + * re-deriving from a queue this function has already emptied. */ const apply = useCallback((action: QueueAction) => { const next = reduceQueue(queuedRef.current, action); queuedRef.current = next.queue; setQueued(next.queue); - return next.run; + // Only on an actual drop, so a settle or a remove that let go of nothing does not hand the + // composer a fresh empty array it has no reason to react to. + if (next.droppedAttachments.length > 0) { + /* + * THE ACTION IS WHAT SAYS WHICH OF THE TWO CAUSES THIS WAS, and this is the only place that + * has both halves. `reduceQueue` reports the same `Attachment[]` whether the files were + * bumped off a merged draft by the cap or carried out of the queue by a message somebody + * removed, so the cause cannot be recovered downstream — and the composer's sentence for one + * of them is false about the other. + */ + for (const attachment of next.droppedAttachments) { + releaseStagedAttachment(attachment); + } + setDroppedAttachments({ + attachments: next.droppedAttachments, + cause: + action.type === "remove" + ? "queued-message-removed" + : "merged-over-cap", + }); + } + return next; }, []); + /** + * WALKING AWAY WITH SOMETHING STILL PARKED IS THE THIRD WAY A ROW LOSES ITS LAST REFERENCE, and + * until this it was the one way that said nothing and gave nothing back. + * + * IT USED TO BE THE FOURTH, and the one that went was a way a row should never have lost its last + * reference at all: a drained turn whose send failed used to release everything it was carrying. + * That one is now a restore — the messages go back in the queue, still holding their rows — so + * the ways out are the two in `apply` above, a removal and the cap's excess, and this. + * + * The other two go through `apply` above. This one goes through React: + * `queue.ts` is candid that the queue "lives and dies with the component holding it" and that + * switching channels "takes anything parked in it with it" — but that paragraph is about the + * person's WORDS, which they watched land on screen and can retype. It was never a statement + * about the staged rows underneath them, and `releaseStagedAttachment` is explicit that a parked + * entry holds the only reference anything has to those. + * + * WHAT IT COSTS TO SKIP, stated because it is smaller than the other two and the fix should be + * priced honestly: the upload cap is scoped by `uploadGroup`, minted per composer mount, so this + * orphan does not refuse anybody's next pick the way a removal's would — the composer that staged + * it is gone and its group with it. It is storage held for up to a day by + * `cull-staged-attachments.ts`, not a 409. Worth releasing anyway, because the row is bytes in a + * table nobody will ever ask for again and the release is two lines. + * + * BEST-EFFORT IN THE STRICTEST SENSE. This runs during teardown, so the requests go out into a + * component that is already gone and nothing here could act on an answer even in principle — + * which is exactly what `releaseStagedAttachment` already is. A tab CLOSING is not this path at + * all and is not chased: the page is going, `fetch` on the way out is not reliable, and the + * sweeper is the honest answer for that one. + * + * The ref rather than the state, for the reason `apply` records: on an unmount that follows a + * transition in the same commit, the state is a render behind and the ref is not. Empty on + * StrictMode's development double-mount, which makes that pass a no-op. + */ + useEffect( + () => () => { + for (const message of queuedRef.current) { + for (const attachment of message.attachments) { + releaseStagedAttachment(attachment); + } + } + }, + [], + ); + + /** + * A RUN BUILT OUT OF THIS QUEUE FAILED, SO THE DRAIN WAITS FOR A TURN BEFORE TRYING AGAIN. + * + * Without it the restore below is a spin. The drain effect reads `queuedRef` rather than the + * state, so the restored messages are visible to it the instant `apply` writes them — and the + * commit that clears `running` schedules that effect with no ordering guarantee against the + * rejection that restores. The two land in either order, so on a server that is refusing every + * request the queue could be re-sent immediately, fail, restore, and be re-sent again, as fast as + * the round trip allows. A retry the person did not ask for is not a retry, it is a loop. + * + * A REF AND NOT STATE, for the reason `queuedRef` is one: the effect that reads this runs on the + * commit that clears `running`, and a state update made in the rejection is a render behind. It + * also must not itself cause a render — nothing on screen changes when a queue is held back; the + * entries are drawn as parked either way. + */ + const heldBack = useRef(false); + + /** + * PUT A FAILED RUN'S MESSAGES BACK WHERE THEY CAME FROM, and hold the drain until somebody asks. + * + * The one answer to a failed send, used by both paths that can produce a run. See the drain's + * `catch` for why restoring rather than releasing, and `heldBack` for why the hold. + */ + const restoreFailedRun = useCallback( + (messages: readonly QueuedMessage[]) => { + if (messages.length === 0) { + return; + } + // Before the restore, not after: the effect reads both off refs, and the guard has to be + // true by the moment the queue is non-empty again rather than one statement later. + heldBack.current = true; + apply({ messages, type: "restore" }); + }, + [apply], + ); + const start = async (draft: ComposerDraft) => { setRunning(true); try { @@ -151,15 +289,44 @@ export function ConversationView({ */ const submit = useCallback( (draft: ComposerDraft, whileBusy: boolean) => { - const run = apply({ + const next = apply({ busy: whileBusy, draft, id: newId(), type: "submit", }); - return run ? startRef.current(run) : undefined; + if (!next.run) { + return undefined; + } + const started = startRef.current(next.run); + /* + * A SEND THAT TOOK THE QUEUE WITH IT AND THEN FAILED LEAVES THE PARKED HALF HELD BY NOBODY, + * which is the same shape the drain effect below answers, and with the same line. + * + * The ordinary send is not this. Its run IS the draft in the box, and the composer's `catch` + * puts those words and those chips straight back — so `restoreIfRunFails` is empty for it and + * this branch never runs. It is only the join, where `reduceQueue` empties the queue into an + * outgoing draft, that produces a run carrying messages the composer never had. + * + * WHICH MESSAGES IS THE QUEUE'S ANSWER AND NOT ONE COMPUTED HERE. By the time this rejects, + * the queue that knew where each message came from is empty; `restoreIfRunFails` was decided + * on the transition that emptied it. Restoring the composer's own as well would queue a + * second copy of the words that are back in somebody's box. + * + * `started` IS WHAT GOES BACK TO THE COMPOSER, not the promise this `catch` derives from it. + * The composer needs the rejection to restore the words, so the failure must still be its to + * handle; the derived promise exists only to hang the restore off, is settled by the `catch` + * itself, and is deliberately dropped. + */ + if (next.restoreIfRunFails.length > 0) { + const carried = next.restoreIfRunFails; + void started.catch(() => { + restoreFailedRun(carried); + }); + } + return started; }, - [apply], + [apply, restoreFailedRun], ); /** @@ -182,21 +349,63 @@ export function ConversationView({ * queue that drained anyway would post one more user turn into a channel the screen has already * said is finished. The cost is that anything parked when that happens stays on screen unrun, * under a notice that explains why, which is the honest half of the trade. + * + * AND IT REFUSES ONCE A DRAIN HAS FAILED, until a turn starts. See `heldBack`. */ useEffect(() => { - if (disabled || inFlight || queuedRef.current.length === 0) { + if (inFlight) { + /* + * A TURN STARTING IS WHAT LETS A HELD-BACK QUEUE GO AGAIN, and it is the only thing that + * does. Every way a turn starts is somebody asking for one — a send, a parked message joined + * to it, a button inside a rendered card — so the retry is always something a person did, + * never this effect trying again on its own. + */ + heldBack.current = false; + return; + } + if (disabled || heldBack.current || queuedRef.current.length === 0) { return; } - const run = apply({ type: "settle" }); - if (!run) { + const next = apply({ type: "settle" }); + if (!next.run) { return; } - void startRef.current(run).catch(() => { - // Swallowed on purpose, and only here. A failed send from the composer throws so the composer - // can put the words back in the box; there is no box to put these back into, and the screen - // already reports a failed turn through its own notice. + const carried = next.restoreIfRunFails; + void startRef.current(next.run).catch(() => { + /* + * Swallowed on purpose, and only here. A failed send from the composer throws so the composer + * can put the words back in the box; the box for these is the queue they came out of, and the + * screen already reports the failed turn through its own notice. + * + * IT USED TO DELETE THE STAGED ROWS INSTEAD, AND THAT WAS DATA LOSS. The reasoning was that + * the queue had emptied to build this draft and nothing retried it, so the rows behind + * `run.attachments` were referenced by nothing and might as well be given back rather than + * waiting for the sweep. The second half of that sentence was never true: `channel-chat.tsx` + * adds the user message to the transcript BEFORE the run and leaves it there when the run + * fails, so the files were referenced by a message the person is looking at. Releasing them + * emptied the tiles under a message that stayed on screen, with nothing said and no way back. + * + * SO THE MESSAGES GO BACK IN THE QUEUE, WHOLE. Their words, their `/` chips and the rows the + * run actually carried return as parked entries — visible in the transcript, carried by the + * next turn, and released only if somebody takes one back by hand. Deleting on an explicit + * removal is the one gesture that has ever justified it; a run that failed is not that. + * + * `restoreIfRunFails` RATHER THAN THE QUEUE THIS DRAINED, WHICH FOR A DRAIN IS NEARLY THE + * SAME LIST AND IS NOT THE SAME CLAIM. The cap may have bumped attachments off the joined + * draft on the way out, and `apply` released those as the run was built; restoring the + * original entries would re-queue messages pointing at rows that are gone. Spelling it as the + * queue's own answer is also what lets `submit` — where the two are further apart still — use + * the identical line. + * + * NOT INSIDE `start`, AND THAT IS LOAD-BEARING. `start` is also called from `submit`, where + * the promise goes back to the composer and the composer restores its own draft. Restoring + * everything a run carries from inside `start` would queue a second copy of the words that + * are back in the box, so the decision belongs to the call sites — this one, and the join in + * `submit`, which answers the same question from the same list. + */ + restoreFailedRun(carried); }); - }, [apply, disabled, inFlight]); + }, [apply, disabled, inFlight, restoreFailedRun]); return (
@@ -228,10 +437,12 @@ export function ConversationView({ { @@ -249,6 +460,30 @@ export function ConversationView({ * turn instead of joining the queue. */ pending={inFlight} + /* + * THE HALF OF THE PER-MESSAGE CAP THE COMPOSER CANNOT SEE. + * + * The composer counts what is on its own strip, and parking a message empties it. The + * server counts every row this person has staged in that composer's `uploadGroup` that + * has not been sent, and parking a message changes nothing about those: `attachedAt` is + * written when the message really goes. So without this the two disagree by exactly the + * size of the queue, and a ninth pick behind eight parked files is accepted here and + * refused on arrival — a 409 the person was given no chance to avoid, phrased as a count + * against a strip they can see is empty. + * + * READ OFF THE QUEUE ON EVERY RENDER RATHER THAN ACCUMULATED, which is what makes it fall + * as well as rise. A running total would be right until somebody took a parked message + * back, and then it would hold slots that nothing occupies — the same refusal, now issued + * by their own client with no round trip to blame it on. + * + * `queued` and not `queuedRef`: this is a render, so the state is the value React is + * drawing from, and the ref exists for the callbacks that cannot wait a render. They + * agree here anyway — `apply` writes both on the same line. + */ + queuedAttachmentCount={queued.reduce( + (total, message) => total + message.attachments.length, + 0, + )} /* * The caller's answer, not `inFlight`. `running` is true from the instant `start` is * entered, which is before `onSubmit` has done anything at all, so a Stop drawn from @@ -261,3 +496,43 @@ export function ConversationView({
); } + +/** + * GIVE BACK THE STAGED ROW BEHIND AN ATTACHMENT THE QUEUE HAS LET GO OF. + * + * The composer hands its staged attachments to the queue as a message is parked and clears its own + * strip in the same breath, so a parked entry holds the only reference anything has to those rows. + * When `reduceQueue` reports one in `droppedAttachments` — a message taken back before it ran, or + * the excess the cap re-check bumped off a drained turn — this is the last chance to release it. + * + * WHAT NOT DOING THIS COSTS. The row stays with `attachedAt IS NULL` until `cull-staged- + * attachments.ts` sweeps it a day later. Until then the upload handler counts it against this + * person's per-channel limit and refuses their ninth pick by naming files that are on nobody's + * screen, with no way back to them. For the cap case that is the exact 409 the on-screen notice + * exists to pre-empt, so leaving the row behind shipped the warning together with its cause. + * + * ONLY A ROW THAT EXISTS. `metadata.attachmentId` is written by `attachmentsConfigFor` from the + * upload response, so an attachment that never finished uploading carries none and there is + * nothing to delete — the same test `composer.tsx` applies before its own DELETE. + * + * BEST-EFFORT, AND DELIBERATELY UNINSPECTED, for the reason `composer.tsx` records against the + * same request: the person did not ask for this, so neither of the two answers it can fail with — + * 404 for a non-uploader, 409 for an attachment already sent — is anything they could act on. + * + * THE SECOND CALLER OF THIS ENDPOINT, and deliberately not shared with the first. `composer.tsx` + * owns the identical two lines for its own strip; a common helper is the right end state and is a + * change to a file this one does not own, so the duplication is written down here rather than + * reached across for. + */ +function releaseStagedAttachment(attachment: Attachment): void { + const metadata = attachment.metadata as + | { attachmentId?: unknown } + | undefined; + if (typeof metadata?.attachmentId !== "string") { + return; + } + void fetch(attachmentUrl(metadata.attachmentId), { + method: "DELETE", + credentials: "include", + }).catch(() => undefined); +} diff --git a/app/src/lib/channels/attachments.ts b/app/src/lib/channels/attachments.ts new file mode 100644 index 000000000..7ff8b2f0b --- /dev/null +++ b/app/src/lib/channels/attachments.ts @@ -0,0 +1,23 @@ +/** + * The attachment limits and classification rules, re-exported from the one place they are declared. + * + * `shared/` is where the server checks the same limits, so a change to a number or a rule changes + * both sides at once. This file exists so the browser code keeps importing through `@/`, and so the + * path to `shared/` is written down once rather than in every composer file that needs it. + */ +export { + ACCEPTED_IMAGE_MIME, + ACCEPTED_TEXT_MIME, + type AttachmentKind, + type AttachmentPart, + type AttachmentSource, + attachmentUrl, + classifyAttachment, + MAX_ATTACHMENTS_PER_MESSAGE, + MAX_EXTRACTED_CHARACTERS, + MAX_FILE_BYTES, + MAX_IMAGE_BYTES, + mediaTypeOf, + namesNoFormat, + shouldClaimPaste, +} from "../../../../shared/attachments"; diff --git a/app/src/routes/__root.tsx b/app/src/routes/__root.tsx index e87526062..8a15bc800 100644 --- a/app/src/routes/__root.tsx +++ b/app/src/routes/__root.tsx @@ -3,6 +3,7 @@ import { Navigate, Outlet, } from "@tanstack/react-router"; +import { useEffect } from "react"; import { ThemeProvider } from "@/components/theme-provider"; import { TooltipProvider } from "@/components/ui/tooltip"; import type { RouterContext } from "../router-context"; @@ -18,7 +19,105 @@ export const Route = createRootRouteWithContext()({ notFoundComponent: () => , }); +/** + * A FILE DROPPED ON A PART OF THIS APP THAT WANTED NO FILE DOES NOT UNLOAD THIS APP. + * + * The browser's default for a file dropped on a document is to NAVIGATE THE TOP-LEVEL DOCUMENT TO + * THAT FILE. Not "ignore it" — replace the page with it. The single-page app unloads and everything + * held in memory goes with it: the sentence somebody was typing, the parked queue on a channel + * (whose teardown effect in `conversation-view.tsx` never runs, because the document is REPLACED + * rather than unmounted, so the rows behind those parked messages stay orphaned until the 24-hour + * sweep), every open dialog, the socket. The person sees their raw PNG on a blank page and presses + * Back. Nothing in the app has to be wrong for this to happen; it is what a browser does when + * nobody claims a drop. + * + * WHY THE ROOT, AND NOT EACH SURFACE THAT MIGHT BE AIMED AT. The composer already guards its own + * form — see `refuseDragOver` in `composer.tsx` — and that comment argued, correctly, that the + * app-wide guard could not be installed by a leaf: several composers can be mounted at once, so a + * `document` listener owned by one of them would be installed once per composer and torn down by + * whichever unmounted first. The leaf approach also cannot reach every surface even in principle. + * The onboarding poster (`_authed/onboarding.tsx`) wraps its decorative composer in + * `pointer-events-none` on purpose, so that composer can never receive a drop at all: the event + * goes straight past it to the document. So do the transcript, the sidebar, the page margin, and + * every screen that will ever be added without thinking about drag and drop. The root is the one + * place where "once" is a fact rather than a hope: `RootComponent` mounts once per app. + * + * WHAT MAKES THIS SAFE TO INSTALL APP-WIDE — `defaultPrevented` IS NOT A HEURISTIC, IT IS THE + * BROWSER'S OWN PREDICATE. An element becomes a drop target only by calling `preventDefault` on + * `dragover`; a handler that does not do that has already declined the drop as far as the browser + * is concerned. This listener is on `document` in the BUBBLE phase, so it runs after every handler + * in the tree has had the event, and it acts only on an event nobody prevented. A future drop + * target therefore cannot be swallowed by this: the very line it must write to work at all is the + * line that makes this guard stand down. Capture phase would be the opposite and would break every + * drop in the app, which is why the phase is load-bearing rather than incidental. + * + * `dropEffect = "none"` ON THE UNCLAIMED `dragover`, for the reason `refuseDragOver` records: a + * prevented `dragover` left at its default effect draws the copy-badge cursor, promising to accept + * a file that is about to be dropped into nothing. "none" is the no-entry cursor and it is the only + * part of this refusal the person sees before they let go. + * + * THIS IS A FLOOR, NOT AN ANSWER. Refusing the drop is all it does — nothing is said, because at + * this level there is nothing true to say: the guard does not know what the person was aiming at. + * A surface that wants to explain itself claims the drop and writes its own sentence, which is what + * the composer does with `RejectedFiles`. Silence about a file nobody can place is a poorer outcome + * than an explanation and a far better one than losing the page. + * + * Exported for `app/tests/root-drop-guard.test.tsx`, which mounts it on its own rather than + * standing up a router. + */ +export function useUnclaimedDropGuard() { + useEffect(() => { + const refuse = (event: DragEvent) => { + // Somebody in the tree already claimed it: the composer's container, or any drop target + // added later. Returning here is what keeps this guard from becoming the thing that breaks + // them. + if (event.defaultPrevented) { + return; + } + /* + * FILES ONLY, AND THIS NARROWING IS THE DIFFERENCE BETWEEN A FLOOR AND A WRECKING BALL. + * + * An editable element — a text input, or the composer's own contenteditable editor — is a + * drop target the BROWSER makes, with no script calling `preventDefault` anywhere. So a + * guard that refused every unclaimed drop would refuse dragging a selected phrase into the + * message box, which is a gesture people use and which nothing in this app would have been + * left to re-implement. + * + * `types` carrying "Files" is how a drag says it holds files rather than text, and it is set + * on `dragover` as well as on `drop` — the browser deliberately exposes the kinds before it + * exposes the contents. A file is also the only payload worth this guard: dropping one is + * what unloads the app. + * + * THE LIMIT THIS LEAVES, NAMED RATHER THAN HIDDEN. A LINK dragged out of another tab and let + * go on the page margin still navigates, because its drag carries no file and refusing it + * here would also refuse dropping that link into the editor as text. Files are the case that + * costs somebody their unsent message; a dragged link is deliberate and rare. + */ + if (!event.dataTransfer?.types.includes("Files")) { + return; + } + event.preventDefault(); + if (event.type === "dragover") { + event.dataTransfer.dropEffect = "none"; + } + }; + /* + * Both, and both are needed. `dragover` is what stops the browser treating the document as a + * plain navigation target and is what fixes the cursor; `drop` is the event that actually + * carries the file, and a browser that never saw a prevented `dragover` — a drop that arrived + * some other way, or a `dragover` an extension stopped — would still navigate on it. + */ + document.addEventListener("dragover", refuse); + document.addEventListener("drop", refuse); + return () => { + document.removeEventListener("dragover", refuse); + document.removeEventListener("drop", refuse); + }; + }, []); +} + function RootComponent() { + useUnclaimedDropGuard(); return (
diff --git a/app/src/routes/_authed/onboarding.tsx b/app/src/routes/_authed/onboarding.tsx index 5c666b685..e1d58b98b 100644 --- a/app/src/routes/_authed/onboarding.tsx +++ b/app/src/routes/_authed/onboarding.tsx @@ -36,6 +36,21 @@ function WelcomeStep() {
+ {/* + * A POSTER OF A COMPOSER, AND `pointer-events-none` IS WHAT MAKES IT ONE. Nothing here is + * meant to be typed in, clicked or dropped on: it is a picture of the thing the person is + * about to get, shown while they read a sentence about it. + * + * THE DROP THAT PASSES STRAIGHT THROUGH IT IS SOMEBODY ELSE'S TO CATCH, WHICH IS WORTH + * SAYING OUT LOUD. The composer guards its own form against a dropped file navigating the + * whole app away (`refuseDragOver` in `composer.tsx`), and that guard cannot fire here: an + * element with no pointer events is never the target of the drop, so the event goes past it + * to the document as if this composer were not on the page. What catches it is + * `useUnclaimedDropGuard` in `routes/__root.tsx`, which refuses every drop nobody claimed — + * the reason that guard lives at the root rather than in the composer, and the reason this + * wrapper does not need to change to be safe. Taking `pointer-events-none` off to "fix" the + * drop would turn the poster back into a live composer with nowhere to upload to. + */}
{ // response is not `ok`. A 500 with no body is the shape a broken server actually sends, and is // exactly what `client()`'s fallback message path exists for. global.fetch = (async () => - new Response(null, { status: 500 })) as typeof fetch; + new Response(null, { status: 500 })) as unknown as typeof fetch; }); afterEach(() => { @@ -313,7 +313,8 @@ test("both /agents sections hold a skeleton while the roster is pending, not an // A fetch that never settles is `isPending` forever — the state each section's loading arm // renders once the router has finished its own (also async) initial match, which is why this // still waits rather than reading `view.container` on the very next line. - global.fetch = (() => new Promise(() => {})) as typeof fetch; + global.fetch = (() => + new Promise(() => {})) as unknown as typeof fetch; const view = renderAgents(failingQueryClient()); diff --git a/app/tests/auth-queries.test.ts b/app/tests/auth-queries.test.ts index c28e15cba..cd9113294 100644 --- a/app/tests/auth-queries.test.ts +++ b/app/tests/auth-queries.test.ts @@ -3,5 +3,11 @@ import { authKeys, currentUserQueryOptions } from "../src/lib/auth/queries"; test("uses a stable key for the current authenticated user", () => { expect(authKeys.currentUser()).toEqual(["auth", "current-user"]); - expect(currentUserQueryOptions().queryKey).toEqual(["auth", "current-user"]); + // Spread before comparing: `queryOptions` brands its key with TanStack's `DataTag` + // phantom symbols, which carry the result and error types and exist only in the type + // system. No literal can satisfy that brand, so the elements are what get compared. + expect([...currentUserQueryOptions().queryKey]).toEqual([ + "auth", + "current-user", + ]); }); diff --git a/app/tests/channel-chat.test.ts b/app/tests/channel-chat.test.ts new file mode 100644 index 000000000..f6fec705b --- /dev/null +++ b/app/tests/channel-chat.test.ts @@ -0,0 +1,196 @@ +import type { Attachment } from "@copilotkit/react-core/v2"; +import { describe, expect, test } from "bun:test"; +import { toMessageContent } from "@/components/channels/channel-chat"; +import { attachmentUrl } from "@/lib/channels/attachments"; + +/** + * `toMessageContent` is the wire format every attachment message is built from — the contract + * between the composer and everything downstream: the AG-UI schema, the server's + * `resolveAttachmentParts`, the transcript projection. It is module-private in `channel-chat.tsx` + * and exported there only for this test; see the comment above its definition for why a narrow + * export was the honest call rather than standing up the whole `useAgent` runtime to reach it + * through `say`/`deliver`. + * + * No DOM is registered here: `toMessageContent` is a pure function, and importing the module that + * defines it does not touch `document`/`window` at import time (only rendering `ChannelChat` itself + * would), so this file needs none of the `happy-dom` scaffolding a rendering test would. + */ + +/** A minimal but complete SDK `Attachment`, overridable per test. */ +function attachment( + overrides: Partial & { id: string }, +): Attachment { + return { + type: "image", + source: { type: "url", value: "https://example.test/placeholder" }, + status: "ready", + metadata: { attachmentId: overrides.id }, + ...overrides, + }; +} + +describe("toMessageContent", () => { + test("text only is sent as a plain string, not a wrapped array", () => { + const content = toMessageContent("hello there", []); + + // `typeof` rather than `toEqual("hello there")`: the whole point is that this is a string and + // not a one-element array that happens to stringify the same way in some assertions. + expect(typeof content).toBe("string"); + expect(content).toBe("hello there"); + }); + + test("text plus one image is [text, ref], in that order", () => { + const image = attachment({ + id: "att_img_1", + type: "image", + metadata: { attachmentId: "att_img_1", filename: "photo.png" }, + }); + + const content = toMessageContent("check this out", [image]); + + expect(Array.isArray(content)).toBe(true); + const parts = content as unknown[]; + expect(parts).toHaveLength(2); + expect(parts[0]).toEqual({ type: "text", text: "check this out" }); + expect(parts[1]).toEqual({ + type: "image", + source: { type: "url", value: attachmentUrl("att_img_1") }, + metadata: { attachmentId: "att_img_1", filename: "photo.png" }, + }); + }); + + test("one image with no text has no leading empty text part", () => { + const image = attachment({ + id: "att_img_2", + type: "image", + metadata: { attachmentId: "att_img_2" }, + }); + + const content = toMessageContent("", [image]); + + expect(Array.isArray(content)).toBe(true); + const parts = content as unknown[]; + // Just the ref: an empty leading text part would be noise the model has to read past. + expect(parts).toHaveLength(1); + expect(parts[0]).toEqual({ + type: "image", + source: { type: "url", value: attachmentUrl("att_img_2") }, + metadata: { attachmentId: "att_img_2" }, + }); + }); + + test("a document attachment carries type: document", () => { + const document = attachment({ + id: "att_doc_1", + type: "document", + metadata: { attachmentId: "att_doc_1", filename: "report.pdf" }, + }); + + const content = toMessageContent("", [document]); + + const parts = content as { type: string }[]; + expect(parts).toHaveLength(1); + expect(parts[0]?.type).toBe("document"); + }); + + /* + * WHAT IS WRITTEN HERE IS WHAT EVERY LATER RENDER READS, so a modality decided from the browser's + * claim is wrong for ever rather than until the next reload. + * + * `attachment.type` is fixed before the upload and never reconciled with what the file turned out + * to be. The server already refuses to trust it — `resolvePart` classifies on its own sniffed + * `mimeType` — but that correction stays on the server and never comes back to the stored message. + * A screenshot the browser mislabelled therefore drew a grey file card over the picture. + */ + test("a mislabelled image is stored as an image, because the bytes say so", () => { + const mislabelled = attachment({ + id: "att_mislabelled", + // What the browser claimed. + type: "document", + // What the server sniffed, which is the only one of the two that saw the bytes. + source: { + type: "url", + value: "https://example.test/placeholder", + mimeType: "image/png", + }, + metadata: { attachmentId: "att_mislabelled", filename: "screenshot" }, + }); + + const content = toMessageContent("", [mislabelled]); + + const parts = content as { type: string }[]; + expect(parts[0]?.type).toBe("image"); + }); + + /* + * The other direction, which is what keeps the rule honest rather than merely image-favouring: a + * corroborated text type overrides a browser claim of `image` just as readily. + */ + test("a mislabelled document is stored as a document, for the same reason", () => { + const mislabelled = attachment({ + id: "att_doc_sniffed", + type: "image", + source: { + type: "url", + value: "https://example.test/placeholder", + mimeType: "text/csv", + }, + metadata: { attachmentId: "att_doc_sniffed", filename: "rows" }, + }); + + const content = toMessageContent("", [mislabelled]); + + const parts = content as { type: string }[]; + expect(parts[0]?.type).toBe("document"); + }); + + /* + * AND WHEN NOTHING CORROBORATED IT, THE CLAIM STANDS. A `data` source's `mimeType` is `file.type` + * — the same claim, wearing the field name of an answer — so only a `url` source, which has been + * past the server, is read. An attachment that somehow reaches here unuploaded is written exactly + * as it always was. + */ + test("with no corroborated type, the declared one is kept", () => { + const unsniffed = attachment({ + id: "att_unsniffed", + type: "document", + source: { type: "url", value: "https://example.test/placeholder" }, + metadata: { attachmentId: "att_unsniffed" }, + }); + + const content = toMessageContent("", [unsniffed]); + + const parts = content as { type: string }[]; + expect(parts[0]?.type).toBe("document"); + }); + + test("the ref's id comes from metadata.attachmentId, not attachment.id", () => { + const image = attachment({ + // The SDK's own client-side upload placeholder — must never reach the wire. + id: "client-placeholder-xyz", + type: "image", + // The id this deployment actually stored the file under. + metadata: { attachmentId: "server-stored-id-123" }, + }); + + const content = toMessageContent("", [image]); + + const [ref] = content as { source: { value: string } }[]; + expect(ref?.source.value).toContain("server-stored-id-123"); + expect(ref?.source.value).not.toContain("client-placeholder-xyz"); + }); + + test("the ref's url is built by attachmentUrl, matching /api/attachments/ exactly", () => { + const image = attachment({ + id: "ignored-client-id", + type: "image", + metadata: { attachmentId: "att_url_check" }, + }); + + const content = toMessageContent("", [image]); + + const [ref] = content as { source: { value: string } }[]; + expect(ref?.source.value).toBe(attachmentUrl("att_url_check")); + expect(ref?.source.value).toBe("/api/attachments/att_url_check"); + }); +}); diff --git a/app/tests/channel-event-patch.test.ts b/app/tests/channel-event-patch.test.ts index 7dba1dbef..cc850820d 100644 --- a/app/tests/channel-event-patch.test.ts +++ b/app/tests/channel-event-patch.test.ts @@ -16,11 +16,13 @@ function channel( agentIds: [], threadId: `thread-${id}`, active: true, + summary: null, lastMessage: null, lastMessageAt: null, lastMessageAgentId: null, createdAt: "2024-01-01T00:00:00.000Z", pinned: false, + lastReadAt: null, ...overrides, }; } diff --git a/app/tests/channel-menu-mutations.test.ts b/app/tests/channel-menu-mutations.test.ts index ddc191023..dd79240e1 100644 --- a/app/tests/channel-menu-mutations.test.ts +++ b/app/tests/channel-menu-mutations.test.ts @@ -37,12 +37,26 @@ function invalidationRecorder() { return { queryClient, invalidated }; } +/* + * The context TanStack Query hands a mutation callback alongside its variables. These tests drive + * the callbacks directly rather than through a MutationObserver, so they have to supply it. Both + * fields are the real thing rather than a stand-in: `meta` is undefined exactly as it is for a + * mutation declared without one, and `mutationKey` is optional and genuinely absent, because none + * of these options factories sets one. + */ +function mutationContext(queryClient: QueryClient) { + return { client: queryClient, meta: undefined }; +} + test("pinning PUTs the flag to the channel's pin route and invalidates the roster", async () => { const seen = capturingFetch(200, { pinned: true }); const { queryClient, invalidated } = invalidationRecorder(); const options = setChannelPinnedMutationOptions(queryClient); - await options.mutationFn?.({ channelId: "channel-1", pinned: true }); + await options.mutationFn?.( + { channelId: "channel-1", pinned: true }, + mutationContext(queryClient), + ); await options.onSuccess?.( undefined as never, { channelId: "channel-1", pinned: true }, @@ -62,7 +76,7 @@ test("deleting sends DELETE to the channel route and invalidates the roster", as const { queryClient, invalidated } = invalidationRecorder(); const options = deleteChannelMutationOptions(queryClient); - await options.mutationFn?.("channel-1"); + await options.mutationFn?.("channel-1", mutationContext(queryClient)); await options.onSuccess?.( undefined as never, "channel-1", @@ -84,7 +98,9 @@ test("a refused delete surfaces the server's sentence", async () => { const { queryClient } = invalidationRecorder(); const options = deleteChannelMutationOptions(queryClient); - await expect(options.mutationFn?.("channel-1")).rejects.toThrow( + await expect( + options.mutationFn?.("channel-1", mutationContext(queryClient)), + ).rejects.toThrow( "This channel is defined by the deployment package, so it cannot be deleted here.", ); }); @@ -118,8 +134,8 @@ test("marking read PUTs the read route and patches lastReadAt in place", async ( } satisfies InfiniteData); const options = markChannelReadMutationOptions(queryClient); - options.onMutate?.("channel-1"); - await options.mutationFn?.("channel-1"); + options.onMutate?.("channel-1", mutationContext(queryClient)); + await options.mutationFn?.("channel-1", mutationContext(queryClient)); expect(seen).toHaveLength(1); expect(seen[0]?.url).toBe("/api/channels/channel-1/read"); @@ -163,7 +179,7 @@ test("a message stamped by a clock ahead of ours still reads as seen after marki } satisfies InfiniteData); const options = markChannelReadMutationOptions(queryClient); - options.onMutate?.("channel-1"); + options.onMutate?.("channel-1", mutationContext(queryClient)); const patched = queryClient.getQueryData>( channelKeys.list(), diff --git a/app/tests/chat-messages.test.ts b/app/tests/chat-messages.test.ts index 028c18a7b..dbef194f7 100644 --- a/app/tests/chat-messages.test.ts +++ b/app/tests/chat-messages.test.ts @@ -1,4 +1,4 @@ -import type { Message } from "@ag-ui/core"; +import type { Message, ToolCall } from "@ag-ui/core"; import { describe, expect, test } from "bun:test"; import { toVisibleChatItems } from "../src/components/channels/chat-messages"; @@ -70,17 +70,16 @@ describe("toVisibleChatItems", () => { // The roles this file already understood, so the addition above is not paid for elsewhere. test("still pairs a tool call with the result that answers it", () => { + const toolCall: ToolCall = { + id: "call-1", + type: "function", + function: { name: "botActivity", arguments: '{"days":7}' }, + }; const called: Message = { id: "assistant-2", role: "assistant", content: "", - toolCalls: [ - { - id: "call-1", - type: "function", - function: { name: "botActivity", arguments: '{"days":7}' }, - }, - ], + toolCalls: [toolCall], }; const answered: Message = { id: "result-1", @@ -93,7 +92,7 @@ describe("toVisibleChatItems", () => { { kind: "tool", id: "call-1", - toolCall: called.toolCalls?.[0], + toolCall, result: "42", }, ]); @@ -166,4 +165,888 @@ describe("toVisibleChatItems", () => { expect(toVisibleChatItems([bad])).toEqual([]); } }); + + /* + * THE SAME DEFENCE, ONE LEVEL DOWN, AND THE CASE ABOVE WAS NOT IT. Non-array content never + * reaches the part loop at all, so it proved nothing about what the loop does with a part that + * is not a part: `[null]` was read for `.type` and `[{ type: "image" }]` for `.source.type`, and + * either one threw a TypeError out of `toVisibleChatItems` — which runs inside `ChatTranscript`'s + * own render, so the throw did not spoil one row, it UNMOUNTED THE WHOLE CHANNEL VIEW. One + * malformed turn anywhere in a channel's history took the conversation with it. + * + * Every shape here is one a live turn can carry: content arrays skip the schema the stored ones + * are parsed by, so a part with a missing `source`, a source with no `value`, or a hole in the + * array is only ever a bad producer away. + */ + test("drops a malformed part instead of throwing out of render", () => { + const malformed: unknown[][] = [ + [null], + [undefined], + [42], + ["a bare string"], + [{}], + [{ type: "image" }], + [{ type: "document" }], + [{ type: "image", source: null }], + [{ type: "image", source: { type: "url" } }], + [{ type: "text" }], + ]; + + for (const content of malformed) { + const bad = { + id: "user-bad", + role: "user", + content, + } as unknown as Message; + expect(() => toVisibleChatItems([bad])).not.toThrow(); + expect(toVisibleChatItems([bad])).toEqual([]); + } + }); + + /* + * And the turn is not thrown away wholesale either: the good part beside the bad one still + * draws, keeping its own index, because that index is what the render key is built from. + */ + test("keeps the sound parts of a turn that also carries a malformed one", () => { + const mixed = { + id: "user-6", + role: "user", + content: [ + null, + { type: "text", text: "the second one is fine" }, + { + type: "image", + source: { type: "url", value: "/api/attachments/att-7" }, + metadata: { attachmentId: "att-7", filename: "shot.png" }, + }, + ], + } as unknown as Message; + + expect(toVisibleChatItems([mixed])).toEqual([ + { + kind: "attachments", + id: "user-6:attachments", + attachments: [ + { + // The PART's index, counted over the whole array — the malformed hole included, since + // dropping it from the count would renumber every file after it. + id: "user-6:2", + attachmentId: "att-7", + url: "/api/attachments/att-7", + filename: "shot.png", + modality: "image", + }, + ], + }, + { + kind: "text", + id: "user-6", + role: "user", + text: "the second one is fine", + }, + ]); + }); + + /* + * Every stored channel message is a plain string, never the array form a live composer produces. + * Adding array-content handling below must not so much as touch this path. + */ + test("projects a string-content user message exactly as before", () => { + const said: Message = { + id: "user-1", + role: "user", + content: "Here is the screenshot you asked for.", + }; + + expect(toVisibleChatItems([said])).toEqual([ + { + kind: "text", + id: "user-1", + role: "user", + text: "Here is the screenshot you asked for.", + }, + ]); + }); + + /* + * The regression this task fixes. A screenshot pasted with no caption is a content array holding + * only an attachment part, so the joined text is empty — the old code returned `[]` for the whole + * message, and a bare screenshot showed as though nothing had been sent. + */ + test("does not drop a user turn that is only an attachment, with no caption", () => { + const screenshotOnly: Message = { + id: "user-2", + role: "user", + content: [ + { + type: "image", + source: { type: "url", value: "/api/attachments/att-1" }, + metadata: { attachmentId: "att-1", filename: "screenshot.png" }, + }, + ], + } as unknown as Message; + + expect(toVisibleChatItems([screenshotOnly])).toEqual([ + { + kind: "attachments", + id: "user-2:attachments", + attachments: [ + { + id: "user-2:0", + attachmentId: "att-1", + url: "/api/attachments/att-1", + filename: "screenshot.png", + modality: "image", + }, + ], + }, + ]); + }); + + // ONE ROW FOR ALL OF THEM, and the caption after it. Three files used to be three stacked rows + // each as wide as the transcript; they are drawn as a single row of thumbnails now, so they are a + // single item to lay out, to animate and to anchor the scroller on. + test("gathers a turn's files into one row, above the caption", () => { + const captioned: Message = { + id: "user-3", + role: "user", + content: [ + { type: "text", text: "Two files, see attached." }, + { + type: "image", + source: { type: "url", value: "/api/attachments/att-2" }, + metadata: { attachmentId: "att-2", filename: "photo.jpg" }, + }, + { + type: "document", + source: { type: "url", value: "/api/attachments/att-3" }, + metadata: { attachmentId: "att-3" }, + }, + ], + } as unknown as Message; + + expect(toVisibleChatItems([captioned])).toEqual([ + { + kind: "attachments", + id: "user-3:attachments", + attachments: [ + { + // The PART's index, so it does not shift when the caption above it is added or removed. + id: "user-3:1", + attachmentId: "att-2", + url: "/api/attachments/att-2", + filename: "photo.jpg", + modality: "image", + }, + { + id: "user-3:2", + attachmentId: "att-3", + url: "/api/attachments/att-3", + modality: "document", + }, + ], + }, + { + kind: "text", + id: "user-3", + role: "user", + text: "Two files, see attached.", + }, + ]); + }); + + // A data-sourced part never reaches the browser as a stored message, but must not throw either. + test("skips an attachment part whose source is not a url", () => { + const dataSourced: Message = { + id: "user-4", + role: "user", + content: [ + { + type: "image", + source: { type: "data", value: "aGVsbG8=", mimeType: "image/png" }, + }, + ], + } as unknown as Message; + + expect(toVisibleChatItems([dataSourced])).toEqual([]); + }); + + // No attachmentId in metadata: falls back to the trailing path segment of the url. + test("derives the attachment id from the url when metadata carries none", () => { + const noMetadata: Message = { + id: "user-5", + role: "user", + content: [ + { + type: "document", + source: { type: "url", value: "/api/attachments/att-9" }, + }, + ], + } as unknown as Message; + + expect(toVisibleChatItems([noMetadata])).toEqual([ + { + kind: "attachments", + id: "user-5:attachments", + attachments: [ + { + id: "user-5:0", + attachmentId: "att-9", + url: "/api/attachments/att-9", + modality: "document", + }, + ], + }, + ]); + }); + + /* + * THE SAME DEFENCE THE USER BRANCH HAS, ON THE BRANCH BESIDE IT. A live assistant turn skips the + * schema exactly as a live user turn does, and this branch trusted `content` to be a string on + * the strength of the type alone: `if (message.content)` is TRUE for `[]` and for `{}`, so both + * were passed down as the `text` of a text item and handed to the markdown renderer, which reads + * them as a string and throws. A throw here unmounts the channel view, same as the user branch's + * did — the two are one flatMap apart. + */ + test("drops an assistant turn whose content is not words", () => { + for (const content of [ + [], + {}, + 42, + [{ type: "text", text: "hi" }], + ] as unknown[]) { + const bad = { + id: "assistant-bad", + role: "assistant", + content, + } as unknown as Message; + expect(() => toVisibleChatItems([bad])).not.toThrow(); + expect(toVisibleChatItems([bad])).toEqual([]); + } + }); + + /* + * And the calls beside it, which are read three fields deep — `toolCall.function.name` — off + * whatever the run put in the array. A hole in it, or a call with no function, threw before the + * transcript could draw a single row. + */ + test("skips a malformed tool call rather than throwing", () => { + const malformed: unknown[] = [ + null, + undefined, + 42, + {}, + { id: "call-x" }, + { id: "call-x", function: null }, + { id: "call-x", function: {} }, + ]; + + for (const toolCall of malformed) { + const bad = { + id: "assistant-bad", + role: "assistant", + content: "", + toolCalls: [toolCall], + } as unknown as Message; + expect(() => toVisibleChatItems([bad])).not.toThrow(); + expect(toVisibleChatItems([bad])).toEqual([]); + } + }); + + test("keeps a sound tool call beside a malformed one", () => { + const mixed = { + id: "assistant-5", + role: "assistant", + content: "", + toolCalls: [ + null, + { + id: "call-4", + type: "function", + function: { name: "botActivity", arguments: "{}" }, + }, + ], + } as unknown as Message; + + expect(toVisibleChatItems([mixed]).map((item) => item.id)).toEqual([ + "call-4", + ]); + }); + + // `toolCalls` that is not a list at all is not iterable, and `for...of` says so by throwing. + test("survives a toolCalls that is not a list", () => { + const bad = { + id: "assistant-6", + role: "assistant", + content: "still talking", + toolCalls: { id: "call-5" }, + } as unknown as Message; + + expect(toVisibleChatItems([bad])).toEqual([ + { + kind: "text", + id: "assistant-6", + role: "assistant", + text: "still talking", + }, + ]); + }); + + /* + * A HOLE IN THE MESSAGE ARRAY, WHICH IS THE ONE THIS FILE HAD NOT DEFENDED. + * + * Every guard above is about a bad PART, or bad `toolCalls`, inside a message that is itself an + * object. A `null` MESSAGE throws earlier than any of them: `isToolResult` reads `message.role` + * in the results-gathering loop that runs before the projection begins, so the whole array is + * lost — not the one hole in it — and none of the per-message care below ever gets to run. + * + * Same stakes as the rest: `toVisibleChatItems` runs inside `ChatTranscript`'s render, so the + * TypeError escapes into React and the channel view unmounts. A history with one hole in it is a + * blank screen instead of a conversation with one turn missing. + * + * `undefined` beside `null` because a sparse array and a dropped element produce different holes + * and only one of them is `null`. + */ + test("drops a hole in the message array instead of throwing out of render", () => { + for (const hole of [null, undefined]) { + const messages = [hole, PROSE] as unknown as Message[]; + + expect(() => toVisibleChatItems(messages)).not.toThrow(); + // And the sound message beside the hole still projects: the hole costs itself and nothing + // else, which is the whole point of skipping it rather than bailing on the array. + expect(toVisibleChatItems(messages)).toEqual([ + { + kind: "text", + id: PROSE.id, + role: "assistant", + text: PROSE.content as string, + }, + ]); + } + }); + + /* + * A hole where a TOOL RESULT would have been, which is the other half of the same loop. + * + * The results pass and the projection pass walk the same array, so a guard added to only one of + * them moves the throw rather than removing it. This pins that a hole sitting beside a real + * call-and-result pair costs neither of them: the call still finds its answer. + */ + test("a hole beside a tool result still lets the call find its answer", () => { + const messages = [ + null, + { + id: "assistant-7", + role: "assistant", + toolCalls: [ + { + id: "call-1", + type: "function", + function: { name: "search", arguments: "{}" }, + }, + ], + }, + { role: "tool", toolCallId: "call-1", content: "the answer" }, + ] as unknown as Message[]; + + expect(toVisibleChatItems(messages)).toEqual([ + { + kind: "tool", + id: "call-1", + toolCall: { + id: "call-1", + type: "function", + function: { name: "search", arguments: "{}" }, + }, + result: "the answer", + }, + ]); + }); + + /* + * `metadata` IS READ OFF THE SAME UNVALIDATED ARRAY AS EVERYTHING ELSE HERE, and was the one + * field taken on trust. It does not throw — `?.` covers a null, and a string or a number yields + * `undefined` for both keys — so the damage is quieter than the crashes the guards above exist + * for: a number lands in `attachmentId`, which is DECLARED `string` and compared for identity by + * `sameAttachmentRow`, and in `filename`, which reaches `title={filename}` and an `alt` template. + * + * A number is used rather than an object because it is the shape that survives furthest: `?.` + * and the truthiness check at the `filename` spread both wave it through. + */ + test("ignores metadata fields that are not strings", () => { + const wrong: Message = { + id: "user-meta", + role: "user", + content: [ + { + type: "image", + source: { type: "url", value: "/api/attachments/att-9" }, + metadata: { attachmentId: 42, filename: 99 }, + }, + ], + } as unknown as Message; + + expect(toVisibleChatItems([wrong])).toEqual([ + { + kind: "attachments", + id: "user-meta:attachments", + attachments: [ + { + id: "user-meta:0", + // Fell back to the url rather than carrying the number: the field is named for an id + // and a number is not one. + attachmentId: "att-9", + url: "/api/attachments/att-9", + // Absent, not `99`. A tile draws "Untitled file" for a name it does not have, which is + // honest; drawing a number is not. + modality: "image", + }, + ], + }, + ]); + }); + + /* + * A metadata that is not an object at all takes the same route, rather than the cast's route. + * The cast said `{ attachmentId?: string } | undefined` about whatever was there, and a string + * `metadata` has an `attachmentId` of `undefined` only by luck of it not being an array index. + */ + test("ignores a metadata that is not an object", () => { + for (const metadata of ["att-nope", 7, [], true]) { + const odd: Message = { + id: "user-odd", + role: "user", + content: [ + { + type: "document", + source: { type: "url", value: "/api/attachments/att-10" }, + metadata, + }, + ], + } as unknown as Message; + + expect(toVisibleChatItems([odd])).toEqual([ + { + kind: "attachments", + id: "user-odd:attachments", + attachments: [ + { + id: "user-odd:0", + attachmentId: "att-10", + url: "/api/attachments/att-10", + modality: "document", + }, + ], + }, + ]); + } + }); + + /* + * THE URL FALLBACK IS AN ID, NOT THE LAST PATH SEGMENT. + * + * `url.split("/").at(-1)` kept everything after the last slash, query string and fragment + * included, so `/api/attachments/?v=2` produced `"?v=2"` in a field named for an id. + * Nothing renders it today, which is exactly why it is worth pinning: it is a wrong value + * sitting quietly in a typed field, waiting for the first reader that builds a url back out of + * it — and the server's own `attachmentIdFor` slices a browser-supplied url the same way, so + * this is the shape of mistake this field is downstream of. + */ + test("the url fallback for an attachment id drops a query string and a fragment", () => { + const urls = [ + "/api/attachments/att-11?v=2", + "/api/attachments/att-11#page=3", + "/api/attachments/att-11?v=2#page=3", + ]; + + for (const url of urls) { + const part: Message = { + id: "user-q", + role: "user", + content: [{ type: "document", source: { type: "url", value: url } }], + } as unknown as Message; + + const [item] = toVisibleChatItems([part]); + expect(item).toMatchObject({ kind: "attachments" }); + /* + * Narrowed on the discriminant rather than cast to a hand-written shape. The projection + * returns `readonly SentAttachment[]`, and the old cast both dropped that `readonly` and + * re-declared a two-field subset of the row — so it would have gone on compiling through a + * rename of any field it did not happen to mention. + */ + if (item.kind !== "attachments") { + throw new Error(`expected an attachments item, got ${item.kind}`); + } + expect(item.attachments[0].attachmentId).toBe("att-11"); + // The url itself is untouched — it is what the tile fetches, and the query string may well + // be load-bearing to whoever put it there. + expect(item.attachments[0].url).toBe(url); + } + }); + + /* + * AN EMPTY TEXT PART IS NOT A BLANK LINE IN SOMEBODY'S MESSAGE. + * + * `readText` returns `""` for a text part carrying an empty string — not `null`, which is what + * the `.filter` drops — so `.join("\n")` put a newline in front of the real caption. A composer + * that sends a text part alongside a screenshot with nothing typed in it produces exactly this, + * and Streamdown renders the result with a leading blank line above the person's own words. + * + * The empty part contributes nothing at either end or in the middle; two real parts either side + * of it are still joined to each other. + */ + test("an empty text part does not become a blank line in the caption", () => { + const cases: [unknown[], string][] = [ + [ + [ + { type: "text", text: "" }, + { type: "text", text: "hi" }, + ], + "hi", + ], + [ + [ + { type: "text", text: "hi" }, + { type: "text", text: "" }, + ], + "hi", + ], + [ + [ + { type: "text", text: "one" }, + { type: "text", text: "" }, + { type: "text", text: "two" }, + ], + "one\ntwo", + ], + ]; + + for (const [content, text] of cases) { + const said = { + id: "user-blank", + role: "user", + content, + } as unknown as Message; + + expect(toVisibleChatItems([said])).toEqual([ + { kind: "text", id: "user-blank", role: "user", text }, + ]); + } + }); + + /* + * And a turn whose only text part is empty is not a text item at all — the same rule the + * string-content branch above already follows, where `""` produces no row rather than an empty + * bubble. + */ + test("a turn whose only text part is empty draws no caption", () => { + const said = { + id: "user-empty", + role: "user", + content: [ + { type: "text", text: "" }, + { + type: "image", + source: { type: "url", value: "/api/attachments/att-12" }, + metadata: { attachmentId: "att-12", filename: "shot.png" }, + }, + ], + } as unknown as Message; + + expect(toVisibleChatItems([said])).toEqual([ + { + kind: "attachments", + id: "user-empty:attachments", + attachments: [ + { + id: "user-empty:1", + attachmentId: "att-12", + url: "/api/attachments/att-12", + filename: "shot.png", + modality: "image", + }, + ], + }, + ]); + }); + + /* + * THE ID IS THE ONE FIELD EVERYTHING DOWNSTREAM IS KEYED ON, so it is the one an unvalidated + * live message must not be trusted for. + * + * `isReadableToolCall` has checked `toolCall.id` from the day it was written, for a reason its + * own comment gives — "the row it builds is keyed on the id". `message.id` is keyed on harder: + * it is the React key, `MessageScrollerItem`'s `messageId`, the grouping key `anchorRowIds` and + * `turnOf` cut apart, and the memo key for the entrance delay. It went unchecked for the reason + * `readAttachmentMetadata`'s comment gives for its own fields — it does not THROW, so nothing + * ever pointed at it. + * + * Nothing throws here either. That is the point: a missing id renders, quietly and wrongly. + */ + test("drops a live message whose id is not a string", () => { + // `""` sits in this list rather than beside it: it is a string, so it passes a `typeof` check, + // and it names nothing — the same reason `readAttachmentMetadata` refuses an empty + // `attachmentId`. Two turns carrying it collide exactly as two carrying a hole do. + const ids: unknown[] = [undefined, null, 42, {}, ""]; + + for (const id of ids) { + const said = { id, role: "user", content: "hi" } as unknown as Message; + expect(toVisibleChatItems([said])).toEqual([]); + } + }); + + /* + * THE QUIET ONE, AND THE REASON THE GUARD IS ON THE MESSAGE RATHER THAN AT EACH READ. A text + * item with `id: undefined` is loud — React logs a duplicate-key warning. The attachments item + * beside it is silent: `` `${message.id}:attachments` `` STRINGIFIES the hole, so two such turns + * both come out as the literal `"undefined:attachments"` and collide on one render key, one + * scroller registration and one memoised entrance delay. Two files from two different turns are + * drawn as one row. + */ + test("two id-less turns do not collide on one attachments row", () => { + const attached = (value: string) => + ({ + role: "user", + content: [{ type: "image", source: { type: "url", value } }], + }) as unknown as Message; + + expect( + toVisibleChatItems([ + attached("/api/attachments/att-a"), + attached("/api/attachments/att-b"), + ]), + ).toEqual([]); + }); + + /* + * AN EMPTY STRING IS A MALFORMED SOURCE, WHICH IS WHAT `readUrlSource`'S OWN DOC ALREADY SAID. + * + * It promised null for "a source that is missing, or carries no `value`", and `""` carries no + * value by any reading — but `typeof value === "string"` let it through. What came out the other + * side was a tile: `attachmentIdFromUrl("")` is `""`, so the row got an `attachmentId` naming + * nothing, and `SentAttachmentTile` refuses a url that does not start with the attachment prefix, + * so the reader was shown "This attachment is unavailable." over a message that never had a file + * the server lost. Every sibling narrowing in this file already refuses `""` — `metadata.filename`, + * `metadata.attachmentId`, `source.mimeType`, `readText` — and for the same reason. + */ + test("an attachment part whose source value is empty draws no tile", () => { + const empty = { + id: "user-empty-source", + role: "user", + content: [{ type: "image", source: { type: "url", value: "" } }], + } as unknown as Message; + + expect(toVisibleChatItems([empty])).toEqual([]); + }); + + /* + * THE GATE IS THE SOURCE URL, NOT `part.type`, WHICH IS THE RULE THE SERVER ALREADY APPLIES. + * + * `attachmentIdFor` in `server/src/channels/attachment-parts.ts` reads the source and deliberately + * never reads `part.type`, with the alternative spelled out in its comment: AG-UI's part union is + * `text | image | audio | video | document | binary`, and a client writing its own content can + * send any of the six naming one of our urls. The server therefore RESOLVES such a part — inlines + * the bytes and stamps `attachedAt` so the sweeper spares it — while this projection dropped it, + * so the file went to the model, survived on the shelf, and was drawn to the person nowhere. + * + * Drawn as a document rather than guessed at: `attachmentModality` maps everything that is not + * `image` to the file card, which is the honest tile for a kind this app has no viewer for. + */ + test("a part type this app does not send still draws the file it names", () => { + const exotic = { + id: "user-audio", + role: "user", + content: [ + { + type: "audio", + source: { type: "url", value: "/api/attachments/att-audio" }, + metadata: { attachmentId: "att-audio", filename: "note.m4a" }, + }, + ], + } as unknown as Message; + + expect(toVisibleChatItems([exotic])).toEqual([ + { + kind: "attachments", + id: "user-audio:attachments", + attachments: [ + { + id: "user-audio:0", + attachmentId: "att-audio", + url: "/api/attachments/att-audio", + filename: "note.m4a", + modality: "document", + }, + ], + }, + ]); + }); + + /* + * AND A TEXT PART IS STILL NOT A TILE, because a well-formed one carries no `source` at all — + * including the three the SERVER substitutes for an attachment it could not send + * (`unavailableNote`, `notIncludedNote`, `unreadableNote` all write `{ type: "text", text }` and + * nothing else). Widening the gate to the source url therefore costs the caption path nothing, + * and this pins that rather than leaving it to the argument. + */ + test("a text part is read as a caption, never as an attachment", () => { + const noted = { + id: "user-note", + role: "user", + content: [ + { type: "text", text: '[attachment "a.pdf" is no longer available]' }, + ], + } as unknown as Message; + + expect(toVisibleChatItems([noted])).toEqual([ + { + kind: "text", + id: "user-note", + role: "user", + text: '[attachment "a.pdf" is no longer available]', + }, + ]); + }); +}); + +/* + * THE MODALITY IS THE SERVER'S ANSWER ABOUT THE BYTES, NOT THE BROWSER'S GUESS ABOUT THE FILE. + * + * These are the client half of the defect `server/src/channels/attachment-parts.ts` was fixed for. + * The chain, verified against the installed SDK rather than assumed: + * + * 1. `useAttachments.processFiles` sets `type: getModalityFromMimeType(file.type)` on the + * placeholder — `file.type` being what the BROWSER claimed before a byte was uploaded, and + * that function maps everything that is not `image/`, `audio/` or `video/` to `"document"`. + * 2. `onUpload` answers with our `uploadToChannel`, whose `mimeType` is the type the SERVER + * earned from `sniffMimeType` over the actual bytes. + * 3. The merge back onto the staged attachment is + * `{ ...att, source, status: "ready", thumbnail, metadata }` — it replaces the SOURCE and + * NEVER the `type`. + * 4. So the stale guess and the fresh answer sit side by side on the staged object: `type` says + * one thing, `source.mimeType` says another. + * + * `composer/picked-files.ts` makes step 1 wrong ON PURPOSE: `screenPickedFiles` deliberately lets + * through a claim that names no format (`application/octet-stream`, `""`) so the server can sniff + * the bytes, which is exactly the case where the guess and the answer disagree. A PNG dragged out + * of an editor is claimed as text, so `type` is `"document"` while the bytes are `image/png` — and + * the transcript drew a grey file card over somebody's screenshot. + * + * `source.mimeType` is not an invented field. It is declared on `AttachmentSource` in + * `shared/attachments.ts`, and it is a first-class optional key on AG-UI's own + * `InputContentUrlSourceSchema`, so it survives the parse a stored message is put through — unlike + * a sibling key on `metadata`, which that file's comment notes would be silently stripped. + * + * WHAT THESE CASES DO AND DO NOT PROVE, STATED UP FRONT SO NOBODY READS THEM AS MORE THAN THEY ARE. + * They pin the RULE this projection applies to a part, and the rule is now right. They do NOT show + * that a real sent message is drawn correctly, because no real sent message reaches here carrying a + * `mimeType` at all: OpenBot does not use the SDK's own send path but `toAttachmentPart` in + * `channel-chat.tsx`, which rebuilds the source as `{ type: "url", value }` and discards the + * `mimeType` `onUpload` returned. Every part below that carries one is therefore a shape this app + * cannot currently produce — deliberately so, because the reader has to be correct BEFORE the one + * line upstream that would start supplying it, or that line would land on a projection that ignores + * it. The surface where the answer really is in hand today is the parked row, and it is pinned for + * real in `transcript-attachments.test.tsx`. + */ +describe("what a sent attachment is drawn as", () => { + /** The one field under test, pulled out of the row the projection builds. */ + function modalityOf(part: unknown): string | undefined { + const said = { + id: "user-modality", + role: "user", + content: [part], + } as unknown as Message; + + const [item] = toVisibleChatItems([said]); + return (item as { attachments?: { modality: string }[] })?.attachments?.[0] + ?.modality; + } + + function part(type: string, mimeType?: string) { + return { + type, + source: { + type: "url", + value: "/api/attachments/att-1", + ...(mimeType === undefined ? {} : { mimeType }), + }, + metadata: { attachmentId: "att-1", filename: "picture.png" }, + }; + } + + /* + * THE DEFECT ITSELF. The model is shown the picture — `resolvePart` decides with + * `classifyAttachment(attachment.mimeType)` off the stored row — and the person who attached it + * was shown a document tile. The wrong drawing is the whole of it: a grey card with a filename + * where somebody's screenshot should be. + * + * This note used to add that the tile "also cost the server a whole-file read out of Postgres on + * every render", because `SentAttachmentTile` probes a document with HEAD and a picture not at + * all. That stopped being true when the attachment route grew a HEAD branch selecting `sizeBytes` + * rather than `bytes`; the mislabelled tile now buys one cheap round trip. Corrected rather than + * cut, because the same claim was repeated in three places and read as current twice. + */ + test("a document part whose bytes the server read as an image is drawn as an image", () => { + expect(modalityOf(part("document", "image/png"))).toBe("image"); + }); + + /* + * AND THE SAME MISTAKE POINTING THE OTHER WAY, which is the louder failure of the two. A text + * file claimed as an image draws an `` at a url the route answers 200 for with text; the + * browser cannot decode it, `onError` fires, and the tile swaps itself for the destructive card + * reading "notes.txt is unavailable." — an accusation that a file was deleted when it is sitting + * right there. Reachable the moment a browser claims `image/png` for something that is not one. + */ + test("an image part whose bytes the server read as text is drawn as a document", () => { + expect(modalityOf(part("image", "text/plain"))).toBe("document"); + }); + + /* + * A PICTURE THIS BROWSER CANNOT DRAW IS NOT A PICTURE, and `classifyAttachment` is asked rather + * than `mimeType.startsWith("image/")` precisely so this case answers correctly. A HEIC is an + * image by media type and an `` renders nothing for it, so the honest tile is the card + * naming the file. Asking `classifyAttachment` also makes the tile agree with the model by + * construction: `resolvePart` gates on that same function returning `"image"`, so a picture is + * drawn to the person exactly when a picture was put in front of the Bot. + */ + test("an image type this app does not accept is a document, not a broken picture", () => { + expect(modalityOf(part("image", "image/heic"))).toBe("document"); + expect(modalityOf(part("document", "image/heic"))).toBe("document"); + }); + + /* + * WITHOUT THE SERVER'S ANSWER, THE DECLARED TYPE IS ALL THERE IS, and it is used rather than + * everything collapsing to a document. `mimeType` is OPTIONAL on both `AttachmentSource` and + * AG-UI's url-source schema, so a part without one is well-formed: every message sent before + * `uploadToChannel` began returning it is one, and those threads are still in the database. + * Falling back keeps them drawing exactly as they do today instead of turning every stored + * screenshot into a file card. + */ + test("a part carrying no server type falls back to the type the browser declared", () => { + expect(modalityOf(part("image"))).toBe("image"); + expect(modalityOf(part("document"))).toBe("document"); + }); + + /* + * The same standard the rest of this file holds `metadata` to, applied to the field beside it: a + * live turn is whatever the run put in the array, so a `mimeType` that is not a non-empty string + * is not an answer. `""` is the one that matters — `classifyAttachment("")` returns + * `"unsupported"`, so trusting it would turn a screenshot into a file card on the strength of a + * field that says nothing. + */ + test("a source mimeType that is not a usable string is not trusted", () => { + for (const bad of ["", 42, null, {}, undefined]) { + expect( + modalityOf({ + type: "image", + source: { + type: "url", + value: "/api/attachments/att-1", + mimeType: bad, + }, + metadata: { attachmentId: "att-1" }, + }), + ).toBe("image"); + } + }); }); diff --git a/app/tests/composer-attachment-lifecycle.test.tsx b/app/tests/composer-attachment-lifecycle.test.tsx new file mode 100644 index 000000000..9a1ca8a7a --- /dev/null +++ b/app/tests/composer-attachment-lifecycle.test.tsx @@ -0,0 +1,361 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + expect, + test, +} from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { + cleanup, + fireEvent, + render, + type RenderResult, + waitFor, +} from "@testing-library/react"; +import { Composer } from "@/components/channels/composer/composer"; +import type { ComposerDraft } from "@/components/channels/composer/draft"; +import { settleReactWork } from "./settle-react-work"; + +/** + * WHAT HAPPENS TO A STAGED FILE AFTER IT IS STAGED: the send it rides on, and the Remove that + * takes it back. + * + * Two defects, both only reachable once `channelId` was threaded through. A send held the strip + * for the whole length of the run, so the same attachment ids could be pressed into a second + * message; and nothing in the app had ever called `DELETE /api/attachments/:id`, so a removed chip + * left its row staged forever and counted against the eight-per-channel cap that later refuses an + * upload by naming files nobody can see. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in + * `afterEach`, matching `composer-attachments-ui.test.tsx` for the reason recorded there: bun walks + * every file into one process, and a document another file tore down mid-run fails invisibly. The + * registration carries a `url` because without one `location` is `about:blank` and the relative + * URLs every one of these requests uses do not resolve. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +const originalFetch = global.fetch; + +afterEach(() => { + global.fetch = originalFetch; +}); + +/** Every `DELETE` this composer sent, by path. */ +let deletes: string[]; +/** Held open by a test that wants an upload still in flight when it looks at the strip. */ +let holdUpload: boolean; +/** A network that drops the delete on the floor, which must change nothing on screen. */ +let deleteFails: boolean; + +beforeEach(() => { + deletes = []; + holdUpload = false; + deleteFails = false; + global.fetch = (async (path: string, init: RequestInit) => { + if (init?.method === "DELETE") { + deletes.push(path); + if (deleteFails) { + // What a browser does when the request never reaches anything: a rejected promise, not a + // response with a status. The 404 and 409 this endpoint really answers with are ordinary + // responses, and are ignored for the same reason this rejection is. + throw new TypeError("Failed to fetch"); + } + return new Response(null, { status: 204 }); + } + const file = (init.body as FormData).get("file") as File; + if (holdUpload) { + // Never settles: the placeholder stays `uploading` for the length of the test. + return await new Promise(() => {}); + } + return new Response( + JSON.stringify({ + id: "stored-id", + name: file.name, + mimeType: "text/plain", + }), + { status: 201, headers: { "content-type": "application/json" } }, + ); + }) as unknown as typeof fetch; +}); + +/** A drop, as the browser delivers one: files hanging off `dataTransfer`. */ +function drop(form: Element, files: File[]) { + fireEvent.drop(form, { + dataTransfer: { files, items: [], types: ["Files"] }, + }); +} + +function notes() { + return new File(["hello"], "notes.txt", { type: "text/plain" }); +} + +/** + * A dropped file, all the way up — and, after a run, a composer with nothing left in flight at all. + * + * `Remove ` on its own is not that: the chip goes on the strip the moment the upload starts, + * so a wait on it can be over while `POST .../attachments` is still outstanding, and every test + * below then depends on a race it never states — the DELETE only goes out for a `ready` + * attachment, and `canSendDraft` will not submit one that is still uploading. See the note on + * `uploaded` in `composer-attachments-ui.test.tsx` for what a test that ends mid-upload does to the + * document. + * + * `canSend` answers both halves at once: it is false while any attachment is `uploading` AND false + * for the whole length of a run (`isBusy`, and nothing below has anything to park), so a live Send + * is an upload that has been answered by a composer that is not mid-send. + */ +async function uploaded( + { getByLabelText, queryByLabelText }: RenderResult, + name: string, +) { + await waitFor(() => { + expect(queryByLabelText(`Remove ${name}`)).not.toBeNull(); + expect((getByLabelText("Send message") as HTMLButtonElement).disabled).toBe( + false, + ); + }); +} + +test("a send in flight takes its attachment off the composer, so a second press cannot send it twice", async () => { + const submitted: ComposerDraft[] = []; + const queued: ComposerDraft[] = []; + let land: () => void = () => {}; + const run = new Promise((resolve) => { + land = resolve; + }); + + const view = render( + queued.push(draft)} + onSubmit={async (draft) => { + submitted.push(draft); + // A real send does not resolve until the whole run does — which is the window the whole + // defect lived in. + await run; + }} + />, + ); + const { container, getByLabelText, queryByLabelText } = view; + + const form = container.querySelector("form") as HTMLFormElement; + drop(form, [notes()]); + await uploaded(view, "notes.txt"); + + fireEvent.submit(form); + await waitFor(() => expect(submitted).toHaveLength(1)); + expect(submitted[0].attachments).toHaveLength(1); + + // The screenshot is in the transcript now. It must not also still be here: it was on screen + // twice for the length of the run, and `canSendDraft` unlocks on attachments alone, so the empty + // text box was not what kept the button from going again. + expect(queryByLabelText("Remove notes.txt")).toBeNull(); + expect((getByLabelText("Send message") as HTMLButtonElement).disabled).toBe( + true, + ); + + // The press that used to queue the identical attachment ids into a second user message. + fireEvent.submit(form); + expect(queued).toEqual([]); + expect(submitted).toHaveLength(1); + + land(); + // `aria-busy` IS `isSubmitting`, so this is the run finishing and the composer re-rendering + // without it — and, unlike a wait for `submitted` to have one entry, it is not a condition that + // was already true before `land()`. The assertion under it used to run before the run had landed + // and so asked nothing. + await waitFor(() => expect(form.getAttribute("aria-busy")).toBe("false")); + // And it does not come back once the run lands, either: the send became a message. + expect(queryByLabelText("Remove notes.txt")).toBeNull(); +}); + +test("a send carries off only what it took, and a failure would have it to hand back", async () => { + /* + * THE TRADE-OFF THIS COMPOSER CHOSE, PINNED FROM THE ONE SIDE A TEST CAN REACH. + * + * The strip is HIDDEN for the length of a run, not consumed — so the ids are still the + * composer's, and the `finally` hands them straight back when a send fails. The failing send + * itself cannot be driven from here: `submitDraft` rethrows into two call sites that both void + * the promise (`handleFormSubmit`, and prompt-area's Enter), which is deliberate — see the + * "swallowed on purpose, and only here" note on `conversation-view`'s drain — and `bun test` + * fails any test in whose tick an unhandled rejection appears. What is reachable is the property + * the restore rests on, and it is the one the other trade-off would have broken: taking the ids + * the send actually carried rather than consuming the queue, so nothing that was never sent is + * swept up with them. + */ + const submitted: ComposerDraft[] = []; + let land: () => void = () => {}; + const run = new Promise((resolve) => { + land = resolve; + }); + + const view = render( + { + submitted.push(draft); + await run; + }} + />, + ); + const { container, queryByLabelText } = view; + + const form = container.querySelector("form") as HTMLFormElement; + drop(form, [notes()]); + await uploaded(view, "notes.txt"); + + fireEvent.submit(form); + await waitFor(() => expect(submitted).toHaveLength(1)); + + // Staged while the run is still going: a correction's file, belonging to the next message and + // never sent by this one. Send is shut for the length of the run, so the chip is all there is to + // wait for here — the wait after `land()` is what closes the upload out. + drop(form, [new File(["later"], "report.txt", { type: "text/plain" })]); + await waitFor(() => + expect(queryByLabelText("Remove report.txt")).not.toBeNull(), + ); + + land(); + // Send comes back only when the run has landed AND nothing is left uploading, so this one wait + // is both — and it is the assertion that report.txt is still here, which is the half of the + // property `consumeAttachments()` would have broken: that sweep takes every ready attachment, + // which at this moment would include a file this send never carried. It replaces a wait for + // notes.txt's chip to be ABSENT, which was already true before `land()` — the send had hidden it + // — and so let everything below run on a composer that had not yet seen the run finish. + await uploaded(view, "report.txt"); + + // And the half that rode on the send is gone, and does not come back with the run. + expect(queryByLabelText("Remove notes.txt")).toBeNull(); + expect(submitted[0].attachments).toHaveLength(1); + expect(deletes).toEqual([]); +}); + +test("removing a staged attachment gives its row back to the server", async () => { + const view = render( + {}} />, + ); + const { container, getByLabelText, queryByLabelText } = view; + + drop(container.querySelector("form") as HTMLFormElement, [notes()]); + // The whole point of the test is what a READY attachment gives back — an `uploading` one has no + // row to reclaim and issues no DELETE at all — so the removal must not be pressed until the + // upload has been answered. + await uploaded(view, "notes.txt"); + + fireEvent.click(getByLabelText("Remove notes.txt")); + + // The id the server stored it under — `metadata.attachmentId` — not the client-side placeholder + // id the strip keys its chips on. + await waitFor(() => expect(deletes).toEqual(["/api/attachments/stored-id"])); + expect(queryByLabelText("Remove notes.txt")).toBeNull(); +}); + +/** + * NOTHING IS INVENTED FOR A ROW THE COMPOSER HAS NOT BEEN TOLD ABOUT YET. + * + * This test used to be named for a guard and pinned neither half of it. `discardAttachment` had two + * checks before its DELETE — `status !== "ready"` and `typeof metadata?.attachmentId === "string"` — + * and against an `uploading` placeholder they are redundant: it has no metadata, so deleting either + * one on its own left the suite green. Only deleting both turned it red. + * + * Both are gone now, and not by being papered over. `discardAttachment` no longer decides anything + * about rows at all — a single reconciler owns that, because the guard version could only ever + * handle the `ready` case and silently leaked the in-flight one (see + * `composer-inflight-removal.test.tsx`, which pins what this composer now does about it). + * + * What is left here is the half that stays true and is worth keeping: while the upload is still + * outstanding there is no id, so no request goes out — in particular not a + * `DELETE /api/attachments/undefined`, which is a different endpoint rather than a broken one. + */ +test("removing an attachment that is still uploading asks the server for nothing", async () => { + holdUpload = true; + const { container, getByLabelText, queryByLabelText } = render( + {}} />, + ); + + drop(container.querySelector("form") as HTMLFormElement, [notes()]); + // The placeholder is on the strip from the moment the upload starts, and there is no row behind + // it yet: `POST .../attachments` has not answered, so there is no id to delete. + await waitFor(() => + expect(queryByLabelText("Remove notes.txt")).not.toBeNull(), + ); + + fireEvent.click(getByLabelText("Remove notes.txt")); + + expect(queryByLabelText("Remove notes.txt")).toBeNull(); + expect(deletes).toEqual([]); +}); + +test("a delete that fails still takes the chip off the strip", async () => { + deleteFails = true; + const view = render( + {}} />, + ); + const { container, getByLabelText, queryByLabelText } = view; + + drop(container.querySelector("form") as HTMLFormElement, [notes()]); + // Ready before the press, for the same reason as the test above: an `uploading` attachment sends + // no DELETE, so the failure this pins would never be reached. + await uploaded(view, "notes.txt"); + + fireEvent.click(getByLabelText("Remove notes.txt")); + + // Best-effort, and the person asked for it gone: the removal is not held open by the request, + // and the sweeper is the backstop for the row this one did not reclaim. + expect(queryByLabelText("Remove notes.txt")).toBeNull(); + await waitFor(() => expect(deletes).toEqual(["/api/attachments/stored-id"])); +}); + +test("a refusal is dismissible, and a send clears it", async () => { + /* + * A REFUSAL WAS THE ONE THING ON THIS COMPOSER WITH NO END. An attachment leaves when it is sent + * or removed and typed words leave when they are sent; the sentence about a file that never got + * in stayed until the tab was closed, so a send landed under a complaint about something that was + * not in it. + * + * Asserted through `role="alert"` rather than the filename, which appears twice in one refusal — + * once as the line's own subject and once inside the reason — and would make every query + * ambiguous. + */ + const submitted: ComposerDraft[] = []; + const view = render( + { + submitted.push(draft); + }} + />, + ); + const { container, getByLabelText, queryByRole } = view; + + const form = container.querySelector("form") as HTMLFormElement; + const svg = () => new File([""], "logo.svg", { type: "image/svg+xml" }); + drop(form, [svg()]); + await waitFor(() => expect(queryByRole("alert")).not.toBeNull()); + + // Closable on its own, for somebody who drops a bad file and then thinks better of the whole + // message rather than sending one. + fireEvent.click(getByLabelText("Dismiss this refusal")); + expect(queryByRole("alert")).toBeNull(); + + // And cleared by a send, which is the other way out. + drop(form, [svg(), notes()]); + await waitFor(() => expect(queryByRole("alert")).not.toBeNull()); + // `canSendDraft` refuses a draft carrying an upload still in flight, so a submit fired before + // notes.txt was answered would be a no-op and the send this test is about would never happen. + await uploaded(view, "notes.txt"); + + fireEvent.submit(form); + await waitFor(() => expect(submitted).toHaveLength(1)); + expect(queryByRole("alert")).toBeNull(); +}); diff --git a/app/tests/composer-attachment-strip.test.tsx b/app/tests/composer-attachment-strip.test.tsx new file mode 100644 index 000000000..51732b70a --- /dev/null +++ b/app/tests/composer-attachment-strip.test.tsx @@ -0,0 +1,123 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { cleanup, render } from "@testing-library/react"; +import { + AttachmentStrip, + type StagedFile, +} from "@/components/channels/composer/attachment-strip"; +import { settleReactWork } from "./settle-react-work"; + +/** + * WHAT A SCREEN READER IS OFFERED BY THE ROW ACROSS THE TOP OF THE COMPOSER, AND WHEN. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in + * `afterEach`, matching `agent-roster-error.test.tsx`: bun walks every file into one process, and a + * document another file tore down mid-run fails invisibly. + * + * The registration carries a `url` for the reason `composer-attachments-ui.test.tsx` records: + * without one `location` is `about:blank` and a thumbnail's relative `src` does not resolve. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +const notes: StagedFile = { + id: "1", + name: "notes.txt", + size: 2048, + loading: false, +}; + +test("the empty strip stays mounted but leaves the accessibility tree", () => { + // Two halves of one bargain, and the file's comment used to claim only the first. + // + // The list is deliberately NOT unmounted while the composer carries nothing: it is the element + // `Collapse` measures, and a box with nothing in it to measure cannot animate its own height. + // + // What that costs is the part the comment got wrong. Height 0 under `overflow-hidden` is a + // visual state, not an accessibility one, so an `aria-label`led `
    ` sitting at zero height is + // still a labelled list a screen reader walks into and announces — "Attachments, list, 0 items" + // — on a composer with no attachments on it. Nothing is drawn; something is still read. + const { container, queryByRole } = render( + {}} />, + ); + + expect(container.querySelector("ul")).toBeTruthy(); + expect(queryByRole("list")).toBeNull(); + expect(queryByRole("list", { name: "Attachments" })).toBeNull(); +}); + +test("the strip is announced once it carries something", () => { + // The other half: hiding the empty list must not hide the full one, or the label stops being + // worth having at the only moment it says anything. + const { getAllByRole, getByRole } = render( + {}} />, + ); + + expect(getByRole("list", { name: "Attachments" })).toBeTruthy(); + expect(getAllByRole("listitem")).toHaveLength(1); +}); + +test("the strip leaves the accessibility tree again when the last one goes", () => { + // Removing the last attachment is the ordinary way back to empty, and it is the path that would + // leave a stale labelled list behind if the hiding were done once at mount. + const { queryByRole, rerender } = render( + {}} />, + ); + + expect(queryByRole("list", { name: "Attachments" })).toBeTruthy(); + + rerender( {}} />); + + expect(queryByRole("list")).toBeNull(); +}); + +test("an image on its own puts the strip back in the accessibility tree", () => { + // `occupied` is either list being non-empty; a strip carrying only a picture is announced too. + const { getAllByRole, getByRole } = render( + {}} + />, + ); + + expect(getByRole("list", { name: "Attachments" })).toBeTruthy(); + expect(getAllByRole("listitem")).toHaveLength(1); +}); + +test("an image still going up is named rather than left a blank box", () => { + // The same pattern as the empty list, one level down. The skeleton is drawn to hold the tile's + // shape while the bytes are in flight, and holding a shape was all it was asked to do — so the + // tile announced nothing at all, and the only thing a screen reader found inside it was a button + // offering to remove something that had never been named. The file tile beside it has said + // "Uploading…" in plain text all along; the picture said nothing. + // + // Named, not announced: no live region here. The file tile does not have one either, and a paste + // of eight images would otherwise interrupt with eight of them. + const { getByRole } = render( + {}} + />, + ); + + expect(getByRole("img", { name: "A cat, uploading" })).toBeTruthy(); +}); + +test("an unnamed image going up falls back to the same word the finished one uses", () => { + const { getByRole } = render( + {}} + />, + ); + + expect(getByRole("img", { name: "Attachment, uploading" })).toBeTruthy(); +}); diff --git a/app/tests/composer-attachments-ui.test.tsx b/app/tests/composer-attachments-ui.test.tsx new file mode 100644 index 000000000..8a53d5f59 --- /dev/null +++ b/app/tests/composer-attachments-ui.test.tsx @@ -0,0 +1,406 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + expect, + test, +} from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { + cleanup, + fireEvent, + render, + type RenderResult, + waitFor, +} from "@testing-library/react"; +import { Composer } from "@/components/channels/composer/composer"; +import { settleReactWork } from "./settle-react-work"; + +/** + * What the composer does with a file, from the outside: the button that opens the picker, the drop + * that gets refused, and the screen that has no channel to upload to and so must go on behaving + * exactly as it did before any of this existed. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in + * `afterEach`, matching `agent-roster-error.test.tsx` for the reason recorded there: bun walks + * every file into one process, and a document another file tore down mid-run fails invisibly. + * + * The registration carries a `url`. Without one `location` is `about:blank`, relative URLs do not + * resolve, and an assertion about an attachment's preview would pass or fail for a reason that has + * nothing to do with the composer. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +const originalFetch = global.fetch; + +afterEach(() => { + global.fetch = originalFetch; +}); + +/** Every upload this composer attempted: where it went, and what it carried. */ +let uploads: { path: string; name: string }[]; + +beforeEach(() => { + uploads = []; + global.fetch = (async (path: string, init: RequestInit) => { + const body = init.body as FormData; + uploads.push({ path, name: (body.get("file") as File).name }); + return new Response( + JSON.stringify({ + id: "attachment-id", + name: (body.get("file") as File).name, + mimeType: "text/plain", + }), + { status: 201, headers: { "content-type": "application/json" } }, + ); + }) as unknown as typeof fetch; +}); + +/** + * The same upload stub, with the one answer that matters here under the test's control: what the + * SERVER says the file is, having read its bytes, as against what the browser claimed when it was + * picked up. `POST /api/channels/:id/attachments` returns a sniffed `mimeType`, and + * `uploadToChannel` puts it on the `url` source it hands back to the SDK. + */ +function serverSniffs(mimeType: string) { + global.fetch = (async (path: string, init: RequestInit) => { + const body = init.body as FormData; + uploads.push({ path, name: (body.get("file") as File).name }); + return new Response( + JSON.stringify({ + id: "attachment-id", + name: (body.get("file") as File).name, + mimeType, + }), + { status: 201, headers: { "content-type": "application/json" } }, + ); + }) as unknown as typeof fetch; +} + +/** A drop, as the browser delivers one: files hanging off `dataTransfer`. */ +function drop(form: Element, files: File[]) { + fireEvent.drop(form, { + dataTransfer: { files, items: [], types: ["Files"] }, + }); +} + +/** + * WAIT FOR A FILE TO BE ALL THE WAY UP, NOT MERELY ON ITS WAY. Written out here because the other + * three composer test files wait on the same thing and cite this note. + * + * Neither of the two obvious conditions is that one. The stub above records a request BEFORE it + * answers it, so a wait on `uploads` is over while `POST .../attachments` is still outstanding; and + * the chip goes on the strip the moment an upload starts, so a wait on `Remove ` can be over + * there too. A test that ends on either one ends with a `fetch` continuation still to come: + * `cleanup` takes the document away, `afterAll` unregisters happy-dom, and React lands the state + * update in a world with no `window`. That is the ` 1 error` this suite used to print next to 0 + * failures. + * + * Send is the part of the screen that knows the difference. `canSendDraft` holds the button shut + * while any attachment is still `uploading`, so a Send that has come back on is an upload that has + * been answered and a composer that has re-rendered on the answer. It is also FALSE when the wait + * begins — the box is empty and nothing is staged — which is what makes it a wait at all, and so + * what makes anything asserted after it a question that was actually asked. + */ +async function uploaded( + { getByLabelText, queryByLabelText }: RenderResult, + name: string, +) { + await waitFor(() => { + expect(queryByLabelText(`Remove ${name}`)).not.toBeNull(); + expect((getByLabelText("Send message") as HTMLButtonElement).disabled).toBe( + false, + ); + }); +} + +test("a composer with a channel offers a file picker behind the plus button", () => { + const { getByLabelText } = render( + {}} />, + ); + + const attach = getByLabelText("Attach a file") as HTMLButtonElement; + expect(attach.disabled).toBe(false); + + // The button is only worth anything if it has an input to open. Clicking it must reach a real + // `` — the one the hook's ref is on — rather than a placeholder. + let opened = false; + const input = document.querySelector( + 'input[type="file"]', + ) as HTMLInputElement; + input.addEventListener("click", () => { + opened = true; + }); + fireEvent.click(attach); + expect(opened).toBe(true); +}); + +/** + * NOT ASSERTED ON AN EMPTY `uploads` ALONE — the same rule the disabled-composer test below states + * and this one used to break. A synchronous `expect(uploads).toEqual([])` is true before anything + * has had a chance to happen, so it proves nothing about the claim in this test's own name. + * + * The composer is given a channel and a second file is dropped; waiting for THAT upload to be + * answered puts the question after the point by which the first — started earlier — would have had + * to appear. Two files go down on the channel-less composer because they prove different halves: + * the SVG is a file the screen WOULD refuse out loud if the drag handlers were installed, so a + * silent drop is what says they are not; the text file is one that WOULD upload, so its absence + * from `uploads` is what says nothing was sent. + */ +test("a composer with no channel keeps the button that says so, and takes no files", async () => { + const view = render( {}} />); + const { container, getByLabelText, queryByLabelText, rerender } = view; + const form = () => container.querySelector("form") as HTMLFormElement; + + // The old affordance, untouched: nothing to offer and it says so, rather than a live button + // that would open a picker with nowhere to upload to. + const placeholder = getByLabelText( + "More message options unavailable", + ) as HTMLButtonElement; + expect(placeholder.disabled).toBe(true); + expect(queryByLabelText("Attach a file")).toBeNull(); + expect(container.querySelector('input[type="file"]')).toBeNull(); + + /* + * THE DROP IS CAUGHT AND ANSWERED, WHICH INVERTS WHAT THESE TWO LINES USED TO PIN. + * + * They read "a file dropped on this composer is the browser's business and not ours" and + * asserted an empty alert. "The browser's business" turned out to mean the browser NAVIGATING + * THE TOP-LEVEL DOCUMENT TO THE FILE — the single-page app unloads and takes the typed message + * with it — because an element with no `dragover` handler is not a drop target at all. See + * `refuseDragOver` in `composer.tsx`, and `composer-drop-guard.test.tsx` for the guard itself. + * + * What this test still owns is the half that has NOT changed, and it is the half its name is + * about: no picker, no upload, nothing staged. Only the silence is gone. + */ + drop(form(), [ + new File([""], "logo.svg", { type: "image/svg+xml" }), + new File(["hello"], "ignored.txt", { type: "text/plain" }), + ]); + expect(container.querySelector('[role="alert"]')?.textContent).toContain( + "no conversation here yet", + ); + + rerender( {}} />); + drop(form(), [new File(["hello"], "kept.txt", { type: "text/plain" })]); + await uploaded(view, "kept.txt"); + + // Only the file dropped once there was somewhere to put it. + expect(uploads.map((upload) => upload.name)).toEqual(["kept.txt"]); + expect(queryByLabelText("Remove ignored.txt")).toBeNull(); + expect(queryByLabelText("Remove logo.svg")).toBeNull(); + /* + * AND THE SVG WAS NEVER SCREENED FOR BEING AN SVG, which is the claim this line still carries + * after the assertion above it changed. `screenPickedFiles` phrases that particular refusal with + * an upper-case "SVG" in it (see "a dropped file the screen refuses says why, in our words" + * below); the filename is lower-case, so the ABSENCE of the upper-case token is what says the + * file was turned down for having nowhere to go rather than for its type. The type screen sits + * downstream of a channel that did not exist, and it must not have run. + */ + const refusal = container.querySelector('[role="alert"]'); + expect(refusal?.textContent).toContain("logo.svg"); + expect(refusal?.textContent).not.toContain("SVG"); +}); + +test("a dropped file the screen refuses says why, in our words", async () => { + const { container, findByRole } = render( + {}} />, + ); + + const form = container.querySelector("form") as HTMLFormElement; + drop(form, [new File([""], "logo.svg", { type: "image/svg+xml" })]); + + const alert = await findByRole("alert"); + expect(alert.textContent).toContain("logo.svg"); + expect(alert.textContent).toContain("SVG"); + // Screened before `processFiles`, so the SDK never sees it: no second refusal in its own + // machine wording, and no upload of a file we had already decided against. + expect(alert.textContent).not.toContain("Supported types:"); + expect(uploads).toEqual([]); +}); + +test("a text file the screen accepts is not refused a second time by the SDK", async () => { + // The two-gate collision, pinned. `processFiles` compares `file.type` to the accept list + // exactly, while our screen (and the server behind it) drop MIME parameters first. A `File` + // really does arrive carrying them — this is the type a browser reports for a text file taken + // off the clipboard, and Bun's own `File` appends it to every text type — and left unreconciled + // the SDK refuses a file the composer had already accepted, in wording nobody wrote for a + // person to read. + const notes = new File(["hello"], "notes.txt", { + type: "text/plain;charset=utf-8", + }); + + const view = render( + {}} />, + ); + const { container } = view; + + drop(container.querySelector("form") as HTMLFormElement, [notes]); + + await uploaded(view, "notes.txt"); + + expect(uploads).toEqual([ + { path: "/api/channels/channel-1/attachments", name: "notes.txt" }, + ]); + // Asked once the SDK has been all the way through the file it was handed, which is the only + // point at which the absence of a second refusal means anything: waiting on `uploads` put this + // question before `processFiles` could have answered it either way. + expect(container.querySelector('[role="alert"]')).toBeNull(); +}); + +/** + * `disabled` IS THE CONVERSATION REFUSING ANOTHER MESSAGE AT ALL, AND IT USED TO GATE ONLY SENDING. + * + * A channel whose coworker was deleted still took files: the drop handlers were installed, the `+` + * button opened the picker, and the input behind it was live. Each one uploaded into a channel that + * can never reply — a row staged server-side, counted against that channel's cap and left for the + * sweeper, with nothing on screen connecting it to a message that cannot be sent. + * + * NOT ASSERTED ON AN EMPTY `uploads` ALONE, which is true before anything has had a chance to + * happen and so proves nothing. The composer is re-enabled and a second file dropped; waiting for + * THAT upload to be answered puts the question after the point by which the first — started + * earlier — would have had to appear. + */ +test("a disabled composer takes no dropped file, and offers no way to pick one", async () => { + const view = render( + {}} />, + ); + const { container, getByLabelText, queryByLabelText, rerender } = view; + const form = () => container.querySelector("form") as HTMLFormElement; + + drop(form(), [new File(["hello"], "refused.txt", { type: "text/plain" })]); + + // Visibly shut rather than merely ignored: a button that opens a picker whose file will be + // dropped on the floor is worse than one that says it cannot. + expect((getByLabelText("Attach a file") as HTMLButtonElement).disabled).toBe( + true, + ); + expect( + (container.querySelector('input[type="file"]') as HTMLInputElement) + .disabled, + ).toBe(true); + + rerender( {}} />); + drop(form(), [new File(["hello"], "kept.txt", { type: "text/plain" })]); + await uploaded(view, "kept.txt"); + + expect(uploads.map((upload) => upload.name)).toEqual(["kept.txt"]); + expect(queryByLabelText("Remove refused.txt")).toBeNull(); + /* + * REFUSED OUT LOUD RATHER THAN SWALLOWED, and this line used to pin the swallow. + * + * The drop is caught now — an uncaught one navigated the whole app to the file, and on a + * `disabled` channel that unload takes the parked queue with it (see `refuseDragOver` in + * `composer.tsx`). A caught file that says nothing is still a file that vanished from under the + * person's cursor, so it gets the same one sentence every other refusal on this composer gets. + * + * The claims this test is named for are untouched by that: `uploads` above proves nothing was + * sent, and `Remove refused.txt` proves nothing was staged. + */ + expect(container.querySelector('[role="alert"]')?.textContent).toContain( + "can no longer take messages", + ); +}); + +/** + * MIME TYPES ARE CASE-INSENSITIVE AND THE SDK'S ACCEPT CHECK IS NOT. + * + * `matchesAcceptFilter` compares `file.type === filter` against a lower-case list, while + * `classifyAttachment` — ours and the server's — lower-cases before it compares. + * `withMediaTypeOnly` exists to keep those two answers the same and did only half the job: it + * dropped the `;charset=` parameter and left the case alone, so a type that differs from the accept + * list only in case went through our screen and was then refused by the SDK in its own machine + * wording, which is the one outcome that function exists to make impossible. + * + * THE TYPE IS FORCED ON RATHER THAN CONSTRUCTED, because it cannot be constructed: `new File(...)` + * ASCII-lower-cases `type` per the Blob spec, so the constructor would quietly repair the very + * thing being tested — and it is also why the parameter-carrying case is not the interesting one. + * `TEXT/PLAIN` with no parameter is: `file.type.split(";")[0].trim()` gives back the string it was + * handed, the old function decided nothing needed doing, and the file went to the SDK untouched. + * A `File` this app never built is exactly what a drop or a clipboard hands over. + */ +test("a text file whose type differs only in case is not refused by the SDK", async () => { + const notes = new File(["hello"], "notes.txt", { type: "text/plain" }); + Object.defineProperty(notes, "type", { value: "TEXT/PLAIN" }); + + const view = render( + {}} />, + ); + const { container } = view; + + drop(container.querySelector("form") as HTMLFormElement, [notes]); + + await uploaded(view, "notes.txt"); + + expect(uploads).toEqual([ + { path: "/api/channels/channel-1/attachments", name: "notes.txt" }, + ]); + expect(container.querySelector('[role="alert"]')).toBeNull(); +}); + +/** + * THE STRIP DRAWS WHAT THE FILE IS, NOT WHAT THE BROWSER CALLED IT. + * + * `file.type` is a guess made from a filename before anything read a byte, and plenty of real + * sources get it wrong: a screenshot dragged out of another app arrives as + * `application/octet-stream` routinely. The server sniffs the bytes and sends back the type it + * found, and both the send path (`toAttachmentPart` in `channel-chat.tsx`) and the parked tiles + * (`parkedTiles` in `chat-transcript.tsx`) already read THAT through the shared + * `attachmentModality`. The composer strip read the browser's claim instead, so one file had two + * answers on three surfaces: a grey file card while it sat in the composer, a thumbnail the instant + * it was parked or sent. The tile visibly changed shape at the moment of sending. + * + * `img[alt=...]` IS THE DISCRIMINATOR because it is the whole difference: an image tile renders an + * `` with the filename as its alt text, a file tile renders an `IconFile` and the name. Both + * carry a `Remove ` button, so that label cannot tell the two apart. + */ +test("a screenshot the browser could not name is drawn as a picture once the server names it", async () => { + serverSniffs("image/png"); + const view = render( + {}} />, + ); + const { container } = view; + + // What a drag out of another application actually hands over: real PNG bytes under a type that + // names no format at all. `screenPickedFiles` lets this one through for exactly that reason — + // the claim is an absence rather than a claim, and only the server can settle it. + drop(container.querySelector("form") as HTMLFormElement, [ + new File(["\x89PNG"], "shot.bin", { type: "application/octet-stream" }), + ]); + + await uploaded(view, "shot.bin"); + + expect(container.querySelector('img[alt="shot.bin"]')).not.toBeNull(); +}); + +/** + * THE OTHER DIRECTION, WHICH IS THE WORSE ONE. A mislabelled image is a grey card where a preview + * should be; a mislabelled TEXT FILE handed to an `` is a broken-image icon, because there is + * no image there and there never was. The browser's claim is wrong in both directions and the + * server's answer settles both. + */ +test("a file the browser called an image is drawn as a file once the server reads it", async () => { + serverSniffs("text/plain"); + const view = render( + {}} />, + ); + const { container } = view; + + drop(container.querySelector("form") as HTMLFormElement, [ + new File(["hello"], "notes.png", { type: "image/png" }), + ]); + + await uploaded(view, "notes.png"); + + // No `` at all: the tile is the card that names the file, which is the only honest thing to + // draw for bytes that cannot be rendered. + expect(container.querySelector("img")).toBeNull(); + expect(container.textContent).toContain("notes.png"); +}); diff --git a/app/tests/composer-attachments.test.ts b/app/tests/composer-attachments.test.ts new file mode 100644 index 000000000..d217ffc61 --- /dev/null +++ b/app/tests/composer-attachments.test.ts @@ -0,0 +1,226 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { + attachmentsConfigFor, + FILE_PICKER_ACCEPT, + uploadToChannel, +} from "@/components/channels/composer/attachments"; +import { MAX_IMAGE_BYTES } from "@/lib/channels/attachments"; + +/** + * `uploadToChannel` is the SDK's `onUpload`, called with a raw `File`; `attachmentsConfigFor` + * assembles the whole `AttachmentsConfig` around it. Both are exercised here against a stubbed + * `global.fetch` rather than a real server, matching `agent-roster-error.test.tsx`'s pattern for + * the one other file in this app that stubs `fetch` directly. + */ + +const originalFetch = global.fetch; + +afterEach(() => { + global.fetch = originalFetch; +}); + +function jsonResponse(body: unknown, status: number): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function file(name: string, mimeType: string): File { + return new File(["stub bytes"], name, { type: mimeType }); +} + +describe("uploadToChannel", () => { + let requests: { path: string; init: RequestInit }[]; + + beforeEach(() => { + requests = []; + }); + + test("a successful upload returns a url source pointing at our endpoint, with metadata", async () => { + global.fetch = (async (path: string, init: RequestInit) => { + requests.push({ path, init }); + return jsonResponse( + { id: "att_1", name: "receipt.png", mimeType: "image/png" }, + 201, + ); + }) as typeof fetch; + + const result = await uploadToChannel( + "chan_1", + "group_1", + )(file("receipt.png", "image/png")); + + expect(requests).toHaveLength(1); + const [sent] = requests; + const body = sent?.init.body; + + expect(sent?.path).toBe("/api/channels/chan_1/attachments"); + expect(sent?.init.method).toBe("POST"); + expect(sent?.init.credentials).toBe("include"); + expect(body).toBeInstanceOf(FormData); + expect((body as FormData).get("file")).toBeInstanceOf(File); + // The composer's own upload group rides with every upload, which is what makes the server's + // per-message cap count the same rows this composer can see. It arrived here as `undefined` + // for as long as these tests called `uploadToChannel` with the arity it had before the group + // existed, so the field went out as the string "undefined" and nothing said so. + expect((body as FormData).get("uploadGroup")).toBe("group_1"); + + expect(result).toEqual({ + type: "url", + value: "/api/attachments/att_1", + mimeType: "image/png", + metadata: { attachmentId: "att_1", filename: "receipt.png" }, + }); + }); + + test("a 415 refusal throws the server's reason, not a generic string", async () => { + global.fetch = (async () => + jsonResponse( + { + error: + "'icon.svg' is an SVG, which can carry scripts and is not accepted.", + }, + 415, + )) as unknown as typeof fetch; + + await expect( + uploadToChannel("chan_1", "group_1")(file("icon.svg", "image/svg+xml")), + ).rejects.toThrow(/SVG/); + }); + + test("a non-JSON error body throws a named fallback, not a JSON parse error", async () => { + global.fetch = (async () => + new Response("Internal Server Error", { + status: 500, + headers: { "content-type": "text/plain" }, + })) as unknown as typeof fetch; + + await expect( + uploadToChannel("chan_1", "group_1")(file("receipt.png", "image/png")), + ).rejects.toThrow(/receipt\.png/); + await expect( + uploadToChannel("chan_1", "group_1")(file("receipt.png", "image/png")), + ).rejects.not.toThrow(/JSON/); + }); + + test("a JSON error body with no `error` key throws the named fallback, not an empty message", async () => { + global.fetch = (async () => + jsonResponse({}, 500)) as unknown as typeof fetch; + + await expect( + uploadToChannel("chan_1", "group_1")(file("receipt.png", "image/png")), + ).rejects.toThrow('Could not upload "receipt.png".'); + }); + + test("a refusal whose `error` is empty throws the named fallback, not an empty refusal", async () => { + // `??` only steps in for `null` and `undefined`, so an `error` of `""` was preferred over the + // fallback and reached the strip as a file refused with no reason beside it at all. + global.fetch = (async () => + jsonResponse({ error: " " }, 415)) as unknown as typeof fetch; + + await expect( + uploadToChannel("chan_1", "group_1")(file("receipt.png", "image/png")), + ).rejects.toThrow('Could not upload "receipt.png".'); + }); + + test("a refusal whose `error` is not a string throws the named fallback", async () => { + global.fetch = (async () => + jsonResponse({ error: 415 }, 415)) as unknown as typeof fetch; + + await expect( + uploadToChannel("chan_1", "group_1")(file("receipt.png", "image/png")), + ).rejects.toThrow('Could not upload "receipt.png".'); + }); + + /** + * THE SUCCESS BODY WAS THE ONE ANSWER NOBODY CHECKED. + * + * The failure path directly above it has been careful since `2f1d68b` — parse, fall back, never + * show a parse error as a refusal. The success path went straight to `as UploadedAttachment` and + * trusted whatever came back, so the two halves of the same response were held to opposite + * standards. A 200 from a proxy, or any handler that answers before the JSON is written, lands + * here. + */ + test("a 2xx body that is not JSON throws the named fallback, not a parse error", async () => { + global.fetch = (async () => + new Response("OK", { + status: 200, + headers: { "content-type": "text/html" }, + })) as unknown as typeof fetch; + + const upload = uploadToChannel("chan_1", "group_1"); + await expect(upload(file("receipt.png", "image/png"))).rejects.toThrow( + 'Could not upload "receipt.png".', + ); + await expect(upload(file("receipt.png", "image/png"))).rejects.not.toThrow( + /JSON/, + ); + }); + + test("a 2xx body with no id is refused rather than made into a chip pointing at nothing", async () => { + // Unchecked, `attachmentUrl(uploaded.id)` produced "/api/attachments/undefined": a tile on the + // strip, a Send button that unlocks, and a message that carries a link to no attachment. + global.fetch = (async () => + jsonResponse( + { name: "receipt.png", mimeType: "image/png" }, + 201, + )) as unknown as typeof fetch; + + await expect( + uploadToChannel("chan_1", "group_1")(file("receipt.png", "image/png")), + ).rejects.toThrow('Could not upload "receipt.png".'); + }); +}); + +describe("attachmentsConfigFor", () => { + test("maxSize is MAX_IMAGE_BYTES", () => { + const config = attachmentsConfigFor( + "chan_1", + "group_1", + () => {}, + () => {}, + ); + expect(config.maxSize).toBe(MAX_IMAGE_BYTES); + }); + + /** + * THE SDK'S `accept` REFUSES NOTHING, ON PURPOSE, AND THAT IS THE CONTRACT NOW. + * + * This used to assert the config carried the eight media types. It did, and that was the defect: + * `processFiles` applies `accept` with an exact `file.type === filter`, so it stood behind + * `screenPickedFiles` as a second, stricter, machine-worded gate and refused the very files that + * screen deliberately passes — a claim naming no format, which the server is supposed to sniff. + * See `attachmentsConfigFor` for the full argument, and `composer-unnamed-mime.test.tsx` for the + * end-to-end proof, which is the level this could only ever have been caught at. + */ + test("accept refuses nothing, leaving screenPickedFiles the only client gate", () => { + const config = attachmentsConfigFor( + "chan_1", + "group_1", + () => {}, + () => {}, + ); + + expect(config.accept).toBe("*/*"); + }); + + /** + * The narrow list still exists; it has moved to the one place it is honest — the `+` button's + * file dialog, which greys files out rather than refusing them. SVG stays off it for the reason + * `ACCEPTED_IMAGE_MIME` states: an SVG served inline from this origin is stored XSS. + */ + test("the file dialog offers the accepted types and their extensions, never SVG", () => { + const offered = FILE_PICKER_ACCEPT.split(","); + + expect(offered).toContain("image/png"); + expect(offered).toContain("text/markdown"); + expect(offered).not.toContain("image/svg+xml"); + expect(offered).not.toContain(".svg"); + // The extensions are the half that makes the dialog agree with the screen: a `.txt` dragged out + // of an editor is claimed as `application/octet-stream`, so a MIME-only list greys out exactly + // the files `screenPickedFiles` goes to some trouble to accept. + expect(offered).toContain(".txt"); + expect(offered).toContain(".md"); + }); +}); diff --git a/app/tests/composer-collapse.test.tsx b/app/tests/composer-collapse.test.tsx new file mode 100644 index 000000000..961fae661 --- /dev/null +++ b/app/tests/composer-collapse.test.tsx @@ -0,0 +1,228 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { act, cleanup, render } from "@testing-library/react"; +import { Suspense, startTransition, use, useState } from "react"; +import { Collapse } from "@/components/channels/composer/collapse"; +import { settleReactWork } from "./settle-react-work"; + +/** + * WHEN THE BOX MEASURES THE THING IT IS ABOUT TO ANIMATE, AND WHEN IT REFUSES TO. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in + * `afterEach`, matching `agent-roster-error.test.tsx`: bun walks every file into one process, and a + * document another file tore down mid-run fails invisibly. The registration carries a `url` for the + * reason `composer-attachments-ui.test.tsx` records. + * + * WHY THESE TESTS WATCH THE MEASUREMENT RATHER THAN THE HEIGHT ON SCREEN. There is no height on + * screen to watch. `motion` binds its frame loop at import time, and every module in this suite is + * imported before `GlobalRegistrator.register()` has put a `window` in the world, so motion's + * animated values are never written to the DOM here: the box reports `height: 0px` forever no + * matter what it was told to animate to. What IS observable is the only thing this component reads + * from the DOM — `offsetHeight` on the content — and reading it is not incidental, it is the whole + * act of taking a measurement. Counting those reads is counting the decisions the component made. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +/** Every `offsetHeight` the component read since the last reset. */ +let measurements = 0; +/** The callbacks the component handed to `ResizeObserver`, for the test to fire on cue. */ +let observers: (() => void)[] = []; + +/** + * happy-dom has no layout engine, so `offsetHeight` is 0 for everything and a real `ResizeObserver` + * never fires. Both are stubbed rather than worked around: the height so a measurement is + * distinguishable from the absence of one, and the observer so the test says exactly when it + * arrives instead of hoping. + */ +function installMeasurementProbe() { + measurements = 0; + observers = []; + + const height = Object.getOwnPropertyDescriptor( + HTMLElement.prototype, + "offsetHeight", + ); + Object.defineProperty(HTMLElement.prototype, "offsetHeight", { + configurable: true, + get() { + measurements += 1; + return 64; + }, + }); + + const realObserver = globalThis.ResizeObserver; + globalThis.ResizeObserver = class { + constructor(callback: () => void) { + observers.push(callback); + } + disconnect() {} + observe() {} + unobserve() {} + } as unknown as typeof ResizeObserver; + + return () => { + if (height) + Object.defineProperty(HTMLElement.prototype, "offsetHeight", height); + globalThis.ResizeObserver = realObserver; + }; +} + +/** A promise that never settles: anything that reads it suspends and stays suspended. */ +const pending = new Promise(() => {}); + +function Blocker() { + use(pending); + return null; +} + +test("the observer is refused while the box is closed", () => { + // The guard this component is built around, stated on its own so the two tests below cannot pass + // by having removed it. A caller may unmount its content on the way closed — `RejectedFiles` + // does, because a `role="alert"` left in the tree is still an alert — and the observer fires for + // that. Taking THAT measurement would overwrite the height we are animating FROM with the zero + // we are animating TO, and the close would have nothing to travel. + const restore = installMeasurementProbe(); + + try { + const { rerender } = render( + +
    content
    +     , + ); + + rerender({null}); + measurements = 0; + for (const fire of observers) fire(); + + expect(measurements).toBe(0); + } finally { + restore(); + } +}); + +test("a render that never commits does not close the guard", async () => { + // THE DEFECT: the guard's ref was assigned during render (`isOpen.current = open`), so it + // recorded what React was CONSIDERING rather than what React had committed. + // + // React is allowed to render a component and throw the work away. That is not exotic — it is + // what every interrupted or suspended transition does. Below, closing the box is wrapped in a + // transition that suspends, so React renders `Collapse` with `open={false}` and then abandons + // the attempt: the content is still mounted, the box is still open, and nothing on screen has + // moved. A ref written during render has already been told otherwise. + // + // The observer then fires for a real change to the still-open content, and the poisoned guard + // turns it away — the box goes on animating to a height its content no longer has. + const restore = installMeasurementProbe(); + let close = () => {}; + + function Harness() { + const [open, setOpen] = useState(true); + close = () => startTransition(() => setOpen(false)); + + return ( + <> + +
    content
    +     + waiting}> + {open ? null : } + + + ); + } + + try { + const { container } = render(); + + await act(async () => { + close(); + }); + + // The transition never landed: what is on screen is still the open box with its content. + expect(container.textContent).toBe("content"); + + measurements = 0; + await act(async () => { + for (const fire of observers) fire(); + }); + + expect(measurements).toBeGreaterThan(0); + } finally { + restore(); + } +}); + +test("opening measures the content in the commit that opened it", () => { + // THE DEFECT: `openHeight` starts at 0 and nothing but the `ResizeObserver` ever moved it. On + // the very first open there has been no measurement to move it with — the observer's own first + // callback landed while the box was still closed, and the guard above correctly refused it — so + // the first `open` flipped `animate` from a height of 0 to a height of 0. The real height only + // arrived on the frame after, once the observer had fired again and its `setState` had landed. + // + // Nobody sees an animation there. They see the composer sit still for a frame and then jump, + // which is the exact hitch this component exists to remove, on the first attachment of every + // session. + // + // The observer is stubbed to silence below, which is the point: with nothing firing it, the only + // way the content gets measured is if opening measures it. Before this was fixed, no measurement + // was ever taken and the strip animated to nothing. + const restore = installMeasurementProbe(); + + try { + const { rerender } = render( + +
    content
    +     , + ); + + measurements = 0; + rerender( + +
    content
    +     , + ); + + expect(measurements).toBeGreaterThan(0); + } finally { + restore(); + } +}); + +test("every later open measures again, not just the first", () => { + // The narrowest fix for the test above would be to measure once, on mount, and that would leave + // the second attachment of a session animating to the height of the first. A composer opens and + // closes this box all day, and what goes in it is a different size every time. + const restore = installMeasurementProbe(); + + try { + const { rerender } = render( + +
    content
    +     , + ); + + rerender( + +
    content
    +     , + ); + rerender({null}); + + measurements = 0; + rerender( + +
    a taller thing
    +     , + ); + + expect(measurements).toBeGreaterThan(0); + } finally { + restore(); + } +}); diff --git a/app/tests/composer-drop-guard.test.tsx b/app/tests/composer-drop-guard.test.tsx new file mode 100644 index 000000000..610a48a12 --- /dev/null +++ b/app/tests/composer-drop-guard.test.tsx @@ -0,0 +1,343 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { + cleanup, + createEvent, + fireEvent, + render, +} from "@testing-library/react"; +import { Composer } from "@/components/channels/composer/composer"; +import { settleReactWork } from "./settle-react-work"; + +/** + * WHAT HAPPENS TO A FILE DROPPED ON A COMPOSER THAT CANNOT ACCEPT IT. + * + * The composer used to spread `{}` in place of its drag handlers whenever `canAttach` was false — + * no `channelId`, or a `disabled` conversation. An element with no `dragover` handler is not a + * drop target at all, so the browser keeps the drop and performs its own default for a file + * dropped on a document: it navigates the top-level document to that file. The single-page app + * unloads and takes the typed sentence and the parked queue with it. See `refuseDragOver` in + * `composer.tsx` for the whole account. + * + * WHY THESE ASSERT ON `defaultPrevented` AND NOT ON A NAVIGATION. happy-dom implements no + * navigation whatsoever — there is no unload to observe, no `location` change, nothing an + * assertion could catch — so a test that tried to watch the symptom would pass identically before + * and after the fix and would be worth nothing. `defaultPrevented` is the CAUSE: it is the exact + * bit a real browser reads to decide whether to keep the drop, and it is the bit the old code + * never set. Pinning the cause is the only honest thing this environment can pin. + * + * `createEvent` + `fireEvent` rather than `fireEvent.drop(...)`, because it keeps a handle on the + * native event after dispatch. React's synthetic `preventDefault` forwards to that native event, + * so `event.defaultPrevented` is a direct read of what the browser would see. The events are + * cancelable by way of testing-library's own defaults for the drag family; an uncancelable event + * would report `false` here forever and quietly turn these into tautologies. + * + * BOTH FORM BRANCHES ARE EXERCISED. `dropZone` is spread onto the compact form AND the full-size + * one, and the full-size branch is what the home screen draws — a screen with no `channelId`, so + * one of the two states under test. `composer-paste.test.tsx` exists because that branch was once + * missed for exactly this kind of prop, and this file does not repeat the mistake. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in + * `afterEach`, matching `composer-dropped-attachments.test.tsx`: bun walks every file into one + * process, and a document another file tore down mid-run fails invisibly. The registration carries + * a `url` for the same reason it does there — without one `location` is `about:blank`. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +/** + * A `dataTransfer` good enough for the handler under test: it reads `.files` and writes + * `.dropEffect`, and nothing here is asked to be a real `DataTransfer`. + */ +function transfer(...files: File[]) { + return { dropEffect: "copy", files, items: [], types: ["Files"] }; +} + +/* + * `createEvent.dragOver` is typed as returning a bare `Event`, which carries no `dataTransfer`. + * testing-library copies the init's `dataTransfer` straight onto the event object it builds, so + * the property really is there at run time: it is the very stub handed in at the call site. This + * names that one fact, rather than asserting the event is a full `DragEvent` carrying a real + * `DataTransfer` -- which is exactly what the stub above documents itself as not being. + */ +function dropEffectOf(event: Event) { + return (event as Event & { dataTransfer: { dropEffect: string } }) + .dataTransfer.dropEffect; +} + +function png(name: string) { + return new File(["x"], name, { type: "image/png" }); +} + +function formIn(container: HTMLElement): HTMLFormElement { + const form = container.querySelector("form"); + if (!form) { + throw new Error("the composer rendered no form to drop onto"); + } + return form; +} + +test("a file dropped on a composer with no channel does not reach the browser", () => { + const { container } = render( {}} />); + const form = formIn(container); + + /* + * THE `dragover` ASSERTION IS THE LOAD-BEARING ONE, and it is the one the old code failed in a + * way happy-dom cannot show. In a real browser an unprevented `dragover` means the element never + * becomes a drop target and the `drop` below would not fire on it AT ALL — the browser would + * take it. happy-dom dispatches whatever it is told to dispatch, so `drop` "arrives" here either + * way; only this line distinguishes a form that would have caught the file from one that would + * have watched the page navigate away. + */ + const dragOver = createEvent.dragOver(form, { dataTransfer: transfer() }); + fireEvent(form, dragOver); + expect(dragOver.defaultPrevented).toBe(true); + + const drop = createEvent.drop(form, { dataTransfer: transfer(png("a.png")) }); + fireEvent(form, drop); + expect(drop.defaultPrevented).toBe(true); +}); + +test("a file dropped on a disabled conversation does not reach the browser", () => { + const { container } = render( + {}} />, + ); + const form = formIn(container); + + const dragOver = createEvent.dragOver(form, { dataTransfer: transfer() }); + fireEvent(form, dragOver); + expect(dragOver.defaultPrevented).toBe(true); + + const drop = createEvent.drop(form, { dataTransfer: transfer(png("b.png")) }); + fireEvent(form, drop); + expect(drop.defaultPrevented).toBe(true); +}); + +test("the full-size composer with no channel guards its drop too", () => { + // Not `compact`: the shape the home screen draws, which is also a screen with no `channelId`. + const { container } = render( {}} />); + const form = formIn(container); + + const dragOver = createEvent.dragOver(form, { dataTransfer: transfer() }); + fireEvent(form, dragOver); + expect(dragOver.defaultPrevented).toBe(true); + + const drop = createEvent.drop(form, { dataTransfer: transfer(png("c.png")) }); + fireEvent(form, drop); + expect(drop.defaultPrevented).toBe(true); + // Refusing the drop is the floor, not the whole answer: the file is still gone, so it is named. + expect(container.querySelector('[role="alert"]')?.textContent).toContain( + "c.png", + ); +}); + +test("a drop with no channel behind it says so, and says what to do instead", async () => { + const { container, findByRole } = render( + {}} />, + ); + + fireEvent( + formIn(container), + createEvent.drop(formIn(container), { + dataTransfer: transfer(png("screenshot.png")), + }), + ); + + const alert = await findByRole("alert"); + expect(alert.textContent).toContain("screenshot.png"); + // The "not yet" sentence: there is a next step, and it is the one the compose screen is for. + expect(alert.textContent).toContain("no conversation here yet"); + // And NOT the other one. Telling somebody on `/channel/new` that their conversation is over + // would be false about a conversation that has not started. + expect(alert.textContent).not.toContain("can no longer take messages"); +}); + +test("a drop on a disabled conversation says that one instead", async () => { + const { container, findByRole } = render( + {}} />, + ); + + fireEvent( + formIn(container), + createEvent.drop(formIn(container), { + dataTransfer: transfer(png("receipt.png")), + }), + ); + + const alert = await findByRole("alert"); + expect(alert.textContent).toContain("receipt.png"); + expect(alert.textContent).toContain("can no longer take messages"); + // No "send this first" advice on a conversation where sending is the thing that cannot happen. + expect(alert.textContent).not.toContain("no conversation here yet"); +}); + +test("two files refused at once produce two lines, one per file", async () => { + const { container, findByRole } = render( + {}} />, + ); + + fireEvent( + formIn(container), + createEvent.drop(formIn(container), { + dataTransfer: transfer(png("one.png"), png("two.png")), + }), + ); + + const alert = await findByRole("alert"); + // Matching every other refusal on this composer: folding a batch into one line loses which file + // the reason was about, and two files dragged from two folders share a name routinely. + expect(container.querySelectorAll('[role="alert"] p')).toHaveLength(2); + expect(alert.textContent).toContain("one.png"); + expect(alert.textContent).toContain("two.png"); +}); + +test("a drop carrying no file is still refused, and says nothing", () => { + const { container } = render( {}} />); + const form = formIn(container); + + // A dragged link or a text selection. The browser navigates for these too, so the default still + // has to be refused — but nothing was attached, there is no filename, and a refusal written here + // would be one invented for a gesture nobody made. + const drop = createEvent.drop(form, { + dataTransfer: { + dropEffect: "copy", + files: [], + items: [], + types: ["text/uri-list"], + }, + }); + fireEvent(form, drop); + + expect(drop.defaultPrevented).toBe(true); + expect(container.querySelector('[role="alert"]')).toBeNull(); +}); + +test("the cursor over a composer that cannot take the file says so before it is let go", () => { + const { container } = render( {}} />); + const form = formIn(container); + + /* + * READ BACK OFF THE EVENT, NOT OFF THE OBJECT HANDED TO `createEvent`, and the difference is not + * a detail. happy-dom implements `DataTransfer`, so testing-library takes the branch that copies + * each property of the init onto a REAL `new DataTransfer()` rather than attaching the literal + * — the handler therefore never sees the object written here, and an assertion against it reads + * an untouched "copy" forever no matter what the composer does. Asserting through the event is + * asserting against the thing the handler was actually given. + */ + const dragOver = createEvent.dragOver(form, { + dataTransfer: transfer(png("d.png")), + }); + fireEvent(form, dragOver); + + /* + * `preventDefault` on `dragover` makes the form a drop target, and a drop target left at the + * default effect draws the same copy-badge cursor the working composer draws — advertising an + * acceptance that is about to be refused. "none" is the no-entry cursor, and it is the only part + * of this refusal the person sees BEFORE they commit to the gesture. The init above says "copy", + * so this passing means the composer changed it rather than that it was never set. + */ + expect(dropEffectOf(dragOver)).toBe("none"); +}); + +/** + * THE BRANCH THAT ACCEPTS FILES IS A DROP TARGET TOO, AND NOTHING USED TO SAY SO. + * + * Every test above this one renders a composer that REFUSES. That left the whole file pinning one + * half of `dropZone`: deleting `onDragOver` from the accepting branch kept all 600 tests green + * while, in a real browser, a file dropped on a perfectly working composer navigated the page away + * — the exact bug this file was written for, on the branch people use every day. An element is a + * drop target only if something calls `preventDefault` on its `dragover`; "it has an `onDrop`" is + * not the same claim and does not imply it. + * + * WHY THE `dropEffect` LINE IS HERE AND NOT JUST THE FIRST ONE. Since `useUnclaimedDropGuard` + * (`routes/__root.tsx`) now refuses every unclaimed drop app-wide, a composer that quietly stopped + * claiming its own `dragover` would no longer navigate the page — the root would catch it — but it + * WOULD start drawing the root's no-entry cursor over a box that is about to accept the file. This + * asserts the effect is untouched at the "copy" the init sets, which is the cursor that says yes. + */ +test("a composer that can take the file claims the drag before the browser does", () => { + const { container } = render( + {}} />, + ); + const form = formIn(container); + + const dragOver = createEvent.dragOver(form, { + dataTransfer: transfer(png("welcome.png")), + }); + fireEvent(form, dragOver); + + expect(dragOver.defaultPrevented).toBe(true); + expect(dropEffectOf(dragOver)).toBe("copy"); +}); + +/** + * THE SECOND DROP LANDS ON THE FIRST REFUSAL, AND THAT IS NOT A CONTRIVED GESTURE. + * + * `RejectedFiles` renders ABOVE the form — deliberately, so the reasons sit next to the box rather + * than below it — and the handlers used to be on the form alone. So the strip was a hole in the + * guard, and the likeliest drop in the whole app aims straight at it: somebody drops a file, reads + * the sentence saying why it was not taken, and drops the retry on the sentence they are reading. + * That one went to the browser, and the page unloaded with the refusal still on screen. + * + * The handlers sit on the container that wraps both now — the same element `containerRef` marks for + * the paste listener, so "inside this composer" means one thing for both doors. + */ +test("a file let go over the refusal it caused is caught as well", async () => { + const view = render( {}} />); + const { container, findByRole } = view; + const form = formIn(container); + + fireEvent( + form, + createEvent.drop(form, { dataTransfer: transfer(png("first.png")) }), + ); + const alert = await findByRole("alert"); + + const second = createEvent.drop(alert, { + dataTransfer: transfer(png("second.png")), + }); + fireEvent(alert, second); + + expect(second.defaultPrevented).toBe(true); + // And it is answered rather than merely swallowed: two files were let go, so two lines say so. + expect(container.querySelectorAll('[role="alert"] p')).toHaveLength(2); + expect(container.textContent).toContain("second.png"); +}); + +/** + * ADVICE THAT CANNOT BE FOLLOWED IS WORSE THAN NONE. + * + * The sentence used to be chosen by `attachmentsEnabled` — is there a channel — which is right for + * three of the four states and wrong for this one. A composer that is BOTH `disabled` and + * channel-less was told "there is no conversation here yet. Send this message first, then attach to + * the one it opens", with the Send button beside it shut. Following that instruction is pressing a + * dead button; the person concludes the app is broken rather than that this conversation is over. + * `disabled` is the question actually being answered: can they do the thing the sentence is about + * to tell them to do. + */ +test("a disabled composer with no channel does not tell people to send first", async () => { + const { container, findByRole, getByLabelText } = render( + {}} />, + ); + const form = formIn(container); + + fireEvent( + form, + createEvent.drop(form, { dataTransfer: transfer(png("late.png")) }), + ); + + const alert = await findByRole("alert"); + expect(alert.textContent).toContain("late.png"); + expect(alert.textContent).toContain("can no longer take messages"); + // The half that made the old sentence a lie, asserted rather than assumed: the control it sent + // them to is shut. + expect((getByLabelText("Send message") as HTMLButtonElement).disabled).toBe( + true, + ); + expect(alert.textContent).not.toContain("Send this message first"); +}); diff --git a/app/tests/composer-dropped-attachments.test.tsx b/app/tests/composer-dropped-attachments.test.tsx new file mode 100644 index 000000000..b83e61d7c --- /dev/null +++ b/app/tests/composer-dropped-attachments.test.tsx @@ -0,0 +1,289 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import type { Attachment } from "@copilotkit/react-core/v2"; +import { cleanup, fireEvent, render, waitFor } from "@testing-library/react"; +import { Composer } from "@/components/channels/composer/composer"; +import { settleReactWork } from "./settle-react-work"; + +/** + * What the composer does with `droppedAttachments`: the files the queue has let go of, which must + * not vanish with nothing said — see `queue.ts`'s `droppedAttachments` and + * `conversation-view.tsx`, the only caller that produces this prop from a real queue. + * + * TWO KINDS OF FILE ARRIVE ON THIS PROP AND THEY GET DIFFERENT SENTENCES. The cap re-check bumps + * the excess off a drained turn; a person taking a queued message back takes everything it was + * carrying with it. `conversation-view.tsx` releases the server-side row either way and hands both + * here to be said out loud, with the cause attached — because the composer's reason for one of them + * is false about the other, and telling somebody about a limit they never reached sends them + * looking for a rule to work around. The last two cases below are that split; the earlier ones are + * the part that holds regardless, which is that the FILE is named. + * + * WHAT THE PROP OWES ITS CALLER, and it is not only a value: the effect that turns this into + * refusal lines is keyed on the OBJECT, so the same object arriving again is a render that reports + * nothing and a fresh object is a new batch of lines. That is a contract on whoever passes it, and + * the two `rerender` cases at the end are what hold it to that in both directions. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in + * `afterEach`, matching `agent-roster-error.test.tsx` for the reason recorded there: bun walks + * every file into one process, and a document another file tore down mid-run fails invisibly. + * + * The registration carries a `url`, matching `composer-attachments-ui.test.tsx`: without one + * `location` is `about:blank` and relative URLs do not resolve. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +/** + * `filename` is OPTIONAL on the SDK's `Attachment` and optional here for the same reason: the + * composer has a fallback for the case where there is no name to print, and a helper that always + * supplies one is a helper that makes that fallback untestable. See the last case in this file. + */ +function attachment(id: string, filename?: string): Attachment { + return { + id, + ...(filename === undefined ? {} : { filename }), + source: { type: "url", value: `https://example.com/${id}.png` }, + status: "ready", + type: "image", + }; +} + +test("a drain that drops attachments shows one line per dropped file, each naming that file", async () => { + const dropped = { + cause: "merged-over-cap", + attachments: [ + attachment("one", "invoice.pdf"), + attachment("two", "receipt.png"), + ], + } as const; + + const { container, findByRole } = render( + {}} />, + ); + + const alert = await findByRole("alert"); + const lines = container.querySelectorAll('[role="alert"] p'); + // One line per dropped file, not one message for the whole batch — collapsing them would leave + // no way to tell which of the two files a single reported reason was about. + expect(lines).toHaveLength(2); + expect(alert.textContent).toContain("invoice.pdf"); + expect(alert.textContent).toContain("receipt.png"); +}); + +test("a drain that drops none shows nothing", () => { + const { container } = render( + {}} + />, + ); + + expect(container.querySelector('[role="alert"]')).toBeNull(); +}); + +test("the dropped reason is distinguishable from a pick-time refusal reason", async () => { + const { container, findByRole } = render( + {}} + />, + ); + + // The dropped reason is already on screen from mount, before any file is ever picked. + const droppedAlert = await findByRole("alert"); + const droppedReason = droppedAlert.textContent ?? ""; + expect(droppedReason).toContain("queued messages were merged"); + + // Now trigger the other kind of refusal: a file refused at pick time, for an unrelated reason. + const form = container.querySelector("form") as HTMLFormElement; + fireEvent.drop(form, { + dataTransfer: { + files: [new File([""], "logo.svg", { type: "image/svg+xml" })], + items: [], + types: ["Files"], + }, + }); + + await waitFor(() => + expect(container.querySelectorAll('[role="alert"] p')).toHaveLength(2), + ); + const lines = Array.from(container.querySelectorAll('[role="alert"] p')).map( + (line) => line.textContent ?? "", + ); + + const pickTimeReason = lines.find((line) => line.includes("logo.svg")); + const dropReason = lines.find((line) => line.includes("invoice.pdf")); + expect(pickTimeReason).toBeDefined(); + expect(dropReason).toBeDefined(); + // Different sentences for a different cause: a file refused on pick names the SVG rule, and a + // file dropped by a drain names the cap and the merge that overran it. + expect(dropReason).not.toBe(pickTimeReason); + expect(dropReason).toContain("queued messages were merged"); + expect(pickTimeReason).toContain("SVG"); + expect(pickTimeReason).not.toContain("queued messages were merged"); +}); + +/** + * TWO CAUSES REACH THIS PROP, AND UNTIL NOW ONLY ONE OF THEM WAS TRUE OF THE SENTENCE. + * + * `reduceQueue` reports a bare `Attachment[]` whichever way the files left, and the composer built + * one hardcoded reason for it — "dropped when queued messages were merged into one: a message can + * carry at most 8 attachments". Once a REMOVED queued message routes its attachments down the same + * channel, that sentence is simply false: nothing was merged and no cap was hit. Somebody took a + * parked message out of the queue and its files went with it, and being told about a cap they never + * reached is worse than the bare fact, because it sends them looking for a limit to work around. + * + * The cause travels with the files now — see `DroppedAttachments` in `composer.tsx` — and + * `conversation-view.tsx` reads it off the queue action, which is the one place that knows. + */ +test("attachments dropped by a removal say that, not that a cap was hit", async () => { + const { findByRole } = render( + {}} + />, + ); + + const alert = await findByRole("alert"); + const reason = alert.textContent ?? ""; + + expect(reason).toContain("invoice.pdf"); + expect(reason).toContain("removed"); + expect(reason).not.toContain("merged"); + expect(reason).not.toContain("at most"); +}); + +test("the two causes do not share a sentence", async () => { + const merged = render( + {}} + />, + ); + const mergedReason = (await merged.findByRole("alert")).textContent ?? ""; + cleanup(); + + const removed = render( + {}} + />, + ); + const removedReason = (await removed.findByRole("alert")).textContent ?? ""; + + expect(mergedReason).not.toBe(removedReason); + expect(mergedReason).toContain("merged"); +}); + +/** + * THE CONTRACT THIS PROP PUTS ON ITS CALLER, WHICH NOTHING HELD IT TO. + * + * The composer keys its dropped-files effect on the prop OBJECT rather than on anything derived + * from it, and says why: "the caller is expected to hand over a fresh one only when a new drop + * actually happened". That is a real requirement and an invisible one. A caller that builds the + * object inline in its JSX passes a new identity on every render, and every render then appends + * the same refusal lines again — a list that grows without bound underneath somebody who is only + * typing. `conversation-view.tsx` holds it in state, so it is safe today; nothing was checking. + * + * Every other case in this file renders once, which is exactly the shape that cannot see this. + */ +test("the same dropped object arriving again reports nothing a second time", async () => { + const dropped = { + attachments: [attachment("one", "invoice.pdf")], + cause: "merged-over-cap", + } as const; + + const { container, findByRole, rerender } = render( + {}} />, + ); + await findByRole("alert"); + expect(container.querySelectorAll('[role="alert"] p')).toHaveLength(1); + + // The identical object, which is what an unrelated re-render looks like from in here: a keystroke, + // a parent refetch, a `pending` flipping. Nothing was dropped, so nothing may be said again. + rerender( + {}} />, + ); + + expect(container.querySelectorAll('[role="alert"] p')).toHaveLength(1); +}); + +test("a fresh dropped object with equal contents reports the drop again", async () => { + const first = { + attachments: [attachment("one", "invoice.pdf")], + cause: "merged-over-cap", + } as const; + + const { container, findByRole, rerender } = render( + {}} />, + ); + await findByRole("alert"); + expect(container.querySelectorAll('[role="alert"] p')).toHaveLength(1); + + // THE OTHER HALF, AND THE REASON THE FIRST CANNOT BE FIXED BY COMPARING CONTENTS. Two drops of + // the same file are two events — drop it, park it, remove the message, do all of it again — and + // a composer that deduplicated on value would swallow the second one silently. Equal contents, + // new object: one more line. + const second = { + attachments: [attachment("one", "invoice.pdf")], + cause: "merged-over-cap", + } as const; + rerender( + {}} />, + ); + + await waitFor(() => + expect(container.querySelectorAll('[role="alert"] p')).toHaveLength(2), + ); +}); + +/** + * A DROPPED FILE WITH NO NAME STILL GETS A LINE, AND THE LINE IS NOT ABOUT `undefined`. + * + * `filename` is optional on the SDK's `Attachment`, so the composer prints `filename ?? "Attachment"` + * — and every other case in this file went through a helper that always supplied one, so the + * fallback had never run. The rendered line is `: `, which without the fallback reads + * "undefined: dropped when queued messages were merged into one…": a sentence naming a file the + * person cannot match to anything they picked, about a file they cannot get back. + */ +test("a dropped attachment with no filename reads as an attachment, not as undefined", async () => { + const { container, findByRole } = render( + {}} + />, + ); + + await findByRole("alert"); + const line = container.querySelector('[role="alert"] p'); + + expect(line?.textContent).toStartWith("Attachment: "); + expect(line?.textContent).not.toContain("undefined"); +}); diff --git a/app/tests/composer-inflight-removal.test.tsx b/app/tests/composer-inflight-removal.test.tsx new file mode 100644 index 000000000..0ce65466e --- /dev/null +++ b/app/tests/composer-inflight-removal.test.tsx @@ -0,0 +1,299 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + expect, + test, +} from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { cleanup, fireEvent, render, waitFor } from "@testing-library/react"; +import { Composer } from "@/components/channels/composer/composer"; +import { settleReactWork } from "./settle-react-work"; + +/** + * A CHIP REMOVED WHILE ITS UPLOAD IS STILL IN THE AIR, WHICH USED TO LEAVE A ROW NOBODY COULD SEE. + * + * `removeAttachment` drops the SDK's placeholder. The upload it belonged to is mid-flight, so when + * it lands the SDK writes its answer onto a placeholder that no longer exists and the write is a + * no-op — but the row is on the server by then, in THIS composer's `uploadGroup`, which is minted + * once per mount and lives as long as the composer does. The screen counts seven, the server counts + * eight, and the next pick comes back `409 You already have 8 attachments waiting to send in this + * channel.` naming files that are on nobody's screen. That is verbatim the failure `uploadGroup` + * was introduced to remove, reached through a Remove button rather than through a closed tab. + * + * The composer cannot ask "was my chip removed?" at the moment the upload lands: `onUpload` is + * handed a `File` and the SDK never says which placeholder the call belongs to. So it reconciles + * instead — every row it uploaded against every row still drawn — and these tests drive that from + * both sides, because a reconciler that deletes too much is worse than the leak it replaces. + * + * A HARNESS OF ITS OWN RATHER THAN `composer-attachment-lifecycle.test.tsx`'s, for two reasons that + * file's fixture cannot give: an upload that can be RELEASED on demand rather than merely held + * open forever, and a distinct server id per file, since the whole question here is which row was + * given back. That file answers `stored-id` to everything, which cannot tell two rows apart. + * + * `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in `afterEach` is this repository's + * pattern, because bun walks every file into one process and a document another file tore down + * mid-run fails invisibly. The registration carries a `url` because without one `location` is + * `about:blank` and the relative upload URL does not resolve. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +const originalFetch = global.fetch; + +afterEach(() => { + global.fetch = originalFetch; +}); + +/** Every `DELETE` this composer sent, by path. */ +let deletes: string[]; +/** Uploads that have started and are waiting to be let go, in the order they started. */ +let held: (() => void)[]; + +/** Let every upload currently in the air answer. */ +function releaseUploads() { + const waiting = held; + held = []; + for (const release of waiting) { + release(); + } +} + +beforeEach(() => { + deletes = []; + held = []; + global.fetch = (async (path: string, init: RequestInit) => { + if (init?.method === "DELETE") { + deletes.push(path); + return new Response(null, { status: 204 }); + } + const file = (init.body as FormData).get("file") as File; + // Held until the test says otherwise, which is what makes "still in flight" a state a test can + // stand in rather than a race it has to win. + await new Promise((resolve) => { + held.push(resolve); + }); + return new Response( + JSON.stringify({ + // Named after the file so an assertion can say WHICH row was given back. That is the whole + // question in the two-file case below. + id: `row-${file.name}`, + name: file.name, + mimeType: "text/plain", + }), + { status: 201, headers: { "content-type": "application/json" } }, + ); + }) as unknown as typeof fetch; +}); + +function drop(form: Element, files: File[]) { + fireEvent.drop(form, { + dataTransfer: { files, items: [], types: ["Files"] }, + }); +} + +function textFile(name: string): File { + return new File(["hello"], name, { type: "text/plain" }); +} + +test("a chip removed mid-upload gives its row back once the upload lands", async () => { + const view = render( + {}} />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + drop(form, [textFile("notes.txt")]); + // The placeholder goes on the strip the moment the upload starts, which is the window this whole + // test lives in: a chip on screen with no row behind it yet. + await waitFor(() => + expect(view.queryByLabelText("Remove notes.txt")).not.toBeNull(), + ); + + fireEvent.click(view.getByLabelText("Remove notes.txt")); + expect(view.queryByLabelText("Remove notes.txt")).toBeNull(); + // Nothing to delete yet, and nothing invented: the row does not exist until the upload answers, + // so a DELETE here would be for an id the composer has never been told. + expect(deletes).toEqual([]); + + releaseUploads(); + + // And now it does exist, with no chip left to stand for it — so it goes back rather than sitting + // out the sweeper's 24-hour window counting against this channel's cap. + await waitFor(() => + expect(deletes).toEqual(["/api/attachments/row-notes.txt"]), + ); +}); + +test("only the removed one is given back when two uploads are in the air", async () => { + /* + * The half a reconciler gets wrong. Two uploads are in flight at once — two drops, two + * concurrent `processFiles` calls, since the SDK's own loop is sequential within one call — and + * only one chip is removed. Giving back both, or giving back the wrong one, would leave a live + * chip on the strip pointing at a row that has just been deleted: a message sent carrying a link + * to nothing, which is worse than the leak. + */ + const view = render( + {}} />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + drop(form, [textFile("gone.txt")]); + drop(form, [textFile("kept.txt")]); + await waitFor(() => { + expect(view.queryByLabelText("Remove gone.txt")).not.toBeNull(); + expect(view.queryByLabelText("Remove kept.txt")).not.toBeNull(); + }); + + fireEvent.click(view.getByLabelText("Remove gone.txt")); + releaseUploads(); + + await waitFor(() => + expect(deletes).toEqual(["/api/attachments/row-gone.txt"]), + ); + // The one nobody touched is still on the strip, and still owns its row. + expect(view.queryByLabelText("Remove kept.txt")).not.toBeNull(); +}); + +test("an upload nobody removed keeps its row", async () => { + /* + * The reconciler's own restraint, pinned on its own. Every test above removes something, so a + * reconciler that simply deleted every row it had ever heard about would pass all of them. + */ + const view = render( + {}} />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + drop(form, [textFile("notes.txt")]); + await waitFor(() => + expect(view.queryByLabelText("Remove notes.txt")).not.toBeNull(), + ); + releaseUploads(); + + // Waited for through the Send button rather than through the chip: the chip is on screen from the + // moment the upload starts, so it settles BEFORE the upload lands and would let this assert + // against a composer that had not yet reconciled anything. `canSendDraft` refuses a draft + // carrying an upload in flight, so a live Send is an upload that has been answered. + await waitFor(() => + expect( + (view.getByLabelText("Send message") as HTMLButtonElement).disabled, + ).toBe(false), + ); + + expect(deletes).toEqual([]); + expect(view.queryByLabelText("Remove notes.txt")).not.toBeNull(); +}); + +test("a message that was sent keeps the rows it carried", async () => { + /* + * The send's half of the same rule as the queue's, below. A landed send takes its chips off the + * strip, which to the reconciler is indistinguishable from somebody removing them — and deleting + * those rows would delete attachments out of a message that has already gone. + * + * `composer-attachment-lifecycle.test.tsx` covers the same ground and cannot see this: its + * fixture answers `stored-id` to every upload, so the row released by the send is the same string + * as every other row and the mistake cancels itself out. Distinct ids per file are the whole + * reason this file has a fixture of its own. + */ + const sent: unknown[] = []; + const view = render( + { + sent.push(draft); + }} + />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + drop(form, [textFile("notes.txt")]); + await waitFor(() => + expect(view.queryByLabelText("Remove notes.txt")).not.toBeNull(), + ); + releaseUploads(); + // `canSendDraft` refuses a draft carrying an upload in flight, so a live Send is an upload that + // has been answered — and a submit fired before that would be a no-op. + await waitFor(() => + expect( + (view.getByLabelText("Send message") as HTMLButtonElement).disabled, + ).toBe(false), + ); + + fireEvent.submit(form); + + await waitFor(() => expect(sent).toHaveLength(1)); + await waitFor(() => + expect(view.queryByLabelText("Remove notes.txt")).toBeNull(), + ); + expect(deletes).toEqual([]); +}); + +test("a parked message keeps the rows it is carrying", async () => { + /* + * The queue takes the chips off the strip without the attachments having been sent, so to the + * reconciler this looks exactly like a removal — and it must not be treated as one. The parked + * message still holds those attachments and will send them when the turn drains; deleting their + * rows here would send it carrying links to nothing. + */ + const queued: unknown[] = []; + const sent: unknown[] = []; + const view = render( + queued.push(draft)} + // A turn is in flight, which is what makes the send park rather than send. + pending + onSubmit={(draft) => { + sent.push(draft); + }} + />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + drop(form, [textFile("notes.txt")]); + await waitFor(() => + expect(view.queryByLabelText("Remove notes.txt")).not.toBeNull(), + ); + releaseUploads(); + await waitFor(() => + expect( + (view.getByLabelText("Queue message") as HTMLButtonElement).disabled, + ).toBe(false), + ); + + fireEvent.submit(form); + + await waitFor(() => expect(queued).toHaveLength(1)); + // The chip has left the strip with the parked message, and the row has NOT been given back. + expect(view.queryByLabelText("Remove notes.txt")).toBeNull(); + expect(deletes).toEqual([]); + + /* + * AND `onSubmit` WAS NOT CALLED, WHICH IS LOAD-BEARING SOMEWHERE ELSE ENTIRELY. + * + * `reduceQueue`'s submit branch has a join for "send now, with messages already parked" and says + * of it: "The two disagreeing is not supposed to be reachable — the drain empties the queue on + * the same edge that frees the composer." That join is the one path on which the cap can bump the + * LIVE draft's attachments, `conversation-view` deletes their rows as `droppedAttachments`, and + * this composer's failure path then hands the same chips back — chips pointing at rows that no + * longer exist. + * + * It is unreachable because of the line asserted here: the composer parks instead of sending + * whenever a turn is in flight, so `conversation-view`'s `submit(draft, false)` — the only caller + * that can pass `busy: false` — cannot fire while anything is parked. The queue is only ever + * non-empty while a turn is in flight, and the drain runs on the same commit that ends it. + * + * So the join is defensive, and this is the assertion that keeps it that way. If the composer + * ever sends while busy, that whole path goes live and the restore in `submitDraft`'s catch is + * where it will show up. + */ + expect(sent).toEqual([]); +}); diff --git a/app/tests/composer-insecure-context.test.tsx b/app/tests/composer-insecure-context.test.tsx new file mode 100644 index 000000000..70ce0859e --- /dev/null +++ b/app/tests/composer-insecure-context.test.tsx @@ -0,0 +1,187 @@ +import type { Attachment } from "@copilotkit/react-core/v2"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + expect, + test, +} from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { cleanup, fireEvent, render, waitFor } from "@testing-library/react"; +import { Composer } from "@/components/channels/composer/composer"; +import { settleReactWork } from "./settle-react-work"; + +/** + * THE COMPOSER ON A DEPLOYMENT THAT IS NOT A SECURE CONTEXT. + * + * `crypto.randomUUID` is a secure-context-only API. `http://localhost` is one, which is why a + * laptop never sees this; a deployment reached at plain `http://
    ` is not, and the property + * is simply ABSENT there — so the call does not degrade, it throws a `TypeError`. `lib/new-id.ts` + * exists for that reason alone and `new-id.test.ts` pins the function itself. + * + * These pin the CALLERS, which is the half that was wrong. Three id-minting sites in this composer + * called `crypto.randomUUID` directly, and all three are on the path that reports a refusal — so on + * an http deployment the composer's answer to a bad file was to break in a different way each time + * and say nothing at all. Every test below removes the function, exactly as that deployment does, + * and asserts the sentence still arrives. + * + * WHY IT IS DONE HERE AND NOT WITH A MOCK. The throw has to come from the real call site inside the + * real render, because what made this dangerous was never the id: it was where the throw landed — + * inside `screenPickedFiles` (taking the whole screening pass), inside the SDK's per-file upload + * loop, and inside a `useEffect` (taking the tree). A stub that returns a fake id would test the + * stub. + * + * THE HARNESS IS THIS REPOSITORY'S, matching `composer-attachment-lifecycle.test.tsx` — + * `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in `afterEach`, because bun walks + * every file into one process and a document another file tore down mid-run fails invisibly. The + * registration carries a `url` because without one `location` is `about:blank` and the relative + * upload URL does not resolve. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +const originalFetch = global.fetch; +let originalRandomUUID: typeof crypto.randomUUID; + +/** + * An origin that is not a secure context, spelled the way `new-id.test.ts` spells it: the property + * is gone, not merely different. + * + * Taken in `beforeEach` rather than once at module scope because bun runs every test file into one + * process, and a sibling that installs its own stub would otherwise be restored over. + */ +beforeEach(() => { + originalRandomUUID = crypto.randomUUID; + Object.defineProperty(crypto, "randomUUID", { + configurable: true, + value: undefined, + }); +}); + +afterEach(() => { + Object.defineProperty(crypto, "randomUUID", { + configurable: true, + value: originalRandomUUID, + }); + global.fetch = originalFetch; +}); + +/** + * The SDK mints its own placeholder ids with `uuid`'s v4, which reads `crypto.getRandomValues` and + * has no secure-context requirement — so removing `randomUUID` leaves uploads themselves working + * and breaks only our own refusal bookkeeping. That is the worst way round, and it is why these + * tests can drive a full upload with the function missing. + */ +function drop(form: Element, files: File[]) { + fireEvent.drop(form, { + dataTransfer: { files, items: [], types: ["Files"] }, + }); +} + +test("a pick-time refusal still names the file with no crypto.randomUUID", async () => { + /* + * `picked-files.ts`'s `reject()`. The throw came out of `screenPickedFiles` itself, which is the + * single door every drag, paste and file dialog goes through — so the FIRST refusable file in a + * gesture aborted the whole pass and the acceptable files beside it were never staged either. + * Both halves are asserted: the sentence about the SVG, and notes.txt staged from the same drop. + */ + const view = render( + {}} />, + ); + global.fetch = (async (_path: string, init: RequestInit) => { + const file = (init.body as FormData).get("file") as File; + return new Response( + JSON.stringify({ id: "stored-1", name: file.name, mimeType: file.type }), + { status: 201, headers: { "content-type": "application/json" } }, + ); + }) as unknown as typeof fetch; + const form = view.container.querySelector("form") as HTMLFormElement; + + drop(form, [ + new File([""], "logo.svg", { type: "image/svg+xml" }), + new File(["hello"], "notes.txt", { type: "text/plain" }), + ]); + + await waitFor(() => expect(view.queryByRole("alert")).not.toBeNull()); + expect( + view.queryByText(/can carry scripts and is not accepted/), + ).not.toBeNull(); + // The rest of the gesture survived the refusal, which is the part the throw used to take with it. + await waitFor(() => + expect(view.queryByLabelText("Remove notes.txt")).not.toBeNull(), + ); +}); + +test("an upload refusal still reaches the strip with no crypto.randomUUID", async () => { + /* + * `recordRejection`, which is the SDK's `onUploadFailed` — called from inside `processFiles`' + * per-file loop. A throw there escaped the loop, so the report of a failed upload itself failed: + * the file left the strip the way a failed upload always does and the sentence never arrived. + */ + global.fetch = (async () => + new Response(JSON.stringify({ error: "That file is not allowed here." }), { + status: 415, + headers: { "content-type": "application/json" }, + })) as unknown as typeof fetch; + + /* + * The SDK `console.error`s every failed upload before it calls `onUploadFailed`, and this test + * fails one on purpose — so without this the suite reports an error for the one thing here that + * is meant to happen, and a real error would be lost in the noise. Silenced the way + * `composer-rejected-files.test.tsx` silences React's key warning, and restored in a `finally` so + * a failing assertion cannot leave the console swallowed for every file after this one. + */ + const realError = console.error; + console.error = () => {}; + try { + const view = render( + {}} />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + drop(form, [new File(["hello"], "notes.txt", { type: "text/plain" })]); + + await waitFor(() => + expect( + view.queryByText(/That file is not allowed here\./), + ).not.toBeNull(), + ); + } finally { + console.error = realError; + } +}); + +test("a queue drop still names its files with no crypto.randomUUID", () => { + /* + * The `droppedAttachments` effect, and the least survivable of the three: an effect body that + * throws propagates out of React's commit, so this rendered nothing at all rather than rendering + * the list of files the queue had just thrown away. + */ + const view = render( + {}} + />, + ); + + expect(view.queryByRole("alert")).not.toBeNull(); + expect(view.queryByText(/invoice\.pdf/)).not.toBeNull(); +}); diff --git a/app/tests/composer-paste.test.tsx b/app/tests/composer-paste.test.tsx new file mode 100644 index 000000000..72b3a25f0 --- /dev/null +++ b/app/tests/composer-paste.test.tsx @@ -0,0 +1,470 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + expect, + test, +} from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { + act, + cleanup, + fireEvent, + render, + type RenderResult, + waitFor, +} from "@testing-library/react"; +import { Composer } from "@/components/channels/composer/composer"; +import { settleReactWork } from "./settle-react-work"; + +/** + * PASTE, THE THIRD AND LAST DOOR A FILE COMES IN THROUGH. + * + * The `+` picker and the drop are already screened; paste was not. The SDK's own listener + * pre-filters the clipboard with an exact `file.type === filter` match and then returns without a + * word when nothing survives it, so a pasted text file went nowhere and said nothing. The composer + * now claims paste ahead of it — see the fourth test, which is the pin for that whole failure. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in + * `afterEach`, matching `composer-attachments-ui.test.tsx` for the reason recorded there: bun walks + * every file into one process, and a document another file tore down mid-run fails invisibly. The + * registration carries a `url` because without one `location` is `about:blank` and relative URLs — + * every attachment preview is one — do not resolve. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +const originalFetch = global.fetch; + +afterEach(() => { + global.fetch = originalFetch; +}); + +/** Every upload this composer attempted, in order, by filename. */ +let uploads: string[]; + +beforeEach(() => { + uploads = []; + global.fetch = (async (_path: string, init: RequestInit) => { + const file = (init.body as FormData).get("file") as File; + uploads.push(file.name); + return new Response( + JSON.stringify({ + id: "attachment-id", + name: file.name, + mimeType: file.type, + }), + { status: 201, headers: { "content-type": "application/json" } }, + ); + }) as unknown as typeof fetch; +}); + +/** + * A clipboard as the browser hands one over: files reachable through `items`, and whatever text + * came with them behind `getData`. Both halves matter — the decision to claim a paste or leave it + * alone is made by comparing the two. + */ +function clipboard({ + files = [], + text = "", +}: { + files?: File[]; + text?: string; +}) { + return { + files, + items: files.map((file) => ({ + kind: "file", + type: file.type, + getAsFile: () => file, + })), + types: [...(files.length > 0 ? ["Files"] : []), "text/plain"], + getData: (type: string) => (type === "text/plain" ? text : ""), + }; +} + +/** The editor inside the composer: the element a paste actually lands on. */ +function editorOf(container: HTMLElement): HTMLElement { + return container.querySelector("[contenteditable]") as HTMLElement; +} + +/** What is typed in the box, with the placeholder and the strip left out of it. */ +function typedText(container: HTMLElement): string { + return editorOf(container).textContent ?? ""; +} + +/** + * A paste into the composer, caret and all. + * + * The caret is not decoration. PromptArea inserts pasted text at the selection and gives up when + * there is not one, so an editor nobody has clicked into would swallow an ordinary text paste for + * a reason that has nothing to do with this composer — and the test that ordinary pastes still + * reach the box would pass while proving nothing. + */ +function pasteInto( + container: HTMLElement, + clipboardData: ReturnType, +) { + const editor = editorOf(container); + editor.focus(); + const caret = document.createRange(); + caret.selectNodeContents(editor); + caret.collapse(true); + const selection = window.getSelection(); + selection?.removeAllRanges(); + selection?.addRange(caret); + fireEvent.paste(editor, { clipboardData }); +} + +/** + * A pasted file, all the way up. + * + * Not `uploads`, and not the chip either: both are true while the request is still outstanding, and + * a test that ends on either leaves the response to land after the document is gone. See the note + * on `uploaded` in `composer-attachments-ui.test.tsx` for the whole mechanism — Send is shut while + * anything is `uploading`, so a Send that has come back on is an upload that has been answered. + */ +async function uploaded( + { getByLabelText, queryByLabelText }: RenderResult, + name: string, +) { + await waitFor(() => { + expect(queryByLabelText(`Remove ${name}`)).not.toBeNull(); + expect((getByLabelText("Send message") as HTMLButtonElement).disabled).toBe( + false, + ); + }); +} + +test("a pasted screenshot is attached, and types nothing into the box", async () => { + const view = render( + {}} />, + ); + const { container } = view; + + const shot = new File(["png"], "screenshot.png", { type: "image/png" }); + pasteInto(container, clipboard({ files: [shot] })); + + await uploaded(view, "screenshot.png"); + + /* + * ONE UPLOAD, AND IT IS NOT `stopPropagation` THAT MAKES IT ONE. + * + * This assertion used to carry a comment claiming it proved the capture-phase `stopPropagation` + * cut the SDK's listener off. It does not, and the way to see that is to delete + * `event.stopPropagation()` from `composer.tsx` and run this suite: it stays green. The SDK's + * listener opens `if (!containerRef.current?.contains(target)) return`, and `containerRef` is the + * hook's own ref, which this composer deliberately never attaches — it holds its own instead, for + * the reason recorded on that ref. So the hook's listener returns on the first line of every + * paste, whether or not anything stopped the event, and one upload is all there could have been. + * + * What this line really pins is that OUR path uploads exactly once. The effect of + * `stopPropagation` is pinned separately, by the last test in this file, which watches a + * bubble-phase `document` listener — the exact shape the SDK installs — rather than inferring it. + */ + expect(uploads).toEqual(["screenshot.png"]); + expect(typedText(container)).toBe(""); + expect(container.querySelector('[role="alert"]')).toBeNull(); +}); + +test("a pasted spreadsheet cell types its text and attaches nothing", async () => { + const { container } = render( + {}} />, + ); + + // A cell copied out of a spreadsheet is BOTH: an `.html` (or `.txt`) file for the formatting and + // the plain text for everybody else. Claiming any paste that carries a file would swallow this + // one whole and leave the person with a paste that visibly did nothing. + pasteInto( + container, + clipboard({ + files: [new File(["7"], "cell.html", { type: "text/html" })], + text: "7", + }), + ); + + await waitFor(() => expect(typedText(container)).toContain("7")); + expect(uploads).toEqual([]); + expect(container.querySelector('[role="alert"]')).toBeNull(); +}); + +test("a pasted file with no text alongside it is attached, not typed", async () => { + const view = render( + {}} />, + ); + const { container } = view; + + pasteInto( + container, + clipboard({ files: [new File(["a,b"], "rows.csv", { type: "text/csv" })] }), + ); + + await uploaded(view, "rows.csv"); + + expect(uploads).toEqual(["rows.csv"]); + // The filename is not a message. A composer that typed it would be putting words in somebody's + // mouth on top of attaching the file they asked for. + expect(typedText(container)).toBe(""); +}); + +test("a pasted `text/plain;charset=utf-8` file is attached", async () => { + // THE REGRESSION PIN FOR THIS WHOLE TASK. The SDK's paste listener compares `file.type` to its + // accept list exactly, and this is the type a browser reports for a text file taken off the + // clipboard — so the file was filtered out, `processFiles` was never reached, and its + // `fileItems.length === 0` early return said nothing to anybody. Nothing uploaded, nothing + // refused, nothing on screen. + const notes = new File(["hello"], "notes.txt", { + type: "text/plain;charset=utf-8", + }); + + const view = render( + {}} />, + ); + const { container } = view; + + pasteInto(container, clipboard({ files: [notes] })); + + await uploaded(view, "notes.txt"); + + expect(uploads).toEqual(["notes.txt"]); + // Asked once the whole upload has been through the SDK, which is the only point at which "and it + // was not refused on the way" is a question rather than a coin toss on timing. + expect(container.querySelector('[role="alert"]')).toBeNull(); + expect(typedText(container)).toBe(""); +}); + +test("with no channel, a pasted file does nothing and no listener is installed", async () => { + const listeners: boolean[] = []; + const addEventListener = document.addEventListener.bind(document); + document.addEventListener = (( + type: string, + listener: EventListener, + options?: boolean | AddEventListenerOptions, + ) => { + if (type === "paste") { + listeners.push(true); + } + addEventListener(type, listener, options); + }) as typeof document.addEventListener; + + try { + const { container } = render( {}} />); + + pasteInto( + container, + clipboard({ + files: [new File(["hello"], "notes.txt", { type: "text/plain" })], + }), + ); + + // Neither ours nor the SDK's: with no `channelId` there is no config, so the hook is disabled + // and this composer is exactly the one every screen had before attachments existed. + expect(listeners).toEqual([]); + await waitFor(() => expect(uploads).toEqual([])); + expect(container.querySelector('[role="alert"]')).toBeNull(); + } finally { + document.addEventListener = addEventListener; + } +}); + +/** + * THE FULL-SIZE BRANCH, WHICH IS THE ONE THE HOME AND ONBOARDING COMPOSERS DRAW. + * + * A clipboard carrying an image AND text is not ours: `shouldClaimPaste` gives text the win, so our + * capture-phase listener declines it and the event goes on to reach PromptArea. Its own rule then + * decides — Microsoft Office markup in the `text/html` means the text is what was copied, anything + * else means the image is — and for every source that is not Word or Excel (Google Sheets, Numbers, + * a screenshot tool that puts a caption on the clipboard too) it takes the second branch, calls + * `onImagePaste` and returns having inserted nothing. + * + * Without that prop the call goes nowhere and the paste has already been `preventDefault`-ed: no + * text typed, no file staged, no refusal shown. It was passed on the compact branch only, so this + * composer — the one the home screen and the onboarding poster render — swallowed such a paste + * whole, which is the one outcome this file exists to make impossible. + * + * ASSERTED ON THE STRIP, NOT ON THE FETCH STUB. The stub records an upload the instant it is + * called, which is before the composer has seen the answer; a tile with the pasted file's name on + * it is the first thing that proves the file actually went through `stageFiles`. + */ +test("a full-size composer stages an image pasted alongside text", async () => { + const { container, queryByLabelText } = render( + {}} />, + ); + + const chart = new File(["png"], "chart.png", { type: "image/png" }); + // `act` around the paste, which no other test in this file needs: this branch is the only one + // that draws PromptArea with `autoGrow`, and the measurement that follows a paste settles in a + // task of its own. Unwrapped, that update lands outside React's test scope and is reported as a + // warning that has nothing to do with what is being asserted. + await act(async () => { + pasteInto(container, clipboard({ files: [chart], text: "Q3 revenue" })); + }); + + await waitFor(() => + expect(queryByLabelText("Remove chart.png")).not.toBeNull(), + ); + // All the way through the upload rather than stalled as a placeholder: the tile only draws an + // `` once `onUpload` has answered with a URL to point it at. + await waitFor(() => + expect( + container.querySelector('img[alt="chart.png"]')?.getAttribute("src"), + ).toBe("/api/attachments/attachment-id"), + ); + expect(uploads).toEqual(["chart.png"]); +}); + +/** + * THE PASTE DOOR ANSWERS TO `disabled` TOO. + * + * Our capture-phase listener was installed on `attachmentsEnabled` alone, so a composer the screen + * had already declared finished went on claiming pastes and uploading what it found in them. Same + * settle as the drop test in `composer-attachments-ui.test.tsx`: the composer is re-enabled and + * pasted into again, and the wait is on that second upload being ANSWERED — the first, started + * earlier, would have had to show up before it. + */ +test("a disabled composer claims no paste and stages nothing", async () => { + /* + * THE CLAIM IS WHAT IS ASSERTED, AND THE UPLOAD COUNT ALONE COULD NOT SEE IT. + * + * This used to watch `uploads` and nothing else. Two independent guards keep a file off a + * disabled composer — `canAttach` keeps the capture-phase listener uninstalled, and `stageFiles` + * re-asks `disabled` at the choke point — and against an upload count they are redundant, so + * either could be deleted on its own and this stayed green. + * + * Worse, the guard it names is the one that could go silently. Re-arm the listener while disabled + * and the paste is `preventDefault`-ed and `stopPropagation`-ed and then dropped on the floor by + * `stageFiles`: no upload, so the old assertion held, and an ordinary text paste into a finished + * channel now types NOTHING. That is a worse outcome than the bug this test was written for. + * + * So the bubble-phase listener from the test below stands in for everything downstream, and the + * assertion is that a disabled composer LET THE PASTE THROUGH untouched. + * + * `stageFiles`' own `disabled` return is deliberately not pinned here, and cannot be from the + * outside: while `canAttach` is correct it closes every door — no capture listener, no drop + * handlers, a disabled file input, and `onImagePaste` passed as `undefined` — so nothing can + * reach `stageFiles` to be turned away by it. It is defence in depth for `onImagePaste`, which is + * PromptArea's call to make and not ours, and its comment says exactly that. + */ + const reached: string[] = []; + const bubbleListener = () => reached.push("paste"); + document.addEventListener("paste", bubbleListener); + + try { + const view = render( + {}} />, + ); + const { container, rerender } = view; + + pasteInto( + container, + clipboard({ + files: [new File(["png"], "refused.png", { type: "image/png" })], + }), + ); + + // Not claimed: the event went past the composer to everything downstream of it, which is what + // keeps an ordinary paste working in a channel that can no longer take a message. + expect(reached).toEqual(["paste"]); + + rerender( {}} />); + pasteInto( + container, + clipboard({ + files: [new File(["png"], "kept.png", { type: "image/png" })], + }), + ); + await uploaded(view, "kept.png"); + + // And re-enabled it claims again, so the paste stops here rather than bubbling on. + expect(reached).toEqual(["paste"]); + expect(uploads).toEqual(["kept.png"]); + expect(container.querySelector('[role="alert"]')).toBeNull(); + } finally { + document.removeEventListener("paste", bubbleListener); + } +}); + +/** + * THE COMPACT BRANCH, WHICH IS THE ONE `channel-chat` DRAWS — SO IT IS WHERE MOST PASTES LAND. + * + * `onImagePaste` is passed twice, and the note on it in `composer.tsx` says every branch has to + * pass it because the full-size one once did not. The test above pins the full-size branch, i.e. + * the direction the historical failure happened in; blanking the prop on the COMPACT branch left + * the whole suite green, on the composer a person actually types into all day. + * + * Same clipboard as the full-size case: an image alongside text, which `shouldClaimPaste` declines + * so that PromptArea's own rule gets it and hands the image to `onImagePaste`. Without the prop + * that call goes nowhere and the paste has already been `preventDefault`-ed — no text, no file, no + * refusal. + * + * No `act` wrapper here, unlike the full-size case: that one is only needed for `autoGrow`'s + * measurement, which the compact branch does not use. + */ +test("a compact composer stages an image pasted alongside text", async () => { + const view = render( + {}} />, + ); + + pasteInto( + view.container, + clipboard({ + files: [new File(["png"], "chart.png", { type: "image/png" })], + text: "Q3 revenue", + }), + ); + + await waitFor(() => + expect(view.queryByLabelText("Remove chart.png")).not.toBeNull(), + ); + expect(uploads).toEqual(["chart.png"]); +}); + +/** + * `stopPropagation`, PINNED DIRECTLY RATHER THAN INFERRED FROM AN UPLOAD COUNT. + * + * `useAttachments` registers its paste handler with a plain `document.addEventListener("paste", h)` + * — bubble phase — so a listener of the same shape is the honest stand-in for it, and unlike the + * real one it is not also gated on a `containerRef` this composer never attaches. A claimed paste + * must not reach it; a declined one must, because that is the whole reason the capture-phase + * listener stops at claiming rather than swallowing everything. + * + * Delete `event.stopPropagation()` from `composer.tsx` and the first half of this goes red, which + * is what the first test in this file was mistakenly credited with doing. + */ +test("a claimed paste is stopped at the capture phase, and a declined one is not", async () => { + const reached: string[] = []; + const bubbleListener = () => reached.push("paste"); + document.addEventListener("paste", bubbleListener); + + try { + const view = render( + {}} />, + ); + const { container } = view; + + pasteInto( + container, + clipboard({ + files: [new File(["png"], "claimed.png", { type: "image/png" })], + }), + ); + // Asked once the claimed paste has been all the way up, so "it never arrived" is a settled + // answer rather than a question asked too early. + await uploaded(view, "claimed.png"); + expect(reached).toEqual([]); + + // Text and no file: `shouldClaimPaste` gives text the win, our listener declines, and the event + // goes on to everything downstream of it — which is what makes an ordinary paste still work. + pasteInto(container, clipboard({ text: "just words" })); + await waitFor(() => expect(typedText(container)).toContain("just words")); + expect(reached).toEqual(["paste"]); + } finally { + document.removeEventListener("paste", bubbleListener); + } +}); diff --git a/app/tests/composer-queue-attachments.test.tsx b/app/tests/composer-queue-attachments.test.tsx new file mode 100644 index 000000000..254d3a0be --- /dev/null +++ b/app/tests/composer-queue-attachments.test.tsx @@ -0,0 +1,851 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + expect, + spyOn, + test, +} from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { + cleanup, + fireEvent, + render, + type RenderResult, + waitFor, +} from "@testing-library/react"; +import * as ReactCoreV2 from "@copilotkit/react-core/v2"; +import { useCallback } from "react"; +import { Composer } from "@/components/channels/composer/composer"; +import { ConversationView } from "@/components/channels/conversation-view"; +import type { ComposerDraft } from "@/components/channels/composer/draft"; +import { + attachmentUrl, + MAX_ATTACHMENTS_PER_MESSAGE, +} from "@/lib/channels/attachments"; +import { settleReactWork } from "./settle-react-work"; + +/** + * WHAT THE QUEUE DOES TO AN ATTACHMENT BEHIND SOMEBODY'S BACK, AT BOTH ENDS OF THE SAME ROW. + * + * The first half of the file is the composer handing a staged attachment TO the queue. The second + * half — from `ConversationView` down — is the queue handing one back, and what has to happen to + * the server-side row when it does. The last test is about the instrument rather than the code. + * + * The queue branch used to call the SDK's `consumeAttachments()`, which sweeps every `ready` + * attachment regardless of which one a send already in flight is riding — so parking a + * correction while a first send was still out took the first send's own attachment with it, and + * the composer's own comment ("the catch hands them straight back") went false the moment a + * queue happened mid-send. The fix removes only what `onQueue` is actually taking. + * + * WHY THE FIRST TEST SPIES ON THE SDK HOOK RATHER THAN DRIVING AN ACTUAL FAILED SEND. The + * regression only shows up on screen once `sending` stops hiding the riding attachment, and the + * only two things that clear `sending` are a successful send — whose own `finally` removes the + * riding ids explicitly either way, masking the bug — and a failed one, whose `catch` this suite + * cannot reach: `submitDraft`'s rejection reaches both call sites (`handleFormSubmit`'s form + * submit and prompt-area's own Enter-key call) as a voided promise neither awaits nor catches, + * so bun's test runner reports the resulting unhandled rejection as a failure of whichever test + * is running when it surfaces — see `composer-attachment-lifecycle.test.tsx`'s second test for + * this repository's own note on the same wall. What is reachable, and what the fix actually + * changes, is which SDK function the queue branch calls — `consumeAttachments()` (sweeps every + * `ready` attachment) versus `removeAttachment()` per id (surgical) — so this spies on both, + * wrapping the real hook rather than replacing it, and asserts on the calls directly. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` + * in `afterEach`, matching `composer-attachment-lifecycle.test.tsx` for the reason recorded + * there: bun walks every file into one process, and a document another file tore down mid-run + * fails invisibly. The registration carries a `url` because without one `location` is + * `about:blank` and the relative URLs every one of these requests uses do not resolve. + */ + +/** + * Captured before the spy is installed, because the spy replaces this very property: calling + * `ReactCoreV2.useAttachments` from inside the implementation below would call the spy again and + * recur forever. + */ +const realUseAttachments = ReactCoreV2.useAttachments; + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +/** Every id `removeAttachment` was called with, and how many times `consumeAttachments` was. */ +let removeAttachmentCalls: string[]; +let consumeAttachmentsCallCount: number; + +/** + * Every distinct function identity the spy has handed a rendering composer, so the test below can + * say that it handed out exactly one of each. See the note on stability under the spy itself. + */ +const handedOut = { + removeAttachment: new Set(), + consumeAttachments: new Set(), +}; + +/** + * A THIN SPY AROUND THE REAL HOOK, NOT A REPLACEMENT FOR IT. Every other field — `attachments`, + * `processFiles`, the drag handlers — passes straight through to the real `useAttachments`, so + * uploads, paste and drag still behave exactly as the real hook makes them; only the two calls + * this file cares about are intercepted, on their way to doing the real thing regardless. + * + * `spyOn` ON THE MODULE NAMESPACE, NOT `mock.module`, BECAUSE ONLY ONE OF THE TWO COMES BACK OFF. + * Both reach composer.tsx — its `import { useAttachments }` is a live binding, so a patch applied + * here lands even though that import already ran. The difference is the undo. Bun has no way to + * unregister a module mock: `mock.restore()` leaves one in place, so the honest reading of what + * this file used to do in `afterAll` — call `mock.module` a second time with a spread snapshot — + * is that it installed a PERMANENT mock rather than restoring anything, and every test file bun + * walked into the same process afterwards imported that snapshot instead of the package. A + * namespace spy is restorable, and `afterAll` genuinely puts the real function back. + * + * THE TWO WRAPPERS ARE MEMOISED, and that is not tidiness. The SDK builds both of these as + * `useCallback(…, [])` — they are stable for the life of the hook, deliberately — and composer.tsx + * depends on that: `submitDraft` lists `removeAttachment` in its dependency array, and every + * prompt-area callback hangs off `submitDraft`. Minting fresh wrappers on each render made the + * spy, not the code under test, the thing that decided how often those memos were rebuilt, which + * is a harness that changes the behaviour it is measuring. Keyed on the SDK's own functions, so + * they stay stable exactly as long as the real ones do. + */ +const useAttachmentsSpy = spyOn(ReactCoreV2, "useAttachments"); + +beforeAll(() => { + useAttachmentsSpy.mockImplementation( + (config: Parameters[0]) => { + const hook = realUseAttachments(config); + const removeAttachment = useCallback( + (id: string) => { + removeAttachmentCalls.push(id); + return hook.removeAttachment(id); + }, + [hook.removeAttachment], + ); + const consumeAttachments = useCallback(() => { + consumeAttachmentsCallCount += 1; + return hook.consumeAttachments(); + }, [hook.consumeAttachments]); + + handedOut.removeAttachment.add(removeAttachment); + handedOut.consumeAttachments.add(consumeAttachments); + + return { ...hook, removeAttachment, consumeAttachments }; + }, + ); +}); + +afterAll(() => { + useAttachmentsSpy.mockRestore(); +}); + +const originalFetch = global.fetch; + +afterEach(() => { + global.fetch = originalFetch; +}); + +/** Every `DELETE` this composer sent, by path. */ +let deletes: string[]; + +/** + * Every upload that actually left the browser, by file name. + * + * The point of a client-side cap is that a file it refuses never becomes a request. Asserting only + * on the reason line would pass just as happily against a composer that uploaded the file and then + * printed the server's refusal, which is the round trip the cap exists to remove — so the absence + * of the POST is the assertion that distinguishes them. + */ +let uploads: string[]; + +beforeEach(() => { + removeAttachmentCalls = []; + consumeAttachmentsCallCount = 0; + handedOut.removeAttachment.clear(); + handedOut.consumeAttachments.clear(); + deletes = []; + uploads = []; + global.fetch = (async (path: string, init: RequestInit) => { + if (init?.method === "DELETE") { + deletes.push(path); + return new Response(null, { status: 204 }); + } + const file = (init.body as FormData).get("file") as File; + uploads.push(file.name); + return new Response( + JSON.stringify({ + // One stored id per file rather than one for the whole suite, so a test that stages more + // than one attachment can say WHICH of them a DELETE was for. Derived from the name + // because every file below is named for the role it plays. + id: `stored-${file.name}`, + name: file.name, + mimeType: "text/plain", + }), + { status: 201, headers: { "content-type": "application/json" } }, + ); + }) as unknown as typeof fetch; +}); + +/** A drop, as the browser delivers one: files hanging off `dataTransfer`. */ +function drop(form: Element, files: File[]) { + fireEvent.drop(form, { + dataTransfer: { files, items: [], types: ["Files"] }, + }); +} + +function notes() { + return new File(["hello"], "notes.txt", { type: "text/plain" }); +} + +function correction() { + return new File(["later"], "correction.txt", { type: "text/plain" }); +} + +/** + * A dropped file, all the way up. + * + * `Remove ` on its own is not that condition — the chip goes on the strip the moment the + * upload starts, so a wait on it can be over while `POST .../attachments` is still outstanding, and + * both tests below then depend on a race they never state. See the note on `uploaded` in + * `composer-attachments-ui.test.tsx` for what a test that ends mid-upload does to the document. + * + * The button is the one thing on screen that knows: `canSendDraft` holds it shut while any + * attachment is still `uploading`. Which of the two it is says something in itself here — this + * composer has an `onQueue`, so mid-run the same button is Queue — and either name being live is + * the same answer, that the upload has been answered and the composer has re-rendered on it. + */ +async function uploaded({ queryByLabelText }: RenderResult, name: string) { + await waitFor(() => { + expect(queryByLabelText(`Remove ${name}`)).not.toBeNull(); + const button = (queryByLabelText("Send message") ?? + queryByLabelText("Queue message")) as HTMLButtonElement | null; + expect(button?.disabled).toBe(false); + }); +} + +test("queuing a correction mid-send removes only what it takes, never sweeps the attachment the send is riding", async () => { + const submitted: ComposerDraft[] = []; + const queued: ComposerDraft[] = []; + let land: () => void = () => {}; + const run = new Promise((resolve) => { + land = resolve; + }); + + const view = render( + queued.push(draft)} + onSubmit={async (draft) => { + submitted.push(draft); + await run; + }} + />, + ); + const { container, queryByLabelText } = view; + + const form = container.querySelector("form") as HTMLFormElement; + + // The first send takes notes.txt off the strip and holds it there for the length of the run. + drop(form, [notes()]); + await uploaded(view, "notes.txt"); + fireEvent.submit(form); + await waitFor(() => expect(submitted).toHaveLength(1)); + expect(queryByLabelText("Remove notes.txt")).toBeNull(); + + // A correction, staged while the first send is still out — the next message's file, not this + // one's. + drop(form, [correction()]); + await uploaded(view, "correction.txt"); + + // Parked, not sent: `isSubmitting` is still true from the send above, so this press queues. + fireEvent.submit(form); + expect(queued).toHaveLength(1); + // Only the correction rode into the queued draft — the first send's attachment was never + // `staged` to begin with, so it was never a candidate for this message. + expect(queued[0].attachments).toHaveLength(1); + expect(queued[0].attachments[0].filename).toBe("correction.txt"); + // Taken off the strip because it was queued, same as any parked attachment. + expect(queryByLabelText("Remove correction.txt")).toBeNull(); + + // THE REGRESSION, PINNED DIRECTLY. The old code called `consumeAttachments()`, which sweeps + // every `ready` attachment regardless of `sending` — including notes.txt, which this send is + // still riding. The fix calls `removeAttachment()` once per id `onQueue` actually took, and + // never the sweep at all. + expect(consumeAttachmentsCallCount).toBe(0); + expect(removeAttachmentCalls).toEqual([queued[0].attachments[0].id]); + + land(); + // `aria-busy` IS `isSubmitting`, so this is the run finishing and the composer re-rendering + // without it. A wait for `submitted` to have one entry was already satisfied before `land()` and + // so let the test end with the run's own state updates still to come. + await waitFor(() => expect(form.getAttribute("aria-busy")).toBe("false")); +}); + +test("removing a ready attachment issues the DELETE through attachmentUrl(), not a hand-typed path", async () => { + const view = render( + {}} />, + ); + const { container, getByLabelText, queryByLabelText } = view; + + drop(container.querySelector("form") as HTMLFormElement, [notes()]); + // Ready, which is the state this test names: an `uploading` attachment has no stored row behind + // it and issues no DELETE at all, so pressing Remove too early would pin nothing. + await uploaded(view, "notes.txt"); + + fireEvent.click(getByLabelText("Remove notes.txt")); + + // Built from the same helper the server-side comment and `chat-transcript.tsx` both insist on: + // a hand-typed literal here is exactly the fifth spelling this repository forbids. + await waitFor(() => + expect(deletes).toEqual([attachmentUrl("stored-notes.txt")]), + ); + expect(queryByLabelText("Remove notes.txt")).toBeNull(); +}); + +/** + * THE OTHER END OF THE SAME ROW: WHAT THE QUEUE DOES WITH AN ATTACHMENT IT LETS GO OF. + * + * The tests above are about the composer handing a staged attachment TO the queue. These are about + * the queue handing one back. The composer clears its own strip as a message is parked, so from + * that moment the parked entry holds the only reference anything has to those rows — and when the + * queue lets one go, nothing but `ConversationView` is left to release it. + * + * `ConversationView` rather than `Composer`, because the queue lives there: it owns `reduceQueue`, + * it is what turns a transition's `droppedAttachments` into `DELETE /api/attachments/:id`, and + * neither of the two ways a row is let go of — a person taking a queued message back, and the cap + * re-check bumping the excess off a drained turn — is reachable from the composer alone. + */ + +/** Every `Remove ` chip currently on the composer's strip. */ +function chips({ container }: RenderResult): string[] { + return Array.from(container.querySelectorAll("[aria-label^='Remove ']")) + .map((button) => button.getAttribute("aria-label") ?? "") + .filter((label) => !label.startsWith("Remove queued message")); +} + +/** Drop `files` on the composer and wait for every one of them to finish uploading. */ +async function dropAndWait(view: RenderResult, files: File[]) { + drop(view.container.querySelector("form") as HTMLFormElement, files); + await waitFor(() => expect(chips(view)).toHaveLength(files.length)); + await waitFor(() => { + const button = (view.queryByLabelText("Send message") ?? + view.queryByLabelText("Queue message")) as HTMLButtonElement | null; + expect(button?.disabled).toBe(false); + }); +} + +function named(name: string) { + return new File(["hello"], name, { type: "text/plain" }); +} + +test("taking a queued message back releases the rows it was carrying", async () => { + const view = render( + {}} + // A turn is in flight, so the press below parks rather than sends. + pending + queueWhileBusy + />, + ); + const { container } = view; + + await dropAndWait(view, [notes()]); + fireEvent.submit(container.querySelector("form") as HTMLFormElement); + + // Parked: off the composer's strip, onto the transcript, and now referenced by nothing else. + await waitFor(() => + expect( + container.querySelector("[aria-label^='Remove queued message']"), + ).not.toBeNull(), + ); + expect(chips(view)).toEqual([]); + expect(deletes).toEqual([]); + + // By attribute prefix rather than by name: this message is a file and nothing else — a + // screenshot with no words is the whole reason `canSendDraft` unlocks on attachments alone — so + // the row's label ends at the colon with nothing after it to match on. + fireEvent.click( + container.querySelector( + "[aria-label^='Remove queued message']", + ) as HTMLButtonElement, + ); + + // THE FINDING. The entry left the queue and took the last reference to a staged row with it. + // Without this the row sits with `attachedAt IS NULL` until the 24-hour sweep, counted against + // this person's per-channel limit and surfacing as a 409 naming a file on nobody's screen. + await waitFor(() => + expect(deletes).toEqual([attachmentUrl("stored-notes.txt")]), + ); + + // AND THE PERSON IS TOLD, which is the other half of the same finding: a file that leaves with + // nothing said about it is the failure this whole apparatus exists to avoid, and the removed + // row is reported through the same `droppedAttachments` channel the cap re-check uses. + const alert = await view.findByRole("alert"); + expect(alert.textContent).toContain("notes.txt"); + + /* + * AND TOLD THE RIGHT THING, WHICH IS A SEPARATE ASSERTION BECAUSE IT WAS A SEPARATE BUG. + * + * `reduceQueue` reports the same `Attachment[]` whichever way a row left, so the cause is read + * off the queue ACTION here in `conversation-view.tsx` — the one place holding both halves — and + * turned into one of `DROPPED_REASON`'s two sentences. Get that ternary backwards and every + * removal blames a cap the person never hit, which `composer.tsx` says out loud is worse than + * saying nothing: it sends them looking for a limit to work around. + * + * This suite used to decline the assertion, on the grounds that a removal deserved its own words + * and that was composer.tsx's change to make. It has been made — `DroppedAttachmentCause` and + * both sentences are live — so declining now leaves the wiring between them unpinned, and + * swapping the two arms of that ternary passed the whole suite. + */ + expect(alert.textContent).toContain("removed along with the queued message"); + expect(alert.textContent).not.toContain("merged"); + expect(alert.textContent).not.toContain("at most"); +}); + +/** + * THIS TEST USED TO DRIVE A DRAIN OVER THE CAP AND CANNOT ANY MORE, WHICH IS THE FIX WORKING. + * + * It parked eight files, dropped a ninth, parked that too, and let the two messages drain into one + * draft of nine — then asserted that the cap re-check released the row it bumped and named the file + * on screen. Every step of that is still the right behaviour of `reduceQueue`, and none of it is + * reachable from here now: the ninth file never gets staged, because the composer screens picks + * against the strip PLUS what is parked, so it is refused before it becomes a row at all. + * + * That is the better failure. A drop the person is told about after the fact, with a row to clean + * up behind it, has been replaced by a pick that never happened — and the drained turn carries the + * eight they chose first either way. + * + * WHAT THAT COSTS IN COVERAGE, SAID OUT LOUD SO NOBODY THINKS THE PIN MOVED BY ITSELF. The cap + * re-check in `joinQueued` is now defensive at both layers — the client will not stage a ninth row + * and the server will not accept one into a single `uploadGroup` — so it keeps its tests where it + * is still reachable rather than here: `queue.test.ts` pins the rule and the `droppedAttachments` + * it reports, and `composer-dropped-attachments.test.tsx` pins the `merged-over-cap` sentence + * against the prop directly. What is left for this file is that the ninth never lands, and that a + * full queue still drains whole. + */ +test("a full queue drains whole, with nothing bumped and nothing to release", async () => { + const view = render( + {}} + pending + queueWhileBusy + />, + ); + const { container, rerender } = view; + const form = container.querySelector("form") as HTMLFormElement; + + await dropAndWait( + view, + Array.from({ length: MAX_ATTACHMENTS_PER_MESSAGE }, (_, index) => + named(`kept-${index}.txt`), + ), + ); + fireEvent.submit(form); + await waitFor(() => expect(chips(view)).toEqual([])); + + // A ninth, refused before it becomes a row — the case the test above is about, repeated here + // only to build the state this one is about: a queue sitting exactly on the cap. + drop(form, [named("overflow.txt")]); + await view.findByRole("alert"); + expect(uploads).not.toContain("overflow.txt"); + + // The turn ends and the parked message drains. + rerender( + {}} + queueWhileBusy + />, + ); + + // Nothing was bumped, so nothing is released. The old version of this expected exactly one + // DELETE here, for a row that no longer gets created. + await waitFor(() => expect(chips(view)).toEqual([])); + expect(deletes).toEqual([]); + expect(uploads).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE); +}); + +test("the spy hands the composer one stable pair of callbacks, not a fresh pair per render", async () => { + // THE HARNESS MEASURING ITSELF, and worth the test. The SDK builds `removeAttachment` and + // `consumeAttachments` as `useCallback(…, [])` — stable for the life of the hook, deliberately — + // and `submitDraft` lists `removeAttachment` in its dependency array. A spy that mints a new + // wrapper on every render silently rebuilds that memo on every render too, so the thing under + // test would be reacting to the instrument rather than to the code. + const view = render( + {}} />, + ); + + // Several renders: the placeholder going on the strip, the upload resolving, the strip + // re-rendering without it. + await dropAndWait(view, [notes()]); + fireEvent.click(view.getByLabelText("Remove notes.txt")); + await waitFor(() => expect(chips(view)).toEqual([])); + + expect(handedOut.removeAttachment.size).toBe(1); + expect(handedOut.consumeAttachments.size).toBe(1); +}); + +/** + * Registered after the `afterAll` that restores, so it runs after it: the real function is back on + * the module and no file bun walks into this process after this one is holding a spy. This is the + * assertion the old `mock.module`-based teardown could not have made — bun has no way to + * unregister a module mock, so re-mocking with a snapshot left one installed for good. + */ +afterAll(() => { + expect(ReactCoreV2.useAttachments).toBe(realUseAttachments); +}); + +/** + * THIS TEST USED TO ASSERT THE DATA LOSS AND CALL IT A RELEASE. + * + * It drove a drained turn whose send failed and expected exactly one DELETE, on the reasoning that + * the queue had emptied to build the draft and nothing pointed at those rows any more. That second + * half was never true. `channel-chat.tsx` adds the user message to the transcript BEFORE the run + * and leaves it there when the run fails, so the rows were pointed at by a message the person was + * looking at, and deleting them emptied the tiles underneath it. + * + * The reason the old shape could not see that is visible in what it renders: `messages={[]}` and an + * `onSubmit` that is a rejecting stub. There is no transcript to contradict, so the deletion looks + * free. The real-path coverage — a real `ChannelChat`, a real agent, a run answered with 503 — is + * in `failed-send-attachments.test.tsx`, which is where the two halves can be seen at once. + * + * WHAT IT IS WORTH KEEPING HERE ANYWAY. The stub submitter is the only way to fail a send without + * a runtime, so this file can still say the narrow thing it is for: the transition hands the queue + * back its own messages, and the rows behind them are not touched. + */ +test("a drained turn whose send fails puts the queue back instead of releasing its rows", async () => { + const attempted: ComposerDraft[] = []; + // Rejects rather than resolves: this is the drained turn failing after the queue has already + // been emptied to build it. + const onSubmit = (draft: ComposerDraft) => { + attempted.push(draft); + return Promise.reject(new Error("the turn failed")); + }; + + const view = render( + , + ); + const { container, rerender } = view; + + await dropAndWait(view, [notes()]); + fireEvent.submit(container.querySelector("form") as HTMLFormElement); + await waitFor(() => expect(chips(view)).toEqual([])); + expect(deletes).toEqual([]); + + // The turn ends, so what was parked drains — and the send for it fails. + rerender( + , + ); + + await waitFor(() => expect(attempted).toHaveLength(1)); + expect(attempted[0].attachments).toHaveLength(1); + + // THE INVERSION. The parked message is back, carrying the same row, and nothing was deleted. + await waitFor(() => + expect( + container.querySelector("[aria-label^='Remove queued message']"), + ).not.toBeNull(), + ); + expect(deletes).toEqual([]); + + // And it is not re-sent on its own. A restored queue that drained itself again would spin + // against a server that is refusing every request; the next turn is what carries it. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(attempted).toHaveLength(1); + + // The one gesture that still releases the row: taking the restored message back by hand. + fireEvent.click( + container.querySelector( + "[aria-label^='Remove queued message']", + ) as HTMLElement, + ); + await waitFor(() => + expect(deletes).toEqual([attachmentUrl("stored-notes.txt")]), + ); +}); + +/** + * THE THIRD WAY A PARKED ROW LOSES ITS LAST REFERENCE, and the one that used to say nothing. + * + * The two above go through the queue: a removal, and the cap re-check. A drained turn whose send + * failed used to be a third — it is not one any more, because a failed run puts its messages back + * rather than deleting what they were carrying; see the test above. This one goes through React. + * The composer clears its strip as a message is parked, so the parked entry is the only thing + * holding those rows — and walking to another channel unmounts the whole conversation, entry and + * all, restored entries included. + * + * `queue.ts` is candid that switching channels "takes anything parked in it with it", but that + * sentence is about the person's WORDS, which they watched land on screen and can retype. The + * staged rows underneath them are what `releaseStagedAttachment` exists for, and nothing was + * releasing them here. + */ +test("walking away with a message still parked releases the rows it was holding", async () => { + const view = render( + {}} + pending + queueWhileBusy + />, + ); + const { container } = view; + + await dropAndWait(view, [notes()]); + fireEvent.submit(container.querySelector("form") as HTMLFormElement); + + await waitFor(() => + expect( + container.querySelector("[aria-label^='Remove queued message']"), + ).not.toBeNull(), + ); + expect(chips(view)).toEqual([]); + expect(deletes).toEqual([]); + + // Another channel, a closed panel, a route change: whatever the gesture, this is what reaches the + // queue — a teardown with something still in it and nobody left to ask. + view.unmount(); + + await waitFor(() => + expect(deletes).toEqual([attachmentUrl("stored-notes.txt")]), + ); +}); + +test("unmounting with an empty queue releases nothing", async () => { + // The guard on the loop above, and not decoration: that teardown runs on EVERY unmount, including + // the ordinary one after a turn has drained, and a version of it that reached for the composer's + // strip rather than for the queue would delete rows behind chips somebody is still looking at. + const view = render( + {}} + queueWhileBusy + />, + ); + + await dropAndWait(view, [notes()]); + view.unmount(); + + await waitFor(() => expect(deletes).toEqual([])); +}); + +/** + * THE CASE THAT MUST NOT RELEASE, which is the whole reason the queue answers "which of these has + * nobody left holding it" rather than a caller assuming "all of them". + * + * An ordinary send carries only what is in the box. When it fails, `composer.tsx` puts the words + * back and hands the chips back with them — `setSending([])` in its `finally` — so those rows are + * referenced by something on screen that somebody can press send on again. Deleting them would + * leave chips pointing at rows that no longer exist. + */ +test("an ordinary send that fails releases nothing, because the chips come back", async () => { + const attempted: ComposerDraft[] = []; + const onSubmit = (draft: ComposerDraft) => { + attempted.push(draft); + return Promise.reject(new Error("the turn failed")); + }; + + const view = render( + , + ); + const { container } = view; + + await dropAndWait(view, [notes()]); + // Nothing is parked and no turn is in flight, so this sends rather than queues. + fireEvent.submit(container.querySelector("form") as HTMLFormElement); + + await waitFor(() => expect(attempted).toHaveLength(1)); + // Back on the strip, which is the fact the assertion after it depends on. + await waitFor(() => expect(chips(view)).toEqual(["Remove notes.txt"])); + expect(deletes).toEqual([]); +}); + +/** + * THE TWO CAPS COUNTING THE SAME SET AGAIN, WHICH TAKES BOTH HALVES AND THIS IS THE CALLER'S. + * + * Parking a message takes its chips off the strip, and until this the number the client screened + * against went with them: `stagedCount` resyncs from the strip, so after a park it read zero. The + * server's cap counts something the park does not touch — every row this person has staged in this + * composer's `uploadGroup` with `attachedAt IS NULL`, which a parked row is until the drained turn + * is sent. So a ninth pick behind eight parked files was accepted here and refused there, which is + * a per-draft cap and a per-group cap disagreeing by exactly the size of the queue. + * + * `composer.tsx` takes the number as `queuedAttachmentCount` and adds it to what it screens + * against. It cannot work the number out itself — the queue is this file's, not the composer's — + * so the composer's half is inert until something passes it, and this is the test that something + * does. Without the prop on the `Composer` below, the ninth file uploads. + */ +test("a ninth pick behind eight parked files is refused here, without a round trip", async () => { + const view = render( + {}} + // A turn is in flight, which is the only condition under which anything is parked at all. + pending + queueWhileBusy + />, + ); + const { container } = view; + const form = container.querySelector("form") as HTMLFormElement; + + await dropAndWait( + view, + Array.from({ length: MAX_ATTACHMENTS_PER_MESSAGE }, (_, index) => + named(`parked-${index}.txt`), + ), + ); + fireEvent.submit(form); + + // Parked: a full message's worth, off the strip and into the queue, with their rows untouched. + await waitFor(() => expect(chips(view)).toEqual([])); + expect(uploads).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE); + + drop(form, [named("ninth.txt")]); + + const alert = await view.findByRole("alert"); + // Our sentence, naming the file and the limit — not the server's, which names a count against a + // strip the person is looking at and can see is empty. + expect(alert.textContent).toContain("ninth.txt"); + expect(alert.textContent).toContain( + `at most ${MAX_ATTACHMENTS_PER_MESSAGE} attachments`, + ); + + // AND IT NEVER LEFT THE BROWSER. This is the half that says the refusal came from here: the + // reason line alone would read the same if the file had been uploaded and refused on arrival. + expect(uploads).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE); + expect(uploads).not.toContain("ninth.txt"); + // And no chip for it, so nothing on screen suggests it is going anywhere. + expect(chips(view)).toEqual([]); +}); + +test("taking the parked message back frees the slots it was holding", async () => { + // THE OTHER DIRECTION, AND THE REASON THE COUNT IS READ FROM THE QUEUE RATHER THAN ACCUMULATED. + // A count that only ever went up would leave somebody who changed their mind about a parked + // message locked out of the slots it had been occupying — refused by their own client this time, + // which is worse than the 409 because there is no round trip to blame it on. + const view = render( + {}} + pending + queueWhileBusy + />, + ); + const { container } = view; + const form = container.querySelector("form") as HTMLFormElement; + + await dropAndWait( + view, + Array.from({ length: MAX_ATTACHMENTS_PER_MESSAGE }, (_, index) => + named(`parked-${index}.txt`), + ), + ); + fireEvent.submit(form); + await waitFor(() => expect(chips(view)).toEqual([])); + + fireEvent.click( + container.querySelector( + "[aria-label^='Remove queued message']", + ) as HTMLButtonElement, + ); + await waitFor(() => + expect(deletes).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE), + ); + + // Nothing parked any more, so the strip is the whole count again and a pick is a pick. + await dropAndWait(view, [named("ninth.txt")]); + + expect(uploads).toContain("ninth.txt"); + expect(chips(view)).toEqual(["Remove ninth.txt"]); +}); + +/** + * THE ONE STATE IN WHICH A QUEUE IS NON-EMPTY AND NO TURN IS IN FLIGHT, which is worth a test of + * its own because an argument made elsewhere leans on it. + * + * `composer-inflight-removal.test.tsx` pins that the composer PARKS rather than sends while a turn + * is in flight, and concludes from it that `reduceQueue`'s submit-join — send now, with messages + * already parked — is unreachable, which is what keeps the cap bumping the live draft's rows a + * defensive path rather than a live one. That conclusion is right, and the reason given for it is + * not the whole reason: it says the queue is only ever non-empty while a turn is in flight, and + * here is the state where it is not. The drain refuses while the conversation is `disabled` — a + * coworker deleted mid-turn — so the turn can end with the queue still full. + * + * What closes it is the OTHER guard, in `submitDraft`: a disabled composer returns before it can + * either send or park. So there is still no way to reach the join, by a second route, and this is + * the test that keeps the second route shut. If `disabled` ever stops gating the drain, the queue + * empties into a channel this screen has already said is finished; if it ever stops gating + * `submitDraft`, the join goes live. + */ +test("a disabled conversation keeps what is parked instead of draining it", async () => { + const attempted: ComposerDraft[] = []; + const onSubmit = (draft: ComposerDraft) => { + attempted.push(draft); + }; + + const view = render( + , + ); + const { container, rerender } = view; + + await dropAndWait(view, [notes()]); + fireEvent.submit(container.querySelector("form") as HTMLFormElement); + await waitFor(() => + expect( + container.querySelector("[aria-label^='Remove queued message']"), + ).not.toBeNull(), + ); + + // The turn ends AND the conversation goes read-only in the same breath, which is what a deleted + // coworker looks like from here. + rerender( + , + ); + + // Still parked, still on screen, and nothing ran: a drain here would post one more user turn into + // a channel the notice under the composer has already said cannot reply. + expect(attempted).toEqual([]); + expect( + container.querySelector("[aria-label^='Remove queued message']"), + ).not.toBeNull(); + // And nothing was released either — the message is still there to be taken back by hand, so its + // rows still have something pointing at them. + expect(deletes).toEqual([]); +}); diff --git a/app/tests/composer-rejected-files-gap.test.tsx b/app/tests/composer-rejected-files-gap.test.tsx new file mode 100644 index 000000000..d59fc33f4 --- /dev/null +++ b/app/tests/composer-rejected-files-gap.test.tsx @@ -0,0 +1,65 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { cleanup, render } from "@testing-library/react"; +import { RejectedFiles } from "@/components/channels/composer/rejected-files"; +import { settleReactWork } from "./settle-react-work"; + +/** + * THE GAP UNDER THE REFUSALS HAS TO BE INSIDE THE BOX THAT GETS MEASURED. + * + * `Collapse` animates its height to `content.offsetHeight` (`collapse.tsx`), and `offsetHeight` is + * the border-box height: it includes padding and excludes margins. The content wrapper has no + * border and no padding of its own, so a bottom margin on the alert inside it collapses straight + * out of the number `Collapse` measures. The animating box then settles 8px shorter than the space + * the alert is meant to occupy, and the gap between the refusal block and the composer directly + * below it is simply not drawn — on the one component here that appears without being asked for, + * pressed against the box somebody is typing in. + * + * `AttachmentStrip` already spends `pb-3` for exactly this reason, on the `ul` inside its own + * `Collapse`. This is the same bargain, spent on the wrapper `Collapse` measures rather than on the + * alert, because the alert has a dashed border and padding inside it would land within that border + * rather than under it. + * + * WHY THIS TEST IS STRUCTURAL AND NOT A MEASUREMENT. happy-dom reports `offsetHeight` as 0 for + * everything and compiles no Tailwind, so the real number cannot be observed from here and neither + * can a computed style. What CAN be pinned is the property the number depends on: the spacing is + * spent as padding on the measured element and not as a margin on its child. That is the whole of + * the defect, and it is a class-name assertion because the alternative is no assertion at all. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +const oneRefusal = [ + { id: "1", name: "logo.svg", reason: "SVGs are not accepted." }, +]; + +test("the gap below the refusals is padding on the measured box, not a margin on the alert", () => { + const { container } = render( + {}} rejected={oneRefusal} />, + ); + + const alert = container.querySelector('[role="alert"]') as HTMLElement; + expect(alert).not.toBeNull(); + + /* + * `Collapse` renders `
    {children}
    ` inside the + * animating box, so the alert's parent IS the element whose `offsetHeight` is measured. Reaching + * for it through the alert rather than by class name is deliberate: the point is that the spacing + * sits on the measured element, whichever element that turns out to be. + */ + const measured = alert.parentElement as HTMLElement; + const measuredClasses = (measured.getAttribute("class") ?? "").split(/\s+/); + const alertClasses = (alert.getAttribute("class") ?? "").split(/\s+/); + + // Padding, on the box `offsetHeight` is read from — so the gap is part of the height animated to. + expect(measuredClasses).toContain("pb-2"); + + // And not a bottom margin anywhere inside it, which is the spelling that gets measured away. + expect(alertClasses).not.toContain("mb-2"); + expect(alertClasses.filter((name) => /^-?mb-/.test(name))).toEqual([]); +}); diff --git a/app/tests/composer-rejected-files.test.tsx b/app/tests/composer-rejected-files.test.tsx new file mode 100644 index 000000000..b20011cb0 --- /dev/null +++ b/app/tests/composer-rejected-files.test.tsx @@ -0,0 +1,193 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { cleanup, fireEvent, render } from "@testing-library/react"; +import { RejectedFiles } from "@/components/channels/composer/rejected-files"; +import { settleReactWork } from "./settle-react-work"; + +/** + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in + * `afterEach`, matching `agent-roster-error.test.tsx`: bun walks every file into one process, and a + * document another file tore down mid-run fails invisibly. + * + * The registration carries a `url`, as every other `.test.tsx` in this directory that renders a + * composer surface does. Without one `location` is `about:blank`, which has no origin: relative + * URLs do not resolve and anything that touches storage or `URL` construction fails for a reason + * that has nothing to do with the component under test. This file did not have one, and the only + * thing that made that harmless was that it happens not to have needed an origin yet. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +/** + * THE REFUSALS AS THEY ARE READ, ONE STRING PER LINE. + * + * Written out because the obvious spelling does not pin what it looks like it pins. + * `getByText(/invoice\.pdf/)` and `getByText(/Unsupported file type/)` both pass against a + * component that has put every reason next to the wrong name: testing-library's default matcher + * reads only an element's DIRECT text children, so `

    invoice.pdf: reason

    ` is two + * separate haystacks — "invoice.pdf" in the span, ": reason" in the paragraph — and asking whether + * each exists somewhere never asks whether they are on the same line. Which file failed for which + * reason is the entire content of this component, so it is the paragraph text that has to be + * asserted, whole and in order. + */ +function lines(container: HTMLElement): string[] { + return Array.from(container.querySelectorAll("p"), (p) => p.textContent); +} + +/** + * Renders while listening for what React says under its breath. + * + * React does not throw on a duplicate `key`, it warns — so a test that only looks at the DOM + * cannot tell a keyed list from an unkeyed one on mount, and mostly cannot on update either, since + * the reconciler's positional fast path papers over the mistake until something moves. The warning + * is the observable, so this catches it. + */ +function renderWatchingReact(element: React.ReactElement) { + const complaints: string[] = []; + const realError = console.error; + console.error = (...args: unknown[]) => { + complaints.push(args.map(String).join(" ")); + }; + + try { + return { ...render(element), complaints }; + } finally { + console.error = realError; + } +} + +test("shows both filenames and both reasons for two rejected files", () => { + const { container } = render( + {}} + rejected={[ + { id: "1", name: "invoice.pdf", reason: "Unsupported file type" }, + { + id: "2", + name: "photo.heic", + reason: "That is not a supported image type", + }, + ]} + />, + ); + + // Each file's own reason must survive next to its OWN name, in the order they were refused. A + // single overwritten error string would report only one of these two refusals, and a reason + // printed against the wrong file is worse than no reason at all — it sends somebody to convert a + // PDF that was never the problem. + expect(lines(container)).toEqual([ + "invoice.pdf: Unsupported file type", + "photo.heic: That is not a supported image type", + ]); +}); + +test("renders both entries when two rejected files share a name", () => { + // The defect this pins: two files dragged from different folders can genuinely share a name, and + // keying the list on `name` alone gives React two children with one key. It does not drop + // either line on mount — which is why asserting only that both lines are there proves nothing — + // it complains, and then loses track of which line is which the first time the list is reordered + // or shortened. The `id` field is what tells them apart, and the complaint is what betrays its + // absence. + const { complaints, container } = renderWatchingReact( + {}} + rejected={[ + { id: "1", name: "screenshot.png", reason: "File is too large" }, + { + id: "2", + name: "screenshot.png", + reason: "Unsupported file type", + }, + ]} + />, + ); + + expect(complaints.filter((line) => line.includes("same key"))).toEqual([]); + expect(lines(container)).toEqual([ + "screenshot.png: File is too large", + "screenshot.png: Unsupported file type", + ]); +}); + +test("the surviving entry keeps its own reason when a same-named one is removed", () => { + // Where a duplicate key stops being a warning and starts being wrong. Dropping the FIRST of two + // identically named refusals leaves the second, and the second's reason has to come with it. + // Keyed on `id`, React matches the remaining entry to the fiber it already had. Keyed on `name`, + // both old children answer to the same key, the first one is what the survivor is matched + // against, and the line that stays is the line that should have gone. + const { container, rerender } = render( + {}} + rejected={[ + { id: "1", name: "screenshot.png", reason: "File is too large" }, + { id: "2", name: "screenshot.png", reason: "Unsupported file type" }, + ]} + />, + ); + + const survivor = container.querySelectorAll("p")[1]; + + rerender( + {}} + rejected={[ + { id: "2", name: "screenshot.png", reason: "Unsupported file type" }, + ]} + />, + ); + + expect(lines(container)).toEqual(["screenshot.png: Unsupported file type"]); + // Same paragraph node, not a lookalike built in its place: identity across an update is what a + // key is FOR, and it is the thing a duplicate key cannot deliver. + expect(container.querySelectorAll("p")[0]).toBe(survivor); +}); + +test("says nothing for an empty list", () => { + // The collapsing box stays mounted — it is what animates the composer's height — but the ALERT + // inside it does not. An empty `role="alert"` is still an alert to a screen reader. + const { container, queryByRole } = render( + {}} rejected={[]} />, + ); + + expect(queryByRole("alert")).toBeNull(); + expect(container.textContent).toBe(""); +}); + +test("the dismiss button hands back every refusal at once, not one line", () => { + // One press clears the block. Dropping eight files can refuse several together, and they are + // read together, so a per-line dismissal would be work with no purpose. + let dismissed = 0; + const { getByLabelText } = render( + { + dismissed += 1; + }} + rejected={[ + { id: "1", name: "logo.svg", reason: "SVGs are not accepted" }, + { id: "2", name: "huge.txt", reason: "Too large" }, + ]} + />, + ); + + fireEvent.click(getByLabelText("Dismiss these 2 refusals")); + + expect(dismissed).toBe(1); +}); + +test("a single refusal is dismissed in the singular", () => { + const { getByLabelText } = render( + {}} + rejected={[ + { id: "1", name: "logo.svg", reason: "SVGs are not accepted" }, + ]} + />, + ); + + expect(getByLabelText("Dismiss this refusal")).toBeTruthy(); +}); diff --git a/app/tests/composer-send-failure.test.tsx b/app/tests/composer-send-failure.test.tsx new file mode 100644 index 000000000..d00e7addb --- /dev/null +++ b/app/tests/composer-send-failure.test.tsx @@ -0,0 +1,147 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { act, cleanup, fireEvent, render } from "@testing-library/react"; +import { Composer } from "@/components/channels/composer/composer"; +import { settleReactWork } from "./settle-react-work"; + +/** + * WHAT A FAILED SEND DOES TO THE WORDS, WHICH UNTIL NOW COULD NOT BE ASKED AT ALL. + * + * `submitDraft` used to rethrow out of its own catch, and it has exactly two callers: + * `handleFormSubmit`, which does `void submitDraft(value)`, and PromptArea's `onSubmit`, which + * calls it and ignores the promise. Neither awaits and neither catches, so a failed send was an + * unhandled rejection — which bun's runner reports as a failure of whichever test is running when + * it surfaces. `composer-attachment-lifecycle.test.tsx` and `composer-queue-attachments.test.tsx` + * each carry a note recording that they gave up on driving a failed send for exactly that reason. + * They are the reason this file exists: with the rethrow gone the path is reachable, and these are + * the two things it has to get right. + * + * NO `fetch` STUB AND NO CHANNEL. Neither test attaches anything, so nothing here goes near the + * network — the failure being driven is the caller's `onSubmit` rejecting, which is the whole of + * what a failed turn looks like from inside the composer. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in + * `afterEach`, matching `composer-paste.test.tsx` for the reason recorded there: bun walks every + * file into one process, and a document another file tore down mid-run fails invisibly. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +/** The editor inside the composer: the element the words actually live in. */ +function editorOf(container: HTMLElement): HTMLElement { + return container.querySelector("[contenteditable]") as HTMLElement; +} + +/** What is typed in the box, with the placeholder and the strip left out of it. */ +function typedText(container: HTMLElement): string { + return editorOf(container).textContent ?? ""; +} + +/** + * A send whose answer this test hands out by hand. + * + * The whole question is what happens BETWEEN the press and the failure, so `onSubmit` has to be a + * promise the test still holds when it starts asking. + */ +function deferredSend() { + let fail!: (error: Error) => void; + const settled = new Promise((_, reject) => { + fail = reject; + }); + return { + onSubmit: () => settled, + /** Fail the send, and let the composer's own `catch` and `finally` run before returning. */ + async reject() { + fail(new Error("the turn could not be started")); + await act(async () => { + await settled.catch(() => undefined); + }); + }, + }; +} + +/** + * Type into the editor the one way this suite can: an ordinary text paste. + * + * The caret is not decoration — PromptArea inserts at the selection and gives up when there is not + * one. A clipboard carrying text and no file is declined by the composer's own capture-phase + * listener (`shouldClaimPaste` gives text the win), so this lands in the editor exactly as typing + * would. + */ +function typeInto(container: HTMLElement, words: string) { + const editor = editorOf(container); + editor.focus(); + const caret = document.createRange(); + caret.selectNodeContents(editor); + caret.collapse(false); + const selection = window.getSelection(); + selection?.removeAllRanges(); + selection?.addRange(caret); + fireEvent.paste(editor, { + clipboardData: { + files: [], + items: [], + types: ["text/plain"], + getData: (type: string) => (type === "text/plain" ? words : ""), + }, + }); +} + +test("a failed send puts the words back rather than throwing at nobody", async () => { + const send = deferredSend(); + const { container, getByLabelText } = render( + , + ); + + await act(async () => { + fireEvent.click(getByLabelText("Send message")); + }); + // Cleared optimistically, which is the state the restore has to undo. + expect(typedText(container)).toBe(""); + + await send.reject(); + + expect(typedText(container)).toContain("ship it"); + // Back to a composer that can be sent from again, rather than one stuck mid-send. + expect((getByLabelText("Send message") as HTMLButtonElement).disabled).toBe( + false, + ); +}); + +/** + * THE EDITOR IS NEVER DISABLED MID-TURN, AND THAT IS DELIBERATE — it is how a correction gets typed + * at a Bot that is already working. So by the time a send fails the box may well hold something + * newer than the message that failed. `setValue(segments)` wrote straight over it: the failed + * message came back and the sentence typed after it was gone, with nothing said about either. + * + * Both are somebody's words, so neither may be dropped. The restored ones go in FRONT of the newer + * ones and the person edits the join. + */ +test("a failed send keeps what was typed while it was in flight", async () => { + const send = deferredSend(); + const { container, getByLabelText } = render( + , + ); + + await act(async () => { + fireEvent.click(getByLabelText("Send message")); + }); + expect(typedText(container)).toBe(""); + + await act(async () => { + typeInto(container, "second"); + }); + expect(typedText(container)).toContain("second"); + + await send.reject(); + + const restored = typedText(container); + expect(restored).toContain("first"); + expect(restored).toContain("second"); + expect(restored.indexOf("first")).toBeLessThan(restored.indexOf("second")); +}); diff --git a/app/tests/composer-unnamed-mime.test.tsx b/app/tests/composer-unnamed-mime.test.tsx new file mode 100644 index 000000000..c535dfab3 --- /dev/null +++ b/app/tests/composer-unnamed-mime.test.tsx @@ -0,0 +1,135 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + expect, + test, +} from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { cleanup, fireEvent, render, waitFor } from "@testing-library/react"; +import { Composer } from "@/components/channels/composer/composer"; +import { settleReactWork } from "./settle-react-work"; + +/** + * A CLAIM THAT NAMES NO FORMAT, ALL THE WAY TO THE UPLOAD — WHICH IS THE ONLY PLACE IT CAN BE SEEN. + * + * `screenPickedFiles` deliberately lets `application/octet-stream` and a blank type through so the + * server can read the bytes (`picked-files.ts`, "THE BROWSER TOLD US NOTHING, SO THE SERVER GETS TO + * LOOK"), and `picked-files.test.ts` pins that. It pins the pure function, and the pure function was + * never the problem: the SDK's `processFiles` applies `AttachmentsConfig.accept` itself with an + * exact `file.type === filter`, so for eighteen months the branch could be — and was — completely + * dead downstream of a unit test that went on passing. Four reviewers found it by reading; nothing + * in the suite could. + * + * So these tests are deliberately NOT about `screenPickedFiles`. They drive a real `Composer` with + * the real `useAttachments`, and they assert on the two things only the whole path can show: that a + * request went out, and that no refusal was drawn. That is the gap, and this is the level it lives + * at. + * + * THE HARNESS IS THIS REPOSITORY'S, matching `composer-upload-group.test.tsx` — `GlobalRegistrator` + * in `beforeAll`/`afterAll` and `cleanup` in `afterEach`, because bun walks every file into one + * process and a document another file tore down mid-run fails invisibly. The registration carries a + * `url` because without one `location` is `about:blank` and the relative upload URL does not + * resolve. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +const originalFetch = global.fetch; + +afterEach(() => { + global.fetch = originalFetch; +}); + +/** The `type` of every file this composer actually tried to upload, in order. */ +let uploaded: { name: string; type: string }[]; + +beforeEach(() => { + uploaded = []; + global.fetch = (async (_path: string, init: RequestInit) => { + const file = (init.body as FormData).get("file") as File; + uploaded.push({ name: file.name, type: file.type }); + return new Response( + JSON.stringify({ + // The server's answer, and the point of the round trip: it read the bytes and named the + // format the browser could not. + id: `stored-${uploaded.length}`, + name: file.name, + mimeType: "text/plain", + }), + { status: 201, headers: { "content-type": "application/json" } }, + ); + }) as unknown as typeof fetch; +}); + +/** A drop, as the browser delivers one: files hanging off `dataTransfer`. */ +function drop(form: Element, files: File[]) { + fireEvent.drop(form, { + dataTransfer: { files, items: [], types: ["Files"] }, + }); +} + +/** + * The two ways a browser says "I have no idea what this is": the generic binary claim it attaches + * to an unfamiliar extension, and no claim at all. `namesNoFormat` treats them identically and so + * does the server, so both have to survive the same journey. + */ +const unnamed = [ + { label: "application/octet-stream", type: "application/octet-stream" }, + { label: "a blank type", type: "" }, +] as const; + +for (const claim of unnamed) { + test(`a file claiming ${claim.label} is uploaded for the server to sniff`, async () => { + const view = render( + {}} />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + drop(form, [new File(["hello"], "notes.txt", { type: claim.type })]); + + // The request is the assertion. Before this was fixed the SDK's `accept` filter refused the + // file first, so nothing was ever sent and the branch that let it through was decoration. + await waitFor(() => expect(uploaded).toHaveLength(1)); + expect(uploaded[0].name).toBe("notes.txt"); + // Handed over exactly as the browser reported it: `withMediaTypeOnly` normalises case and + // strips parameters, and neither of these has either, so the server sees what we saw. + expect(uploaded[0].type).toBe(claim.type); + + // And no refusal is drawn for it — not ours, and above all not the SDK's + // `File "notes.txt" is not accepted. Supported types: …`, which is the machine sentence + // `stageFiles` promises a screened file can never produce. + await waitFor(() => + expect(view.queryByLabelText("Remove notes.txt")).not.toBeNull(), + ); + expect(view.queryByRole("alert")).toBeNull(); + }); +} + +/** + * The other half, and the reason `accept` cannot simply be deleted from the design: opening the + * gate for unnamed claims must not open it for named ones. An SVG says exactly what it is, and the + * screen still refuses it in the product's own words rather than passing it to the SDK to refuse in + * the SDK's. + */ +test("a file that does name its format is still refused here, in our words", async () => { + const view = render( + {}} />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + drop(form, [new File([""], "logo.svg", { type: "image/svg+xml" })]); + + await waitFor(() => expect(view.queryByRole("alert")).not.toBeNull()); + expect( + view.queryByText(/can carry scripts and is not accepted/), + ).not.toBeNull(); + expect(view.queryByText(/Supported types:/)).toBeNull(); + expect(uploaded).toEqual([]); +}); diff --git a/app/tests/composer-upload-group.test.tsx b/app/tests/composer-upload-group.test.tsx new file mode 100644 index 000000000..dad08a2ed --- /dev/null +++ b/app/tests/composer-upload-group.test.tsx @@ -0,0 +1,234 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + expect, + test, +} from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { cleanup, fireEvent, render, waitFor } from "@testing-library/react"; +import { Composer } from "@/components/channels/composer/composer"; +import { MAX_ATTACHMENTS_PER_MESSAGE } from "@/lib/channels/attachments"; +import { settleReactWork } from "./settle-react-work"; + +/** + * THE TWO HALVES OF ONE CAP, AND THE TICK IN WHICH THE CLIENT'S HALF USED TO MISCOUNT. + * + * The per-message cap has two enforcers. This file pins the client's: that every upload carries the + * key the server counts by, and that two gestures landing in the same tick cannot between them + * stage more than the cap allows. + * + * THE HARNESS IS THIS REPOSITORY'S, matching `composer-attachment-lifecycle.test.tsx` — + * `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in `afterEach`, because bun walks + * every file into one process and a document another file tore down mid-run fails invisibly. The + * registration carries a `url` because without one `location` is `about:blank` and the relative + * upload URL does not resolve. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +const originalFetch = global.fetch; + +afterEach(() => { + global.fetch = originalFetch; +}); + +/** The `uploadGroup` field of every upload this composer sent, in order. */ +let groups: (string | null)[]; + +beforeEach(() => { + groups = []; + global.fetch = (async (_path: string, init: RequestInit) => { + const body = init.body as FormData; + const file = body.get("file") as File; + const group = body.get("uploadGroup"); + groups.push(typeof group === "string" ? group : null); + return new Response( + JSON.stringify({ + id: `stored-${groups.length}`, + name: file.name, + mimeType: "text/plain", + }), + { status: 201, headers: { "content-type": "application/json" } }, + ); + }) as unknown as typeof fetch; +}); + +/** A drop, as the browser delivers one: files hanging off `dataTransfer`. */ +function drop(form: Element, files: File[]) { + fireEvent.drop(form, { + dataTransfer: { files, items: [], types: ["Files"] }, + }); +} + +function textFiles(prefix: string, count: number): File[] { + return Array.from( + { length: count }, + (_, index) => + new File(["hello"], `${prefix}-${index}.txt`, { type: "text/plain" }), + ); +} + +/** + * TWO GESTURES, ONE TICK — a drop landing while another is still being screened. + * + * `stageFiles` used to screen against `staged.length`, a number captured at render, and it is + * async: neither drop below has re-rendered the composer by the time the other reads it, so both + * read zero and both accepted a full batch. Ten uploads went out against a cap of eight, and the + * server refused the last two — a refusal the person had been given no chance to avoid. + * + * The uploads actually attempted are what is counted, not only the chips: the cap exists to bound + * what reaches the server, and the chips are waited on first only because they settle after the + * uploads do, which is what gives a surplus a chance to be seen rather than merely missed. + */ +test("two batches staged in one tick cannot exceed the per-message cap", async () => { + const view = render( + {}} />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + // No await between them: this is the whole point. Five and five is ten, and the cap is eight. + drop(form, textFiles("first", 5)); + drop(form, textFiles("second", 5)); + + // The chips settle after the uploads do, so waiting on them is what lets a batch that should + // have been refused go out first and be counted — an over-count shows up here as a surplus + // rather than as a wait that merely has not finished yet. + await waitFor(() => + expect(view.queryAllByLabelText(/^Remove /)).toHaveLength( + MAX_ATTACHMENTS_PER_MESSAGE, + ), + ); + + expect(groups).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE); + // And the two that did not fit are refused in our words, naming each file, rather than by a + // server 409 the person was given no chance to avoid. + // + // Built from the constant rather than written out, because the literal `8` two lines under + // `MAX_ATTACHMENTS_PER_MESSAGE` is the same number in two spellings: change the cap and this + // regex quietly stops matching the sentence it is checking, and the failure reads as a + // rejection-count problem rather than as a stale literal. + expect( + view.queryAllByText( + new RegExp(`at most ${MAX_ATTACHMENTS_PER_MESSAGE} attachments`), + ), + ).toHaveLength(2); +}); + +/** + * One key per composer, on every upload it makes — which is what lets the server count the same set + * this composer can see instead of every unsent row in the channel. + */ +test("every upload from one composer carries the same upload group", async () => { + const view = render( + {}} />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + drop(form, textFiles("first", 1)); + await waitFor(() => expect(groups).toHaveLength(1)); + drop(form, textFiles("second", 1)); + await waitFor(() => expect(groups).toHaveLength(2)); + + const [first, second] = groups; + expect(typeof first).toBe("string"); + expect((first as string).length).toBeGreaterThan(0); + expect(second).toBe(first); +}); + +/** + * WHAT IS PARKED COUNTS TOO, BECAUSE THE SERVER IS STILL COUNTING IT. + * + * Parking a message takes its chips off the strip, but `attachedAt` is written only when the + * message is really sent — so those rows stay `attached_at IS NULL` in this composer's + * `uploadGroup` for the whole life of the turn, and the server's cap counts exactly that set. With + * the strip empty the client screened against zero, accepted the pick, and let the server refuse it + * with a 409 the person had been given no chance to avoid. + * + * The count arrives as a prop because the composer cannot see the queue: `conversation-view.tsx` + * owns it. This drives the prop directly, which is the whole of the composer's half of the + * contract — the caller's half is one `reduce` there. + */ +test("attachments parked in the queue are counted against the per-message cap", async () => { + const view = render( + {}} + // A full cap's worth already parked, and nothing at all on this composer's strip. + queuedAttachmentCount={MAX_ATTACHMENTS_PER_MESSAGE} + />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + drop(form, textFiles("ninth", 1)); + + // Refused here, in our words, naming the file — rather than uploaded and then refused by the + // server in its own. + await waitFor(() => + expect( + view.queryAllByText( + new RegExp(`at most ${MAX_ATTACHMENTS_PER_MESSAGE} attachments`), + ), + ).toHaveLength(1), + ); + expect(groups).toEqual([]); +}); + +/** + * And the prop is an addend, not an override: a composer told about parked files still counts what + * is on its own strip. Seven parked plus one staged is the cap, so the next pick is the ninth. + */ +test("parked attachments are counted alongside the ones on the strip", async () => { + const view = render( + {}} + queuedAttachmentCount={MAX_ATTACHMENTS_PER_MESSAGE - 1} + />, + ); + const form = view.container.querySelector("form") as HTMLFormElement; + + // The eighth overall, and the first this composer can see: accepted. + drop(form, textFiles("eighth", 1)); + await waitFor(() => expect(groups).toHaveLength(1)); + + // The ninth: refused, without the strip ever having held more than one chip. + drop(form, textFiles("ninth", 1)); + await waitFor(() => + expect( + view.queryAllByText( + new RegExp(`at most ${MAX_ATTACHMENTS_PER_MESSAGE} attachments`), + ), + ).toHaveLength(1), + ); + expect(groups).toHaveLength(1); +}); + +/** Two composers are two sessions, and the cap they are counted against is per session. */ +test("a second composer mints a group of its own", async () => { + const first = render( + {}} />, + ); + drop(first.container.querySelector("form") as HTMLFormElement, [ + new File(["hello"], "a.txt", { type: "text/plain" }), + ]); + await waitFor(() => expect(groups).toHaveLength(1)); + + const second = render( + {}} />, + ); + drop(second.container.querySelector("form") as HTMLFormElement, [ + new File(["hello"], "b.txt", { type: "text/plain" }), + ]); + await waitFor(() => expect(groups).toHaveLength(2)); + + expect(groups[1]).not.toBe(groups[0]); +}); diff --git a/app/tests/computer-reset-clears-activity.test.ts b/app/tests/computer-reset-clears-activity.test.ts index 53e2be294..327909025 100644 --- a/app/tests/computer-reset-clears-activity.test.ts +++ b/app/tests/computer-reset-clears-activity.test.ts @@ -13,6 +13,17 @@ const queryClient = { invalidateQueries: async () => undefined, } as unknown as QueryClient; +/* + * The context TanStack Query hands a mutation callback alongside its variables. These tests drive + * the callbacks directly rather than through a MutationObserver, so they have to supply it. Both + * fields are the real thing rather than a stand-in: `meta` is undefined exactly as it is for a + * mutation declared without one, and `mutationKey` is optional and genuinely absent, because none + * of these options factories sets one. + */ +function mutationContext(queryClient: QueryClient) { + return { client: queryClient, meta: undefined }; +} + beforeEach(() => { globalThis.fetch = (async () => new Response(null, { status: 200 })) as unknown as typeof fetch; @@ -33,7 +44,7 @@ afterEach(() => { async function run(action: "stop" | "reset") { const options = setComputerStateMutationOptions(queryClient); const variables = { action, botId: "general-assistant" } as const; - await options.mutationFn?.(variables); + await options.mutationFn?.(variables, mutationContext(queryClient)); await options.onSuccess?.( undefined as never, variables, diff --git a/app/tests/failed-send-attachments.test.tsx b/app/tests/failed-send-attachments.test.tsx new file mode 100644 index 000000000..90bf1d5b8 --- /dev/null +++ b/app/tests/failed-send-attachments.test.tsx @@ -0,0 +1,526 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + expect, + test, +} from "bun:test"; +import type { Message, RunAgentInput } from "@ag-ui/core"; +import { RunAgentInputSchema } from "@ag-ui/core"; +import { CopilotKitProvider, useCopilotKit } from "@copilotkit/react-core/v2"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { type InfiniteData, QueryClientProvider } from "@tanstack/react-query"; +import { + cleanup, + fireEvent, + render, + type RenderResult, + waitFor, +} from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { ChannelChat } from "@/components/channels/channel-chat"; +import { + attachmentUrl, + MAX_ATTACHMENTS_PER_MESSAGE, +} from "@/lib/channels/attachments"; +import { + type AgentChannel, + type ChannelPage, + type ChannelSummary, + channelKeys, +} from "@/lib/channels/queries"; +import { queryClient } from "@/query-client"; +import { settleReactWork } from "./settle-react-work"; + +/** + * WHAT A FAILED SEND DOES TO THE FILES IT WAS CARRYING, DRIVEN THROUGH THE REAL CHANNEL. + * + * The composer suites reach this area through a fake `onSubmit` and `messages={[]}`, which is + * exactly the state in which deleting a failed send's rows looks harmless: there is no transcript + * to contradict, so a test can watch the DELETE go out and call it a release. The whole point of + * this file is that the transcript IS there. `ChannelChat` is what turns a draft into a message — + * it adds the user turn to `agent.messages` BEFORE the run, so by the time the run fails the + * screen is already showing a message whose attachments point at those rows — and only a test + * that goes through it can see the two halves at once. + * + * So everything below the HTTP boundary is real: the provider, the agent, `say`/`deliver`, the + * conversation view's queue, and the composer. Only `fetch` is a fixture, and it is the fixture + * that decides the two failures under test — a run that answers 503, and an upload that answers + * with a stored row id. + * + * TEXT FILES RATHER THAN THE REVIEWER'S SCREENSHOT, and nothing in the path cares: a staged row is + * a staged row, `attachmentUrl` addresses both the same way, and the release, the cap and the + * queue all count them identically. An image tile would additionally ask happy-dom to load an + * ``, which is a second fixture for a detail this file is not about. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll`, `cleanup` in + * `afterEach`, and `settleReactWork()` before the document goes away — see that helper for what a + * scheduler callback landing after `unregister()` does to an otherwise green run. + */ + +const NativeResponse = globalThis.Response; + +const channel: AgentChannel = { + id: "failed-send-channel", + name: "Failed send", + agentIds: ["failed-send-bot"], + threadId: "failed-send-thread", + active: true, + lastMessageAt: "2026-09-09T00:00:00.000Z", +}; + +const opening = { + id: "opening", + role: "assistant", + content: "Stored opening", +} satisfies Message; + +/** The run inputs that actually left the browser, in order. */ +let runs: RunAgentInput[]; +/** Every `DELETE /api/attachments/:id` the screen sent, by path. */ +let deletes: string[]; +/** Every upload that actually left the browser, by file name. */ +let uploads: string[]; +/** + * What each run answers, by the order it was started. A test installs one entry per run it expects; + * anything past the end finishes normally, so a stray run is visible as an extra entry in `runs` + * rather than as a hang. + */ +let runAnswers: ((input: RunAgentInput) => Promise)[]; +let core: ReturnType["copilotkit"] | undefined; +let originalFetch: typeof fetch; + +function CoreProbe() { + core = useCopilotKit().copilotkit; + return null; +} + +function sse(events: unknown[]) { + return new NativeResponse( + events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join(""), + { headers: { "content-type": "text/event-stream" } }, + ); +} + +function finished(input: RunAgentInput) { + return sse([ + { type: "RUN_STARTED", threadId: input.threadId, runId: input.runId }, + { type: "RUN_FINISHED", threadId: input.threadId, runId: input.runId }, + ]); +} + +/** A response this test hands over when it decides to, not when the fixture is called. */ +function deferred() { + let settle: (response: Response) => void = () => { + throw new Error("Deferred response not initialized"); + }; + const promise = new Promise((resolve) => { + settle = resolve; + }); + return { promise, settle }; +} + +beforeAll(() => { + GlobalRegistrator.register({ url: "http://localhost/" }); + originalFetch = globalThis.fetch; + globalThis.fetch = Object.assign( + async (input: Parameters[0], init?: RequestInit) => { + const url = new URL( + typeof input === "string" + ? input + : input instanceof URL + ? input.href + : input.url, + "http://localhost", + ); + if (init?.method === "DELETE") { + deletes.push(url.pathname); + return new NativeResponse(null, { status: 204 }); + } + if (url.pathname === "/api/agents") + return NativeResponse.json({ agents: [] }); + if (url.pathname === "/api/plugins/for/failed-send-bot") + return NativeResponse.json({ skills: [], tools: [] }); + if (url.pathname.endsWith("/info")) + return NativeResponse.json({ + version: "fixture", + agents: { + "failed-send-bot": { description: "Fixture", capabilities: {} }, + }, + mode: "sse", + telemetryDisabled: true, + }); + if (url.pathname.endsWith("/connect")) + return sse([ + { type: "RUN_STARTED", threadId: channel.threadId, runId: "join" }, + { type: "MESSAGES_SNAPSHOT", messages: [opening] }, + { type: "RUN_FINISHED", threadId: channel.threadId, runId: "join" }, + ]); + if (url.pathname.endsWith("/run")) { + const request = + input instanceof Request ? input : new Request(url, init); + const body = RunAgentInputSchema.parse(await request.json()); + const answer = runAnswers[runs.length]; + runs.push(body); + return answer ? await answer(body) : finished(body); + } + if (/\/api\/channels\/[^/]+\/attachments$/.test(url.pathname)) { + const body = init?.body as FormData; + const file = body.get("file") as File; + uploads.push(file.name); + return NativeResponse.json( + { + // One stored id per file, derived from its name, so a DELETE can be attributed to the + // file it was for rather than to "the attachment". + id: `stored-${file.name}`, + name: file.name, + mimeType: "text/plain", + }, + { status: 201 }, + ); + } + if (/\/api\/channels\/[^/]+\/(activity|busy)$/.test(url.pathname)) + return new NativeResponse(null, { status: 204 }); + if (/\/threads\/[^/]+\/messages$/.test(url.pathname)) + return NativeResponse.json({ messages: [opening] }); + throw new Error(`Unexpected fixture request: ${url.pathname}`); + }, + { + preconnect() { + throw new Error("Unexpected fixture preconnect"); + }, + }, + ); +}); + +beforeEach(() => { + runs = []; + deletes = []; + uploads = []; + runAnswers = []; +}); + +afterEach(() => { + cleanup(); + queryClient.clear(); + core = undefined; +}); + +afterAll(async () => { + await settleReactWork(); + globalThis.fetch = originalFetch; + GlobalRegistrator.unregister(); +}); + +function cacheChannel() { + const summary: ChannelSummary = { + ...channel, + summary: null, + lastMessage: null, + lastMessageAgentId: "failed-send-bot", + lastMessageAt: channel.lastMessageAt, + createdAt: "2026-09-09T00:00:00.000Z", + pinned: false, + lastReadAt: null, + }; + queryClient.setQueryData>(channelKeys.list(), { + pages: [{ channels: [summary], nextCursor: null }], + pageParams: [""], + }); +} + +async function mounted() { + cacheChannel(); + const view = render( + + + + + + , + ); + await view.findByText(opening.content); + return view; +} + +function currentAgent() { + const agent = core?.getAgent(`channel:${channel.id}`); + if (!agent) throw new Error("Mounted channel agent is not registered"); + return agent; +} + +function form({ container }: RenderResult) { + return container.querySelector("form") as HTMLFormElement; +} + +/** A drop, as the browser delivers one: files hanging off `dataTransfer`. */ +function drop(target: Element, files: File[]) { + fireEvent.drop(target, { + dataTransfer: { files, items: [], types: ["Files"] }, + }); +} + +function named(name: string) { + return new File([name], name, { type: "text/plain" }); +} + +/** + * A dropped file, all the way up. See `composer-attachments-ui.test.tsx` for the full note: the + * chip appears when the upload STARTS, so only the send button coming back on says it has landed. + */ +async function uploaded(view: RenderResult, names: readonly string[]) { + await waitFor(() => { + for (const name of names) { + expect(view.queryByLabelText(`Remove ${name}`)).not.toBeNull(); + } + const button = (view.queryByLabelText("Send message") ?? + view.queryByLabelText("Queue message")) as HTMLButtonElement | null; + expect(button?.disabled).toBe(false); + }); +} + +/** Every attachment chip on the composer's strip, by the name on its Remove button. */ +function chips({ container }: RenderResult) { + return Array.from(container.querySelectorAll("[aria-label^='Remove ']")) + .map((element) => element.getAttribute("aria-label") ?? "") + .filter((label) => !label.startsWith("Remove queued message")) + .map((label) => label.slice("Remove ".length)); +} + +/** What the last user turn on the agent is carrying, as the wire sees it. */ +function lastUserContent() { + const messages = currentAgent().messages; + const last = [...messages] + .reverse() + .find((message) => message.role === "user"); + return last?.content; +} + +/** + * FINDING 1. A drained turn whose send fails must not delete rows the transcript is still showing. + * + * The reviewer's reproduction, step for step: park a file mid-turn, let the queue drain when the + * turn ends, and fail the drained request. `deliver` has already added the user message — with the + * attachment's URL in it — to `agent.messages` by the time the run is attempted, and nothing + * removes it when the run rejects. So a release here deletes the row behind a message that is on + * screen and stays on screen: a broken attachment in the transcript, which is data loss and not a + * rough edge. + */ +test("a drained turn whose send fails keeps the rows behind the message the transcript still shows", async () => { + const firstRun = deferred(); + runAnswers = [ + // The turn the file is parked behind: held open until this test ends it. + () => firstRun.promise, + // The drained turn: a transient 503 before anything is stamped. + async () => new NativeResponse("upstream unavailable", { status: 503 }), + ]; + + const view = await mounted(); + const user = userEvent.setup({ document: view.container.ownerDocument }); + + await user.type( + view.getByRole("textbox", { name: "Message" }), + "Get started", + ); + await user.click(view.getByRole("button", { name: "Send message" })); + await waitFor(() => expect(runs).toHaveLength(1)); + + // A correction typed at a Bot that is already working, with a file under it. + drop(form(view), [named("plan.txt")]); + await uploaded(view, ["plan.txt"]); + await user.type( + view.getByRole("textbox", { name: "Message" }), + "Use the attached plan", + ); + await user.click(view.getByRole("button", { name: "Queue message" })); + await waitFor(() => + expect( + view.container.querySelector("[aria-label^='Remove queued message']"), + ).not.toBeNull(), + ); + expect(deletes).toEqual([]); + + // The turn ends, the queue drains, and the drained send fails. + firstRun.settle(finished(runs[0] as RunAgentInput)); + await waitFor(() => expect(runs).toHaveLength(2)); + + // The drained message reached the wire carrying the row, which is what makes the release below + // wrong: the same message is now in `agent.messages` and drawn in the transcript. + await waitFor(() => + expect(JSON.stringify(lastUserContent())).toContain( + attachmentUrl("stored-plan.txt"), + ), + ); + expect(view.getAllByText("Use the attached plan").length).toBeGreaterThan(0); + + // THE FINDING. Nothing may delete a row that a message on this screen still points at. The + // release that used to happen here left the transcript showing an attachment whose bytes were + // gone, with nothing said and no way back to it. + await settleReactWork(); + expect(deletes).toEqual([]); + + // And the failed turn is retryable rather than merely undeleted: its words and its file are back + // in the queue, where the next turn carries them and a Remove is the only thing that drops them. + // + // Read off the queued ROW rather than off the button's label, which names the words only: the + // claim is that the file came back with them, and the file is a tile inside that row. + const parked = view.container + .querySelector("[aria-label^='Remove queued message']") + ?.closest("[data-slot='message']"); + expect(parked).not.toBeNull(); + expect(parked?.textContent).toContain("Use the attached plan"); + expect(parked?.textContent).toContain("plan.txt"); +}); + +/** + * FINDING 2. Restoring a failed send beside newly staged files must not produce a sendable draft + * over the per-message cap. + * + * `sending` hides the outgoing files from `staged`, so the cap screening counts zero while a send + * is out and a second full batch is accepted behind it. When the response then fails, clearing + * `sending` puts both batches on one strip — and `canSendDraft` asks about upload status and + * emptiness, not about the cap, so that doubled draft stayed sendable and the retry shipped twice + * the server's per-message budget. + */ +test("a failed send restored beside a second batch cannot be retried over the per-message cap", async () => { + const firstRun = deferred(); + runAnswers = [() => firstRun.promise]; + + const view = await mounted(); + const first = Array.from( + { length: MAX_ATTACHMENTS_PER_MESSAGE }, + (_, index) => named(`first-${index}.txt`), + ); + const second = Array.from( + { length: MAX_ATTACHMENTS_PER_MESSAGE }, + (_, index) => named(`second-${index}.txt`), + ); + + drop(form(view), first); + await uploaded( + view, + first.map((file) => file.name), + ); + fireEvent.submit(form(view)); + await waitFor(() => expect(runs).toHaveLength(1)); + // The send took them off the strip, which is what frees the client-side count for the batch + // below — the server has stamped them by now, so it will not refuse these either. + expect(chips(view)).toEqual([]); + + drop(form(view), second); + await uploaded( + view, + second.map((file) => file.name), + ); + expect(uploads).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE * 2); + + // The response fails, so the first batch comes back beside the second. + firstRun.settle(new NativeResponse("upstream unavailable", { status: 503 })); + await waitFor(() => + expect(chips(view)).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE * 2), + ); + + // THE FINDING. Sixteen attachments is not a message this deployment accepts, and the button that + // sends it must say so rather than letting the retry discover it as a rejected request. + const send = view.getByLabelText("Send message") as HTMLButtonElement; + expect(send.disabled).toBe(true); + expect( + view.getByText(new RegExp(`at most ${MAX_ATTACHMENTS_PER_MESSAGE}`, "i")), + ).toBeTruthy(); + + // Nothing was dropped to get there: every file is still on the strip and still deletable by the + // person who picked it, and no row was released behind their back. + expect(deletes).toEqual([]); + expect(runs).toHaveLength(1); + + // AND IT IS A GATE, NOT A DEAD END. Taking the second batch off by hand — the gesture that HAS + // always justified releasing a row — puts the draft back inside the cap and Send comes back on. + for (const file of second) { + fireEvent.click(view.getByLabelText(`Remove ${file.name}`)); + } + await waitFor(() => { + expect(chips(view)).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE); + expect( + (view.getByLabelText("Send message") as HTMLButtonElement).disabled, + ).toBe(false); + }); + await waitFor(() => + expect(deletes).toEqual( + second.map((file) => attachmentUrl(`stored-${file.name}`)), + ), + ); +}); + +/** + * THE OTHER HALF OF "RETRYABLE", THROUGH THE SAME REAL PATH. A restored queue entry that nothing + * ever carries is just an undeleted row with a Remove button on it. + * + * The retry is the next turn, and it has to be a turn somebody asked for: a queue that re-sent + * itself the instant it came back would spin against a server that is refusing every request. So + * the restored message waits, and the next thing sent takes it along — ahead of the newer words, + * because it was typed first and the whole reason this queue exists is that a correction must not + * be read after the sentence correcting it. + */ +test("the next turn carries a restored message, with its file, ahead of the newer words", async () => { + const firstRun = deferred(); + runAnswers = [ + () => firstRun.promise, + async () => new NativeResponse("upstream unavailable", { status: 503 }), + ]; + + const view = await mounted(); + const user = userEvent.setup({ document: view.container.ownerDocument }); + + await user.type( + view.getByRole("textbox", { name: "Message" }), + "Get started", + ); + await user.click(view.getByRole("button", { name: "Send message" })); + await waitFor(() => expect(runs).toHaveLength(1)); + + drop(form(view), [named("plan.txt")]); + await uploaded(view, ["plan.txt"]); + await user.type( + view.getByRole("textbox", { name: "Message" }), + "Use the attached plan", + ); + await user.click(view.getByRole("button", { name: "Queue message" })); + + // The turn ends, the queue drains, and the drained send fails. + firstRun.settle(finished(runs[0] as RunAgentInput)); + await waitFor(() => expect(runs).toHaveLength(2)); + await waitFor(() => + expect( + view.container.querySelector("[aria-label^='Remove queued message']"), + ).not.toBeNull(), + ); + + // Nothing ran on its own while it sat there. The restore is a retry somebody can take, not one + // this screen keeps attempting. + await settleReactWork(); + expect(runs).toHaveLength(2); + + // The next thing sent takes it along. + await user.type( + view.getByRole("textbox", { name: "Message" }), + "And the summary too", + ); + await user.click(view.getByRole("button", { name: "Send message" })); + await waitFor(() => expect(runs).toHaveLength(3)); + + const sent = runs[2]?.messages.at(-1); + expect(JSON.stringify(sent?.content)).toContain( + attachmentUrl("stored-plan.txt"), + ); + // In the order they were typed: the message that failed, then the one typed after it. + expect(JSON.stringify(sent?.content)).toContain( + "Use the attached plan\\nAnd the summary too", + ); + + // Carried, not copied: the queue gave it up to the run, so nothing is left waiting to run again. + expect( + view.container.querySelector("[aria-label^='Remove queued message']"), + ).toBeNull(); + expect(deletes).toEqual([]); +}); diff --git a/app/tests/plugin-grants.test.ts b/app/tests/plugin-grants.test.ts index f18ee1aaa..4b6b5ed1b 100644 --- a/app/tests/plugin-grants.test.ts +++ b/app/tests/plugin-grants.test.ts @@ -45,6 +45,17 @@ function invalidationRecorder() { return { queryClient, invalidated }; } +/* + * The context TanStack Query hands a mutation callback alongside its variables. These tests drive + * the callbacks directly rather than through a MutationObserver, so they have to supply it. Both + * fields are the real thing rather than a stand-in: `meta` is undefined exactly as it is for a + * mutation declared without one, and `mutationKey` is optional and genuinely absent, because none + * of these options factories sets one. + */ +function mutationContext(queryClient: QueryClient) { + return { client: queryClient, meta: undefined }; +} + test("one grant is one POST of the three things it joins", async () => { const seen = capturingFetch(200, {}); @@ -88,12 +99,15 @@ test("granting one on its own still carries its refetch", async () => { const { queryClient, invalidated } = invalidationRecorder(); const options = setPluginGrantMutationOptions(queryClient); - await options.mutationFn?.({ - agentId: "agent-1", - granted: true, - kind: "mcp", - ref: "notion/search", - }); + await options.mutationFn?.( + { + agentId: "agent-1", + granted: true, + kind: "mcp", + ref: "notion/search", + }, + mutationContext(queryClient), + ); await options.onSuccess?.( undefined as never, { diff --git a/app/tests/root-drop-guard.test.tsx b/app/tests/root-drop-guard.test.tsx new file mode 100644 index 000000000..e381bd6c7 --- /dev/null +++ b/app/tests/root-drop-guard.test.tsx @@ -0,0 +1,190 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { + cleanup, + createEvent, + fireEvent, + render, +} from "@testing-library/react"; +import { useUnclaimedDropGuard } from "@/routes/__root"; +import { settleReactWork } from "./settle-react-work"; + +/** + * A FILE DROPPED WHERE NOTHING WAS LISTENING DOES NOT TAKE THE PAGE WITH IT. + * + * The browser's default for a file dropped on a document is to navigate the top-level document to + * that file, which unloads the app and everything it was holding. `useUnclaimedDropGuard` in + * `routes/__root.tsx` is the app-wide floor under that; its docblock has the whole account, and + * this file pins the two halves that make it both effective and safe. + * + * WHY THESE ASSERT ON `defaultPrevented` AND NOT ON A NAVIGATION, said again here because it is the + * thing a reader will doubt: happy-dom implements no navigation whatsoever — there is no unload to + * observe, no `location` change, nothing an assertion could catch — so a test that watched for the + * symptom would pass identically with and without the guard and would be worth nothing. + * `defaultPrevented` is the CAUSE: it is the exact bit a real browser reads to decide whether to + * keep the drop for itself, and it is the bit that was never set before this guard existed. + * + * `createEvent` + `fireEvent` rather than `fireEvent.drop(...)`, to keep a handle on the native + * event after dispatch — matching `composer-drop-guard.test.tsx`, which explains the choice at + * length. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in + * `afterEach`: bun walks every file into one process, and a document another file tore down + * mid-run fails invisibly. The registration carries a `url` for the same reason it does there — + * without one `location` is `about:blank`. + */ + +beforeAll(() => GlobalRegistrator.register({ url: "http://localhost/" })); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +/** The guard on its own, with no router around it: the hook is the whole of the behaviour. */ +function Guarded({ children }: { children?: React.ReactNode }) { + useUnclaimedDropGuard(); + return
    {children}
    ; +} + +/** + * A `dataTransfer` good enough for the guard: it reads `.files` never, and writes `.dropEffect`. + * `dropEffect` starts at "copy" so that a test asserting "none" is asserting that the guard CHANGED + * it rather than that nobody ever set it. + */ +function transfer() { + return { dropEffect: "copy", files: [], items: [], types: ["Files"] }; +} + +/* + * `createEvent.dragOver` is typed as returning a bare `Event`, which carries no `dataTransfer`. + * testing-library copies the init's `dataTransfer` straight onto the event object it builds, so + * the property really is there at run time: it is the very stub handed in at the call site. This + * names that one fact, rather than asserting the event is a full `DragEvent` carrying a real + * `DataTransfer` -- which is exactly what the stub above documents itself as not being. + */ +function dropEffectOf(event: Event) { + return (event as Event & { dataTransfer: { dropEffect: string } }) + .dataTransfer.dropEffect; +} + +test("a drop on a page region nobody claimed is refused rather than navigated to", () => { + render(); + + const drop = createEvent.drop(document.body, { dataTransfer: transfer() }); + fireEvent(document.body, drop); + + expect(drop.defaultPrevented).toBe(true); +}); + +test("the cursor over unclaimed page says the app will not take the file", () => { + render(); + + const dragOver = createEvent.dragOver(document.body, { + dataTransfer: transfer(), + }); + fireEvent(document.body, dragOver); + + /* + * `preventDefault` is what stops the navigation; `dropEffect` is the only part of the refusal + * that reaches the person BEFORE they let go. A prevented `dragover` left at the default effect + * draws the copy badge — promising to accept a file that is about to land in nothing. + * + * Read back off the event rather than off the object handed to `createEvent`: happy-dom + * implements `DataTransfer`, so testing-library copies the init onto a real one and the handler + * never sees the literal written above. + */ + expect(dragOver.defaultPrevented).toBe(true); + expect(dropEffectOf(dragOver)).toBe("none"); +}); + +test("a drop something in the tree has already claimed is left entirely alone", () => { + /* + * THE HALF THAT MAKES AN APP-WIDE GUARD SAFE TO HAVE. A drop target — today's composer, or any + * surface added later — becomes one by calling `preventDefault` on `dragover`. The guard listens + * in the bubble phase and stands down on an event that is already prevented, so the very line a + * drop target must write in order to work at all is the line that makes the guard ignore it. + * Without that check this guard would quietly draw a no-entry cursor over every working drop + * target in the app, which is exactly the failure the composer's comment warned a document-level + * listener could cause. + */ + const claimed: string[] = []; + const { getByTestId } = render( + + {/** biome-ignore lint/a11y/noStaticElementInteractions: a stand-in drop target, not a control. */} +
    { + event.preventDefault(); + if (event.dataTransfer) { + event.dataTransfer.dropEffect = "copy"; + } + }} + onDrop={(event) => { + event.preventDefault(); + claimed.push("drop"); + }} + /> + , + ); + + const target = getByTestId("claimant"); + const dragOver = createEvent.dragOver(target, { dataTransfer: transfer() }); + fireEvent(target, dragOver); + const drop = createEvent.drop(target, { dataTransfer: transfer() }); + fireEvent(target, drop); + + // The claimant's own answer survives the guard: it still gets the drop, and the cursor still + // says the file is welcome. + expect(claimed).toEqual(["drop"]); + expect(dropEffectOf(dragOver)).toBe("copy"); +}); + +test("the guard leaves with the app rather than outliving it", () => { + const { unmount } = render(); + unmount(); + + const drop = createEvent.drop(document.body, { dataTransfer: transfer() }); + fireEvent(document.body, drop); + + /* + * A listener left on `document` after unmount is the leak the composer's comment named as the + * reason a leaf must not own this — installed once per mount, torn down by whichever unmounted + * first. The root mounts once, but the teardown is what makes that claim checkable, and a test + * process that walks many files into one document is exactly where a stray listener would start + * answering for somebody else's test. + */ + expect(drop.defaultPrevented).toBe(false); +}); + +/** + * A DRAGGED PHRASE IS NOT THIS GUARD'S BUSINESS, AND THE DISTINCTION IS LOAD-BEARING. + * + * An editable element — a text input, or the composer's own contenteditable editor — is a drop + * target the BROWSER makes, with no script preventing anything. So an app-wide guard that refused + * every unclaimed drop would refuse dragging a selected phrase into the message box: a gesture + * people use, that nothing in this app implements and so nothing in this app could give back. The + * guard asks whether the drag carries FILES, which is the payload that unloads the page, and lets + * every text drag past untouched. + * + * The cost of that narrowing is named in the guard's own comment: a dragged LINK let go on the page + * margin still navigates. That is the trade, not an oversight. + */ +test("a dragged phrase carrying no file is left to the browser to handle", () => { + render(); + + const dragOver = createEvent.dragOver(document.body, { + dataTransfer: { + dropEffect: "copy", + files: [], + items: [], + types: ["text/plain"], + }, + }); + fireEvent(document.body, dragOver); + + // Untouched on both counts: the browser still decides, and the cursor is not overwritten with + // the no-entry badge over an input that would have taken the text. + expect(dragOver.defaultPrevented).toBe(false); + expect(dropEffectOf(dragOver)).toBe("copy"); +}); diff --git a/app/tests/serve.test.ts b/app/tests/serve.test.ts index 23ed7a734..61d71bdd4 100644 --- a/app/tests/serve.test.ts +++ b/app/tests/serve.test.ts @@ -4,6 +4,7 @@ import { createServer, type Socket } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { + BunWebSocket, fileFor, isApiCall, isClientRoute, @@ -335,7 +336,7 @@ async function startProxy(upstreamPort: number) { async function failedHandshake(url: string, headers: HeadersInit = {}) { const events: string[] = []; - const socket = new WebSocket(url, { headers }); + const socket = new BunWebSocket(url, { headers }); try { await new Promise((resolve, reject) => { const timeout = setTimeout( @@ -357,8 +358,8 @@ async function failedHandshake(url: string, headers: HeadersInit = {}) { } async function connectWebSocket(url: string, headers: HeadersInit) { - return new Promise((resolve, reject) => { - const socket = new WebSocket(url, { headers }); + return new Promise((resolve, reject) => { + const socket = new BunWebSocket(url, { headers }); const timeout = setTimeout(() => { socket.close(); reject(new Error("websocket did not open")); @@ -382,7 +383,7 @@ async function connectWebSocket(url: string, headers: HeadersInit) { }); } -async function nextSocketMessage(socket: WebSocket) { +async function nextSocketMessage(socket: BunWebSocket) { return new Promise((resolve, reject) => { const timeout = setTimeout(() => { socket.close(); @@ -558,7 +559,7 @@ describe("upstream websocket handshake", () => { }, }); const proxy = await startProxy(upstream.port!); - const socket = new WebSocket( + const socket = new BunWebSocket( `ws://127.0.0.1:${proxy.port}/api/events?mode=delayed`, ); try { diff --git a/app/tests/settle-react-work.ts b/app/tests/settle-react-work.ts new file mode 100644 index 000000000..8ed4e4643 --- /dev/null +++ b/app/tests/settle-react-work.ts @@ -0,0 +1,48 @@ +import { act } from "react"; + +/** + * DRAIN REACT BEFORE TAKING THE DOCUMENT AWAY. + * + * Every React test file here registers happy-dom in `beforeAll` and unregisters it in `afterAll`, + * because bun walks every test file into one process and a document another file tore down mid-run + * fails invisibly. `cleanup` in `afterEach` unmounts what a test rendered — but unmounting does not + * retract a callback React's scheduler has ALREADY posted. If one is still posted when `afterAll` + * runs `GlobalRegistrator.unregister()`, it fires against a `document` that no longer exists, and + * bun reports it as `# Unhandled error between tests`: zero failing tests, a non-zero exit, and a + * stack in `scheduler.development.js` carrying the name of whichever file bun happened to reach + * next. Await this first and the queue is empty before the document goes away. + * + * WHY `act` AND NOT A `setTimeout`. Both halves of that question have a mechanical answer. + * + * First, the primitive. `scheduler` posts its host callback with `setImmediate` when one exists and + * only falls back to `MessageChannel` when it does not (see the branch at the foot of + * `scheduler.development.js`), and bun defines `setImmediate` — so in this runtime the pending work + * is an immediate, not a message. `setImmediate` callbacks run in the order they were registered, + * so a yield that is itself an immediate is ordered strictly AFTER every immediate already posted. + * A `setTimeout(0)` is a different queue and carries no such ordering against a pending immediate: + * it usually lands after, which is exactly the kind of "usually" that only fails on CI. React's + * `act` yields through its own `enqueueTask`, which resolves to `module.require("timers") + * .setImmediate` — the same primitive the scheduler posts on, so the ordering is by construction. + * + * Second, the number of turns. One turn is not enough no matter which primitive it uses, because + * the work that runs during a turn can schedule more: a settling upload resolves on a microtask + * after our yield was already queued, and the render it triggers is posted BEHIND us. What is + * needed is a fixed point, not a fixed count — and that is precisely what `act` computes. While an + * act scope is open React routes every newly scheduled callback into the act queue instead of the + * scheduler, and `recursivelyFlushAsyncActWork` alternates flushing that queue with yielding a + * macrotask until a yield comes back with the queue still empty. So the loop ends when React has + * nothing left rather than after some count somebody guessed. `flushActQueue` also runs each + * callback's continuation to completion, so work the scheduler would have sliced across several + * 5ms budgets finishes inside one flush. + * + * The one thing this cannot reach is a render that was ALREADY sliced by the scheduler before the + * drain opened: that continuation is re-posted by the scheduler rather than into the act queue. + * Reaching that needs a single render to exceed the scheduler's 5ms budget, which a tree of a few + * dozen nodes does not do — and a test that did would be telling us something worth hearing anyway. + * + * `act` needs `IS_REACT_ACT_ENVIRONMENT`, which `@testing-library/react` sets when it is imported; + * every caller of this renders through it, so the flag is on by the time `afterAll` runs. + */ +export async function settleReactWork(): Promise { + await act(async () => {}); +} diff --git a/app/tests/transcript-attachments.test.tsx b/app/tests/transcript-attachments.test.tsx new file mode 100644 index 000000000..dc428dd09 --- /dev/null +++ b/app/tests/transcript-attachments.test.tsx @@ -0,0 +1,1012 @@ +import type { Message, UserMessage } from "@ag-ui/core"; +import type { Attachment } from "@copilotkit/react-core/v2"; +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { + act, + cleanup, + fireEvent, + render, + waitFor, +} from "@testing-library/react"; +import { + ChatTranscript, + LightboxPicture, + sameAttachmentRow, + TranscriptAttachments, +} from "@/components/channels/chat-transcript"; +import type { QueuedMessage } from "@/components/channels/composer"; +import { attachmentUrl } from "@/lib/channels/attachments"; +import { settleReactWork } from "./settle-react-work"; + +/** + * `toVisibleChatItems` (chat-messages.ts) gathers the attachment parts of a user turn into one + * `{ kind: "attachments" }` item; this pins what the transcript draws for it, which until this + * feature was an explicit `null` — a screenshot pasted with no caption rendered as nothing at all. + * + * It also pins the two ways an attachment-only turn has to behave like a typed one — the Thinking + * indicator and the scroll anchor — and, in the same breath, that a CAPTIONED turn still anchors + * exactly once. The scroller jumps to the end when it finds two anchors among the rows appended + * together, so the naive "every attachment is an anchor" rule fixes the first case by breaking the + * ordinary one, and only a test that counts anchors notices. + * + * URLS ARE RELATIVE HERE BECAUSE THAT IS THE ONLY KIND A SENT MESSAGE CARRIES. `shared/attachments.ts` + * says so, and the transcript now refuses anything else rather than fetching it. + * + * THE HARNESS IS THIS REPOSITORY'S. `GlobalRegistrator` in `beforeAll`/`afterAll` and `cleanup` in + * `afterEach`, matching `agent-roster-error.test.tsx`: bun walks every file into one process, and a + * document another file tore down mid-run fails invisibly. + * + * WITH ONE ADDITION: an origin. happy-dom defaults to `about:blank`, against which a relative `src` + * does not resolve at all, and the image element fires `error` before anything has been asserted — + * so every attachment would test as unavailable and the tests below would agree with a transcript + * that draws nothing. + */ + +/** + * AND A STUBBED `fetch`, because a document tile now ASKS whether its row is still there. + * + * A document draws no ``, so nothing about it can fail to load and nothing tells it the row + * behind it was deleted; the tile probes the attachment route instead. Every test here that draws + * a document therefore makes a request, and left unstubbed each one would be a real socket to a + * server that is not running — slow, and answering "gone" for the wrong reason. The default answer + * is the ordinary one: the file is still there. + */ +/* + * ANSWERED ON A MICROTASK, NOT SYNCHRONOUSLY, so that a `probeAnswer` which THROWS becomes a + * rejected promise rather than an exception thrown out of `fetch` itself. No real `fetch` throws at + * the call site — an offline browser rejects — and the probe's `.catch` is written for the real + * shape, so answering synchronously here would have the one test about an unreachable server take + * a path production never takes, straight through the effect and into React. + * + * The request is still RECORDED synchronously, which is what lets the tests below use one document's + * probe as a clock for another's absence. + */ +let probeAnswer: () => Response | Promise = () => + new Response(null, { status: 200 }); +let probes: { url: string; method?: string }[] = []; +let realFetch: typeof fetch; + +beforeAll(() => { + GlobalRegistrator.register({ url: "http://localhost/" }); + realFetch = globalThis.fetch; + globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + probes.push({ + url: String(input), + ...(init?.method ? { method: init.method } : {}), + }); + return Promise.resolve().then(() => probeAnswer()); + }) as typeof fetch; +}); + +afterEach(() => { + cleanup(); + probeAnswer = () => new Response(null, { status: 200 }); + probes = []; +}); + +afterAll(async () => { + /* + * REACT FIRST, BEFORE THE DOCUMENT GOES AWAY. A probe answered during the last test leaves React + * work scheduled on a macrotask, and the scheduler reaches for `window` when it runs — after + * `unregister`, that is an error reported against whichever file bun happens to be running by + * then, which is a failure with somebody else's name on it. `settle-react-work.ts` says why one + * turn of `setTimeout` was the wrong instrument for waiting on it. + */ + await settleReactWork(); + globalThis.fetch = realFetch; + GlobalRegistrator.unregister(); +}); + +let idCounter = 0; + +/** A user turn, with whatever content a test needs — a caption, an attachment, or both. */ +function userMessage(content: UserMessage["content"]): Message { + idCounter += 1; + return { id: `user-${idCounter}`, role: "user", content }; +} + +function textPart(text: string) { + return { type: "text" as const, text }; +} + +function imagePart(url: string, filename: string) { + return { + type: "image" as const, + source: { type: "url" as const, value: url }, + metadata: { attachmentId: "att-image", filename }, + }; +} + +function documentPart(url: string, filename: string) { + return { + type: "document" as const, + source: { type: "url" as const, value: url }, + metadata: { attachmentId: "att-document", filename }, + }; +} + +function renderTranscript( + messages: readonly Message[], + props: { + busy?: boolean; + onRemoveQueued?: (id: string) => void; + queued?: readonly QueuedMessage[]; + } = {}, +) { + return render(); +} + +/** A file staged on the composer, as the SDK hands it over: settled, and pointing at its own row. */ +function staged(id: string, filename: string): Attachment { + return { + id, + type: "image", + source: { type: "url", value: attachmentUrl(id) }, + filename, + status: "ready", + }; +} + +/** A message typed while the Bot had the turn, with whatever was staged on the draft beside it. */ +function parked(text: string, attachments: Attachment[]): QueuedMessage { + return { id: "queued-1", text, commandIds: [], attachments }; +} + +/** + * Which rows the scroller would treat as the place to scroll to, in the order it walks them. + * + * Read off the DOM rather than off a return value because that attribute IS the contract: the + * primitive looks for `data-scroll-anchor="true"` among the newly appended children and nothing + * else, so asserting on anything closer to the component would pass while the scroller misbehaved. + */ +function anchoredRows(container: HTMLElement): (string | null)[] { + return Array.from( + container.querySelectorAll('[data-scroll-anchor="true"]'), + ).map((row) => row.getAttribute("data-message-id")); +} + +test("an image attachment renders as an img carrying the attachment url", () => { + const url = attachmentUrl("att-image"); + const { getByRole } = renderTranscript([ + userMessage([imagePart(url, "shot.png")]), + ]); + + const img = getByRole("img", { name: /attachment/i }); + expect(img.getAttribute("src")).toBe(url); +}); + +test("a broken image is replaced by a stated absence, not a broken-image glyph", () => { + const { getByRole, getByText, queryByRole } = renderTranscript([ + userMessage([imagePart(attachmentUrl("att-image"), "gone.png")]), + ]); + + const img = getByRole("img", { name: /attachment/i }); + fireEvent.error(img); + + // The element is gone entirely — this reader is never left staring at the browser's own + // broken-image box, which says nothing about what actually happened. + expect(queryByRole("img")).toBeNull(); + expect(getByText(/unavailable/i)).toBeTruthy(); +}); + +test("an off-site attachment url is never fetched, it is reported missing", () => { + // An absolute url cannot have come from this app's composer, and drawing it would have the + // reader's browser announce to a third party that they opened this channel. + const { getByText, queryByRole } = renderTranscript([ + userMessage([imagePart("https://example.com/shot.png", "shot.png")]), + ]); + + expect(queryByRole("img")).toBeNull(); + expect(getByText("shot.png is unavailable.")).toBeTruthy(); +}); + +test("a message that is only an attachment still appears", () => { + // The regression this whole feature rests on: before this change, `chat-transcript.tsx` had an + // explicit `null` for the attachment branch, so a document pasted with no caption vanished from + // the transcript entirely. + const { getByText } = renderTranscript([ + userMessage([documentPart(attachmentUrl("att-document"), "report.pdf")]), + ]); + + expect(getByText("report.pdf")).toBeTruthy(); +}); + +test("a turn that is only an attachment is still waited on", () => { + // Somebody who pastes a screenshot and sends it is watching the same spot under it as somebody + // who typed a question, and without this they watched it stay empty. + const { getByRole } = renderTranscript( + [userMessage([imagePart(attachmentUrl("att-image"), "shot.png")])], + { busy: true }, + ); + + expect(getByRole("status").textContent).toBe("Thinking"); +}); + +test("a turn that is only an attachment anchors the scroller on itself", () => { + const message = userMessage([ + imagePart(attachmentUrl("att-image"), "shot.png"), + ]); + const { container } = renderTranscript([message]); + + expect(anchoredRows(container)).toEqual([`${message.id}:attachments`]); +}); + +test("two files sent together anchor their turn exactly once", () => { + const message = userMessage([ + imagePart(attachmentUrl("att-image"), "one.png"), + imagePart(attachmentUrl("att-image"), "two.png"), + ]); + const { container } = renderTranscript([message]); + + // Two files are ONE row now, so there is only one thing that could be an anchor. Kept anyway: + // the assertion that survives the grouping is the one worth having if the grouping is ever undone. + expect(anchoredRows(container)).toEqual([`${message.id}:attachments`]); +}); + +test("a caption and its attachment anchor their turn exactly once", () => { + const message = userMessage([ + textPart("does this look right?"), + imagePart(attachmentUrl("att-image"), "shot.png"), + ]); + const { container } = renderTranscript([message]); + + // ONE, and it is the PICTURES, which is the row the caption now sits under. A second anchor among + // rows appended together makes the scroller give up and jump to the end, which would quietly cost + // every captioned turn the anchoring it has today — the exact price of marking the caption an + // anchor too. + expect(anchoredRows(container)).toEqual([`${message.id}:attachments`]); +}); + +test("a plain string message renders exactly as before", () => { + const { getByText } = renderTranscript([ + userMessage("when does the offer expire?"), + ]); + + expect(getByText("when does the offer expire?")).toBeTruthy(); +}); + +test("two files sent together are drawn as one row, not two", () => { + const { container } = renderTranscript([ + userMessage([ + imagePart(attachmentUrl("att-image"), "one.png"), + imagePart(attachmentUrl("att-image"), "two.png"), + ]), + ]); + + // The thing the grouping buys, stated as the DOM: two tiles inside a single list, rather than + // two rows each as wide as the transcript with a picture alone on each. + const lists = container.querySelectorAll("ul"); + expect(lists).toHaveLength(1); + expect(lists[0].querySelectorAll("li")).toHaveLength(2); +}); + +test("a thumbnail is a crop, and what opens it is a button rather than a link", () => { + /* + * WHAT THIS CANNOT ASSERT, SAID PLAINLY RATHER THAN LEFT AS A GAP. The picture opens in a dialog + * now, and none of that is observable here: Base UI portals its popup and under happy-dom the + * portal never mounts, while `aria-expanded` on the trigger stays `false` even though + * `onOpenChange(true)` demonstrably fires — the primitive's own state does not settle without a + * frame this environment never delivers. Both were checked before this comment was written. + * + * So the dialog was verified in Chrome instead — centred at its own aspect ratio, close button at + * the viewport's top right, and closing on the button, on Escape and on a click outside — and + * what is pinned here is the contract around it that a unit test CAN see: the tile is a crop, so + * something has to open the whole picture, and the thing that does is a button on this page + * rather than a link out of it. + */ + const url = attachmentUrl("att-image"); + const { getByRole, getByLabelText, queryByRole } = renderTranscript([ + userMessage([imagePart(url, "shot.png")]), + ]); + + expect(queryByRole("dialog")).toBeNull(); + // `object-cover` is what makes it a square crop, and therefore what makes the dialog necessary. + expect(getByRole("img", { name: /attachment/i }).className).toContain( + "object-cover", + ); + + const trigger = getByLabelText("Open shot.png"); + expect(trigger.tagName).toBe("BUTTON"); + // Not an anchor: it opens something here, and a middle-click must not offer a tab of raw bytes. + expect(trigger.getAttribute("href")).toBeNull(); + expect(trigger.getAttribute("aria-haspopup")).toBe("dialog"); +}); + +/* + * THE OTHER HALF OF THE TILE'S `onError`, ON THE PICTURE THE TILE OPENS. + * + * The tile swaps a broken-image box for a sentence, and the full-size picture behind it did not — + * it had no `onError` at all — so a file deleted between the tile painting and the reader clicking + * it opened a dialog containing exactly the browser placeholder the tile path exists to avoid. + * + * The tile cannot answer for this: its own `` has already loaded, and a loaded image does not + * fire `error` again because the bytes behind it went away. Nor can the document probe, which + * deliberately never asks about a picture. + * + * RENDERED DIRECTLY, NOT THROUGH THE TRIGGER, and the test named "a thumbnail is a crop" above + * records why: Base UI portals the popup and under happy-dom that portal never mounts, so there is + * no way to reach this element by clicking. The component is exported for exactly this, the same + * reason `sameAttachmentRow` is. + */ +test("the opened picture states its absence rather than showing a broken frame", () => { + const { getByRole, getByText, queryByRole } = render( + , + ); + + fireEvent.error(getByRole("img")); + + // The is gone entirely, exactly as it is in the tile: no browser placeholder is left for + // the reader to interpret. + expect(queryByRole("img")).toBeNull(); + expect(getByText("gone.png is unavailable.")).toBeTruthy(); + // A note, never an alert — this answers something the reader just did, it does not interrupt. + expect(getByRole("note")).toBeTruthy(); +}); + +/* A picture whose name never arrived still gets a sentence rather than a blank dialog. */ +test("an unnamed picture that will not open still says what happened", () => { + const { getByRole, getByText } = render( + , + ); + + fireEvent.error(getByRole("img")); + + expect(getByText("This attachment is unavailable.")).toBeTruthy(); +}); + +/* + * And the ordinary case, which a fix to the above can break silently: a picture that loads is drawn, + * at its own aspect ratio rather than cropped. `object-contain` is what makes the dialog worth + * opening at all, given the tile is `object-cover`. + */ +test("the opened picture is drawn whole, not cropped like its tile", () => { + const url = attachmentUrl("att-image"); + const { getByRole } = render( + , + ); + + const img = getByRole("img"); + expect(img.getAttribute("src")).toBe(url); + expect(img.getAttribute("alt")).toBe("shot.png"); + expect(img.className).toContain("object-contain"); +}); + +test("an off-site document url is reported missing rather than drawn as a file card", () => { + /* + * THE SAME RULE AS THE PICTURE ABOVE, and it has to be, because the lie a document tells is the + * worse one: a picture that cannot be drawn at least looks wrong, while a card naming a file the + * app cannot serve reads as an intact attachment sitting right there. The reader is told the + * thing is present when it is not. + */ + const { getByText, queryByText } = renderTranscript([ + userMessage([documentPart("https://example.com/report.pdf", "report.pdf")]), + ]); + + expect(getByText("report.pdf is unavailable.")).toBeTruthy(); + // Not the file card: no "Attachment" caption, and the name is not offered as an intact one. + expect(queryByText("Attachment")).toBeNull(); +}); + +/* + * A DELETED ROW IS THE CASE THE OFF-SITE RULE ABOVE DOES NOT COVER, and it was the one that shipped + * broken. `unavailable` was `failedToLoad || !url.startsWith(...)`, and `failedToLoad` is set only + * by an image's `onError` — an event a document, which renders no ``, can never receive. So a + * document whose row had been deleted kept a url of exactly the right shape and drew as an intact + * file card: the reader was told the file was sitting right there while the server was answering + * 404 for it. Verified in a browser before it was written down here. + */ +test("a document whose row is gone is reported missing rather than named", async () => { + probeAnswer = () => new Response(null, { status: 404 }); + + const { findByText, queryByText } = renderTranscript([ + userMessage([documentPart(attachmentUrl("att-document"), "report.pdf")]), + ]); + + expect(await findByText("report.pdf is unavailable.")).toBeTruthy(); + // Not the file card: the name is no longer offered as an intact one, caption and all. + expect(queryByText("Attachment")).toBeNull(); +}); + +/* + * The other half of that, and the half a fix can break without anybody noticing: a document whose + * row is still there keeps its card. "Everything is unavailable" passes the test above. + */ +test("a document whose row is still there keeps its card", async () => { + const { getByText, queryByText } = renderTranscript([ + userMessage([documentPart(attachmentUrl("att-document"), "report.pdf")]), + ]); + + await waitFor(() => expect(probes).toHaveLength(1)); + + expect(getByText("report.pdf")).toBeTruthy(); + expect(getByText("Attachment")).toBeTruthy(); + expect(queryByText("report.pdf is unavailable.")).toBeNull(); +}); + +/* + * WHAT THE PROBE IS ALLOWED TO BE: a bodyless request for the file's own url. `HEAD` because the + * question is whether the row exists and the answer is the status line — pulling a whole PDF back + * through the browser to learn it is still there would cost the reader the file's bytes on every + * transcript that mentions it. + */ +test("the probe asks for the head of the attachment url, not its bytes", async () => { + const url = attachmentUrl("att-document"); + renderTranscript([userMessage([documentPart(url, "report.pdf")])]); + + await waitFor(() => expect(probes).toHaveLength(1)); + expect(probes[0].url).toBe(url); + expect(probes[0].method).toBe("HEAD"); +}); + +/* + * AND WHAT IT MUST NEVER BE: a request to somebody else's server. The off-site rule exists so that + * drawing a transcript cannot announce to a third party that this person opened this channel, and + * a probe is a request like any other — an existence check sent there would leak exactly the fact + * the rule was written to keep. An off-site document is already known to be unavailable without + * asking anybody. + */ +/* + * NOT ASSERTED BY WAITING ON AN EMPTY LIST, which is what this did and which proves nothing: the + * callback `waitFor` retries does not throw on the very first check, so `await waitFor(() => + * expect(probes).toEqual([]))` is over before a macrotask has run and the `await` reads as patience + * it never bought. It said "not probed synchronously" while its name said "never" — a probe moved + * behind a microtask, an `IntersectionObserver` or an idle callback would have slid straight past. + * + * A SERVABLE DOCUMENT IS DRAWN BESIDE IT AS THE CLOCK. Waiting for THAT one's probe to arrive puts + * the question after the point by which the off-site one — mounted in the same commit — would have + * had to appear, so an empty result now means the request was not made rather than not made yet. + */ +test("an off-site document is never probed", async () => { + const kept = attachmentUrl("att-document"); + const { getByText } = renderTranscript([ + userMessage([ + documentPart("https://example.com/report.pdf", "report.pdf"), + documentPart(kept, "kept.pdf"), + ]), + ]); + + expect(getByText("report.pdf is unavailable.")).toBeTruthy(); + + await waitFor(() => expect(probes).toHaveLength(1)); + // One probe, and it is the servable file's. Nothing was sent to example.com. + expect(probes.map((probe) => probe.url)).toEqual([kept]); +}); + +/* + * An image is not probed either: it fetches its own url to draw itself, and `onError` is that same + * request's answer. A probe beside it would ask the server for the same file twice per picture. + * + * Same clock as the test above, and for the same reason. + */ +test("an image is not probed, its own load already answers", async () => { + const kept = attachmentUrl("att-document"); + renderTranscript([ + userMessage([ + imagePart(attachmentUrl("att-image"), "shot.png"), + documentPart(kept, "kept.pdf"), + ]), + ]); + + await waitFor(() => expect(probes).toHaveLength(1)); + expect(probes.map((probe) => probe.url)).toEqual([kept]); +}); + +/* + * ONLY 404 MEANS THE FILE IS GONE, AND THE PROBE USED TO READ EVERY OTHER FAILURE AS ONE. + * + * `!response.ok` is every status outside 200-299, and exactly one of them says what the tile then + * says. The route this asks (`server/src/channels/attachments.ts`) deliberately collapses "no such + * row", "channel deleted" and "not yours" into 404 so that probing ids learns nothing — that is the + * status, and the only status, that means "there is no file here for you". + * + * Everything else is a different fact about the REQUEST, not about the file: + * + * - 401 is the session having expired while the channel sat open. Every document tile in the + * transcript flipped at once to a red card asserting, in the file's own name, that somebody's + * files had been deleted — when all that happened is that they need to sign in again. + * - 500 or 503 is the database or the server having a bad moment. It is also PERMANENT for that + * mount: the effect's deps are the url and whether to ask, both stable, so nothing ever asks + * again and the tile goes on claiming deletion until the component remounts. + * - 304 is the strongest possible proof of PRESENCE the route can give — the row was found and the + * membership join passed — and `Response.ok` is false for it. + * + * This is the same lie the probe was added to stop, pointing the other way, and it is the louder + * one: it is drawn in the destructive vocabulary and it names the file. A card that stays intact + * when the answer is unclear is the honest failure, and it matches what the picture beside it does + * — an `` given a 401 or a 500 shows a broken image, not a sentence asserting deletion. + */ +test("only a 404 makes a document missing, not any other failed answer", async () => { + for (const status of [304, 401, 403, 500, 503]) { + probes = []; + probeAnswer = () => new Response(null, { status }); + + const { getByText, queryByText, unmount } = renderTranscript([ + userMessage([documentPart(attachmentUrl("att-document"), "report.pdf")]), + ]); + + await waitFor(() => expect(probes).toHaveLength(1)); + + // The card is intact: the name, the caption, and no accusation. + expect(getByText("report.pdf")).toBeTruthy(); + expect(getByText("Attachment")).toBeTruthy(); + expect(queryByText("report.pdf is unavailable.")).toBeNull(); + + unmount(); + } +}); + +/* + * TWO TILES SHOWING THE SAME FILE ASK ABOUT IT ONCE. + * + * One turn carrying the same file twice is a shape this projection explicitly supports — the tile + * key is the PART index so that it can — and it is not the only way two tiles land on one url: a + * message parked mid-turn draws its files again beside the sent row. + * + * THE SAVING IS THE ROUND TRIP, AND IT USED TO BE THE FILE. This note said the second probe was "a + * second whole-file read out of Postgres", because a HEAD was then answered by Hono running the GET + * handler in full, bytes and all, before dropping the body. The route has since grown a HEAD branch + * that selects `sizeBytes` and never `bytes`, so what is deduped now is a cheap metadata query. + * + * The case still holds, on a footing that does not depend on the old cost: the route serves + * `private, no-cache`, so the browser is required to revalidate rather than answer one tile's probe + * out of the other's, and two tiles are two components with two effects that know nothing of each + * other. Nothing but `probesInFlight` coalesces them, and this is what pins that it does. + */ +test("the same file drawn twice is asked about once", async () => { + const url = attachmentUrl("att-document"); + const { getAllByText } = renderTranscript([ + userMessage([ + documentPart(url, "report.pdf"), + documentPart(url, "report.pdf"), + ]), + ]); + + // Both tiles are really there — the saving is in the asking, not in the drawing. + expect(getAllByText("report.pdf")).toHaveLength(2); + + await waitFor(() => expect(probes).toHaveLength(1)); + // And no straggler arrives behind it once the shared answer has settled. + await act(async () => {}); + expect(probes).toHaveLength(1); +}); + +/* + * But a SETTLED answer is not kept: the next mount asks again. Caching "still there" across mounts + * is the one change that would cut the cost of the common case, and it is also the lie this whole + * probe exists to stop — a file deleted while the tab is open would go on drawing as an intact card + * until the page was reloaded. The cheap answer is not worth the honest one. + */ +test("a later mount asks again rather than reusing a settled answer", async () => { + const url = attachmentUrl("att-document"); + + const first = renderTranscript([ + userMessage([documentPart(url, "report.pdf")]), + ]); + await waitFor(() => expect(probes).toHaveLength(1)); + first.unmount(); + + renderTranscript([userMessage([documentPart(url, "report.pdf")])]); + await waitFor(() => expect(probes).toHaveLength(2)); +}); + +/* + * And a probe that never arrived at all is still not a deleted file — the rule the comment on + * `useDocumentIsGone` already stated for a REJECTED fetch, pinned here beside the statuses so a + * change to one cannot quietly take the other with it. + */ +test("a probe that never arrives leaves the card alone", async () => { + probeAnswer = () => { + throw new Error("offline"); + }; + + const { getByText, queryByText } = renderTranscript([ + userMessage([documentPart(attachmentUrl("att-document"), "report.pdf")]), + ]); + + await waitFor(() => expect(probes).toHaveLength(1)); + await act(async () => {}); + + expect(getByText("report.pdf")).toBeTruthy(); + expect(queryByText("report.pdf is unavailable.")).toBeNull(); +}); + +/* + * WHAT A MISSING FILE IS ALLOWED TO INTERRUPT, WHICH IS NOTHING. `role="alert"` is an ASSERTIVE + * live region: it cuts across whatever a screen reader is currently saying, which is right for + * something that just happened in answer to what somebody did — `Stopped` is exactly that — and + * wrong for history. Opening a channel whose old turns carry three deleted files fired three + * interruptions before the reader had heard the first sentence of the conversation, and none of + * them was news: those files went missing long before this page was opened. + * + * The absence still has to READ as an absence, so the sentence stays exactly as it was and the + * tile stays a thing a screen reader stops on. It simply waits its turn. + */ +test("missing attachments are stated, not announced over what is being read", () => { + const { container, getByText } = renderTranscript([ + userMessage([ + imagePart("https://example.com/one.png", "one.png"), + imagePart("https://example.com/two.png", "two.png"), + imagePart("https://example.com/three.png", "three.png"), + ]), + ]); + + /* + * Read off the DOM, like `anchoredRows` above and for the same reason: the attribute IS the + * contract a screen reader reads, and `getAllByRole` walks the whole transcript computing roles + * for every node in it — which under this harness takes minutes rather than milliseconds. + */ + const role = (name: string) => + container.querySelectorAll(`[role="${name}"]`).length; + + // Three files, three tiles, and not one interruption between them. + expect(role("alert")).toBe(0); + expect(role("note")).toBe(3); + expect(getByText("one.png is unavailable.")).toBeTruthy(); + expect(getByText("three.png is unavailable.")).toBeTruthy(); +}); +/* + * The two wordings, both pinned, because they are the ones a reader is left with when everything + * else about the file is gone. A filename-less attachment is the ordinary case for anything pasted + * rather than picked — the composer has no name to send — so this branch is not an edge. + */ +test("an unnamed missing attachment still says what it is", () => { + const { getByText } = renderTranscript([ + userMessage([ + { + type: "image" as const, + source: { type: "url" as const, value: "https://example.com/x.png" }, + metadata: { attachmentId: "att-image" }, + }, + ]), + ]); + + expect(getByText("This attachment is unavailable.")).toBeTruthy(); +}); + +test("an unnamed document that is present is still drawn as a file", async () => { + const { getByText } = renderTranscript([ + userMessage([ + { + type: "document" as const, + source: { type: "url" as const, value: attachmentUrl("att-document") }, + metadata: { attachmentId: "att-document" }, + }, + ]), + ]); + + await waitFor(() => expect(probes).toHaveLength(1)); + expect(getByText("Untitled file")).toBeTruthy(); + expect(getByText("Attachment")).toBeTruthy(); +}); + +/* + * A PARKED FILE WAS ON NO SURFACE IN THE APP AT ALL, which is the part that makes this worse than a + * missing row. Parking a message consumes the draft, so the composer's own strip is cleared in the + * same beat — and the queued line drew `message.text` and nothing else. Somebody who attached a + * screenshot and typed a correction while the Bot was working watched the file disappear from the + * composer and never appear anywhere else, with nothing on screen to say it was still coming. + */ +test("a file parked with a message is still on screen", () => { + const { container, getByText } = renderTranscript([], { + queued: [parked("this one instead", [staged("att-image", "shot.png")])], + }); + + const image = container.querySelector("img"); + expect(image?.getAttribute("src")).toBe(attachmentUrl("att-image")); + // Still parked, and still takeable back: the row is drawn with the message, not instead of it. + expect(getByText("this one instead")).toBeTruthy(); + expect(getByText("Queued")).toBeTruthy(); +}); + +/* + * AND AN ATTACHMENT-ONLY PARKED MESSAGE IS NOT AN EMPTY BUBBLE. A screenshot pasted mid-turn with + * no words is the ordinary way this feature gets used, and it drew a muted bubble containing + * nothing — which reads as a message somebody sent by mistake rather than as a file waiting its + * turn. + */ +test("a parked message that is only a file draws the file, not an empty bubble", () => { + const { container } = renderTranscript([], { + queued: [parked("", [staged("att-image", "shot.png")])], + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe( + attachmentUrl("att-image"), + ); + expect(container.querySelector('[data-slot="bubble"]')).toBeNull(); +}); + +/* + * The button that takes it back is named after what it deletes — and with no words to name it by, + * the files are what it is. "Remove queued message: " was the label before, which tells somebody + * reading by name alone that there is something to remove and nothing whatever about what. + */ +test("taking back a parked file names the file", () => { + const { getByLabelText } = renderTranscript([], { + onRemoveQueued: () => {}, + queued: [parked("", [staged("att-image", "shot.png")])], + }); + + expect(getByLabelText("Remove queued message: shot.png")).toBeTruthy(); +}); + +/* + * THE MEMO ON THE ROW OF TILES MISSED EVERY SINGLE TIME, and the default comparison is why: + * `toVisibleChatItems` is deliberately not memoised — the agent hands back the same array and + * mutates it, so a `useMemo` keyed on it never invalidates and a reply never appears — which means + * the `attachments` array is a NEW array on every render of the transcript, and `Object.is` on two + * different arrays is false however identical their contents. So every chunk of a streaming answer + * re-rendered every tile in the history, each one carrying an image `Dialog` with it. That is + * exactly the churn the memoised message rows above it exist to stop, and this row opted out of it + * by accident. + * + * Compared field by field rather than by identity, because the fields are what the tiles draw. + */ +const ONE_FILE = [ + { + id: "user-1:0", + attachmentId: "att-image", + url: attachmentUrl("att-image"), + filename: "shot.png", + modality: "image" as const, + }, +]; + +test("a rebuilt but unchanged row of files compares equal", () => { + // Same values, different objects, different array: what every render after the first hands over. + const rebuilt = ONE_FILE.map((file) => ({ ...file })); + + expect( + sameAttachmentRow( + { attachments: ONE_FILE, delay: 0 }, + { attachments: rebuilt, delay: 0 }, + ), + ).toBe(true); +}); + +test("a row that actually changed does not compare equal", () => { + const changed = [ + { ...ONE_FILE[0], id: "user-1:1" }, + { ...ONE_FILE[0], attachmentId: "att-other" }, + { ...ONE_FILE[0], url: attachmentUrl("att-other") }, + { ...ONE_FILE[0], filename: "other.png" }, + { ...ONE_FILE[0], filename: undefined }, + { ...ONE_FILE[0], modality: "document" as const }, + ]; + + for (const file of changed) { + expect( + sameAttachmentRow( + { attachments: ONE_FILE, delay: 0 }, + { attachments: [file], delay: 0 }, + ), + ).toBe(false); + } + + // A file added, a file taken away, and the entrance delay itself — all of them redraw. + expect( + sameAttachmentRow( + { attachments: ONE_FILE, delay: 0 }, + { attachments: [], delay: 0 }, + ), + ).toBe(false); + expect( + sameAttachmentRow( + { attachments: ONE_FILE, delay: 0 }, + { attachments: [...ONE_FILE, ONE_FILE[0]], delay: 0 }, + ), + ).toBe(false); + expect( + sameAttachmentRow( + { attachments: ONE_FILE, delay: 0 }, + { attachments: ONE_FILE, delay: 0.04 }, + ), + ).toBe(false); +}); + +/* + * And that it is the comparison the row is actually memoised WITH — a correct function nobody + * passed to `memo` buys nothing, and that is precisely the state this row was in. `compare` is the + * field `React.memo` keeps its comparator in. + */ +test("the row of tiles is memoised with that comparison", () => { + expect( + (TranscriptAttachments as unknown as { compare?: unknown }).compare, + ).toBe(sameAttachmentRow); +}); + +/* + * A PARKED FILE IS THE ONE SURFACE IN THIS BROWSER THAT KNOWS WHAT IT IS HOLDING. + * + * `attachment.type` is the SDK's `getModalityFromMimeType(file.type)`, decided from the browser's + * claim before a byte was uploaded and never revisited when the upload replies — the merge back is + * `{ ...att, source, status: "ready", thumbnail, metadata }`, which replaces the source and leaves + * `type` alone. `attachment.source.mimeType` is what OUR `onUpload` returned, and that is + * `body.mimeType`: the type the server earned from `sniffMimeType` over the actual bytes. + * + * `composer/picked-files.ts` makes the two disagree deliberately, by passing a claim that names no + * format so the server is the one that decides. So a PNG dragged out of an editor arrives here as + * `type: "document"` with `source.mimeType: "image/png"`. + * + * AND THIS IS THE HALF THAT CAN BE PUT RIGHT FROM HERE. `QueuedMessage.attachments` is + * `Attachment[]` — the staged object itself, source and all. A SENT turn has been through + * `toAttachmentPart` (`channel-chat.tsx`), which rebuilds the source as `{ type: "url", value }` + * and drops the `mimeType`, so the sent row has nothing better than the guess to go on. The two + * rows can therefore genuinely disagree until that one line forwards it. + */ + +/** A staged file as the SDK hands it over, with the guess and the server's answer set apart. */ +function stagedAs( + id: string, + filename: string, + type: "image" | "document", + mimeType?: string, +): Attachment { + return { + id, + type, + source: { + type: "url", + value: attachmentUrl(id), + ...(mimeType === undefined ? {} : { mimeType }), + }, + filename, + status: "ready", + }; +} + +/* + * THE DEFECT, ON THE SURFACE THAT HOLDS THE EVIDENCE. A screenshot the browser called text drew a + * grey card with a filename on it, in front of somebody who had just attached a picture. + * + * The `` is asserted by SRC and the card by its caption, because "not a document" is not the + * claim being made — a tile that rendered nothing at all would satisfy that. + */ +test("a parked picture the browser mislabelled is drawn as a picture", () => { + const { container, queryByText } = renderTranscript([], { + queued: [ + parked("", [stagedAs("att-shot", "shot.png", "document", "image/png")]), + ], + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe( + attachmentUrl("att-shot"), + ); + // Not the file card: that caption is the fixed word every document tile carries. + expect(queryByText("Attachment")).toBeNull(); +}); + +/* + * THE SAME MISTAKE POINTING THE OTHER WAY, AND IT IS THE LOUDER ONE. A text file the browser called + * an image draws an `` at a url the route answers 200 for with text; nothing decodes, `onError` + * fires, and the tile replaces itself with the destructive card reading "notes.txt is unavailable." + * — swearing in the file's own name that it was deleted while it sits on the server intact. + */ +test("a parked file the browser called a picture is drawn as a file", () => { + const { container, getByText } = renderTranscript([], { + queued: [ + parked("", [stagedAs("att-notes", "notes.txt", "image", "text/plain")]), + ], + }); + + expect(container.querySelector("img")).toBeNull(); + expect(getByText("notes.txt")).toBeTruthy(); + expect(getByText("Attachment")).toBeTruthy(); +}); + +/* + * A PICTURE THIS BROWSER CANNOT DRAW IS NOT A PICTURE. `classifyAttachment` is asked rather than + * `startsWith("image/")` exactly so this answers correctly: a HEIC is an image by media type and no + * `` here renders one, so the honest tile is the card naming the file rather than a box that + * silently fails to paint. + */ +test("a parked image type this app cannot draw stays a file card", () => { + const { container, getByText } = renderTranscript([], { + queued: [ + parked("", [stagedAs("att-heic", "photo.heic", "image", "image/heic")]), + ], + }); + + expect(container.querySelector("img")).toBeNull(); + expect(getByText("photo.heic")).toBeTruthy(); +}); + +/* + * WITHOUT THE SERVER'S ANSWER THE GUESS IS STILL USED, rather than everything collapsing to a file + * card. `mimeType` is optional on the source, and a staged attachment that never went through our + * `onUpload` has none — which is the shape every other parked test in this file uses, and they must + * go on drawing exactly as they did. + */ +test("a parked file with no server type falls back to the declared one", () => { + const { container } = renderTranscript([], { + queued: [parked("", [stagedAs("att-shot", "shot.png", "image")])], + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe( + attachmentUrl("att-shot"), + ); +}); + +/* + * AND THE PROBE FOLLOWS THE DRAWING. `SentAttachmentTile` asks the route whether a row is still + * there for a DOCUMENT and never for a picture, so getting the modality right stops a request as + * well as a wrong tile. + * + * HOW MUCH THAT REQUEST COSTS IS NOT WHAT THIS PINS, and the figure this note used to quote is out + * of date: it said a HEAD was answered by Hono running the GET in full — "the whole file out of + * Postgres" — so a mislabelled screenshot "bought a megabyte read on every render". The route now + * answers a HEAD from `sizeBytes` without touching the bytes. What the case is actually good for is + * unchanged and does not rest on the price: a picture's own load is the answer, so asking again is + * asking a question that has already been answered. + * + * NOT ASSERTED BY WAITING ON AN EMPTY LIST, which proves nothing here for the reason the off-site + * test above sets out at length: `waitFor` returns on its first check when the callback does not + * throw, so an empty array reads as "not probed yet" rather than "not probed". A servable document + * is parked beside it as the clock — waiting for THAT one's probe puts the question after the point + * by which a probe for the picture, mounted in the same commit, would have had to appear. + */ +test("a parked picture drawn from its bytes is not probed either", async () => { + const kept = attachmentUrl("att-clock"); + renderTranscript([], { + queued: [ + parked("", [ + stagedAs("att-shot", "shot.png", "document", "image/png"), + stagedAs("att-clock", "clock.pdf", "document"), + ]), + ], + }); + + await waitFor(() => expect(probes).toHaveLength(1)); + // One probe, and it is the real document's. Nothing was asked about the screenshot. + expect(probes.map((probe) => probe.url)).toEqual([kept]); +}); + +/* + * THE PARKED BLOCK COMES FIRST IN THE DOM, AND THAT IS A REQUIREMENT RATHER THAN AN ACCIDENT. + * + * It is drawn last — CSS `order` puts it under the transcript — so the obvious tidy-up is to write + * it where it is drawn and delete the `order` classes. That tidy-up is silently destructive, which + * is exactly why it is pinned here instead of trusted to the comment beside it. + * + * The scroller identifies a newly appended row POSITIONALLY. On each content change it takes + * `Array.from(content.children)` minus the spacer, compares the length with the previous length, + * and on growth scans from the OLD LENGTH FORWARD for the next `data-scroll-anchor="true"`. That + * only finds the row if the row is last. With the parked block moved below `items.map`, every + * appended row lands one slot short of the end, the scan meets the block instead, finds no anchor + * and gives up — and a new turn stops aligning to the top of the viewport. No existing test sees + * it, because the anchor ATTRIBUTE is still on the right row; it is the row's INDEX that broke. + * + * So this asserts the ordering the scroller needs, in the terms the scroller reads it in: among the + * children of the content element, the block sits ahead of every transcript row. + * + * IT PINS A COST TOO, AND KNOWINGLY. Focus order and the reading order of the enclosing + * `role="log"` follow the DOM, not `order`, so a keyboard user meets the queue's Remove buttons + * before the conversation and a screen reader hears parked messages ahead of it. That debt is + * described in full at the block itself. This test does not bless it — it records that the naive + * repair is not available, so that whoever pays it properly changes the scroller's row detection + * rather than only this markup, and has a failing test to tell them which one they changed. + */ +test("the parked block precedes the transcript rows the scroller counts", () => { + const { container } = renderTranscript( + [userMessage("first"), userMessage("second")], + { queued: [parked("hold on", [])] }, + ); + + const content = container.querySelector( + '[data-slot="message-scroller-content"]', + ); + if (!content) throw new Error("no scroller content element"); + + const children = Array.from(content.children); + // The wrapper is `display: contents`, so it is not a flex item — but it IS a child, which is the + // list the scroller walks. It has to be the first of them. + const parkedBlock = children.findIndex((child) => + child.textContent?.includes("hold on"), + ); + const firstRow = children.findIndex((child) => + child.hasAttribute("data-message-id"), + ); + + expect(parkedBlock).toBe(0); + expect(firstRow).toBeGreaterThan(parkedBlock); +}); diff --git a/app/tsconfig.json b/app/tsconfig.json index 0c68ed165..8a446e9bb 100644 --- a/app/tsconfig.json +++ b/app/tsconfig.json @@ -10,6 +10,7 @@ }, "include": [ "src", + "tests", "vite.config.ts", "../shared/handoff-markers.ts", "../shared/listen-port.ts" diff --git a/bunfig.toml b/bunfig.toml index b599e7d8d..8645489bd 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -1,4 +1,8 @@ [test] # See server/scripts/test-preload.ts: makes the module graph deterministic so a test file cannot fail to # import depending on the order the suite happened to be walked. +# +# This applies to `bun test` run from the repository root. Bun reads bunfig.toml from the current +# working directory only, so server/bunfig.toml repeats the declaration for `bun test` run from +# inside server/, and server/tests/bunfig-preload.test.ts keeps the two in agreement. preload = ["./server/scripts/test-preload.ts"] diff --git a/charts/openbot/README.md b/charts/openbot/README.md index 70d5429ec..90fc5c0e3 100644 --- a/charts/openbot/README.md +++ b/charts/openbot/README.md @@ -157,9 +157,46 @@ RDS has `rds.force_ssl` on by default, and Cloud SQL and Azure Database do the s migration fails with `no pg_hba.conf entry for host ... no encryption`, which names the host and the user and not the actual problem. -**The migrating role has to be able to create and drop the `vector` extension.** The first migration -creates it and a later one drops it again. On a managed database, create it once as the -administrative role; `CREATE EXTENSION IF NOT EXISTS` then passes for an ordinary user. +**The migrating role has to OWN the `vector` extension, not just be able to see it.** The first +migration runs `CREATE EXTENSION IF NOT EXISTS vector` and migration `0010` runs +`DROP EXTENSION IF EXISTS "vector"` once the document index is gone. `DROP EXTENSION` is an ownership +check, and `IF EXISTS` does not waive it — it only makes a *missing* extension not an error. So a +role that can see an extension somebody else owns gets through the create and fails the drop, and the +migrations Job stops with: + +``` +must be owner of extension vector +``` + +In Postgres an extension's owner is whoever ran `CREATE EXTENSION`, and there is no +`ALTER EXTENSION ... OWNER TO` to hand it over afterwards. The advice that used to stand here — have +the administrative role create it once so `CREATE EXTENSION IF NOT EXISTS` passes for an ordinary +user — is therefore exactly what produces the failure: it makes the admin the owner and the migrating +role a bystander. + +Do one of these instead: + +- **Let the migrating role create it.** Simplest, and it needs no extra step: allow that role to run + `CREATE EXTENSION` (`GRANT rds_superuser` on RDS, `cloudsqlsuperuser` on Cloud SQL, `azure_pg_admin` + on Azure Database, plus whatever extension allow-list the vendor keeps), then leave the extension + absent and let migration `0000` create it. The migrating role owns it and `0010` drops it cleanly. +- **Pre-create it AS the migrating role.** Where that grant is not on offer, the administrative role + can still do it on the other role's behalf, which is the whole trick: + + ```sql + SET ROLE openbot_migrator; -- the role in DATABASE_URL + CREATE EXTENSION IF NOT EXISTS vector; + RESET ROLE; + ``` + + Ownership follows the role that ran the statement, so this is equivalent to the first option. + +If you have already installed with the extension owned by somebody else, drop and recreate it under +the migrating role before upgrading — `DROP EXTENSION vector;` as the owner, then the block above. +Nothing of yours is in it: `vector` existed for the `embedding` column on `chunks`, which `0010` +drops in the same transaction. A deployment that added a vector column of its own is the one case +where that is not true, and `0010` is written to fail rather than take it; that deployment should +keep the extension and apply only the table drops by hand. ## The five targets @@ -312,6 +349,21 @@ server to be recognised as the worker rather than an arbitrary caller — and is turning it on with no secret set is a CronJob whose every run is refused. See the routines refusal below, and [docs/routines.md](../../docs/routines.md). +A third CronJob **deletes data, is on by default, and is the only one of the three that does**: +`attachments.culler` sweeps staged attachments that were never sent. A file uploaded into the +composer is stored the moment it is pasted or dropped, before anybody presses send — so closing the +tab, or changing your mind, leaves bytes in the database that no message will ever point at. The +sweep removes those, hourly (`attachments.culler.schedule`), once they are older than +`attachments.culler.olderThanHours` — **24 hours by default**. + +It is on by default where the other two are off, because it needs nothing but the database that +every deployment already has, and because the alternative is a table of blobs that only grows. What +it will never remove is an attachment that was sent: those are stamped when the message they ride on +goes out, and the sweep asks only for unstamped rows. The window matters, though — a person who +uploads a file, leaves it in the composer overnight and comes back to send it will find it gone. +Raise `olderThanHours` if that is your deployment's shape, or set `attachments.culler.enabled: +false` to keep every staged row for ever and reclaim them some other way. + ## NetworkPolicy, and whether your cluster enforces one Off by default, because a NetworkPolicy on a cluster whose CNI does not enforce one is a resource @@ -331,6 +383,23 @@ the policy on with an external database and no `networkPolicy.extraEgress` is re enforcing cluster it would fence the API off from its own database, which reads as the database being down. +**`computers.mode: sandbox` now also requires `networkPolicy.kubernetesApiCidr`.** Two of these +policies carry a rule for the Kubernetes API server, which is where a per-Bot computer is asked for, +and the service range it answers on belongs to the cluster rather than to this release: + +```sh +kubectl get svc kubernetes -o jsonpath='{.spec.clusterIP}' # then name the range it sits in +``` + +Usually `172.20.0.0/16` on EKS and `10.96.0.0/12` on GKE and kubeadm. Leaving it empty used to be +allowed and meant "unscoped", which was not a looser version of the rule but the absence of one: an +egress rule with ports and no destination matches everything in Kubernetes, so the default handed +out 443 and 6443 to the private ranges the policy beside it goes to the trouble of excepting — and +on the computer culler, whose only other egress is DNS and the database, it was that pod's entire +reach. If your release has `networkPolicy.enabled` and `computers.mode: sandbox`, the next +`helm upgrade` stops with a message naming this value. Nothing in the cluster changes when it does; +set the range and run it again, and the policy is narrow for the first time. + A Bot's computer is allowed 80 and 443 to public addresses and nothing else, which is what stops a browser reaching the cluster, the database, or the cloud's credential endpoint. A per-Bot egress proxy is therefore two settings rather than one: the variable that names it, and the rule that lets diff --git a/charts/openbot/ci/eks-sandbox-values.yaml b/charts/openbot/ci/eks-sandbox-values.yaml index 95d860a71..514d46b3a 100644 --- a/charts/openbot/ci/eks-sandbox-values.yaml +++ b/charts/openbot/ci/eks-sandbox-values.yaml @@ -94,6 +94,13 @@ computers: # enforces it a wrong rule is an outage, so turning it on stays a deployment's decision. networkPolicy: enabled: true + # The only sandbox target, so the only one that renders the two policies carrying a rule for the + # Kubernetes API server — and the only one that has to name its service range. `172.20.0.0/16` is + # what EKS gives a cluster unless it was created with `--service-ipv4-cidr`. Named here rather than + # defaulted in the chart because the range is the cluster's and not the release's: the same line + # would be an outage on GKE. Leaving it out is now refused, which is the point of setting it here — + # the chart used to render a destination-less rule instead, permitting 443 and 6443 everywhere. + kubernetesApiCidr: 172.20.0.0/16 # This target's database is RDS rather than a pod, so egress to it has to be named. The chart # refuses to render without this, which is how the omission was found: rendering the policies in CI # at all is what exercised the refusal. diff --git a/charts/openbot/templates/_helpers.tpl b/charts/openbot/templates/_helpers.tpl index 5f9a7e270..c81d614d2 100644 --- a/charts/openbot/templates/_helpers.tpl +++ b/charts/openbot/templates/_helpers.tpl @@ -58,6 +58,123 @@ app.kubernetes.io/component: {{ .component }} {{- printf "%s-%s" (include "openbot.fullname" .root) .component | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +The same name, under the shorter limit Kubernetes puts on a CronJob. + +FIFTY-TWO, NOT SIXTY-THREE. A CronJob is the one workload whose name is not the whole budget: the +controller names each Job it creates `-`, so the API server refuses a CronJob +whose own name leaves no room for that suffix — "must be no more than 52 characters". Sixty-three is +the right limit for every other object this chart writes and the wrong one here, and the failure is +not a truncated name, it is `helm install` rejected outright. + +Reached at a 43-character release name, which is an ordinary length for a name that says the +environment and the region. All three of this chart's CronJobs were built on the 63-character helper +and all three were rejected together. + +THE RELEASE NAME IS TRUNCATED, NOT THE WHOLE STRING, so the component survives. Cutting the joined +name at 52 would give a long release two CronJobs called the same thing — `...-routines` and +`...-culler` both ending as the first 52 characters of the release name — which is a release that +cannot install for a second, stranger reason. Trimming the prefix instead keeps the suffix that says +which sweep this is, which is the part a person reads. +*/}} +{{- define "openbot.cronJobName" -}} +{{- $room := int (max 1 (sub 51 (len .component))) -}} +{{- $prefix := include "openbot.fullname" .root | trunc $room | trimSuffix "-" -}} +{{- printf "%s-%s" $prefix .component | trunc 52 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Whether the staged-attachment sweep runs. + +ONE ANSWER FOR TWO TEMPLATES, because the CronJob and the NetworkPolicy that fences it must agree: +a sweep with no policy is the one pod left unfenced on a cluster that enforces them, and a policy +with no sweep is a resource selecting nothing. They were two copies of the same expression, which is +the shape that drifts. + +GUARDED AT BOTH LEVELS, AND DEFAULTED TO ON. `attachments` is a key this chart did not have before, +and `helm upgrade --reuse-values` takes the previous release's computed values rather than merging +the new chart's defaults, so on every existing deployment the whole map is absent — and on a release +installed between the two, `culler` is present without `enabled`. `(.Values.attachments).culler.enabled` +parenthesises one level of that and reads the next two bare: with `enabled` missing the sweep and its +policy silently did not render at all, and with `culler` missing the render died on a nil pointer, +which fails the install rather than the feature. + +`kindIs "invalid"` rather than `| default true`, for the reason `commonEnv` gives above: sprig's +`default` substitutes on EMPTY, and `false` is empty, so `| default true` would switch the sweep back +on for the deployment that had deliberately switched it off. + +IT ANSWERS THE SAME QUESTION ITS SIBLINGS DO, WHICH IT USED NOT TO. This used to hand the value back +untouched for its callers to compare against the string `"true"`, and a string comparison is not what +`if .Values.routines.enabled` next door does. `--set attachments.culler.enabled=1` reaches a template +as the integer 1, and `=yes` reaches it as the string "yes". Go's templating calls both of those +true, so the routines CronJob renders for either — while this returned "1" or "yes", matched neither +caller, and rendered NEITHER the CronJob NOR the NetworkPolicy that fences it. No error, no resource, +and an operator with every reason to believe the sweep was on. Both spellings were driven through +`helm template` before this changed and after. The answer is now the template engine's own notion of +truth, which is the one the rest of the chart was already using. + +THE ONE VALUE IT REFUSES RATHER THAN HONOURS, because agreeing with the siblings here would have been +a regression rather than a fix. That same notion of truth calls the non-empty string "false" TRUE, so +`--set-string attachments.culler.enabled=false` would start the sweep for somebody who had just +written the word false. The old string comparison happened to get that one case right, and a fix is +not allowed to take a correct behaviour away. There is no reading of `--set-string ...=false` that +means ON and no safe way to guess, so it fails the render with a message naming `--set` instead. That +is a narrower rule than it looks: only a STRING spelling a falsehood ever reaches it, and `--set`, +which parses `false` into a boolean, cannot produce one. +*/}} +{{- define "openbot.attachmentsCullerEnabled" -}} +{{- $culler := (.Values.attachments | default dict).culler | default dict -}} +{{- $enabled := $culler.enabled -}} +{{- if kindIs "invalid" $enabled -}} +true +{{- else if and (kindIs "string" $enabled) (has (lower $enabled) (list "false" "no" "off" "n" "0")) -}} +{{- fail (printf "attachments.culler.enabled is the string %q, and this chart will not guess which way you meant it. Helm's templating reads every non-empty string as true, so honouring it would turn the staged-attachment sweep ON, which is the opposite of what it spells. Pass a boolean instead: --set attachments.culler.enabled=false, or enabled: false in a values file. --set-string is what made it a string." $enabled) -}} +{{- else if $enabled -}} +true +{{- else -}} +false +{{- end -}} +{{- end -}} + +{{/* +The service range the Kubernetes API server answers on, or a refusal to render a policy without it. + +ONE ANSWER FOR TWO TEMPLATES, for the same reason as the sweep gate above: the API server's policy +and the computer culler's both need this rule, and both had it wrong in exactly the same way. + +WHY THIS REFUSES INSTEAD OF DEFAULTING. Both policies used to write the rule as +`- {{ with .Values.networkPolicy.kubernetesApiCidr }}to: ...{{ end }}` and let the empty default fall +straight through the `with`. What fell out was an egress rule carrying ports and NO PEER AT ALL, and +in Kubernetes that is neither a narrow rule nor an inert one: an empty or absent `to` matches every +destination. So the shipped default granted 443 and 6443 to everything, which cancelled the `10/8`, +`172.16/12`, `192.168/16` and `169.254/16` exceptions the rule one line above it spells out. On the +culler, whose only other egress is DNS and the database, that peerless rule WAS its entire reach: a +pod holding the database credential could open an HTTPS socket to any address in the cluster or on +the internet. Rendered and read back before any of this was believed. + +THE TWO ALTERNATIVES, AND WHY NEITHER. Rendering no rule at all when nobody has named a CIDR is safe +and silent, and silent is the whole problem: on a cluster that enforces policy the API server can no +longer ask for a Bot's computer, so every browser action fails and the deployment looks broken rather +than fenced — which is the exact failure the comment two rules above this one warns about. Picking a +default CIDR is worse: the range belongs to the cluster and not to the release, so `172.20.0.0/16` is +right on EKS and an outage on GKE, and a wrong CIDR is that outage with a plausible-looking values +file standing behind it. Refusing is the only one of the three that cannot be wrong quietly, and it +is what this chart does everywhere else a value is unknowable and load-bearing. `helm upgrade` +renders before it applies anything, so a release that hits this keeps running exactly as it was while +its operator runs the single command in the message. + +SCOPED TO THE POLICIES THAT NEED IT. Reached only from inside `networkPolicy.enabled` and +`computers.mode: sandbox`, so a deployment with no policies, or with `mode: shared`, never has to +name it. Nothing else in the chart consults it. +*/}} +{{- define "openbot.kubernetesApiCidr" -}} +{{- $cidr := .Values.networkPolicy.kubernetesApiCidr -}} +{{- if not $cidr -}} +{{- fail "networkPolicy.kubernetesApiCidr is required when networkPolicy.enabled is true and computers.mode is sandbox. It is the service range the Kubernetes API server answers on, which is where a per-Bot computer is asked for, and this chart cannot know it: the range belongs to the cluster rather than to this release. Find the address with: kubectl get svc kubernetes -o jsonpath='{.spec.clusterIP}' - then name the range it sits in, usually 172.20.0.0/16 on EKS and 10.96.0.0/12 on GKE and kubeadm. It was previously allowed to be empty, which rendered an egress rule with no destination at all: that permitted 443 and 6443 to every address rather than to the API server, so setting this narrows the policy that was already meant to be narrow." -}} +{{- end -}} +{{- $cidr -}} +{{- end -}} + {{- define "openbot.serviceAccountName" -}} {{- if .Values.serviceAccount.create -}} {{- default (include "openbot.fullname" .) .Values.serviceAccount.name -}} diff --git a/charts/openbot/templates/attachments/cronjob.yaml b/charts/openbot/templates/attachments/cronjob.yaml new file mode 100644 index 000000000..87f16198a --- /dev/null +++ b/charts/openbot/templates/attachments/cronjob.yaml @@ -0,0 +1,202 @@ +{{- if eq (include "openbot.attachmentsCullerEnabled" .) "true" }} +{{- $component := "attachments-culler" -}} +{{/* +Deleting attachments somebody staged and then never sent. + +A CronJob rather than a timer in the API, for the same reason the routines sweep and the computer +culler beside it are: an interval in the server fires in every replica, so five replicas would each +run the same delete. Unlike the computer culler this needs no lease and no claimed-by-this-pod +bookkeeping — deleting a staged attachment twice is harmless, because the second sweep finds nothing +left to delete. `attachedAt` is set the moment an attachment is actually sent, so a null attachment +old enough that the tab is long closed is not spoken for by anything and is safe to delete. + +`concurrencyPolicy: Forbid` on top, because a schedule that overlaps itself is the cross-replica +problem in one workload rather than across several. + +EVERY VALUE BELOW IS READ THROUGH `$culler`, which is `attachments.culler` with both levels made +safe, for the reason `openbot.attachmentsCullerEnabled` gives: `--reuse-values` leaves the whole map +absent on an existing release, and one parenthesis only covers one level of it. +*/}} +{{- $culler := (.Values.attachments | default dict).culler | default dict -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + {{- /* Fifty-two characters, not sixty-three; see `openbot.cronJobName`. */}} + name: {{ include "openbot.cronJobName" (dict "root" . "component" $component) }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} +spec: + {{- $schedule := "0 * * * *" -}} + {{- if not (kindIs "invalid" $culler.schedule) -}} + {{- $schedule = $culler.schedule -}} + {{- end }} + {{- /* + WHAT THE `kindIs "invalid"` GUARD ABOVE DOES NOT CATCH, checked here rather than left to the API + server. + + That guard exists to tell an ABSENT value from a set one, because `--reuse-values` leaves a key + this release adds absent and `| default` would substitute on empty as well. It does its job: an + empty string is set, so it is taken. And an empty schedule is not a CronJob that runs on some + default — the API server refuses the object outright ("empty spec.schedule"), so the whole + release fails to apply, some way past the point where anybody is still reading values. + + Refused here instead, naming the value, in keeping with `templates/validation.yaml`: an install + that stops and says which key is wrong beats one that stops with a schema error about an object + nobody hand-wrote. It lives in this template rather than in that one because the value only + matters when this CronJob is rendered at all — a deployment with the culler switched off should + not be blocked by a schedule it will never use. + */}} + {{- if not (trim (toString $schedule)) }} + {{- fail "attachments.culler.schedule is empty, and an empty schedule is not a default — Kubernetes rejects the CronJob and the release fails to apply. Remove the key to get the hourly default, or set a cron expression such as \"0 * * * *\"." }} + {{- end }} + # The fallback matches values.yaml. Hourly rather than the five-minute schedule the other two + # CronJobs use: nothing here is time-sensitive the way a due routine or an idle browser is, and the + # retention window below is measured in hours, not minutes. + schedule: {{ $schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + startingDeadlineSeconds: 120 + jobTemplate: + spec: + backoffLimit: 1 + {{- /* + A CEILING ON ONE SWEEP, because `Forbid` above turns a hung one into a permanent stop. + + Without it a wedged run is never killed, so Forbid suppresses every later sweep and staged + attachments accumulate for good, with only a Running job as evidence. + + SETTABLE, though a backlog no longer needs it to be raised. The sweep deletes in bounded + batches, each its own transaction, so a run this ceiling kills has still reclaimed every + batch it committed and the next run starts from there — a deployment arriving with a month + of abandoned uploads drains over several sweeps rather than rolling back the same doomed + statement every hour. Raising it just makes that take fewer runs. + + Defaulted through `kindIs "invalid"` rather than `| default`, because sprig substitutes on + EMPTY and zero is empty, and because `--reuse-values` leaves a key this release adds absent. + */}} + {{- $deadline := 300 -}} + {{- if not (kindIs "invalid" $culler.activeDeadlineSeconds) -}} + {{- $deadline = $culler.activeDeadlineSeconds -}} + {{- end }} + {{- /* + AND ZERO IS THE ONE VALUE THE GUARD ABOVE WAS WRITTEN FOR AND STILL LETS THROUGH. + + `kindIs "invalid"` is there because zero is empty to sprig and `| default` would replace it. + It keeps a deliberate zero — and a deliberate zero is the worst number in this field. + Kubernetes validates `activeDeadlineSeconds` as merely non-negative, so the object applies + cleanly; the Job controller then compares the run's age against it and finds every run, + at every age, already over. Each hourly Job is killed as `DeadlineExceeded` before it + deletes a row, `backoffLimit: 1` retires it, and the sweep stops — with a green install, a + CronJob that is plainly scheduled, and Jobs that are plainly being created. Nothing here + looks broken except that `attachments` never gets smaller. + + Refused rather than rounded up to something, because a ceiling is the operator's call and a + chart that quietly substituted its own would be lying about a number they set on purpose. + + A fraction is refused for the reason a reader will not guess: the field is an integer, so + `0.5` renders `activeDeadlineSeconds: 0.5` and the API server rejects the whole object on a + type error. Comparing the value with its own integer conversion is what catches that. + */}} + {{- $deadlineSeconds := int64 $deadline -}} + {{- if or (le $deadlineSeconds 0) (ne (printf "%v" $deadline) (printf "%v" $deadlineSeconds)) }} + {{- fail (printf "attachments.culler.activeDeadlineSeconds must be a whole number of seconds of at least 1; got %v. Zero is accepted by Kubernetes and then kills every sweep as DeadlineExceeded before it deletes anything, which stops the culler without anything looking broken." $deadline) }} + {{- end }} + activeDeadlineSeconds: {{ $deadlineSeconds }} + template: + metadata: + labels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 12 }} + {{- if .Values.postgresql.enabled }} + {{- /* + The bundled database admits pods carrying this label and nothing else, which this chart pins on. + Anything that opens the database needs it, and only the API server had it: this would have been + refused on any cluster that actually enforces a NetworkPolicy. Not caught by hand, because the + cluster it was driven on ships enforcement switched off. + */}} + {{ .Release.Name }}-postgresql-client: "true" + {{- end }} + spec: + restartPolicy: Never + serviceAccountName: {{ include "openbot.serviceAccountName" . }} + {{- /* This sweep only opens the database; it never asks the cluster for anything, so it gets no token. */}} + automountServiceAccountToken: false + {{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml . | indent 12 }} + {{- end }} + containers: + - name: attachments-culler + image: {{ include "openbot.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + workingDir: /app/server + {{- $olderThanHours := 24 -}} + {{- if not (kindIs "invalid" $culler.olderThanHours) -}} + {{- $olderThanHours = $culler.olderThanHours -}} + {{- end }} + {{- /* + A WINDOW THE SCRIPT ITSELF REFUSES, caught before it is handed to it. + + `scripts/cull-staged-attachments.ts` throws on an argument that is not a positive + number, because a window of zero or less is "delete staged attachments that are + newer than now" — either nothing, or a request to take a file out from under the + composer that is still holding it. Rendered into this command it becomes an hourly + Job that exits non-zero on its first line, retried once and recorded as failed, for + as long as the release stands: a crash loop measured in days, whose message is in a + log nobody has opened, about a value sitting in plain sight in the values file. + + `float64`, NOT `int64`, because fractions are meant here: `0.5` is thirty minutes, + the script builds its cutoff by multiplying an interval precisely so that works, and + there is a test that a half-hour window means thirty minutes. Only the sign is + wrong, so only the sign is checked. + */}} + {{- if le (float64 $olderThanHours) 0.0 }} + {{- fail (printf "attachments.culler.olderThanHours must be greater than 0; got %v. The sweep refuses a window that is not a positive number of hours, so this renders a CronJob that fails on every run. A fraction is allowed and means a fraction of an hour: 0.5 is thirty minutes." $olderThanHours) }} + {{- end }} + command: ["/usr/local/bin/bun", "scripts/cull-staged-attachments.ts", {{ $olderThanHours | quote }}] + {{- /* + NOT `openbot.commonEnv`, WHICH IS THE API SERVER'S ENVIRONMENT. + + This pod deletes rows from one table. Through the shared helper it also carried the + licence token, the model API key, the computer token, the managed-Bot token and the + worker's shared secret — five credentials for a workload that opens a database + connection and closes it, in the pod with the least reason of any to hold them. + + AND THEN, FOR A WHILE, IT CARRIED FIVE OTHERS. What replaced the helper was not the + sweep's needs but `loadConfig`'s boot contract: the script built the whole + `DeploymentConfig` before deleting anything, and that refuses to return without the + encryption key, the three Intelligence addressing values and a complete identity + provider — so this block held `KEY_ENCRYPTION_KEY`, the Intelligence API key, an + OAuth client secret and the session-signing secret purely to keep a `DELETE` from + dying at start-up. The narrow fix was named here and has now landed one directory + away: `server/scripts/cull-staged-attachments.ts` reads `DATABASE_URL` itself, the + way `scripts/migrate.ts` always has. What is below is what the sweep uses. + + A variable added to `loadConfig`'s required set therefore no longer has to be added + here. If this CronJob ever starts crash-looping while the API server is healthy, the + cause is that script reaching for the deployment config again, and the answer is + there rather than another secret in this list. + */}} + env: +{{ include "openbot.databaseUrlEnv" . | indent 16 }} + {{- /* + Neither of these is required by the sweep. They are what any container of this + image is run with, and `LOG_LEVEL` is the operator's own setting for it. + */}} + - name: NODE_ENV + value: production + {{- if .Values.config.logLevel }} + - name: LOG_LEVEL + value: {{ .Values.config.logLevel | quote }} + {{- end }} + {{- /* The operator's own environment, last so it wins, as it does on the server. */}} + {{- with .Values.config.extraEnv }} +{{ toYaml . | indent 16 }} + {{- end }} + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi +{{- end }} diff --git a/charts/openbot/templates/computer/culler-cronjob.yaml b/charts/openbot/templates/computer/culler-cronjob.yaml index 997d9e21f..8a61ebd16 100644 --- a/charts/openbot/templates/computer/culler-cronjob.yaml +++ b/charts/openbot/templates/computer/culler-cronjob.yaml @@ -16,7 +16,8 @@ one workload rather than across several. apiVersion: batch/v1 kind: CronJob metadata: - name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + {{- /* Fifty-two characters, not sixty-three; see `openbot.cronJobName`. */}} + name: {{ include "openbot.cronJobName" (dict "root" . "component" $component) }} labels: {{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} spec: diff --git a/charts/openbot/templates/networkpolicy.yaml b/charts/openbot/templates/networkpolicy.yaml index ef80cb8fb..28cc3e839 100644 --- a/charts/openbot/templates/networkpolicy.yaml +++ b/charts/openbot/templates/networkpolicy.yaml @@ -97,15 +97,20 @@ spec: The Kubernetes API server, which is where a per-Bot computer is asked for. Its own rule because it sits on the private network the rule above cuts out, so nothing else - here reaches it. Unscoped unless a deployment says otherwise: the API server answers on a - ClusterIP from the service range, and a chart cannot know that range at template time. Name it - in `networkPolicy.kubernetesApiCidr` and this narrows to it. + here reaches it. The API server answers on a ClusterIP from the service range, and a chart + cannot know that range at template time, so `networkPolicy.kubernetesApiCidr` names it and + the chart refuses to render this policy until something does. + + THAT REFUSAL REPLACED A DEFAULT THAT UNDID THE RULE ABOVE. This was written to fall through a + `with` when the value was empty, which left an egress rule holding ports and no peer — and an + egress rule with no `to` matches EVERY destination, so the shipped default permitted 443 and + 6443 to all of the `10/8`, `172.16/12`, `192.168/16` and `169.254/16` that the rule directly + above goes to the trouble of excepting. See `openbot.kubernetesApiCidr` in `_helpers.tpl` for + why this refuses rather than rendering nothing or guessing a range. */}} - - {{- with .Values.networkPolicy.kubernetesApiCidr }} - to: + - to: - ipBlock: - cidr: {{ . }} - {{- end }} + cidr: {{ include "openbot.kubernetesApiCidr" . }} ports: - port: 443 protocol: TCP @@ -184,7 +189,7 @@ spec: {{- if and .Values.networkPolicy.enabled (eq .Values.computers.mode "sandbox") .Values.computers.sandbox.culler.enabled }} --- {{- $culler := "culler" -}} -{{/* A pod no policy selects keeps the cluster default, so this one was the release's only unfenced one. */}} +{{/* A pod no policy selects keeps the cluster default, so this one needs its own, same as the routines sweep below and the attachments culler after it. */}} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -216,12 +221,21 @@ spec: - port: 5432 protocol: TCP {{- end }} - {{- /* Empty kubernetesApiCidr permits these ports to any address, as the server's rule above does. */}} - - {{- with .Values.networkPolicy.kubernetesApiCidr }} - to: + {{- /* + The Kubernetes API server, which is the only thing this sweep asks anything of besides the + database: it lists Sandboxes and suspends the idle ones. + + THE SAME RULE THE SERVER'S POLICY ABOVE CARRIES, AND IT USED TO BE WRONG IN A WORSE WAY HERE. + The comment that stood on this line said an empty `kubernetesApiCidr` permits these ports to + any address "as the server's rule above does", and treated that as a note rather than as the + hole it was. On the server it cancelled an exception list; on this pod, whose only other egress + is DNS and the database, it WAS the whole of the egress — a CronJob holding the database + credential with an unrestricted HTTPS socket, which is the opposite of what a policy on this + pod is for. `openbot.kubernetesApiCidr` now refuses an empty value for both of them. + */}} + - to: - ipBlock: - cidr: {{ . }} - {{- end }} + cidr: {{ include "openbot.kubernetesApiCidr" . }} ports: - port: 443 protocol: TCP @@ -233,7 +247,7 @@ spec: {{- end }} {{- end }} -{{- if and .Values.networkPolicy.enabled .Values.routines.enabled }} +{{- if and .Values.networkPolicy.enabled (.Values.routines).enabled }} --- {{- $routines := "routines" -}} {{/* Same reason as the culler; this pod holds no token and reaches only the database and the API server. */}} @@ -281,3 +295,54 @@ spec: {{ toYaml . | indent 4 }} {{- end }} {{- end }} + +{{- if and .Values.networkPolicy.enabled (eq (include "openbot.attachmentsCullerEnabled" .) "true") }} +--- +{{- $attachmentsCuller := "attachments-culler" -}} +{{/* +What the attachments culler may reach. + +Same shape as the routines sweep and the computer culler above: a CronJob nothing else selects, so +without its own policy it would keep the cluster default rather than a fence. It holds the database +credential, which is exactly the shape of pod a NetworkPolicy exists for. + +No Kubernetes-API rule, unlike the routines sweep and the computer culler: this pod's spec sets +`automountServiceAccountToken: false` and its script only ever opens the database, so there is +nothing here that would use a token even if one leaked in. +*/}} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" $attachmentsCuller) }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $attachmentsCuller) | indent 4 }} +spec: + podSelector: + matchLabels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $attachmentsCuller) | indent 6 }} + policyTypes: + - Ingress + - Egress + # No ingress rules, which under an Ingress policyType denies all of it. + ingress: [] + egress: + # DNS, or the database hostname does not resolve and the sweep reads as the database being down. + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + {{- if .Values.postgresql.enabled }} + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: postgresql + ports: + - port: 5432 + protocol: TCP + {{- end }} + {{- /* An external database, for the reason given on the culler's policy above. */}} + {{- with (default .Values.networkPolicy.extraEgress .Values.networkPolicy.attachmentsCullerExtraEgress) }} +{{ toYaml . | indent 4 }} + {{- end }} +{{- end }} diff --git a/charts/openbot/templates/routines/cronjob.yaml b/charts/openbot/templates/routines/cronjob.yaml index 4cd48e4b5..4e5aa6c47 100644 --- a/charts/openbot/templates/routines/cronjob.yaml +++ b/charts/openbot/templates/routines/cronjob.yaml @@ -14,7 +14,8 @@ problem in one workload rather than across several. apiVersion: batch/v1 kind: CronJob metadata: - name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + {{- /* Fifty-two characters, not sixty-three; see `openbot.cronJobName`. */}} + name: {{ include "openbot.cronJobName" (dict "root" . "component" $component) }} labels: {{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} spec: diff --git a/charts/openbot/values.yaml b/charts/openbot/values.yaml index 39b37f949..0c7570b2c 100644 --- a/charts/openbot/values.yaml +++ b/charts/openbot/values.yaml @@ -267,6 +267,42 @@ routines: # Inside the 15-minute floor the tools enforce, so a firing waits at most one tick. schedule: "*/5 * * * *" +# Deleting attachments somebody staged and then never sent. `attachedAt` stays null until an +# attachment is actually sent, and the upload endpoint counts exactly those null rows against two +# separate caps: eight per composer session, which is the per-message limit the composer's own screen +# enforces, and 32 per PERSON across every channel and every session at once, which is the one that +# makes this sweep load-bearing. +# +# The second cap is why never running this is worse than growth. The per-session eight is counted +# over a bucket the client names in its own request, so a new tab is a new bucket and it bounds a +# client that plays along and nothing else. The per-person 32 is not: it is every unsent row that +# person holds anywhere, and the refusal they see on the 33rd file tells them anything still unsent +# is cleared within a day, which is a promise made on this sweep's behalf. With the sweep off, a +# changed mind is a permanent dead end naming files that are no longer on anybody's screen, and +# anybody who reaches 32 can attach nothing, in any channel, for good. +attachments: + culler: + # On by default, unlike `routines` above: this needs nothing but the database, which every + # deployment already has, so there is no failure mode where enabling it does more harm than the + # rows it was written to remove. Nothing but the database is meant literally — the script reads + # `DATABASE_URL` and no other variable, which is why this CronJob carries no secret at all. + enabled: true + # Hourly, not the five-minute schedule the other two CronJobs use: nothing here is time-sensitive + # the way a due routine or an idle browser is. + schedule: "0 * * * *" + # How long a staged attachment is kept before the sweep deletes it. Matches the script's own + # fallback, so leaving this out and never running the CronJob behave alike. + # + # A fraction is a fraction of an hour: `0.5` is thirty minutes. Be sure that is what you want — + # this deletes files a person picked in the composer and has not sent, and a window shorter than + # somebody's train of thought deletes one out from under them while the tab is still open. + olderThanHours: 24 + # A ceiling on one sweep, because `concurrencyPolicy: Forbid` turns a hung one into a permanent + # stop. A deployment arriving with a large backlog does not need it raised: the sweep deletes in + # bounded batches, each its own transaction, so a run stopped at the ceiling keeps what it + # committed and the next one carries on from there. Raising it drains that backlog in fewer runs. + activeDeadlineSeconds: 300 + database: # Used when `postgresql.enabled` is false. A URL, or a secret holding one. # @@ -414,11 +450,20 @@ networkPolicy: extraEgress: [] extraIngress: [] # `computers.mode: sandbox` only. The service range the Kubernetes API server answers on, so the - # rule that lets the API ask for a Bot's computer can name it instead of being left open. + # rule that lets the API ask for a Bot's computer can name it. + # + # REQUIRED once `enabled` is true and `computers.mode` is sandbox, and empty here because the range + # is the cluster's rather than the release's, so there is nothing this chart could put in its place. + # `kubectl get svc kubernetes -o jsonpath='{.spec.clusterIP}'` shows which one yours is on; on EKS + # it is usually 172.20.0.0/16, on GKE and kubeadm 10.96.0.0/12. # - # Empty means unscoped, which is the only thing a chart can do by default: the range is the - # cluster's, not the release's. `kubectl get svc kubernetes -o jsonpath='{.spec.clusterIP}'` shows - # which one yours is on; on EKS it is usually 172.20.0.0/16, on GKE and kubeadm 10.96.0.0/12. + # Empty USED to mean unscoped, and unscoped was not a looser version of this rule — it was the + # absence of one. An egress rule with ports and no `to` matches every destination in Kubernetes, so + # the default granted 443 and 6443 to all of `10/8`, `172.16/12`, `192.168/16` and `169.254/16`, + # which are the four ranges the rule beside it exists to cut out. It is refused rather than skipped + # because skipping it is an outage nobody is told about: on a cluster that enforces policy the API + # server can no longer ask for a computer and every Bot action fails. A `helm upgrade` that stops + # here has changed nothing in the cluster; set this and run it again. kubernetesApiCidr: "" # Where a Bot's computer may reach beyond the public internet. A deployment whose Bots must reach # an internal site adds it here, one address at a time, rather than reopening the private ranges. @@ -428,6 +473,8 @@ networkPolicy: cullerExtraEgress: [] # The same, for the routines sweep. routinesExtraEgress: [] + # The same, for the attachments culler. + attachmentsCullerExtraEgress: [] podSecurityContext: runAsNonRoot: false diff --git a/docs/deployment.md b/docs/deployment.md index 36c84126b..ba2d76a9b 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -51,9 +51,39 @@ worker service beside the API, and `worker/` (the looping local variant) is not sweep itself is: `bun scripts/fire-routines.ts` from `/app/server`, one pass then exit, which is what the Helm chart's CronJob runs from this same image. So a one-container deployment needs something outside the container to run it on a schedule — an external cron, a platform scheduled job, or a -second container of this image with that command — with `DATABASE_URL`, `SERVER_INTERNAL_URL` and -`WORKER_SHARED_SECRET` set. Until something does, a routine is stored, its next run time is computed, -the Routines page shows it, and it never fires. See [routines.md](routines.md). +second container of this image started with `--entrypoint sh` (without it the command arrives as a +`CMD`, and this image's entrypoint boots a whole second server before it runs one; see +[Migrations](#migrations)) — with `SERVER_INTERNAL_URL` and +`WORKER_SHARED_SECRET` set **on top of this server's whole environment**, not instead of it. That +sweep builds the same configuration the API server does before it looks for a due routine, so it +refuses to start without the encryption key, the Intelligence values and an identity provider, +exactly as the server does: give it the same env file and add those two. Until something does, a +routine is stored, its next run time is computed, the Routines page shows it, and it never fires. +See [routines.md](routines.md). + +**The staged-attachment sweep.** Same shape as the routines schedule, with a consequence worth +stating on its own: a file dropped into the composer is stored before it is sent, and nothing in +this image reclaims the ones that never were. The sweep is `bun scripts/cull-staged-attachments.ts` +from `/app/server`, one pass then exit, which the Helm chart runs hourly and which deletes unsent +attachments older than 24 hours — the window is the script's one positional argument, so +`bun scripts/cull-staged-attachments.ts 72` keeps them for three days, and a fraction is a fraction +of an hour. It needs only `DATABASE_URL` — no encryption key, no identity provider, nothing else +this image is configured with — so unlike the routines sweep above, an external cron can run it +with one variable set. A second container of this image still needs `--entrypoint sh`, for the +reason under [Migrations](#migrations). + +Until something does, abandoned uploads accumulate in `attachments` up to a ceiling that is one +person's: **32 unsent files each**, counted across every channel and every composer session at once +and refused at the upload endpoint. A file is at most 8 MiB, so that is 256 MiB of staged blobs per +person who uploads, and that is the number to size storage against. It is **not** the eight files +the composer refuses a ninth on: that cap is counted over a bucket the client names in its own +request, so it bounds a client that plays along and nothing else, which is exactly why the +per-person ceiling was added behind it. + +That ceiling is also why never running this sweep is worse than growth. The refusal a person sees on +their 33rd staged file tells them anything still unsent is cleared within a day — which is a promise +made on this sweep's behalf. With nothing running it, the files are never cleared, and anybody who +reaches 32 can attach nothing, in any channel, for good. ## Minimum size @@ -116,10 +146,27 @@ would race, and a failed migration should stop a deploy rather than leave a half serving traffic. ```sh -docker run --rm --env-file .env openbot \ - sh -c "cd /app/server && bun x drizzle-kit migrate --config=drizzle.config.ts" +docker run --rm --env-file .env --entrypoint sh openbot \ + -c "cd /app/server && bun scripts/migrate.ts" ``` +**`--entrypoint sh`, and it is the load-bearing part of that command.** This image's entrypoint is +`/init`, which is s6's, and anything after the image name is a `CMD` — which s6 runs *after* it has +started everything in the image. Without the override, `docker run … openbot sh -c "… migrate.ts"` +brings up the API and Chromium against the database you have not migrated yet, and only then +migrates it: a second server on an unmigrated schema, which is the race this whole section exists to +avoid, in the one command meant to avoid it. Replacing the entrypoint runs the migration and nothing +else. The Helm chart's migration Job is the same thing said in Kubernetes' terms — it sets +`command:`, which overrides an image's entrypoint rather than appending to it — which is why that +path was never wrong and this one was. + +`scripts/migrate.ts`, not `drizzle-kit migrate`. The CLI is a development dependency and this image +is built with `bun install --production`, so it is not in there; it also needs esbuild to read its +TypeScript config. Asked to migrate here it exits 1 without saying why, and the deployment comes up +against an empty database. The script uses the migrator inside `drizzle-orm`, which is a runtime +dependency, and keeps the same journal, so a database migrated by either is migrated. It is what +this image's own start-up path and the Helm chart's migration Job both run. + ## Replicas The page snapshot a Bot resolves element references against lives in Postgres, so a second replica diff --git a/scripts/check-new-values-keys.ts b/scripts/check-new-values-keys.ts index f9e950e3c..d80f7624a 100644 --- a/scripts/check-new-values-keys.ts +++ b/scripts/check-new-values-keys.ts @@ -277,7 +277,21 @@ const targetValues = parse(await Bun.file(valuesFile).text()) as { externalSecrets?: { enabled?: boolean; data?: unknown[] }; }; function enableFor(component: string): string[] { - if (component === "culler") return ["--set", "computers.mode=sandbox"]; + // On by default and needing nothing but the database, so nothing has to be switched on for it. + if (component === "attachments-culler") return []; + // Sandbox mode is what the computer culler belongs to, and on a target that also has + // `networkPolicy.enabled` it drags in the two policies carrying a rule for the Kubernetes API + // server. That rule's CIDR is required rather than defaulted — an empty one used to render an + // egress rule with no destination, which permitted 443 and 6443 everywhere — so flipping the mode + // on `self-hosted`, which ships the policies on, now has to name a range too. Any range: nothing + // here reads it, this is a fallback check for `activeDeadlineSeconds` and not a policy check. + if (component === "culler") + return [ + "--set", + "computers.mode=sandbox", + "--set", + "networkPolicy.kubernetesApiCidr=10.96.0.0/12", + ]; const on = ["--set", "routines.enabled=true"]; if (!targetValues.externalSecrets?.enabled) { return [ @@ -298,6 +312,9 @@ function enableFor(component: string): string[] { /** One step of a dotted path through parsed YAML, without asserting a shape it may not have. */ function at(value: unknown, key: string): unknown { + // A list step, because one fallback below is a positional command ARGUMENT rather than a field, + // and its path therefore has to index `containers` and `command`. + if (Array.isArray(value)) return value[Number(key)]; return value !== null && typeof value === "object" ? (value as Record)[key] : undefined; @@ -340,6 +357,33 @@ const fieldFallbacks: ReadonlyArray<{ component: "culler", field: ["spec", "jobTemplate", "spec", "activeDeadlineSeconds"], }, + { + path: "attachments.culler.schedule", + component: "attachments-culler", + field: ["spec", "schedule"], + }, + { + path: "attachments.culler.activeDeadlineSeconds", + component: "attachments-culler", + field: ["spec", "jobTemplate", "spec", "activeDeadlineSeconds"], + }, + // The retention window is a positional argument to `cull-staged-attachments.ts`, so it lands in + // `command`, where neither the env-var check nor a `spec.*` path can see it. + { + path: "attachments.culler.olderThanHours", + component: "attachments-culler", + field: [ + "spec", + "jobTemplate", + "spec", + "template", + "spec", + "containers", + "0", + "command", + "2", + ], + }, ]; const chartValuesTree = parse(rawChartValues) as unknown; diff --git a/server/bunfig.toml b/server/bunfig.toml new file mode 100644 index 000000000..5fcdbe56f --- /dev/null +++ b/server/bunfig.toml @@ -0,0 +1,12 @@ +[test] +# The same declaration as the bunfig.toml at the repository root, with the path written relative to +# this directory. +# +# Bun reads bunfig.toml from the current working directory and nowhere else, so the root one applies +# to `bun test server/` run from the root and not to `bun test` run from here. Without this file the +# preload never loads from inside `server`, and the suite silently loses whichever test files lost +# the race described in scripts/test-preload.ts. +# +# Both files name the same script, so its contents cannot drift. What could drift is this list, and +# tests/bunfig-preload.test.ts fails if the two stop agreeing. +preload = ["./scripts/test-preload.ts"] diff --git a/server/drizzle/0030_daffy_nighthawk.sql b/server/drizzle/0030_daffy_nighthawk.sql new file mode 100644 index 000000000..42e6a64f4 --- /dev/null +++ b/server/drizzle/0030_daffy_nighthawk.sql @@ -0,0 +1,14 @@ +CREATE TABLE "attachments" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "channel_id" text NOT NULL, + "uploaded_by" text NOT NULL, + "name" text NOT NULL, + "mime_type" text NOT NULL, + "size_bytes" integer NOT NULL, + "bytes" "bytea" NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "attached_at" timestamp with time zone +); +--> statement-breakpoint +ALTER TABLE "attachments" ADD CONSTRAINT "attachments_channel_id_channels_id_fk" FOREIGN KEY ("channel_id") REFERENCES "public"."channels"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "attachments" ADD CONSTRAINT "attachments_uploaded_by_users_id_fk" FOREIGN KEY ("uploaded_by") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/server/drizzle/0031_hesitant_squadron_sinister.sql b/server/drizzle/0031_hesitant_squadron_sinister.sql new file mode 100644 index 000000000..3eb5523d8 --- /dev/null +++ b/server/drizzle/0031_hesitant_squadron_sinister.sql @@ -0,0 +1,2 @@ +CREATE INDEX "attachments_channel_idx" ON "attachments" USING btree ("channel_id");--> statement-breakpoint +CREATE INDEX "attachments_staged_idx" ON "attachments" USING btree ("created_at") WHERE "attachments"."attached_at" is null; \ No newline at end of file diff --git a/server/drizzle/0032_rainy_lake.sql b/server/drizzle/0032_rainy_lake.sql new file mode 100644 index 000000000..31349be14 --- /dev/null +++ b/server/drizzle/0032_rainy_lake.sql @@ -0,0 +1,2 @@ +ALTER TABLE "attachments" ADD COLUMN "upload_group" text;--> statement-breakpoint +CREATE INDEX "attachments_upload_group_idx" ON "attachments" USING btree ("channel_id","uploaded_by","upload_group") WHERE "attachments"."attached_at" is null; \ No newline at end of file diff --git a/server/drizzle/0033_attachments_uploaded_by_index.sql b/server/drizzle/0033_attachments_uploaded_by_index.sql new file mode 100644 index 000000000..d69700822 --- /dev/null +++ b/server/drizzle/0033_attachments_uploaded_by_index.sql @@ -0,0 +1 @@ +CREATE INDEX "attachments_uploaded_by_idx" ON "attachments" USING btree ("uploaded_by"); \ No newline at end of file diff --git a/server/drizzle/meta/0030_snapshot.json b/server/drizzle/meta/0030_snapshot.json new file mode 100644 index 000000000..a11c3a757 --- /dev/null +++ b/server/drizzle/meta/0030_snapshot.json @@ -0,0 +1,3238 @@ +{ + "id": "73d89d85-d1c9-4eff-9ecf-b01fc788ebc2", + "prevId": "d9659673-30fd-40f7-817c-ad175baa211d", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_provider_account_idx": { + "name": "accounts_provider_account_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "agent_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_package_id_deployment_packages_id_fk": { + "name": "agents_package_id_deployment_packages_id_fk", + "tableFrom": "agents", + "tableTo": "deployment_packages", + "columnsFrom": [ + "package_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.attachments": { + "name": "attachments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bytes": { + "name": "bytes", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "attached_at": { + "name": "attached_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "attachments_channel_id_channels_id_fk": { + "name": "attachments_channel_id_channels_id_fk", + "tableFrom": "attachments", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "attachments_uploaded_by_users_id_fk": { + "name": "attachments_uploaded_by_users_id_fk", + "tableFrom": "attachments", + "tableTo": "users", + "columnsFrom": [ + "uploaded_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_events": { + "name": "audit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiator_kind": { + "name": "initiator_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'person'" + }, + "initiator_id": { + "name": "initiator_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_events_created_at_idx": { + "name": "audit_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_type_time_idx": { + "name": "audit_events_type_time_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_actor_time_idx": { + "name": "audit_events_actor_time_idx", + "columns": [ + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_target_time_idx": { + "name": "audit_events_target_time_idx", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_initiator_time_idx": { + "name": "audit_events_initiator_time_idx", + "columns": [ + { + "expression": "initiator_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_agents": { + "name": "channel_agents", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_agents_channel_id_channels_id_fk": { + "name": "channel_agents_channel_id_channels_id_fk", + "tableFrom": "channel_agents", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_agents_agent_id_agents_id_fk": { + "name": "channel_agents_agent_id_agents_id_fk", + "tableFrom": "channel_agents", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_agents_channel_id_agent_id_pk": { + "name": "channel_agents_channel_id_agent_id_pk", + "columns": [ + "channel_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_memberships": { + "name": "channel_memberships", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_memberships_channel_id_channels_id_fk": { + "name": "channel_memberships_channel_id_channels_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_memberships_user_id_users_id_fk": { + "name": "channel_memberships_user_id_users_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_memberships_channel_id_user_id_pk": { + "name": "channel_memberships_channel_id_user_id_pk", + "columns": [ + "channel_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channels": { + "name": "channels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggested_prompts": { + "name": "suggested_prompts", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "allowed_groups": { + "name": "allowed_groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary_at": { + "name": "summary_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message": { + "name": "last_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_message_at": { + "name": "last_message_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message_agent_id": { + "name": "last_message_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "channels_recent_activity_idx": { + "name": "channels_recent_activity_idx", + "columns": [ + { + "expression": "COALESCE(\"last_message_at\", \"created_at\") DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "channels_awaiting_summary_idx": { + "name": "channels_awaiting_summary_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"channels\".\"summary\" is null and \"channels\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "channels_package_id_deployment_packages_id_fk": { + "name": "channels_package_id_deployment_packages_id_fk", + "tableFrom": "channels", + "tableTo": "deployment_packages", + "columnsFrom": [ + "package_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "channels_last_message_agent_id_agents_id_fk": { + "name": "channels_last_message_agent_id_agents_id_fk", + "tableFrom": "channels", + "tableTo": "agents", + "columnsFrom": [ + "last_message_agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credentials": { + "name": "credentials", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "credential_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_value": { + "name": "encrypted_value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credentials_active_key_idx": { + "name": "credentials_active_key_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credentials\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_packages": { + "name": "deployment_packages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "loaded_at": { + "name": "loaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployment_packages_tenant_id_unique": { + "name": "deployment_packages_tenant_id_unique", + "nullsNotDistinct": false, + "columns": [ + "tenant_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.intelligence_channel_mappings": { + "name": "intelligence_channel_mappings", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "intelligence_channel_mappings_thread_idx": { + "name": "intelligence_channel_mappings_thread_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "intelligence_channel_mappings_user_id_users_id_fk": { + "name": "intelligence_channel_mappings_user_id_users_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "intelligence_channel_mappings_channel_id_channels_id_fk": { + "name": "intelligence_channel_mappings_channel_id_channels_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "intelligence_channel_mappings_user_id_channel_id_pk": { + "name": "intelligence_channel_mappings_user_id_channel_id_pk", + "columns": [ + "user_id", + "channel_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.revoked_access": { + "name": "revoked_access", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_providers": { + "name": "sso_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sso_providers_user_id_users_id_fk": { + "name": "sso_providers_user_id_users_id_fk", + "tableFrom": "sso_providers", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "nullsNotDistinct": false, + "columns": [ + "provider_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_instructions": { + "name": "user_instructions", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_instructions_user_id_users_id_fk": { + "name": "user_instructions_user_id_users_id_fk", + "tableFrom": "user_instructions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_pk": { + "name": "user_roles_user_id_role_pk", + "columns": [ + "user_id", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "groups": { + "name": "groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "onboarding_step": { + "name": "onboarding_step", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verifications": { + "name": "verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_policy": { + "name": "action_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deny": { + "name": "deny", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "allow": { + "name": "allow", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_page_frame": { + "name": "computer_page_frame", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "computer_page_frame_captured_idx": { + "name": "computer_page_frame_captured_idx", + "columns": [ + { + "expression": "captured_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "computer_page_frame_computer_id_tool_call_id_pk": { + "name": "computer_page_frame_computer_id_tool_call_id_pk", + "columns": [ + "computer_id", + "tool_call_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_snapshot": { + "name": "computer_snapshot", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "elements": { + "name": "elements", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "taken_at": { + "name": "taken_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "session": { + "name": "session", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_preferences": { + "name": "agent_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_preferences_user_id_users_id_fk": { + "name": "agent_preferences_user_id_users_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_preferences_agent_id_agents_id_fk": { + "name": "agent_preferences_agent_id_agents_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_preferences_user_id_agent_id_pk": { + "name": "agent_preferences_user_id_agent_id_pk", + "columns": [ + "user_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_profiles": { + "name": "agent_profiles", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_description": { + "name": "role_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_seed": { + "name": "avatar_seed", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "agent_visibility", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "callback_token_hash": { + "name": "callback_token_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "callback_token_issued_at": { + "name": "callback_token_issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_profiles_visibility_deleted_idx": { + "name": "agent_profiles_visibility_deleted_idx", + "columns": [ + { + "expression": "visibility", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_profiles_agent_id_agents_id_fk": { + "name": "agent_profiles_agent_id_agents_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_profiles_owner_user_id_users_id_fk": { + "name": "agent_profiles_owner_user_id_users_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routine_runs": { + "name": "routine_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "routine_id": { + "name": "routine_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "routine_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "routine_runs_by_routine_idx": { + "name": "routine_runs_by_routine_idx", + "columns": [ + { + "expression": "routine_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routine_runs_routine_id_routines_id_fk": { + "name": "routine_runs_routine_id_routines_id_fk", + "tableFrom": "routine_runs", + "tableTo": "routines", + "columnsFrom": [ + "routine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routines": { + "name": "routines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instruction": { + "name": "instruction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cron": { + "name": "cron", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "routines_due_idx": { + "name": "routines_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "routines_by_owner_idx": { + "name": "routines_by_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routines_owner_user_id_users_id_fk": { + "name": "routines_owner_user_id_users_id_fk", + "tableFrom": "routines", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "routines_agent_id_agents_id_fk": { + "name": "routines_agent_id_agents_id_fk", + "tableFrom": "routines", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_exclusions": { + "name": "component_exclusions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "withheld_by": { + "name": "withheld_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_exclusions_component_name_components_name_fk": { + "name": "component_exclusions_component_name_components_name_fk", + "tableFrom": "component_exclusions", + "tableTo": "components", + "columnsFrom": [ + "component_name" + ], + "columnsTo": [ + "name" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "component_exclusions_agent_id_agents_id_fk": { + "name": "component_exclusions_agent_id_agents_id_fk", + "tableFrom": "component_exclusions", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_exclusions_component_name_agent_id_pk": { + "name": "component_exclusions_component_name_agent_id_pk", + "columns": [ + "component_name", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_functions": { + "name": "component_functions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "function_name": { + "name": "function_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_functions_component_name_components_name_fk": { + "name": "component_functions_component_name_components_name_fk", + "tableFrom": "component_functions", + "tableTo": "components", + "columnsFrom": [ + "component_name" + ], + "columnsTo": [ + "name" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_functions_component_name_function_name_pk": { + "name": "component_functions_component_name_function_name_pk", + "columns": [ + "component_name", + "function_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.components": { + "name": "components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'first-party'" + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tools_refreshed_at": { + "name": "tools_refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_servers_credential_id_credentials_id_fk": { + "name": "mcp_servers_credential_id_credentials_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_tools": { + "name": "mcp_tools", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "input_schema": { + "name": "input_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_tools_server_id_mcp_servers_id_fk": { + "name": "mcp_tools_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_tools", + "tableTo": "mcp_servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_tools_server_id_name_pk": { + "name": "mcp_tools_server_id_name_pk", + "columns": [ + "server_id", + "name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_user_credentials": { + "name": "mcp_user_credentials", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_user_credentials_user_idx": { + "name": "mcp_user_credentials_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_user_credentials_server_id_mcp_servers_id_fk": { + "name": "mcp_user_credentials_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "mcp_servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_user_id_users_id_fk": { + "name": "mcp_user_credentials_user_id_users_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_credential_id_credentials_id_fk": { + "name": "mcp_user_credentials_credential_id_credentials_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_user_credentials_server_id_user_id_pk": { + "name": "mcp_user_credentials_server_id_user_id_pk", + "columns": [ + "server_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_grants": { + "name": "plugin_grants", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "plugin_grants_agent_idx": { + "name": "plugin_grants_agent_idx", + "columns": [ + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "plugin_grants_agent_id_agents_id_fk": { + "name": "plugin_grants_agent_id_agents_id_fk", + "tableFrom": "plugin_grants", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "plugin_grants_kind_ref_agent_id_pk": { + "name": "plugin_grants_kind_ref_agent_id_pk", + "columns": [ + "kind", + "ref", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandboxed_components": { + "name": "sandboxed_components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_html": { + "name": "draft_html", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_css": { + "name": "draft_css", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_js_functions": { + "name": "draft_js_functions", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_argument_schema": { + "name": "draft_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_html": { + "name": "published_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_css": { + "name": "published_css", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_js_functions": { + "name": "published_js_functions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_argument_schema": { + "name": "published_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "sample_arguments": { + "name": "sample_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "authored_by": { + "name": "authored_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_tools": { + "name": "skill_tools", + "schema": "", + "columns": { + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "declared_by": { + "name": "declared_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_tools_ref_idx": { + "name": "skill_tools_ref_idx", + "columns": [ + { + "expression": "ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_tools_skill_id_skills_id_fk": { + "name": "skill_tools_skill_id_skills_id_fk", + "tableFrom": "skill_tools", + "tableTo": "skills", + "columnsFrom": [ + "skill_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "skill_tools_skill_id_ref_pk": { + "name": "skill_tools_skill_id_ref_pk", + "columns": [ + "skill_id", + "ref" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skills": { + "name": "skills", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'yours'" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skills_slug_key": { + "name": "skills_slug_key", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skills_owner_idx": { + "name": "skills_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skills_owner_user_id_users_id_fk": { + "name": "skills_owner_user_id_users_id_fk", + "tableFrom": "skills", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.work_items": { + "name": "work_items", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_at": { + "name": "run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "work_items_claimable_idx": { + "name": "work_items_claimable_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "work_items_kind_key_pk": { + "name": "work_items_kind_key_pk", + "columns": [ + "kind", + "key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.agent_type": { + "name": "agent_type", + "schema": "public", + "values": [ + "built_in", + "remote_ag_ui", + "remote_mastra" + ] + }, + "public.credential_kind": { + "name": "credential_kind", + "schema": "public", + "values": [ + "model", + "connector", + "agent", + "mcp", + "mcp_oauth_client", + "mcp_user_token" + ] + }, + "public.role": { + "name": "role", + "schema": "public", + "values": [ + "admin", + "user" + ] + }, + "public.agent_visibility": { + "name": "agent_visibility", + "schema": "public", + "values": [ + "public", + "private" + ] + }, + "public.routine_run_status": { + "name": "routine_run_status", + "schema": "public", + "values": [ + "succeeded", + "failed", + "skipped" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/server/drizzle/meta/0031_snapshot.json b/server/drizzle/meta/0031_snapshot.json new file mode 100644 index 000000000..9ad77744a --- /dev/null +++ b/server/drizzle/meta/0031_snapshot.json @@ -0,0 +1,3270 @@ +{ + "id": "652c371e-a254-4efe-8b34-f1c7a2982184", + "prevId": "73d89d85-d1c9-4eff-9ecf-b01fc788ebc2", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_provider_account_idx": { + "name": "accounts_provider_account_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "agent_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_package_id_deployment_packages_id_fk": { + "name": "agents_package_id_deployment_packages_id_fk", + "tableFrom": "agents", + "tableTo": "deployment_packages", + "columnsFrom": [ + "package_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.attachments": { + "name": "attachments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bytes": { + "name": "bytes", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "attached_at": { + "name": "attached_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "attachments_channel_idx": { + "name": "attachments_channel_idx", + "columns": [ + { + "expression": "channel_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "attachments_staged_idx": { + "name": "attachments_staged_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"attachments\".\"attached_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "attachments_channel_id_channels_id_fk": { + "name": "attachments_channel_id_channels_id_fk", + "tableFrom": "attachments", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "attachments_uploaded_by_users_id_fk": { + "name": "attachments_uploaded_by_users_id_fk", + "tableFrom": "attachments", + "tableTo": "users", + "columnsFrom": [ + "uploaded_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_events": { + "name": "audit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiator_kind": { + "name": "initiator_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'person'" + }, + "initiator_id": { + "name": "initiator_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_events_created_at_idx": { + "name": "audit_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_type_time_idx": { + "name": "audit_events_type_time_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_actor_time_idx": { + "name": "audit_events_actor_time_idx", + "columns": [ + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_target_time_idx": { + "name": "audit_events_target_time_idx", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_initiator_time_idx": { + "name": "audit_events_initiator_time_idx", + "columns": [ + { + "expression": "initiator_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_agents": { + "name": "channel_agents", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_agents_channel_id_channels_id_fk": { + "name": "channel_agents_channel_id_channels_id_fk", + "tableFrom": "channel_agents", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_agents_agent_id_agents_id_fk": { + "name": "channel_agents_agent_id_agents_id_fk", + "tableFrom": "channel_agents", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_agents_channel_id_agent_id_pk": { + "name": "channel_agents_channel_id_agent_id_pk", + "columns": [ + "channel_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_memberships": { + "name": "channel_memberships", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_memberships_channel_id_channels_id_fk": { + "name": "channel_memberships_channel_id_channels_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_memberships_user_id_users_id_fk": { + "name": "channel_memberships_user_id_users_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_memberships_channel_id_user_id_pk": { + "name": "channel_memberships_channel_id_user_id_pk", + "columns": [ + "channel_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channels": { + "name": "channels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggested_prompts": { + "name": "suggested_prompts", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "allowed_groups": { + "name": "allowed_groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary_at": { + "name": "summary_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message": { + "name": "last_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_message_at": { + "name": "last_message_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message_agent_id": { + "name": "last_message_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "channels_recent_activity_idx": { + "name": "channels_recent_activity_idx", + "columns": [ + { + "expression": "COALESCE(\"last_message_at\", \"created_at\") DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "channels_awaiting_summary_idx": { + "name": "channels_awaiting_summary_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"channels\".\"summary\" is null and \"channels\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "channels_package_id_deployment_packages_id_fk": { + "name": "channels_package_id_deployment_packages_id_fk", + "tableFrom": "channels", + "tableTo": "deployment_packages", + "columnsFrom": [ + "package_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "channels_last_message_agent_id_agents_id_fk": { + "name": "channels_last_message_agent_id_agents_id_fk", + "tableFrom": "channels", + "tableTo": "agents", + "columnsFrom": [ + "last_message_agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credentials": { + "name": "credentials", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "credential_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_value": { + "name": "encrypted_value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credentials_active_key_idx": { + "name": "credentials_active_key_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credentials\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_packages": { + "name": "deployment_packages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "loaded_at": { + "name": "loaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployment_packages_tenant_id_unique": { + "name": "deployment_packages_tenant_id_unique", + "nullsNotDistinct": false, + "columns": [ + "tenant_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.intelligence_channel_mappings": { + "name": "intelligence_channel_mappings", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "intelligence_channel_mappings_thread_idx": { + "name": "intelligence_channel_mappings_thread_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "intelligence_channel_mappings_user_id_users_id_fk": { + "name": "intelligence_channel_mappings_user_id_users_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "intelligence_channel_mappings_channel_id_channels_id_fk": { + "name": "intelligence_channel_mappings_channel_id_channels_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "intelligence_channel_mappings_user_id_channel_id_pk": { + "name": "intelligence_channel_mappings_user_id_channel_id_pk", + "columns": [ + "user_id", + "channel_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.revoked_access": { + "name": "revoked_access", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_providers": { + "name": "sso_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sso_providers_user_id_users_id_fk": { + "name": "sso_providers_user_id_users_id_fk", + "tableFrom": "sso_providers", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "nullsNotDistinct": false, + "columns": [ + "provider_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_instructions": { + "name": "user_instructions", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_instructions_user_id_users_id_fk": { + "name": "user_instructions_user_id_users_id_fk", + "tableFrom": "user_instructions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_pk": { + "name": "user_roles_user_id_role_pk", + "columns": [ + "user_id", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "groups": { + "name": "groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "onboarding_step": { + "name": "onboarding_step", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verifications": { + "name": "verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_policy": { + "name": "action_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deny": { + "name": "deny", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "allow": { + "name": "allow", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_page_frame": { + "name": "computer_page_frame", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "computer_page_frame_captured_idx": { + "name": "computer_page_frame_captured_idx", + "columns": [ + { + "expression": "captured_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "computer_page_frame_computer_id_tool_call_id_pk": { + "name": "computer_page_frame_computer_id_tool_call_id_pk", + "columns": [ + "computer_id", + "tool_call_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_snapshot": { + "name": "computer_snapshot", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "elements": { + "name": "elements", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "taken_at": { + "name": "taken_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "session": { + "name": "session", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_preferences": { + "name": "agent_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_preferences_user_id_users_id_fk": { + "name": "agent_preferences_user_id_users_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_preferences_agent_id_agents_id_fk": { + "name": "agent_preferences_agent_id_agents_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_preferences_user_id_agent_id_pk": { + "name": "agent_preferences_user_id_agent_id_pk", + "columns": [ + "user_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_profiles": { + "name": "agent_profiles", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_description": { + "name": "role_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_seed": { + "name": "avatar_seed", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "agent_visibility", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "callback_token_hash": { + "name": "callback_token_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "callback_token_issued_at": { + "name": "callback_token_issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_profiles_visibility_deleted_idx": { + "name": "agent_profiles_visibility_deleted_idx", + "columns": [ + { + "expression": "visibility", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_profiles_agent_id_agents_id_fk": { + "name": "agent_profiles_agent_id_agents_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_profiles_owner_user_id_users_id_fk": { + "name": "agent_profiles_owner_user_id_users_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routine_runs": { + "name": "routine_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "routine_id": { + "name": "routine_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "routine_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "routine_runs_by_routine_idx": { + "name": "routine_runs_by_routine_idx", + "columns": [ + { + "expression": "routine_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routine_runs_routine_id_routines_id_fk": { + "name": "routine_runs_routine_id_routines_id_fk", + "tableFrom": "routine_runs", + "tableTo": "routines", + "columnsFrom": [ + "routine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routines": { + "name": "routines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instruction": { + "name": "instruction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cron": { + "name": "cron", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "routines_due_idx": { + "name": "routines_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "routines_by_owner_idx": { + "name": "routines_by_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routines_owner_user_id_users_id_fk": { + "name": "routines_owner_user_id_users_id_fk", + "tableFrom": "routines", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "routines_agent_id_agents_id_fk": { + "name": "routines_agent_id_agents_id_fk", + "tableFrom": "routines", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_exclusions": { + "name": "component_exclusions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "withheld_by": { + "name": "withheld_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_exclusions_component_name_components_name_fk": { + "name": "component_exclusions_component_name_components_name_fk", + "tableFrom": "component_exclusions", + "tableTo": "components", + "columnsFrom": [ + "component_name" + ], + "columnsTo": [ + "name" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "component_exclusions_agent_id_agents_id_fk": { + "name": "component_exclusions_agent_id_agents_id_fk", + "tableFrom": "component_exclusions", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_exclusions_component_name_agent_id_pk": { + "name": "component_exclusions_component_name_agent_id_pk", + "columns": [ + "component_name", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_functions": { + "name": "component_functions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "function_name": { + "name": "function_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_functions_component_name_components_name_fk": { + "name": "component_functions_component_name_components_name_fk", + "tableFrom": "component_functions", + "tableTo": "components", + "columnsFrom": [ + "component_name" + ], + "columnsTo": [ + "name" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_functions_component_name_function_name_pk": { + "name": "component_functions_component_name_function_name_pk", + "columns": [ + "component_name", + "function_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.components": { + "name": "components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'first-party'" + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tools_refreshed_at": { + "name": "tools_refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_servers_credential_id_credentials_id_fk": { + "name": "mcp_servers_credential_id_credentials_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_tools": { + "name": "mcp_tools", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "input_schema": { + "name": "input_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_tools_server_id_mcp_servers_id_fk": { + "name": "mcp_tools_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_tools", + "tableTo": "mcp_servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_tools_server_id_name_pk": { + "name": "mcp_tools_server_id_name_pk", + "columns": [ + "server_id", + "name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_user_credentials": { + "name": "mcp_user_credentials", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_user_credentials_user_idx": { + "name": "mcp_user_credentials_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_user_credentials_server_id_mcp_servers_id_fk": { + "name": "mcp_user_credentials_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "mcp_servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_user_id_users_id_fk": { + "name": "mcp_user_credentials_user_id_users_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_credential_id_credentials_id_fk": { + "name": "mcp_user_credentials_credential_id_credentials_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_user_credentials_server_id_user_id_pk": { + "name": "mcp_user_credentials_server_id_user_id_pk", + "columns": [ + "server_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_grants": { + "name": "plugin_grants", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "plugin_grants_agent_idx": { + "name": "plugin_grants_agent_idx", + "columns": [ + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "plugin_grants_agent_id_agents_id_fk": { + "name": "plugin_grants_agent_id_agents_id_fk", + "tableFrom": "plugin_grants", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "plugin_grants_kind_ref_agent_id_pk": { + "name": "plugin_grants_kind_ref_agent_id_pk", + "columns": [ + "kind", + "ref", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandboxed_components": { + "name": "sandboxed_components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_html": { + "name": "draft_html", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_css": { + "name": "draft_css", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_js_functions": { + "name": "draft_js_functions", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_argument_schema": { + "name": "draft_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_html": { + "name": "published_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_css": { + "name": "published_css", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_js_functions": { + "name": "published_js_functions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_argument_schema": { + "name": "published_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "sample_arguments": { + "name": "sample_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "authored_by": { + "name": "authored_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_tools": { + "name": "skill_tools", + "schema": "", + "columns": { + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "declared_by": { + "name": "declared_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_tools_ref_idx": { + "name": "skill_tools_ref_idx", + "columns": [ + { + "expression": "ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_tools_skill_id_skills_id_fk": { + "name": "skill_tools_skill_id_skills_id_fk", + "tableFrom": "skill_tools", + "tableTo": "skills", + "columnsFrom": [ + "skill_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "skill_tools_skill_id_ref_pk": { + "name": "skill_tools_skill_id_ref_pk", + "columns": [ + "skill_id", + "ref" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skills": { + "name": "skills", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'yours'" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skills_slug_key": { + "name": "skills_slug_key", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skills_owner_idx": { + "name": "skills_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skills_owner_user_id_users_id_fk": { + "name": "skills_owner_user_id_users_id_fk", + "tableFrom": "skills", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.work_items": { + "name": "work_items", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_at": { + "name": "run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "work_items_claimable_idx": { + "name": "work_items_claimable_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "work_items_kind_key_pk": { + "name": "work_items_kind_key_pk", + "columns": [ + "kind", + "key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.agent_type": { + "name": "agent_type", + "schema": "public", + "values": [ + "built_in", + "remote_ag_ui", + "remote_mastra" + ] + }, + "public.credential_kind": { + "name": "credential_kind", + "schema": "public", + "values": [ + "model", + "connector", + "agent", + "mcp", + "mcp_oauth_client", + "mcp_user_token" + ] + }, + "public.role": { + "name": "role", + "schema": "public", + "values": [ + "admin", + "user" + ] + }, + "public.agent_visibility": { + "name": "agent_visibility", + "schema": "public", + "values": [ + "public", + "private" + ] + }, + "public.routine_run_status": { + "name": "routine_run_status", + "schema": "public", + "values": [ + "succeeded", + "failed", + "skipped" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/server/drizzle/meta/0032_snapshot.json b/server/drizzle/meta/0032_snapshot.json new file mode 100644 index 000000000..00df45b82 --- /dev/null +++ b/server/drizzle/meta/0032_snapshot.json @@ -0,0 +1,3304 @@ +{ + "id": "8e4ac6d9-6460-44bb-a7ae-a430c8a5cfa2", + "prevId": "652c371e-a254-4efe-8b34-f1c7a2982184", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_provider_account_idx": { + "name": "accounts_provider_account_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "agent_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_package_id_deployment_packages_id_fk": { + "name": "agents_package_id_deployment_packages_id_fk", + "tableFrom": "agents", + "tableTo": "deployment_packages", + "columnsFrom": [ + "package_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.attachments": { + "name": "attachments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bytes": { + "name": "bytes", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "attached_at": { + "name": "attached_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "upload_group": { + "name": "upload_group", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "attachments_channel_idx": { + "name": "attachments_channel_idx", + "columns": [ + { + "expression": "channel_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "attachments_upload_group_idx": { + "name": "attachments_upload_group_idx", + "columns": [ + { + "expression": "channel_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "uploaded_by", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "upload_group", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"attachments\".\"attached_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "attachments_staged_idx": { + "name": "attachments_staged_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"attachments\".\"attached_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "attachments_channel_id_channels_id_fk": { + "name": "attachments_channel_id_channels_id_fk", + "tableFrom": "attachments", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "attachments_uploaded_by_users_id_fk": { + "name": "attachments_uploaded_by_users_id_fk", + "tableFrom": "attachments", + "tableTo": "users", + "columnsFrom": [ + "uploaded_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_events": { + "name": "audit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiator_kind": { + "name": "initiator_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'person'" + }, + "initiator_id": { + "name": "initiator_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_events_created_at_idx": { + "name": "audit_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_type_time_idx": { + "name": "audit_events_type_time_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_actor_time_idx": { + "name": "audit_events_actor_time_idx", + "columns": [ + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_target_time_idx": { + "name": "audit_events_target_time_idx", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_initiator_time_idx": { + "name": "audit_events_initiator_time_idx", + "columns": [ + { + "expression": "initiator_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_agents": { + "name": "channel_agents", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_agents_channel_id_channels_id_fk": { + "name": "channel_agents_channel_id_channels_id_fk", + "tableFrom": "channel_agents", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_agents_agent_id_agents_id_fk": { + "name": "channel_agents_agent_id_agents_id_fk", + "tableFrom": "channel_agents", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_agents_channel_id_agent_id_pk": { + "name": "channel_agents_channel_id_agent_id_pk", + "columns": [ + "channel_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_memberships": { + "name": "channel_memberships", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_memberships_channel_id_channels_id_fk": { + "name": "channel_memberships_channel_id_channels_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_memberships_user_id_users_id_fk": { + "name": "channel_memberships_user_id_users_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_memberships_channel_id_user_id_pk": { + "name": "channel_memberships_channel_id_user_id_pk", + "columns": [ + "channel_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channels": { + "name": "channels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggested_prompts": { + "name": "suggested_prompts", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "allowed_groups": { + "name": "allowed_groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary_at": { + "name": "summary_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message": { + "name": "last_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_message_at": { + "name": "last_message_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message_agent_id": { + "name": "last_message_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "channels_recent_activity_idx": { + "name": "channels_recent_activity_idx", + "columns": [ + { + "expression": "COALESCE(\"last_message_at\", \"created_at\") DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "channels_awaiting_summary_idx": { + "name": "channels_awaiting_summary_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"channels\".\"summary\" is null and \"channels\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "channels_package_id_deployment_packages_id_fk": { + "name": "channels_package_id_deployment_packages_id_fk", + "tableFrom": "channels", + "tableTo": "deployment_packages", + "columnsFrom": [ + "package_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "channels_last_message_agent_id_agents_id_fk": { + "name": "channels_last_message_agent_id_agents_id_fk", + "tableFrom": "channels", + "tableTo": "agents", + "columnsFrom": [ + "last_message_agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credentials": { + "name": "credentials", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "credential_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_value": { + "name": "encrypted_value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credentials_active_key_idx": { + "name": "credentials_active_key_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credentials\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_packages": { + "name": "deployment_packages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "loaded_at": { + "name": "loaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployment_packages_tenant_id_unique": { + "name": "deployment_packages_tenant_id_unique", + "nullsNotDistinct": false, + "columns": [ + "tenant_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.intelligence_channel_mappings": { + "name": "intelligence_channel_mappings", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "intelligence_channel_mappings_thread_idx": { + "name": "intelligence_channel_mappings_thread_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "intelligence_channel_mappings_user_id_users_id_fk": { + "name": "intelligence_channel_mappings_user_id_users_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "intelligence_channel_mappings_channel_id_channels_id_fk": { + "name": "intelligence_channel_mappings_channel_id_channels_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "intelligence_channel_mappings_user_id_channel_id_pk": { + "name": "intelligence_channel_mappings_user_id_channel_id_pk", + "columns": [ + "user_id", + "channel_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.revoked_access": { + "name": "revoked_access", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_providers": { + "name": "sso_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sso_providers_user_id_users_id_fk": { + "name": "sso_providers_user_id_users_id_fk", + "tableFrom": "sso_providers", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "nullsNotDistinct": false, + "columns": [ + "provider_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_instructions": { + "name": "user_instructions", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_instructions_user_id_users_id_fk": { + "name": "user_instructions_user_id_users_id_fk", + "tableFrom": "user_instructions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_pk": { + "name": "user_roles_user_id_role_pk", + "columns": [ + "user_id", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "groups": { + "name": "groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "onboarding_step": { + "name": "onboarding_step", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verifications": { + "name": "verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_policy": { + "name": "action_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deny": { + "name": "deny", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "allow": { + "name": "allow", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_page_frame": { + "name": "computer_page_frame", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "computer_page_frame_captured_idx": { + "name": "computer_page_frame_captured_idx", + "columns": [ + { + "expression": "captured_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "computer_page_frame_computer_id_tool_call_id_pk": { + "name": "computer_page_frame_computer_id_tool_call_id_pk", + "columns": [ + "computer_id", + "tool_call_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_snapshot": { + "name": "computer_snapshot", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "elements": { + "name": "elements", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "taken_at": { + "name": "taken_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "session": { + "name": "session", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_preferences": { + "name": "agent_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_preferences_user_id_users_id_fk": { + "name": "agent_preferences_user_id_users_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_preferences_agent_id_agents_id_fk": { + "name": "agent_preferences_agent_id_agents_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_preferences_user_id_agent_id_pk": { + "name": "agent_preferences_user_id_agent_id_pk", + "columns": [ + "user_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_profiles": { + "name": "agent_profiles", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_description": { + "name": "role_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_seed": { + "name": "avatar_seed", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "agent_visibility", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "callback_token_hash": { + "name": "callback_token_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "callback_token_issued_at": { + "name": "callback_token_issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_profiles_visibility_deleted_idx": { + "name": "agent_profiles_visibility_deleted_idx", + "columns": [ + { + "expression": "visibility", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_profiles_agent_id_agents_id_fk": { + "name": "agent_profiles_agent_id_agents_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_profiles_owner_user_id_users_id_fk": { + "name": "agent_profiles_owner_user_id_users_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routine_runs": { + "name": "routine_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "routine_id": { + "name": "routine_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "routine_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "routine_runs_by_routine_idx": { + "name": "routine_runs_by_routine_idx", + "columns": [ + { + "expression": "routine_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routine_runs_routine_id_routines_id_fk": { + "name": "routine_runs_routine_id_routines_id_fk", + "tableFrom": "routine_runs", + "tableTo": "routines", + "columnsFrom": [ + "routine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routines": { + "name": "routines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instruction": { + "name": "instruction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cron": { + "name": "cron", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "routines_due_idx": { + "name": "routines_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "routines_by_owner_idx": { + "name": "routines_by_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routines_owner_user_id_users_id_fk": { + "name": "routines_owner_user_id_users_id_fk", + "tableFrom": "routines", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "routines_agent_id_agents_id_fk": { + "name": "routines_agent_id_agents_id_fk", + "tableFrom": "routines", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_exclusions": { + "name": "component_exclusions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "withheld_by": { + "name": "withheld_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_exclusions_component_name_components_name_fk": { + "name": "component_exclusions_component_name_components_name_fk", + "tableFrom": "component_exclusions", + "tableTo": "components", + "columnsFrom": [ + "component_name" + ], + "columnsTo": [ + "name" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "component_exclusions_agent_id_agents_id_fk": { + "name": "component_exclusions_agent_id_agents_id_fk", + "tableFrom": "component_exclusions", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_exclusions_component_name_agent_id_pk": { + "name": "component_exclusions_component_name_agent_id_pk", + "columns": [ + "component_name", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_functions": { + "name": "component_functions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "function_name": { + "name": "function_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_functions_component_name_components_name_fk": { + "name": "component_functions_component_name_components_name_fk", + "tableFrom": "component_functions", + "tableTo": "components", + "columnsFrom": [ + "component_name" + ], + "columnsTo": [ + "name" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_functions_component_name_function_name_pk": { + "name": "component_functions_component_name_function_name_pk", + "columns": [ + "component_name", + "function_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.components": { + "name": "components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'first-party'" + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tools_refreshed_at": { + "name": "tools_refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_servers_credential_id_credentials_id_fk": { + "name": "mcp_servers_credential_id_credentials_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_tools": { + "name": "mcp_tools", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "input_schema": { + "name": "input_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_tools_server_id_mcp_servers_id_fk": { + "name": "mcp_tools_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_tools", + "tableTo": "mcp_servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_tools_server_id_name_pk": { + "name": "mcp_tools_server_id_name_pk", + "columns": [ + "server_id", + "name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_user_credentials": { + "name": "mcp_user_credentials", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_user_credentials_user_idx": { + "name": "mcp_user_credentials_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_user_credentials_server_id_mcp_servers_id_fk": { + "name": "mcp_user_credentials_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "mcp_servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_user_id_users_id_fk": { + "name": "mcp_user_credentials_user_id_users_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_credential_id_credentials_id_fk": { + "name": "mcp_user_credentials_credential_id_credentials_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_user_credentials_server_id_user_id_pk": { + "name": "mcp_user_credentials_server_id_user_id_pk", + "columns": [ + "server_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_grants": { + "name": "plugin_grants", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "plugin_grants_agent_idx": { + "name": "plugin_grants_agent_idx", + "columns": [ + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "plugin_grants_agent_id_agents_id_fk": { + "name": "plugin_grants_agent_id_agents_id_fk", + "tableFrom": "plugin_grants", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "plugin_grants_kind_ref_agent_id_pk": { + "name": "plugin_grants_kind_ref_agent_id_pk", + "columns": [ + "kind", + "ref", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandboxed_components": { + "name": "sandboxed_components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_html": { + "name": "draft_html", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_css": { + "name": "draft_css", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_js_functions": { + "name": "draft_js_functions", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_argument_schema": { + "name": "draft_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_html": { + "name": "published_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_css": { + "name": "published_css", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_js_functions": { + "name": "published_js_functions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_argument_schema": { + "name": "published_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "sample_arguments": { + "name": "sample_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "authored_by": { + "name": "authored_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_tools": { + "name": "skill_tools", + "schema": "", + "columns": { + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "declared_by": { + "name": "declared_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_tools_ref_idx": { + "name": "skill_tools_ref_idx", + "columns": [ + { + "expression": "ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_tools_skill_id_skills_id_fk": { + "name": "skill_tools_skill_id_skills_id_fk", + "tableFrom": "skill_tools", + "tableTo": "skills", + "columnsFrom": [ + "skill_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "skill_tools_skill_id_ref_pk": { + "name": "skill_tools_skill_id_ref_pk", + "columns": [ + "skill_id", + "ref" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skills": { + "name": "skills", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'yours'" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skills_slug_key": { + "name": "skills_slug_key", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skills_owner_idx": { + "name": "skills_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skills_owner_user_id_users_id_fk": { + "name": "skills_owner_user_id_users_id_fk", + "tableFrom": "skills", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.work_items": { + "name": "work_items", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_at": { + "name": "run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "work_items_claimable_idx": { + "name": "work_items_claimable_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "work_items_kind_key_pk": { + "name": "work_items_kind_key_pk", + "columns": [ + "kind", + "key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.agent_type": { + "name": "agent_type", + "schema": "public", + "values": [ + "built_in", + "remote_ag_ui", + "remote_mastra" + ] + }, + "public.credential_kind": { + "name": "credential_kind", + "schema": "public", + "values": [ + "model", + "connector", + "agent", + "mcp", + "mcp_oauth_client", + "mcp_user_token" + ] + }, + "public.role": { + "name": "role", + "schema": "public", + "values": [ + "admin", + "user" + ] + }, + "public.agent_visibility": { + "name": "agent_visibility", + "schema": "public", + "values": [ + "public", + "private" + ] + }, + "public.routine_run_status": { + "name": "routine_run_status", + "schema": "public", + "values": [ + "succeeded", + "failed", + "skipped" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/server/drizzle/meta/0033_snapshot.json b/server/drizzle/meta/0033_snapshot.json new file mode 100644 index 000000000..d9775c371 --- /dev/null +++ b/server/drizzle/meta/0033_snapshot.json @@ -0,0 +1,3319 @@ +{ + "id": "01066a89-2e9c-40f4-b445-2a646e68fdbf", + "prevId": "8e4ac6d9-6460-44bb-a7ae-a430c8a5cfa2", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_provider_account_idx": { + "name": "accounts_provider_account_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "agent_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_package_id_deployment_packages_id_fk": { + "name": "agents_package_id_deployment_packages_id_fk", + "tableFrom": "agents", + "tableTo": "deployment_packages", + "columnsFrom": [ + "package_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.attachments": { + "name": "attachments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bytes": { + "name": "bytes", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "attached_at": { + "name": "attached_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "upload_group": { + "name": "upload_group", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "attachments_channel_idx": { + "name": "attachments_channel_idx", + "columns": [ + { + "expression": "channel_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "attachments_uploaded_by_idx": { + "name": "attachments_uploaded_by_idx", + "columns": [ + { + "expression": "uploaded_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "attachments_upload_group_idx": { + "name": "attachments_upload_group_idx", + "columns": [ + { + "expression": "channel_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "uploaded_by", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "upload_group", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"attachments\".\"attached_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "attachments_staged_idx": { + "name": "attachments_staged_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"attachments\".\"attached_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "attachments_channel_id_channels_id_fk": { + "name": "attachments_channel_id_channels_id_fk", + "tableFrom": "attachments", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "attachments_uploaded_by_users_id_fk": { + "name": "attachments_uploaded_by_users_id_fk", + "tableFrom": "attachments", + "tableTo": "users", + "columnsFrom": [ + "uploaded_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_events": { + "name": "audit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiator_kind": { + "name": "initiator_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'person'" + }, + "initiator_id": { + "name": "initiator_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_events_created_at_idx": { + "name": "audit_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_type_time_idx": { + "name": "audit_events_type_time_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_actor_time_idx": { + "name": "audit_events_actor_time_idx", + "columns": [ + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_target_time_idx": { + "name": "audit_events_target_time_idx", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_initiator_time_idx": { + "name": "audit_events_initiator_time_idx", + "columns": [ + { + "expression": "initiator_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_agents": { + "name": "channel_agents", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_agents_channel_id_channels_id_fk": { + "name": "channel_agents_channel_id_channels_id_fk", + "tableFrom": "channel_agents", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_agents_agent_id_agents_id_fk": { + "name": "channel_agents_agent_id_agents_id_fk", + "tableFrom": "channel_agents", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_agents_channel_id_agent_id_pk": { + "name": "channel_agents_channel_id_agent_id_pk", + "columns": [ + "channel_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_memberships": { + "name": "channel_memberships", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_memberships_channel_id_channels_id_fk": { + "name": "channel_memberships_channel_id_channels_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_memberships_user_id_users_id_fk": { + "name": "channel_memberships_user_id_users_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_memberships_channel_id_user_id_pk": { + "name": "channel_memberships_channel_id_user_id_pk", + "columns": [ + "channel_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channels": { + "name": "channels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggested_prompts": { + "name": "suggested_prompts", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "allowed_groups": { + "name": "allowed_groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary_at": { + "name": "summary_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message": { + "name": "last_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_message_at": { + "name": "last_message_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message_agent_id": { + "name": "last_message_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "channels_recent_activity_idx": { + "name": "channels_recent_activity_idx", + "columns": [ + { + "expression": "COALESCE(\"last_message_at\", \"created_at\") DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "channels_awaiting_summary_idx": { + "name": "channels_awaiting_summary_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"channels\".\"summary\" is null and \"channels\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "channels_package_id_deployment_packages_id_fk": { + "name": "channels_package_id_deployment_packages_id_fk", + "tableFrom": "channels", + "tableTo": "deployment_packages", + "columnsFrom": [ + "package_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "channels_last_message_agent_id_agents_id_fk": { + "name": "channels_last_message_agent_id_agents_id_fk", + "tableFrom": "channels", + "tableTo": "agents", + "columnsFrom": [ + "last_message_agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credentials": { + "name": "credentials", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "credential_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_value": { + "name": "encrypted_value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credentials_active_key_idx": { + "name": "credentials_active_key_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credentials\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_packages": { + "name": "deployment_packages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "loaded_at": { + "name": "loaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployment_packages_tenant_id_unique": { + "name": "deployment_packages_tenant_id_unique", + "nullsNotDistinct": false, + "columns": [ + "tenant_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.intelligence_channel_mappings": { + "name": "intelligence_channel_mappings", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "intelligence_channel_mappings_thread_idx": { + "name": "intelligence_channel_mappings_thread_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "intelligence_channel_mappings_user_id_users_id_fk": { + "name": "intelligence_channel_mappings_user_id_users_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "intelligence_channel_mappings_channel_id_channels_id_fk": { + "name": "intelligence_channel_mappings_channel_id_channels_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "intelligence_channel_mappings_user_id_channel_id_pk": { + "name": "intelligence_channel_mappings_user_id_channel_id_pk", + "columns": [ + "user_id", + "channel_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.revoked_access": { + "name": "revoked_access", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_providers": { + "name": "sso_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sso_providers_user_id_users_id_fk": { + "name": "sso_providers_user_id_users_id_fk", + "tableFrom": "sso_providers", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "nullsNotDistinct": false, + "columns": [ + "provider_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_instructions": { + "name": "user_instructions", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_instructions_user_id_users_id_fk": { + "name": "user_instructions_user_id_users_id_fk", + "tableFrom": "user_instructions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_pk": { + "name": "user_roles_user_id_role_pk", + "columns": [ + "user_id", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "groups": { + "name": "groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "onboarding_step": { + "name": "onboarding_step", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verifications": { + "name": "verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_policy": { + "name": "action_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deny": { + "name": "deny", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "allow": { + "name": "allow", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_page_frame": { + "name": "computer_page_frame", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "computer_page_frame_captured_idx": { + "name": "computer_page_frame_captured_idx", + "columns": [ + { + "expression": "captured_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "computer_page_frame_computer_id_tool_call_id_pk": { + "name": "computer_page_frame_computer_id_tool_call_id_pk", + "columns": [ + "computer_id", + "tool_call_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_snapshot": { + "name": "computer_snapshot", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "elements": { + "name": "elements", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "taken_at": { + "name": "taken_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "session": { + "name": "session", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_preferences": { + "name": "agent_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_preferences_user_id_users_id_fk": { + "name": "agent_preferences_user_id_users_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_preferences_agent_id_agents_id_fk": { + "name": "agent_preferences_agent_id_agents_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_preferences_user_id_agent_id_pk": { + "name": "agent_preferences_user_id_agent_id_pk", + "columns": [ + "user_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_profiles": { + "name": "agent_profiles", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_description": { + "name": "role_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_seed": { + "name": "avatar_seed", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "agent_visibility", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "callback_token_hash": { + "name": "callback_token_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "callback_token_issued_at": { + "name": "callback_token_issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_profiles_visibility_deleted_idx": { + "name": "agent_profiles_visibility_deleted_idx", + "columns": [ + { + "expression": "visibility", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_profiles_agent_id_agents_id_fk": { + "name": "agent_profiles_agent_id_agents_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_profiles_owner_user_id_users_id_fk": { + "name": "agent_profiles_owner_user_id_users_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routine_runs": { + "name": "routine_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "routine_id": { + "name": "routine_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "routine_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "routine_runs_by_routine_idx": { + "name": "routine_runs_by_routine_idx", + "columns": [ + { + "expression": "routine_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routine_runs_routine_id_routines_id_fk": { + "name": "routine_runs_routine_id_routines_id_fk", + "tableFrom": "routine_runs", + "tableTo": "routines", + "columnsFrom": [ + "routine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routines": { + "name": "routines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instruction": { + "name": "instruction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cron": { + "name": "cron", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "routines_due_idx": { + "name": "routines_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "routines_by_owner_idx": { + "name": "routines_by_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routines_owner_user_id_users_id_fk": { + "name": "routines_owner_user_id_users_id_fk", + "tableFrom": "routines", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "routines_agent_id_agents_id_fk": { + "name": "routines_agent_id_agents_id_fk", + "tableFrom": "routines", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_exclusions": { + "name": "component_exclusions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "withheld_by": { + "name": "withheld_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_exclusions_component_name_components_name_fk": { + "name": "component_exclusions_component_name_components_name_fk", + "tableFrom": "component_exclusions", + "tableTo": "components", + "columnsFrom": [ + "component_name" + ], + "columnsTo": [ + "name" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "component_exclusions_agent_id_agents_id_fk": { + "name": "component_exclusions_agent_id_agents_id_fk", + "tableFrom": "component_exclusions", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_exclusions_component_name_agent_id_pk": { + "name": "component_exclusions_component_name_agent_id_pk", + "columns": [ + "component_name", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_functions": { + "name": "component_functions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "function_name": { + "name": "function_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_functions_component_name_components_name_fk": { + "name": "component_functions_component_name_components_name_fk", + "tableFrom": "component_functions", + "tableTo": "components", + "columnsFrom": [ + "component_name" + ], + "columnsTo": [ + "name" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_functions_component_name_function_name_pk": { + "name": "component_functions_component_name_function_name_pk", + "columns": [ + "component_name", + "function_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.components": { + "name": "components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'first-party'" + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tools_refreshed_at": { + "name": "tools_refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_servers_credential_id_credentials_id_fk": { + "name": "mcp_servers_credential_id_credentials_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_tools": { + "name": "mcp_tools", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "input_schema": { + "name": "input_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_tools_server_id_mcp_servers_id_fk": { + "name": "mcp_tools_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_tools", + "tableTo": "mcp_servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_tools_server_id_name_pk": { + "name": "mcp_tools_server_id_name_pk", + "columns": [ + "server_id", + "name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_user_credentials": { + "name": "mcp_user_credentials", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_user_credentials_user_idx": { + "name": "mcp_user_credentials_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_user_credentials_server_id_mcp_servers_id_fk": { + "name": "mcp_user_credentials_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "mcp_servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_user_id_users_id_fk": { + "name": "mcp_user_credentials_user_id_users_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_credential_id_credentials_id_fk": { + "name": "mcp_user_credentials_credential_id_credentials_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_user_credentials_server_id_user_id_pk": { + "name": "mcp_user_credentials_server_id_user_id_pk", + "columns": [ + "server_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_grants": { + "name": "plugin_grants", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "plugin_grants_agent_idx": { + "name": "plugin_grants_agent_idx", + "columns": [ + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "plugin_grants_agent_id_agents_id_fk": { + "name": "plugin_grants_agent_id_agents_id_fk", + "tableFrom": "plugin_grants", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "plugin_grants_kind_ref_agent_id_pk": { + "name": "plugin_grants_kind_ref_agent_id_pk", + "columns": [ + "kind", + "ref", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandboxed_components": { + "name": "sandboxed_components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_html": { + "name": "draft_html", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_css": { + "name": "draft_css", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_js_functions": { + "name": "draft_js_functions", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_argument_schema": { + "name": "draft_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_html": { + "name": "published_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_css": { + "name": "published_css", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_js_functions": { + "name": "published_js_functions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_argument_schema": { + "name": "published_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "sample_arguments": { + "name": "sample_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "authored_by": { + "name": "authored_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_tools": { + "name": "skill_tools", + "schema": "", + "columns": { + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "declared_by": { + "name": "declared_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_tools_ref_idx": { + "name": "skill_tools_ref_idx", + "columns": [ + { + "expression": "ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_tools_skill_id_skills_id_fk": { + "name": "skill_tools_skill_id_skills_id_fk", + "tableFrom": "skill_tools", + "tableTo": "skills", + "columnsFrom": [ + "skill_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "skill_tools_skill_id_ref_pk": { + "name": "skill_tools_skill_id_ref_pk", + "columns": [ + "skill_id", + "ref" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skills": { + "name": "skills", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'yours'" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skills_slug_key": { + "name": "skills_slug_key", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skills_owner_idx": { + "name": "skills_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skills_owner_user_id_users_id_fk": { + "name": "skills_owner_user_id_users_id_fk", + "tableFrom": "skills", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.work_items": { + "name": "work_items", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_at": { + "name": "run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "work_items_claimable_idx": { + "name": "work_items_claimable_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "work_items_kind_key_pk": { + "name": "work_items_kind_key_pk", + "columns": [ + "kind", + "key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.agent_type": { + "name": "agent_type", + "schema": "public", + "values": [ + "built_in", + "remote_ag_ui", + "remote_mastra" + ] + }, + "public.credential_kind": { + "name": "credential_kind", + "schema": "public", + "values": [ + "model", + "connector", + "agent", + "mcp", + "mcp_oauth_client", + "mcp_user_token" + ] + }, + "public.role": { + "name": "role", + "schema": "public", + "values": [ + "admin", + "user" + ] + }, + "public.agent_visibility": { + "name": "agent_visibility", + "schema": "public", + "values": [ + "public", + "private" + ] + }, + "public.routine_run_status": { + "name": "routine_run_status", + "schema": "public", + "values": [ + "succeeded", + "failed", + "skipped" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/server/drizzle/meta/_journal.json b/server/drizzle/meta/_journal.json index d118bb44c..f94b8bf8f 100644 --- a/server/drizzle/meta/_journal.json +++ b/server/drizzle/meta/_journal.json @@ -211,6 +211,34 @@ "when": 1788911713422, "tag": "0029_mastra_agent_type", "breakpoints": true + }, + { + "idx": 30, + "version": "7", + "when": 1788989179914, + "tag": "0030_daffy_nighthawk", + "breakpoints": true + }, + { + "idx": 31, + "version": "7", + "when": 1788990260165, + "tag": "0031_hesitant_squadron_sinister", + "breakpoints": true + }, + { + "idx": 32, + "version": "7", + "when": 1789059537115, + "tag": "0032_rainy_lake", + "breakpoints": true + }, + { + "idx": 33, + "version": "7", + "when": 1789066859018, + "tag": "0033_attachments_uploaded_by_index", + "breakpoints": true } ] } diff --git a/server/scripts/cull-staged-attachments.ts b/server/scripts/cull-staged-attachments.ts new file mode 100644 index 000000000..088b348a3 --- /dev/null +++ b/server/scripts/cull-staged-attachments.ts @@ -0,0 +1,194 @@ +/** + * One sweep: delete attachments somebody staged and never sent. + * + * Attach a file, change your mind and close the tab, and that row sits in `attachments` forever — + * no message ever pointed back at it, just bytes with an `attached_at` that stayed null. + * `attachedAt` is set the moment an attachment is actually sent, so a null attachment old enough + * that the tab is long closed is not spoken for by anything and is safe to delete. + * + * Run from a CronJob rather than from a timer inside the API, for the reason the computer culler + * beside it states: every replica would fire its own timer and each would decide, independently, + * to sweep. Unlike that culler, this needs no lease and no claimed-by-this-pod bookkeeping — + * deleting a staged attachment twice is harmless, because the second sweep finds nothing left to + * delete. + * + * Exits non-zero only when the sweep itself could not run: no database to reach, or an argument + * that could not be parsed. A staged attachment that is a minute past the window and swept on the + * next run instead has lost nothing. + */ +import { and, isNull, lt, sql } from "drizzle-orm"; +import { createDatabase, type Database } from "../src/db/client"; +import { attachments } from "../src/db/schema"; + +/** How long a staged attachment is kept, absent a CLI argument saying otherwise. */ +const DEFAULT_OLDER_THAN_HOURS = 24; + +/** + * How many rows one statement may delete. + * + * Big enough that an ordinary sweep — a handful of abandoned uploads an hour — is a single + * statement and the loop below runs twice, and small enough that the row locks, the WAL record and + * the returned id array of one statement all stay bounded no matter how far behind the sweep is. + */ +const DEFAULT_BATCH_SIZE = 1_000; + +/** + * Delete every attachment that was staged and never sent, and is old enough that nobody is coming + * back for it. + * + * `isNull(attachments.attachedAt)` rather than a truthiness check on the column, on purpose: a + * truthiness check in application code would treat every falsy value as staged, but the column + * being compared is a timestamp, so the only value that trips it is one that was never set at all + * — which sounds safe until the query is written the same way and a driver hands back `null` for + * every row where the comparison itself was mis-stated, sweeping every attachment ever sent in the + * deployment rather than none of them. `IS NULL` is also the exact predicate + * `attachments_staged_idx` was built on, so this delete is an index scan rather than a sequential + * one. + * + * IN BATCHES, NOT IN ONE STATEMENT. This used to be a single unbounded `DELETE ... RETURNING id`, + * which has two failure modes that only appear on the deployment least able to absorb them — one + * that arrives here with a backlog, because the sweep was switched off, or the release predates it. + * The transaction holds a row lock on every doomed row for its whole length, and the driver + * materialises one id per deleted row purely so this function can read `.length`. Cut off by the + * CronJob's `activeDeadlineSeconds` at any point, that statement rolls back entirely and the next + * run redoes the same doomed work, forever. Batching turns the ceiling from a wedge into a pause: + * each batch is its own transaction when this runs on a pool, so a sweep that is killed halfway has + * still deleted everything it got through, and the next one starts from there. + * + * `for update skip locked` inside the subquery so a row somebody is sending RIGHT NOW — the update + * that sets `attachedAt` holds a lock on it — is stepped over rather than waited on. Under + * `read committed` the delete would re-check the predicate and skip such a row anyway; skipping it + * up front means one slow send cannot hold the whole sweep. A batch shortened by skipped rows ends + * the loop early, which costs nothing: those rows are still staged, still past the window, and the + * next sweep takes them. + * + * THE BATCH SIZE IS CHECKED RATHER THAN TRUSTED, and `??` is the reason it has to be. That operator + * defaults an ABSENT value and a null one, and nothing else: `batchSize: 0` is a number, so it is + * taken, and the loop above cannot come out of it. `limit 0` returns no rows, the delete removes + * none, and the termination test is `0 < 0` — false — so the sweep issues that same pair of + * statements for as long as the process lives. Only a caller of this function can reach it; the CLI + * below never passes a batch size, which is why the CronJob has never wedged on this and why the + * tests that drive the CLI could not have caught it. + * + * The neighbouring values are each wrong in their own way, all measured against this deployment's + * Postgres 16 through this same query builder rather than reasoned about: + * + * - `0.5` is the same endless loop wearing a friendlier face. `LIMIT` takes a bigint and a float8 is + * rounded to reach one, so a half becomes `limit 0` — a value nobody would read as "no rows" that + * behaves exactly like zero. + * - A NEGATIVE SIZE IS THE WORST OF THEM, and not for the reason it looks like. Postgres refuses + * `LIMIT -1` outright, but this never reaches Postgres as a limit at all: drizzle emits no `limit` + * clause whatsoever for a negative one, so the statement becomes the single unbounded + * `DELETE ... RETURNING id` over the entire backlog that the paragraph above exists to prevent — + * every doomed row locked for one transaction, every id materialised — and the loop still never + * ends, because no `batch.length` is ever `< -1`. `NaN` takes that same clause-dropping path, + * drizzle's guard being `>= 0` and every comparison with `NaN` being false. + * - A fraction at or above one does terminate, and is refused anyway. `1.5` rounds up to `limit 2`, + * so each statement deletes two rows and is then tested with `2 < 1.5`: a full batch that reads as + * a short one, ending the sweep one batch early on every run. A ceiling on the row locks and the + * WAL of one statement is not a number to accept a rounding of. + * + * A refusal rather than a silent correction — no clamping to 1, no rounding up. Every one of these + * is a caller saying something it does not mean about a statement that DELETES, and the clamped + * sweep would do bounded, plausible, wrong work while the caller kept believing its own number. + * + * `${hours}::float8 * interval '1 hour'` RATHER THAN `make_interval(hours => ${hours})`. The + * argument is an operator's number of hours, from a chart value or a command line, and + * `make_interval` takes an `int`: `make_interval(hours => 0.5)` is not a rounding, it is + * `function make_interval(hours => double precision) does not exist` — verified against Postgres 16 + * — so `attachments.culler.olderThanHours: 0.5` in the chart made EVERY hourly sweep die, with the + * error naming a function nobody set. Multiplying an interval accepts the same integers with the + * same result and gives a fractional window the meaning it obviously has: 0.5 is thirty minutes. + */ +export async function cullStagedAttachments( + database: Database, + options: { olderThanHours: number; batchSize?: number }, +): Promise { + const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE; + if (!Number.isSafeInteger(batchSize) || batchSize < 1) { + throw new RangeError( + "cullStagedAttachments needs a batch size that is a whole number of at least 1; " + + // `String`, not `JSON.stringify`: the two values most worth naming here, `NaN` and + // `Infinity`, are both `null` once JSON has been through them. + `got ${String(batchSize)}. Anything else either deletes the whole backlog in one ` + + "unbounded statement or leaves the sweep looping on batches it can never finish.", + ); + } + const cutoff = sql`now() - ${options.olderThanHours}::float8 * interval '1 hour'`; + + let deleted = 0; + for (;;) { + const doomed = database + .select({ id: attachments.id }) + .from(attachments) + .where( + and(isNull(attachments.attachedAt), lt(attachments.createdAt, cutoff)), + ) + // Oldest first, so a sweep that is cut off has reclaimed the rows nobody could still want. + .orderBy(attachments.createdAt) + .limit(batchSize) + .for("update", { skipLocked: true }); + + const batch = await database + .delete(attachments) + .where(sql`${attachments.id} in ${doomed}`) + .returning({ id: attachments.id }); + + deleted += batch.length; + if (batch.length < batchSize) { + return deleted; + } + } +} + +if (import.meta.main) { + const [, , rawHours] = process.argv; + const olderThanHours = Number(rawHours ?? DEFAULT_OLDER_THAN_HOURS); + if (!Number.isFinite(olderThanHours) || olderThanHours <= 0) { + throw new Error( + "cull-staged-attachments takes an optional number of hours as " + + `its one argument; got ${JSON.stringify(rawHours)}.`, + ); + } + + /* + * `DATABASE_URL`, AND NOTHING ELSE. NOT `loadConfig`. + * + * This used to call `loadConfig(process.env)` and read one field of the result, + * `config.databaseUrl`. That builds the whole `DeploymentConfig` first, which refuses to return + * without `KEY_ENCRYPTION_KEY`, the three Intelligence addressing values, and a complete identity + * provider or `OPENBOT_SINGLE_USER`. So a sweep that deletes rows and touches no ciphertext, + * no model and no session died at start-up with `KEY_ENCRYPTION_KEY must be configured`, and the + * documented external-cron path in `docs/deployment.md` — "it needs only DATABASE_URL" — was + * simply false. It worked under Helm only because that CronJob injected five credentials it had + * no use for, in the pod with the least reason of any to hold them. + * + * NO VALIDATION IS LOST BY DROPPING `loadConfig`, which is the thing to check before believing + * this: `loadConfig` does `required(environment, "DATABASE_URL")`, a non-empty-after-trim test and + * no more. Every check that makes a connection string legible — a `%` in the password that starts + * no escape, a URL with no host, a URL naming no database — lives in `addressOf` inside + * `createDatabase`, below, and applies to exactly this call. + * + * REJECTED: teaching `config.ts` a narrower loader. `loadConfig` is the API server's boot + * contract and the two sibling cron scripts genuinely want it — `fire-routines.ts` needs + * `workerSharedSecret`, `cull-idle-computers.ts` needs `computer.idleAfterMs`. The convention this + * follows instead is `scripts/migrate.ts`, the other script whose whole need is a database: it + * reads `DATABASE_URL` from the environment and says so in one sentence when it is missing. + */ + const databaseUrl = process.env.DATABASE_URL?.trim(); + if (!databaseUrl) { + throw new Error( + "DATABASE_URL must be configured before sweeping staged attachments", + ); + } + const database = createDatabase(databaseUrl); + + try { + const deleted = await cullStagedAttachments(database, { + olderThanHours, + }); + console.info(JSON.stringify({ type: "attachment-cull", deleted })); + } finally { + await database.$client.end({ timeout: 5 }); + } +} diff --git a/server/src/app.ts b/server/src/app.ts index f5762844a..44e205ade 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -1,6 +1,8 @@ import type { Hono as HonoApp, MiddlewareHandler } from "hono"; import { Hono } from "hono"; +import { bodyLimit } from "hono/body-limit"; import { serveStatic } from "hono/bun"; +import { MAX_IMAGE_BYTES } from "../../shared/attachments"; import { authoriseAgentCall, sameToken } from "./agents/callback-token"; import type { BotAccessCheck } from "./agents/profile-policy"; import type { AgentProfileStore } from "./agents/profile-store"; @@ -23,6 +25,10 @@ import { requireAdmin, } from "./auth/guards"; import type { IdentityProviderStore } from "./auth/identity-provider-store"; +import { + createAttachmentRoutes, + createChannelAttachmentRoutes, +} from "./channels/attachments"; import type { ChannelEventHub } from "./channels/events"; import { type ChannelStore, createChannelRoutes } from "./channels/routes"; import type { ThreadIdentity } from "./channels/thread-identity"; @@ -38,6 +44,7 @@ import type { PolicyStore } from "./computer/policy-store"; import { createComputerRoutes } from "./computer/routes"; import { configuredAuthProviders, type DeploymentConfig } from "./config"; import type { CredentialAdminService, CredentialInput } from "./credentials"; +import type { Database } from "./db/client"; import { createIntelligenceClient } from "./intelligence-client"; import type { OnboardingStore } from "./people/onboarding"; import type { PeopleStore } from "./people/store"; @@ -55,6 +62,39 @@ import { type UserInstructionsStore, } from "./user-instructions"; +/** + * How much of a multipart body is boundary, headers and other fields rather than file. + * + * Generous on purpose. Measured against what the composer actually sends — one `file` part and one + * `uploadGroup` field — the framing is 360 bytes for a short filename and 614 for a 255-character + * one; a filename full of non-ASCII percent-encodes to a few times that and is still nowhere near + * this. 64 KiB is therefore an allowance no honest request can exhaust, and it raises the amount of + * memory a hostile request can pin by 0.8%, which was never the number that mattered. + */ +const MULTIPART_FRAMING_ALLOWANCE = 64 * 1024; + +/** + * The ceiling on the whole POST body of a channel attachment upload. + * + * THIS IS NOT `MAX_IMAGE_BYTES`, AND THE DIFFERENCE IS THE POINT. Every other gate on this path — + * the composer's pre-check, `attachmentsConfigFor`'s `maxSize`, the handler's own 413 — measures + * THE FILE. This one measures THE ENVELOPE: `bodyLimit` runs before anything has parsed the + * multipart body, so all it can count is bytes on the wire, file and framing together. + * + * Set to `MAX_IMAGE_BYTES` exactly, those two units were silently treated as one, and the ~360 + * bytes of boundary and headers wrapped around a file at the documented ceiling were enough to push + * the body over it: an 8,388,608-byte image — the exact number the composer publishes as the limit — + * was refused 413, while 8,388,308 bytes went through. A limit nobody can reach is a limit that is + * wrong, so the envelope's ceiling is the file's ceiling plus room for the envelope. + * + * The slack costs nothing it was protecting against. A body between the two numbers is still read + * into memory, and then still refused by the handler once `file.size` is a thing anybody can look + * at — which is where a text upload, whose real limit is `MAX_FILE_BYTES`, is refused too. What the + * door exists to stop is the 2GB body, and it still does. + */ +export const UPLOAD_BODY_LIMIT_BYTES = + MAX_IMAGE_BYTES + MULTIPART_FRAMING_ALLOWANCE; + /** * One row for something an administrator did to somebody's access. * @@ -221,6 +261,17 @@ export function createApp( * shown an empty box, and the obvious thing to do with an empty box is fill it in again. */ userInstructions?: UserInstructionsStore, + /** + * The database behind a channel's staged and sent files: upload, fetch, delete. + * + * Appended last, like everything above it: these are positional, so inserting one anywhere else + * silently shifts every existing call site's arguments by one. + * + * Absent leaves the routes unmounted rather than mounted and refusing every call, the same + * degraded shape every other optional store here takes: a deployment that never built the + * database has no door for this at all, not a locked one. + */ + attachmentDatabase?: Database, ) { const app = new Hono<{ Variables: AppVariables }>(); @@ -1047,6 +1098,68 @@ export function createApp( ); } + if (attachmentDatabase) { + /* + * `bodyLimit` sits in front of the upload route itself, not beside the mount below: the handler + * in channels/attachments.ts calls `file.arrayBuffer()` before it has looked at a single byte of + * size, so an unbounded body is read into memory in full before anything gets the chance to + * refuse it. A person (or an attacker) posting a 2GB body would have it buffered in RAM before + * the 413 the handler already knows how to return. `MAX_IMAGE_BYTES` is the largest thing this + * route could ever legitimately accept — a text upload is refused smaller, inside the handler, + * once the sniffed type is known — so refusing anything larger at the door costs nothing a real + * upload was ever going to use. + * + * The ceiling is `UPLOAD_BODY_LIMIT_BYTES` and not `MAX_IMAGE_BYTES` itself because THIS GATE + * MEASURES A DIFFERENT THING FROM EVERY OTHER ONE. See that constant. + */ + const channelAttachments = new Hono<{ Variables: AppVariables }>(); + channelAttachments.use( + "*", + bodyLimit({ + maxSize: UPLOAD_BODY_LIMIT_BYTES, + /* + * THE REFUSAL AT THE DOOR HAS TO LOOK LIKE THE HANDLER'S OWN. + * + * hono's default `onError` answers with the plain string "Payload Too Large". The composer + * (app/src/components/channels/composer/attachments.ts) reads `{ error }` off every failed + * upload and falls back to a generic `Could not upload ""` when the body will not + * parse as JSON — so the default body cost the person the one sentence that would have told + * them what went wrong, on the single refusal where the reason is both knowable and + * actionable. This is the same `{ error }` shape and the same number the handler's own 413 + * names, so the two paths are indistinguishable from the outside. + * + * THE FILENAME AND THE KIND ARE BOTH DELIBERATELY ABSENT, and for the same reason: nothing + * has parsed the multipart body at this point, which is the entire reason this middleware + * runs ahead of the handler. The handler's sentences can say `'notes.txt' is larger than the + * 1MB limit for files` because by then it has sniffed the bytes. This one cannot, and must + * not guess — a 9MB text file refused here as being over "the 8MB limit for images" would + * send somebody off to shrink it to 7MB, whereupon the handler would refuse it a second time + * with a different number. So the sentence names the only thing that is true of every body + * this gate rejects: none of them can be under the largest ceiling the route has. + */ + onError: (context) => + context.json( + { + // The same rounding as `megabytes` in channels/attachments.ts, so the door and the + // handler name one limit in one voice. + error: `That upload is larger than the ${(MAX_IMAGE_BYTES / (1024 * 1024)).toFixed(0)}MB limit.`, + }, + 413, + ), + }), + ); + channelAttachments.route( + "/", + createChannelAttachmentRoutes(attachmentDatabase, requireUser), + ); + app.route("/api/channels", channelAttachments); + + app.route( + "/api/attachments", + createAttachmentRoutes(attachmentDatabase, requireUser), + ); + } + if (routineStore) { app.route("/api/routines", createRoutineRoutes(routineStore, requireUser)); } diff --git a/server/src/channels/attachment-mime.ts b/server/src/channels/attachment-mime.ts new file mode 100644 index 000000000..7435e5f6c --- /dev/null +++ b/server/src/channels/attachment-mime.ts @@ -0,0 +1,271 @@ +import { + ACCEPTED_IMAGE_MIME, + ACCEPTED_TEXT_MIME, + mediaTypeOf, + namesNoFormat, +} from "../../../shared/attachments"; + +/** + * The text claims this function will hand back under their own name. + * + * Built from `ACCEPTED_TEXT_MIME` rather than listed again, because the two + * lists have to be the same list: a name returned here that `classifyAttachment` + * does not accept is a file refused for a reason nobody can read, and a name + * accepted there but missing here is a text file the byte guess relabels as + * `text/plain` — a `.csv` silently becoming a `.txt`. + * + * THIS IS NOT DEAD CODE, AND THE BRANCH IT GUARDS IS NOT REDUNDANT WITH THE + * `namesNoFormat` CHECK FURTHER DOWN. Two review rounds have now called it + * dead, so the refutation is written down here rather than rediscovered a + * third time. + * + * It is REACHED by every text claim: nothing between the top of + * `sniffMimeType` and its use filters those out. `bytes.length === 0` returns + * first, and `sniffImageType` returns first, but a `.csv` claiming `text/csv` + * is neither empty nor an image, so it arrives here. The test "recognized + * claims win outright once no image signature matches" walks that path. + * + * It also CHANGES THE ANSWER, which is the half that looks redundant and is + * not. The tempting reading is that `if (!namesNoFormat(normalizedClaim)) + * return normalizedClaim` below would return `text/plain` for a `text/plain` + * claim anyway, so this branch merely arrives at the same place early. That + * is true for bytes that ARE text and false for bytes that are not, and the + * false case is the whole point: delete this branch and a stripped executable + * claiming `text/plain` falls through to that line, is returned as + * `text/plain`, and is stored and served as accepted text from this app's own + * origin on the client's word alone. With the branch it becomes + * `application/octet-stream` and is refused. The test "bytes that are not + * UTF-8 at all, claimed text/plain, are not text" fails the moment this is + * removed. + */ +const MIME_BY_LOWER_CLAIM = new Set(ACCEPTED_TEXT_MIME); + +function hasSignature( + bytes: Uint8Array, + signature: number[], + offset = 0, +): boolean { + if (bytes.length < offset + signature.length) return false; + return signature.every((byte, i) => bytes[offset + i] === byte); +} + +function sniffImageType(bytes: Uint8Array): string | null { + if (hasSignature(bytes, [0x89, 0x50, 0x4e, 0x47])) return "image/png"; + if (hasSignature(bytes, [0xff, 0xd8, 0xff])) return "image/jpeg"; + if (hasSignature(bytes, [0x47, 0x49, 0x46, 0x38])) return "image/gif"; + if ( + hasSignature(bytes, [0x52, 0x49, 0x46, 0x46]) && + hasSignature(bytes, [0x57, 0x45, 0x42, 0x50], 8) + ) { + return "image/webp"; + } + return null; +} + +// TextDecoder in "fatal" mode throws on invalid UTF-8 instead of substituting +// U+FFFD, which is what lets the checks below tell "genuinely UTF-8 text" +// apart from "bytes that happen to decode without error but were never text." +// Both callers depend on that: the guess at the end of `sniffMimeType`, and +// the corroboration of a text claim before it. +const strictUtf8Decoder = new TextDecoder("utf-8", { fatal: true }); + +function isValidUtf8(bytes: Uint8Array): boolean { + try { + strictUtf8Decoder.decode(bytes); + return true; + } catch { + return false; + } +} + +/** + * A file's `type` field is whatever the uploading client said it was, and + * this app turns around and serves that string back as the `Content-Type` + * header on its own origin. This function *resolves* that claim rather than + * trusting it, but how much the bytes can settle differs sharply between the + * two families of type this app accepts, and it is worth being exact about + * which is which. + * + * For the four ACCEPTED IMAGE types the bytes decide outright. Each one has a + * magic number in `sniffImageType`, so the bytes name the format, and the + * claim is only ever a tie-breaker that the bytes can overrule — a JPEG + * claiming `image/png` comes back `image/jpeg`. + * + * For the four ACCEPTED TEXT types the bytes decide much less. There is no + * signature that distinguishes Markdown from CSV from JSON from prose; every + * one of them is just UTF-8. So the bytes can only answer whether the file is + * text AT ALL, never which text format it is, and the format that comes back + * is the CLIENT'S CLAIM — corroborated by the UTF-8 check, but not verified. + * A `.csv` uploaded as `text/markdown` is stored and served as Markdown, and + * nothing here can tell. What the corroboration buys is narrower than + * verification and still worth having: bytes that are not text cannot wear a + * text name, so a binary blob can no longer be stored and served as + * `text/plain` on this origin just by saying so. + * + * The corroboration is real content or nothing: a zero-byte file is refused + * outright, because "these bytes decode as UTF-8" is trivially true of no + * bytes and would otherwise wave an empty file through as accepted text. + * + * A claim that names nothing (blank, or a generic + * "I don't know what this is" like `application/octet-stream`) is + * discarded and the bytes are sniffed instead. But a claim that names a + * specific format — `image/svg+xml`, `text/html` — comes back by name on + * purpose, even though it is not verified against the bytes, because the + * caller needs that name to refuse it (an inline SVG is valid UTF-8 text, + * and laundering it into `text/plain` here would erase the one signal + * that lets the caller block it). Identifying a format is not the same as + * authorizing it: the caller MUST run this function's result through + * `classifyAttachment` before anything is stored or served. + * + * The exceptions to "a specific claim comes back by name" are the eight + * claims naming a type this app ACCEPTS. Naming a format the caller will + * refuse is harmless; naming one it will store and serve is not, so those + * eight names have to be earned from the bytes rather than asserted — the + * four image names in full, from a signature, and the four text names as far + * as bytes can go, from the UTF-8 check. Those two lists are + * `ACCEPTED_IMAGE_MIME`, imported from `shared/attachments.ts` because the + * composer screens against the same one, and `MIME_BY_LOWER_CLAIM`, at the top + * of this file. + */ +export function sniffMimeType(bytes: Uint8Array, claimed: string): string { + /* + * A file with no bytes corroborates nothing, and every accepted answer + * below is earned from content: an image signature, or bytes that decode + * as UTF-8. The second of those is trivially true of an empty file — + * `isValidUtf8(new Uint8Array(0))` is `true` — so without this line an + * empty upload came back `text/plain` and was stored and served as an + * accepted text attachment, whether it claimed a text type or claimed + * nothing at all. Refused once here rather than in each branch, so no + * later path can hand back an accepted name for a file that isn't there. + */ + if (bytes.length === 0) return "application/octet-stream"; + + const sniffedImage = sniffImageType(bytes); + if (sniffedImage) return sniffedImage; + + /* + * `mediaTypeOf`, NOT A LOCAL COPY OF WHAT IT DOES. + * + * This line used to be `claimed.toLowerCase().split(";")[0].trim()` — + * character for character the body of `mediaTypeOf`, and the two stayed in + * agreement only by coincidence. `shared/attachments.ts` calls that form + * "THE ONLY FORM ANYTHING HERE COMPARES" and its doc names this function as + * a caller that "already normalises the same two ways", which was a claim + * about a duplicate rather than a reference to the original. + * + * The cost of the duplicate is the drift the shared file exists to prevent. + * RFC 2045 also allows quoted parameters and whitespace before the `;`, so + * if `mediaTypeOf` ever grows a third step the composer would start reading + * a file one way and this server another — one side taking a file the other + * turns away, which is exactly the failure the top-of-file note describes. + * + * `namesNoFormat` below normalises again internally. That is deliberate + * redundancy, not waste: it is exported for callers holding a raw claim, so + * it cannot assume it has been through here first, and the operation is + * idempotent. + */ + const normalizedClaim = mediaTypeOf(claimed); + + /* + * A text claim this app accepts is corroborated the only way text can be: + * the bytes are asked whether this is text at all, not which text format + * it is, because no signature tells Markdown from CSV from JSON. So the + * claim survives the check rather than being replaced by it. + * + * Unverified is not the same as untested. Before this check, `text/plain` + * was returned on the client's word alone, which meant any bytes at all — + * a stripped executable, an encrypted blob — could be stored and served + * from this origin under a text name simply by claiming one. Bytes that + * are not text now fail here. + * + * The failure drops to the generic "just bytes" type, which + * `classifyAttachment` refuses, exactly as the accepted-image branch below + * does and for the same reason: falling through to the guesses further + * down would let a refused claim try its luck under a different name. + */ + if (MIME_BY_LOWER_CLAIM.has(normalizedClaim)) { + return isValidUtf8(bytes) ? normalizedClaim : "application/octet-stream"; + } + + /* + * An image claim this app accepts is the one claim the bytes have to + * corroborate. `sniffImageType` knows a signature for every member of + * `ACCEPTED_IMAGE_MIME`, so reaching this line with such a claim means + * the bytes are NOT that image — and returning the name anyway is what + * let arbitrary bytes (an SVG, most of all) be labelled `image/png` and + * come back as an accepted image. The SVG refusal in + * `shared/attachments.ts` is keyed on this function's answer, so a claim + * that is never checked is a refusal that can be renamed around. + * + * The claim is dropped for the generic "just bytes" type, which + * `classifyAttachment` refuses, rather than falling through to the + * UTF-8 guess below — an SVG decodes as valid UTF-8, and turning it into + * `text/plain` would store and serve the very file that was being + * smuggled, just under a different label. + * + * Only the ACCEPTED names are dropped. An image claim this app does not + * accept (`image/svg+xml`, `image/heic`) still comes back verbatim below, + * because the caller refuses it by name and that name is what makes the + * refusal say something useful. The coupling runs one way: adding a type + * to `ACCEPTED_IMAGE_MIME` without adding its signature to + * `sniffImageType` refuses every file of that type outright — a loud + * failure, which is the right direction for this to break in. + */ + if ((ACCEPTED_IMAGE_MIME as readonly string[]).includes(normalizedClaim)) { + return "application/octet-stream"; + } + + // A claim shaped like a real MIME type (e.g. "image/svg+xml") names a + // specific format the caller may need to refuse by name — an inline SVG + // is valid UTF-8 text, but laundering it into "text/plain" here would + // erase the one signal ("this claims to be SVG") the caller needs to + // block it. A claim that names nothing (blank, not shaped like a MIME + // type, or a generic placeholder such as "application/octet-stream") + // falls through to content-guessing instead. `namesNoFormat` is that + // question, and it lives in `shared/attachments.ts` because the composer + // asks the identical one before it refuses a pick: a list kept twice is a + // list that drifts, and the drift here is a file one side takes and the + // other turns away. + if (!namesNoFormat(normalizedClaim)) { + return normalizedClaim; + } + + if (isValidUtf8(bytes)) return "text/plain"; + + /* + * Neither an image signature, a trusted claim, nor guessable UTF-8 text. + * + * THE CLAIM IS NOT HANDED BACK HERE, AND THE REASON IS THE LINE ABOVE THIS + * BLOCK RATHER THAN ANYTHING ABOUT THESE BYTES. The only way to reach this + * line is for `namesNoFormat` to have already judged the claim to name no + * format — blank, not shaped like a MIME type, or one of the generic + * placeholders. Returning it would be returning a non-answer under the + * pretence that it is a media type. + * + * That mattered because this function's answer is read aloud. `attachments.ts` + * builds `'x.bin' is not a file type this app can read ().` from it, + * and a browser that claimed nothing at all made `normalizedClaim` the empty + * string — so the sentence came out `... can read ().`, a parenthetical that + * names nothing because there was nothing to name. The generic sentence it + * was meant to improve on was better than that. + * + * `application/octet-stream` is the name this file already uses for "these + * are just bytes" in three other places (the empty file above, the + * uncorroborated text claim, the uncorroborated image claim), so the caller + * sees one name for one situation rather than a different spelling of "I + * don't know" for each browser — `application/unknown` and + * `binary/octet-stream` reach here too, and used to produce three different + * refusals for the same unreadable file. + * + * REJECTED: leaving this alone and having `attachments.ts` drop the + * parenthetical when the string is empty. That puts the repair in the caller + * and leaves the hole here for the next caller to fall into, and this + * function's contract is better stated as "always returns a media type" than + * as "returns a media type, or sometimes not, mind how you print it". + * + * A claim that DOES name a format never reaches this line — it returned + * verbatim above, which is what keeps `image/svg+xml` and `text/html` + * refusable by name. + */ + return "application/octet-stream"; +} diff --git a/server/src/channels/attachment-parts.ts b/server/src/channels/attachment-parts.ts new file mode 100644 index 000000000..15f72ad32 --- /dev/null +++ b/server/src/channels/attachment-parts.ts @@ -0,0 +1,609 @@ +import { + attachmentUrl, + classifyAttachment, + MAX_EXTRACTED_CHARACTERS, +} from "../../../shared/attachments"; + +/** + * What `load` hands back for an attachment id: the bytes and the metadata + * needed to put them in front of the model, nothing about how they were + * stored. + */ +export type StoredAttachment = { + mimeType: string; + name: string; + bytes: Buffer; +}; + +/** + * `attachmentUrl("")` rather than a hand-typed literal, so this file cannot + * drift from the one place (`shared/attachments.ts`) that defines the URL + * shape a stored message actually carries. + */ +const ATTACHMENT_URL_PREFIX = attachmentUrl(""); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +/** + * The attachment id a part points at, or null if this part is not one of + * ours — already-inline content (`source.type === "data"`), a plain text + * part with no `source` at all, or a URL that does not name an attachment. + * Any of those pass through untouched. + * + * THE SOURCE URL IS THE ONLY GATE, AND `part.type` IS DELIBERATELY NOT READ. + * AG-UI's part union is `text | image | audio | video | document | binary`, + * so a client that writes its own message content can send any of the six + * naming one of our URLs; the `AttachmentPart` type in `shared/attachments.ts` + * only says what OUR composer emits. Gating on `image`/`document` here was the + * alternative and is worse: an `audio` part naming a real attachment would + * then keep its `/api/attachments/` source, which no model provider goes + * and fetches, so the file would be silently absent from a turn that claims to + * carry it — and never stamped `attachedAt`, so the sweeper would reclaim it a + * day later. Resolving every part that names one of our ids and deciding the + * modality from the STORED MIME TYPE instead (see `resolvePart`) leaves the + * declared type with nothing to lie about. + */ +function attachmentIdFor(part: unknown): string | null { + if (!isRecord(part) || !isRecord(part.source)) return null; + const source = part.source; + if (source.type !== "url" || typeof source.value !== "string") return null; + if (!source.value.startsWith(ATTACHMENT_URL_PREFIX)) return null; + return source.value.slice(ATTACHMENT_URL_PREFIX.length); +} + +/** + * Every attachment id a message's content names, in the order the parts carry them and each one + * only once. + * + * Exported so the caller that knows WHICH message it is holding — `inlineAttachments` in + * `copilot.ts`, the one place that can tell the message being asked about from the history behind + * it — can say those ids went out in a send. Resolving a part cannot make that statement itself: + * every message in the thread is resolved on every turn, and only one of them is the send. + * + * Content that is not an array of parts, or an array naming no attachment, is an empty list rather + * than an error: that is almost every message in almost every thread. + */ +export function attachmentIdsIn(content: unknown): string[] { + if (!Array.isArray(content)) return []; + const ids = new Set(); + for (const part of content) { + const id = attachmentIdFor(part); + if (id !== null) ids.add(id); + } + return [...ids]; +} + +/** + * Cuts extracted text at `MAX_EXTRACTED_CHARACTERS` and says so, rather + * than either sending the whole file (which can blow the context window on + * its own, see `shared/attachments.ts`) or dropping the overflow silently, + * which would leave the model answering questions about a file it read + * only part of with no sign that happened. + * + * AND NEVER THROUGH THE MIDDLE OF A CHARACTER. `slice` counts UTF-16 code + * units, so a limit landing between the two halves of a surrogate pair — and + * every emoji, every astral-plane glyph, every CJK extension character is one + * pair — left a lone high surrogate as the last code unit of the text. That is + * not a character: it means nothing on its own, `JSON.stringify` emits it as a + * bare `\ud83d` escape, and a provider handed one either rejects the request + * outright or silently substitutes U+FFFD. Either way a file whose + * hundred-and-twenty-thousandth code unit happens to land inside a glyph + * damages a turn for a reason having nothing to do with what the file says. + * `withinFilenameLimit` in `channels/attachments.ts` guards exactly this hazard + * on exactly this kind of cut, and says so in a comment; the cut here — applied + * to far more bytes, far more often — did not. + * + * DROPPING THE ORPHAN RATHER THAN REACHING FOR ITS PAIR, because the pair's + * other half sits AT `MAX_EXTRACTED_CHARACTERS`, past the limit, and a + * truncation that goes one code unit over the stated bound to stay whole is a + * stranger rule than one that stops one short of it. The orphan also cannot be + * a lone surrogate that was already in the file: `toString("utf8")` turns every + * malformed sequence into U+FFFD, so the only thing that can leave a high + * surrogate last here is a pair this slice just split. + * + * The note names how many characters were actually KEPT rather than naming the + * constant. The two differ by one, exactly when this guard fires, and a note + * that named the constant either way would be describing a cut that did not + * happen — which is the class of failure this whole function exists to avoid. + */ +function extractDocumentText(bytes: Buffer): string { + const full = bytes.toString("utf8"); + if (full.length <= MAX_EXTRACTED_CHARACTERS) return full; + const sliced = full.slice(0, MAX_EXTRACTED_CHARACTERS); + const last = sliced.charCodeAt(sliced.length - 1); + const splitAPair = last >= 0xd800 && last <= 0xdbff; + const cut = splitAPair ? sliced.slice(0, -1) : sliced; + return `${cut}\n\n[attachment truncated at ${cut.length} characters]`; +} + +/** + * What a turn does about an attachment this deployment can no longer load. + * + * "fail" for the message being asked about, "note" for everything behind + * it. See {@link resolveAttachmentParts} for why those are different + * answers to the same missing row. + */ +export type MissingAttachment = "fail" | "note"; + +/** + * The name to call an attachment in a note the model reads. + * + * `metadata.filename` when the part carries one, because that is the name the + * person saw when they attached it and the name they will use if they ask + * about it again. The id is the fallback: less use to a reader, but it is + * what the stored part always has. + */ +function displayName(part: Record, id: string): string { + const metadata = isRecord(part.metadata) ? part.metadata : undefined; + return typeof metadata?.filename === "string" && metadata.filename.length > 0 + ? metadata.filename + : id; +} + +/** + * How a failure names an attachment: the name the person gave it, and the id behind it. + * + * The two errors in this file used to name the raw uuid alone, which is the one identifier the + * person who attached the file has never seen — they picked `photo.png` out of a file dialog and + * the uuid was minted by the upload route afterwards. So the display name leads. The id stays, + * because it is what an operator correlating a failure against a row or a log line needs, and it is + * the only one of the two guaranteed to be unique. + * + * THE PARENTHETICAL IS DROPPED WHEN IT WOULD REPEAT ITSELF. `displayName` falls back to the id for + * a part carrying no `metadata.filename`, and those parts are common — anything not written by our + * own composer. `Attachment "abc" (id "abc")` reads like a bug in the sentence rather than a fact + * about the file. + */ +function namedForFailure(part: Record, id: string): string { + const name = displayName(part, id); + return name === id ? `"${id}"` : `"${name}" (id "${id}")`; +} + +/** The text part that stands in for a vanished attachment. */ +function unavailableNote(part: Record, id: string): unknown { + return { + type: "text", + text: `[attachment "${displayName(part, id)}" is no longer available]`, + }; +} + +/** + * The text part that stands in for an attachment this run had no room left for. + * + * A DIFFERENT SENTENCE FROM {@link unavailableNote}, ON PURPOSE. "No longer + * available" is a statement about the deployment: the row is gone and asking + * again will not bring it back. This one is about this turn only — the file is + * still there, and a question about it directly makes it the message being + * asked about, which is the first thing the budget is spent on and so is served + * whole — or refused outright, never quietly reduced to this note. Telling + * somebody their file was deleted when it was not is the kind of wrong answer + * that gets acted on. + */ +function notIncludedNote(part: Record, id: string): unknown { + return { + type: "text", + text: `[attachment "${displayName(part, id)}" from an earlier message was not included in this turn]`, + }; +} + +/** + * The refusal for an attachment on the message being asked about that this run has no room for. + * + * A THROW WHERE HISTORY GETS {@link notIncludedNote}, AND THAT ASYMMETRY IS THE WHOLE POINT. + * `MAX_INLINED_BYTES_PER_RUN` used to bound only the half of a run that could degrade: both places + * that stopped spending tested for `onMissing === "note"`, and the message being asked about is + * resolved under `"fail"`, so nothing at all bounded the one message a browser had just written. + * A member naming two hundred previously-sent 8 MiB attachments in a single message inlined about + * 1.6 GiB, plus its base64 on top, in one turn — precisely the heap exhaustion the budget exists to + * prevent, arriving through the one door it left open. + * + * THE FIX IS NOT TO MAKE THAT MESSAGE CUTTABLE. Cutting it would silently drop files out of the + * message somebody is asking a question ABOUT, which is the exact failure the strict `"fail"` mode + * was written to prevent: an answer given confidently about a file the model never received, with + * nothing in the transcript saying so. The guarantee worth keeping is "the asked message is served + * in full, or the turn fails loudly"; all that was missing is that "in full" be a BOUNDED quantity. + * Past the bound, the turn fails loudly. A refusal naming the problem is an answer somebody can act + * on, and silent truncation is not. + * + * IT NAMES THE FILE, THE ID, THE LIMIT AND A WAY OUT, because unlike every other failure in this + * file this one is about the person's own most recent action, which they can still change: the + * message is still in front of them. The display name is what they will recognise, the id is what + * an operator reading a log can grep for, and the limit is what turns "too big" into a number. + */ +function tooMuchToInline( + part: Record, + id: string, + limit: number, +): never { + throw new Error( + `Attachment ${namedForFailure(part, id)} could not be included: this message's attachments come to more than the ${limit} bytes one turn may put in front of the model. Send fewer files, or ask about them across more than one message.`, + ); +} + +/** + * The text part that stands in for a stored file nothing here knows how to read. + * + * Unreachable through today's upload route, which runs `classifyAttachment` + * over the sniffed type before it stores anything. It becomes reachable the + * day a type leaves `ACCEPTED_IMAGE_MIME` or `ACCEPTED_TEXT_MIME` while rows + * of it are still in the table, and the two alternatives are both worse: + * `toString("utf8")` on a PDF hands the model a page of mojibake it will + * happily summarise, and throwing would fail every future turn in the channel + * over a file the person cannot re-attach either — the composer would refuse + * the same type at pick time. A note keeps the one property that matters, + * that nothing answers as though the file were in front of it, and names the + * type so a person reading the transcript can tell what happened. + */ +function unreadableNote( + part: Record, + id: string, + mimeType: string, +): unknown { + return { + type: "text", + text: `[attachment "${displayName(part, id)}" is a ${mimeType} file, which cannot be put in front of the model]`, + }; +} + +/** + * How many STORED bytes one run may put in front of the model. + * + * `MAX_IMAGE_BYTES` bounds one file; until this existed nothing bounded a + * turn. History is replayed in full on every turn, so a channel that has seen + * four messages of eight 8 MB images cost every later turn ~256 MB read out of + * `bytea` and ~340 MB of base64 on top of it, all live at once — and the way + * that fails is not a refusal anybody can read, it is the pod's heap, which + * takes every other person's in-flight run down with it. + * + * 32 MiB of stored bytes is about 43 MB once base64'd. Four files at the + * `MAX_IMAGE_BYTES` ceiling, or thirty-two at the `MAX_FILE_BYTES` one: well + * past what a conversation refers back to, and far short of what exhausts a + * process. + * + * SPENT NEWEST-FIRST, WHICH IS WHY THE ASKED MESSAGE IS SERVED WHOLE. + * `inlineAttachments` walks the history backwards, so the message being asked + * about is charged first and is never the one cut; what runs out is the room + * left for the messages behind it, which the model has already been shown once + * in the turn they arrived. + * + * AND IT BOUNDS THAT MESSAGE TOO, BY REFUSING IT RATHER THAN CUTTING IT. + * Charged first is not the same as bounded, though for a while this comment was + * read that way: both places that stopped spending tested for + * `onMissing === "note"`, so this number bounded only the half of a run that + * could degrade, and a message a browser wrote naming two hundred + * previously-sent 8 MiB files inlined every one of them. A message past this + * limit now fails its turn with a sentence a person can act on — see + * {@link tooMuchToInline} — which keeps what the strict mode is for, that the + * asked message arrives in full or not at all, while giving "in full" a + * ceiling. + * + * WHAT IT COUNTS IS PARTS EMITTED, NOT DISTINCT FILES READ — AND THE DIFFERENCE + * IS NOT A ROUNDING ERROR. For a while the charge was deduplicated by id: a + * `charged` set meant the second and every later part naming one id was both + * free and exempt from the cut, on the reasoning that one id is fetched once so + * it should be billed once. The premise is true and the conclusion does not + * follow. `resolveAttachmentParts` emits a base64 part for EVERY OCCURRENCE of + * an id — it must, because two parts cannot share one object — so what a run + * holds live is one encoded copy per PART, and a bound that counts distinct ids + * is not measuring the quantity it exists to bound. + * + * The measured failure: forty parts naming one stored 1,024-byte image, against + * a budget of 1,024, produced one read, forty inlined parts, 40,960 decoded + * bytes — and `remaining` sitting at zero, reporting a budget spent exactly to + * its limit. At the 8 MiB upload ceiling a hundred references to one file come + * to roughly 1.04 GiB of base64 against a 32 MiB budget. A repeated id was not + * an exotic input either: it is what quoting the same chart twice in a message + * looks like, and it cost nothing to write. + * + * SO THE CHARGE RUNS PER PART AND THE MEMO STAYS. `loadOnce` still fetches one + * id once — deduplicating the READ was never the bug and saves a real database + * round trip — but every part that gets encoded draws the budget down by the + * stored bytes it is about to encode, and once the room is gone no id is + * exempt from being cut. Charging per part is the honest number: those bytes + * really are base64-ed into the run that many times. + * + * DEDUPLICATING THE OUTPUT INSTEAD WAS THE OTHER WAY TO MAKE THE TWO NUMBERS + * AGREE, AND IS REJECTED. Emitting one part per distinct id would make "one + * charge per id" true by making one copy the only copy, but it changes what the + * model is handed — a message that names a file at two points in its content + * means to refer to it at both — and it breaks the rule that whatever runs after + * this owns the parts it was given, which `attachment-parts.test.ts` pins with + * `expect(result[0]).not.toBe(result[1])`. Bounding the output is this budget's + * job; rewriting the message is not. + * + * It lives HERE and not in `shared/attachments.ts` beside the other limits on + * purpose. Those are limits two sides have to agree on — the composer refuses + * a file and the server refuses it again — and that file's whole argument is + * the drift between the two. This one is neither: the composer has no say in + * how much of a thread a turn replays, and nothing in a browser can observe + * it. + */ +export const MAX_INLINED_BYTES_PER_RUN = 32 * 1024 * 1024; + +/** + * What one run has left to spend, threaded through every message in it. + * + * A mutable object rather than a number passed back and forth, because the + * spending is across messages and not within one: `inlineAttachments` hands + * the same object to each message in turn and each draws it down. A caller + * with nothing to bound — a unit test, one message resolved on its own — + * passes nothing and gets the unbounded behaviour this had before. + */ +export type InlineBudget = { + /** Bytes still unspent. Drawn down once per INLINED PART, not once per distinct id. */ + remaining: number; + /** + * What `remaining` started at, carried only so that a refusal can name it. + * + * The message being asked about is refused rather than cut when it does not fit + * ({@link tooMuchToInline}), and a refusal that cannot say what the limit WAS is a failure with + * no action behind it. By the time one is raised `remaining` has already been drawn down by the + * parts of that message that did fit, so it is no longer the number to quote; this is. Set from + * `remaining` rather than from {@link MAX_INLINED_BYTES_PER_RUN} so a caller that constructs a + * smaller budget — every test here does — gets a sentence about the budget it actually passed. + */ + limit: number; +}; + +export function newInlineBudget( + remaining: number = MAX_INLINED_BYTES_PER_RUN, +): InlineBudget { + return { remaining, limit: remaining }; +} + +async function resolvePart( + part: Record, + id: string, + load: (id: string) => Promise, + onMissing: MissingAttachment, + budget: InlineBudget | undefined, +): Promise { + /* + * CUT BEFORE THE LOAD, NOT AFTER IT. The read out of `bytea` is most of what + * this budget exists to bound, so a part that cannot fit must not be fetched + * to find that out. Once the budget reaches zero every later part costs one + * comparison and no database round trip at all. + * + * `onMissing` DECIDES WHAT RUNNING OUT MEANS, NOT WHETHER THE BUDGET APPLIES, + * and that distinction is the fix for what this used to do. The test was + * `onMissing === "note" && budget !== undefined` — one flag standing for both + * questions — so the message being asked about, resolved under `"fail"`, was + * never stopped at all and no ceiling existed on what a browser-written + * message could inline. The budget now applies to every part under it. What + * differs is the answer when it runs out: `"note"` is history and degrades + * into text saying the file was left out of this turn, so the conversation + * still runs; `"fail"` is the message being asked about and does not degrade + * in either direction — it is served in full, or the turn is refused naming + * the file. See {@link notIncludedNote} and {@link tooMuchToInline} for why + * those must be different answers rather than one. + */ + const noRoomLeft = budget !== undefined && budget.remaining <= 0; + /* + * AND AN ID SEEN ON AN EARLIER PART GETS NO EXEMPTION HERE. This read + * `noRoomLeft && !charged.has(id)` for a while, on the reasoning that a second + * mention of a file already paid for costs nothing to include. It costs a + * whole second copy of its base64, live at the same time as the first; see + * {@link MAX_INLINED_BYTES_PER_RUN} for the arithmetic and for the failure + * that exemption let through. Once the room is gone, every later part is cut + * or refused, whatever id it names. + */ + if (noRoomLeft) { + if (onMissing === "note") return notIncludedNote(part, id); + tooMuchToInline(part, id, budget.limit); + } + + const attachment = await load(id); + /* + * Dropping a part whose attachment vanished would let the Bot answer + * confidently about an image or file it never actually received, and + * neither the person who attached it nor the person reading the answer + * could tell that is what happened. A turn that fails outright is + * recoverable; an answer about a file nobody sent is not. So "fail" + * throws, naming the id, instead of silently continuing without it. + * + * "note" is not that same silence. The part is replaced by text that + * says the file is gone, so the model is told there was an attachment + * and told it cannot see it, which is the one thing dropping the part + * would have hidden. What it is not allowed to do is answer as though + * the file were there. + * + * The refusal leads with the name the person gave the file and keeps the id + * behind it; see {@link namedForFailure} for why that order. + * + * AND IT DOES NOT PRETEND TO KNOW WHICH OF FOUR THINGS HAPPENED. `load` + * answers `null` for a row the sweeper reclaimed, for a file belonging to + * somebody this asker cannot see, for one belonging to another channel, and + * for an id that never named a row at all — four different situations, with + * four different things to do about them, flattened into one absent value by + * the `(id) => Promise` seam this function is handed. + * Naming one of them would be a guess printed as a fact, so this names the + * ones it could be and leaves the choice to the reader, who has the context + * to make it. Telling them apart properly means a richer result from the + * loader in `channels/attachments.ts`, which is a change on the other side of + * this seam and not one this sentence can make. + */ + if (!attachment) { + if (onMissing === "note") return unavailableNote(part, id); + throw new Error( + `Attachment ${namedForFailure(part, id)} could not be loaded: it may have been deleted, or it may belong to another channel or to somebody whose files you cannot see.`, + ); + } + + if (budget) { + /* + * The same fork as above, one step later, for the file whose size could not + * be known until it was read. In history a file that does not fit takes the + * budget to zero rather than leaving a sliver behind: half an image is not a + * smaller image, and a remainder left lying about would tempt one more read + * out of every later part instead of stopping the reads here. On the message + * being asked about there is nothing to zero, because the turn ends here. + */ + if (attachment.bytes.length > budget.remaining) { + if (onMissing === "note") { + budget.remaining = 0; + return notIncludedNote(part, id); + } + tooMuchToInline(part, id, budget.limit); + } + budget.remaining = Math.max(0, budget.remaining - attachment.bytes.length); + } + + /* + * THE STORED MIME TYPE DECIDES, NOT `part.type`. + * + * `mimeType` is `sniffMimeType`'s answer, earned from the bytes when the + * file was uploaded. `part.type` is the browser's claim, fixed from + * `file.type` BEFORE that upload happened and never reconciled with what + * came back. The two disagree in a way that reaches the model: a PNG whose + * browser claim was `text/plain` is sniffed and stored as `image/png` while + * the sent part still says `document`, and reading `part.type` there ran a + * PNG through `toString("utf8")` and captioned the mojibake + * `Attached file "photo.txt":`. The same read let any of AG-UI's other part + * types — `binary`, `audio` — base64 a whole text file, around + * `MAX_EXTRACTED_CHARACTERS` entirely. + * + * `classifyAttachment` is the same function the upload route decided to + * accept the row with, so this asks the stored bytes the identical question + * that let them be stored, and the answer cannot be moved by anything a + * client writes. + */ + const kind = classifyAttachment(attachment.mimeType); + + if (kind === "text") { + const text = extractDocumentText(attachment.bytes); + return { + type: "text", + text: `Attached file "${attachment.name}":\n\n${text}`, + }; + } + + if (kind !== "image") return unreadableNote(part, id, attachment.mimeType); + + /* + * `type: "image"` is asserted rather than inherited. A stored PNG that + * arrived on a `document` part has to reach the provider AS an image, or the + * one thing this whole path exists for — the model actually seeing the + * picture — does not happen. + */ + return { + ...part, + type: "image", + source: { + type: "data", + value: attachment.bytes.toString("base64"), + mimeType: attachment.mimeType, + }, + }; +} + +/** + * Swaps a stored attachment reference for content the model can actually + * read, resolving every part in `content` whose source is a + * `/api/attachments/` URL against `load` — whatever type that part + * declares itself to be, and into whatever the STORED bytes turn out to be. + * See `attachmentIdFor` for why the declared type is not a gate, and + * `resolvePart` for why it does not choose the modality either. + * + * Returns `content` BY IDENTITY when no part needs resolving. Every message + * in a thread passes through here on every turn and almost none carry an + * attachment, so that early return is load-bearing, not an optimization: + * callers that compare the result to the input (e.g. to decide whether a + * cache entry changed) depend on getting the same reference back. + * + * `onMissing` DEFAULTS TO "fail", so a caller that has not thought about + * which message this is gets the strict answer. It is the message being + * asked about that must fail: an unloadable attachment there is one the + * answer was supposed to be about. + * + * For an older message it must not. History is replayed in full on every + * turn, so a row that vanished once would fail this channel's every future + * turn, for ever, with no recovery but starting another channel — the same + * shape as the dangling tool call in `agents/history-sanitize.ts`, found in + * production twice: a permanent failure grown out of transient damage, and + * nothing the person did wrong. That file's answer is this one's. History is + * CONTEXT for a turn, not a transaction to resume; the attachment is already + * permanently gone and there is nothing to fetch; so the choice is between a + * conversation that can never run again and the same conversation with one + * old file marked missing. + * + * `budget`, when given, is how much this RUN may still inline; see + * {@link MAX_INLINED_BYTES_PER_RUN}. Absent means unbounded, which is what + * this did before the budget existed and what a single-message caller wants. + * + * IT BOUNDS THIS MESSAGE WHICHEVER `onMissing` SAYS — what changes is the answer when it runs out. + * Under `"note"` the parts that did not fit become text saying so and the call returns; under + * `"fail"` the call REJECTS, because that mode is the message being asked about and a person's own + * question is not something to quietly serve half of. A caller passing `"fail"` with a budget is + * therefore asking for "all of it or an error", which is what {@link tooMuchToInline} spells out. + * + * PARTS RESOLVE ONE AT A TIME, not through `Promise.all`. Two reasons, and the + * first is correctness: a budget spent by whichever load happened to settle + * first would cut a different part on each run over the same thread. The + * second is the peak — `Promise.all` over eight parts holds eight files and + * their base64 at once, which is the shape that exhausts a heap. What is given + * up is concurrency across a handful of small reads, on the rare message that + * carries a file at all. + * + * ONE READ PER DISTINCT ID, ONE CHARGE PER PART. The same id on two parts of one + * message is loaded once — `loadOnce` below — and encoded twice, because two + * parts must not share one object: whatever runs after this is entitled to treat + * the parts it was handed as its own. Those two copies are two charges against + * the budget, because they are two base64 strings live at once. + * + * THE TWO HALVES OF THAT SENTENCE MUST NOT BE COLLAPSED INTO ONE. This once read + * "one read per distinct id, and one charge", with a `charged` set making the + * second half true, and the result was a budget that bounded nothing a repeated + * id could do to it: forty parts naming one 1 KiB file inlined 40 KiB under a + * 1 KiB budget. Deduplicating the read is a saving; deduplicating the charge is a + * hole. See {@link MAX_INLINED_BYTES_PER_RUN} for the full arithmetic, and + * `attachment-parts.test.ts`, which asserts the memo and the per-part charge as + * separate claims and measures the bound on DECODED OUTPUT BYTES rather than on + * `budget.remaining` — the counter read zero while forty copies went out. + * + * THE MEMO IS WITHIN ONE MESSAGE, WHICH IS ONE CALL OF THIS FUNCTION, AND NOT + * ACROSS THE RUN. It is built here, so it does not outlive the message, and an id + * quoted in two messages of one thread is read twice. That is the intended scope: + * a run-scoped memo would pin every distinct attachment's `Buffer` live for the + * whole backward walk, and what is live at the peak is the thing this budget + * exists to bound. Trading a rare second read for unbounded buffer retention is + * the wrong way round. + */ +export async function resolveAttachmentParts( + content: unknown, + load: (id: string) => Promise, + onMissing: MissingAttachment = "fail", + budget?: InlineBudget, +): Promise { + if (!Array.isArray(content)) return content; + + const ids = content.map((part) => attachmentIdFor(part)); + if (ids.every((id) => id === null)) return content; + + const reads = new Map>(); + const loadOnce = (id: string): Promise => { + const already = reads.get(id); + if (already) return already; + const reading = load(id); + reads.set(id, reading); + return reading; + }; + + const resolved: unknown[] = []; + for (const [index, part] of content.entries()) { + const id = ids[index]; + resolved.push( + id === null + ? part + : await resolvePart( + part as Record, + id, + loadOnce, + onMissing, + budget, + ), + ); + } + return resolved; +} diff --git a/server/src/channels/attachments.ts b/server/src/channels/attachments.ts new file mode 100644 index 000000000..19c5e4797 --- /dev/null +++ b/server/src/channels/attachments.ts @@ -0,0 +1,1807 @@ +import { + and, + eq, + exists, + inArray, + isNotNull, + isNull, + notExists, + or, + sql, +} from "drizzle-orm"; +import type { Context, MiddlewareHandler } from "hono"; +import { Hono } from "hono"; +import type { AppVariables } from "../auth/guards"; +import type { Database } from "../db/client"; +import { + attachments, + channelMemberships, + channels, + intelligenceChannelMappings, +} from "../db/schema"; +import { + classifyAttachment, + MAX_ATTACHMENTS_PER_MESSAGE, + MAX_FILE_BYTES, + MAX_IMAGE_BYTES, + namesNoFormat, +} from "../../../shared/attachments"; +import type { StoredAttachment } from "./attachment-parts"; +import { sniffMimeType } from "./attachment-mime"; + +function megabytes(bytes: number): string { + return `${(bytes / (1024 * 1024)).toFixed(0)}MB`; +} + +const UUID_SHAPE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * Whether a value even has the shape Postgres's `uuid` column could accept. + * + * `attachments.id` is a `uuid`, and every id in this file arrives as text somebody else chose: a + * path param off a URL, or a slice of a message part. Comparing text that is not uuid-shaped + * against that column raises `22P02` and throws before any row can fail to match, so each of the + * three entry points below asks this first and answers "no such attachment" itself. + */ +function isUuidShaped(value: string): boolean { + return UUID_SHAPE.test(value); +} + +/** + * What a turn running in one conversation may touch: files that belong to THAT conversation's + * channel. + * + * A CONDITION, NOT A LOOKUP THE CALLER DOES. The obvious shape is to resolve the thread to a + * channel id first and pass that id in beside the actor. That was rejected twice over. It is two + * statements where one will do, so under READ COMMITTED the mapping can change between them; and, + * far more importantly, it makes the scope a thing a caller REMEMBERS — a second `string` argument + * beside `actorId`, which the next person to add a call site can pass the wrong value for, or the + * right value from the wrong run, and get no complaint from anything. As a correlated subquery the + * scope is part of the statement that reads the row: there is no way to ask for an attachment + * without asking this at the same time, because it is the same query. + * + * The two callers take `threadId` in a NAMED FIELD for the same reason. `actorId` and `threadId` + * are both `string`, adjacent, and positionally swappable with no type error and no test failure — + * the swap would simply widen the scope back to what it was. An object parameter cannot be + * transposed. + * + * AND IT DEGRADES WHEN THE THREAD MAPS TO NO CHANNEL, which is the half that took evidence to get + * right rather than reasoning. The tempting shape is an INNER JOIN on + * `intelligence_channel_mappings`: no mapping, no attachment. That is wrong, because three real + * surfaces run turns on threads this deployment deliberately keeps no channel for: + * + * - A FORWARD AGENT HOP. `handoff-delivery.ts` mints a scratch thread of the addressed Bot's own + * (an Intelligence thread has exactly one agent, so a second Bot cannot answer inside the + * first's conversation) and then seeds it with the ASKING channel's history, attachment parts + * and all. Under an inner join every one of those files would resolve to null on a thread that + * maps to nothing, and because history is resolved with `onMissing: "note"` the addressed Bot + * would be told `[attachment "x" is no longer available]` about files that exist and that the + * same person may read. It would not even fail loudly: a hop's last user message is the + * synthetic instruction `handoff-delivery.ts` appends, so the `"fail"` branch that exists to + * catch exactly this never covers a hop's history. + * - THE DIRECT `/bot` CHAT, whose thread comes from `POST /api/threads/mint` — "a conversation + * this deployment keeps no channel for", in that route's own words. Its runs go through the + * ordinary runtime endpoint and so reach this loader. + * - A BACKWARDS HOP relaying into either of those. + * + * So the rule is: if this thread belongs to a channel, the file must belong to that same channel; + * if it belongs to no channel, this adds nothing and the membership join above remains the whole of + * the check. That is strictly narrower than what was here before and never narrower than a working + * surface needs, which is the only combination that closes the gap without breaking a hop. + * + * NOTE WHAT IT STILL STOPS, because the degrade reads more permissive than it is. The case the + * reviewers reproduced is a person in channels A and B naming an A-file on a turn in B. B's thread + * IS mapped, so the degrade branch does not apply, the subquery yields B, and A ≠ B refuses. The + * branch only opens for a thread nobody is shown. + */ +function inTheTurnsChannel(database: Database, threadId: string) { + const channelOfThisThread = () => + database + .select({ channelId: intelligenceChannelMappings.channelId }) + .from(intelligenceChannelMappings) + .where(eq(intelligenceChannelMappings.threadId, threadId)); + return or( + inArray(attachments.channelId, channelOfThisThread()), + notExists(channelOfThisThread()), + ); +} + +/** + * The file behind an attachment reference, read for one person's turn. A PURE READ: nothing about + * being shown a file says anybody sent it, and this function writes nothing. + * + * PER ACTOR, AND THE JOIN IS THE CHECK — the live channel and the actor's membership on it, which + * is the join the upload route leads with and the one `GET /api/attachments/:id` below uses. The id + * this is called with came out of `input.messages`, which is the browser's: `resolveAttachmentParts` + * (attachment-parts.ts) reads it out of a user message's own content, and nothing between the wire + * and here rejects a message that was never in the thread — the runtime hands the input to the agent + * and only filters what it PERSISTS. So a signed-in person can put an `/api/attachments/` part + * for a channel they are not in on a message they compose themselves, and no uuid needs guessing to + * do it: somebody removed from a channel still holds its ids in their local transcript. This function + * therefore cannot assume the asker is entitled to what they named, and checks membership itself + * rather than inferring it from where the id came from. + * + * Null when no row is visible to this actor — no such attachment, an id that could not name one, + * a channel that has since been deleted, or not theirs to see, the same answer for all four as on + * the fetch route. `resolvePart` turns that into a failed turn naming the id, which is the correct + * outcome: a turn that refers to an attachment the asker cannot see must not proceed to a model + * that would read the file back to them. + * + * THIS USED TO STAMP `attachedAt`, and the reasoning was that a load past the membership join is + * the closest thing to evidence of a send there is. It is not close enough, in two directions at + * once. This function is called for the message being asked about AND for every attachment in the + * history behind it, on every turn — so a read-time stamp said "sent" about every file anybody had + * ever been shown, including one that is still staged in somebody's composer. And it is scoped to + * MEMBERSHIP, because members are meant to see each other's sent files, so any member could freeze + * a colleague's staged row by naming its id in a message of their own: that row then answered the + * colleague's own withdrawal with a 409 for ever and the sweeper would never reclaim it either. + * + * The send writes the column instead. It goes out through AG-UI rather than through either router + * in this file, so the write is made where the send is actually known to have happened — in + * `inlineAttachments` (copilot.ts), which is the one place that can tell the message being asked + * about from the history behind it — through {@link markAttachmentsSent}. + * + * IT DOES READ `attachedAt`, THOUGH, WHICH IS NEW. A staged row is a file nobody has shared with + * anybody yet, so it is its uploader's alone until a send says otherwise; a sent one belongs to the + * conversation and every member of the channel may read it. The asymmetry used to run one way only: + * the comment below explains at length why a reader must not WRITE a colleague's staged row, and + * then let any member read one. + * + * AND SCOPED TO THE TURN'S OWN CHANNEL, which it did not used to be. See + * {@link inTheTurnsChannel} for the shape and for why it degrades on an unmapped thread instead of + * refusing. What it closes: somebody in channels A and B could put an id from A on a message they + * compose in B, and this returned the bytes. They already hold that file, so it was never + * escalation — but it left a message in B whose file lives in a channel B has nothing to do with, + * and the day A is deleted that message's attachment is permanently broken in B while the row is + * still there. The run carries the thread id, so the channel behind it is knowable here now. + */ +export async function loadAttachmentForTurn( + database: Database, + turn: { actorId: string; threadId: string }, + id: string, +): Promise { + /* + * The same uuid-shape guard as the two routes below, and it belongs here rather than at either + * caller because this id is the one nothing shaped. A path param at least came off a route + * pattern; this one comes out of `attachmentIdFor` (attachment-parts.ts), which slices whatever + * follows `/api/attachments/` in a browser-supplied message part — so a query string, a second + * path segment, and the empty string all arrive here as "ids". + * + * The consequence is worse here than on a route, too. A 500 is one failed request; this throw + * lands inside `resolvePart`, which only degrades a NULL into the `onMissing: "note"` text. A + * throw goes straight past that degradation, and history is replayed on every turn, so a single + * malformed part would fail this channel's every future turn for ever, with no recovery but + * starting another channel. An id no `uuid` column could hold is an id no row has: null. + */ + if (!isUuidShaped(id)) return null; + + /* + * `channels` is in the join, not only `channelMemberships`, and the difference is a channel that + * has been deleted. Deletion here is soft — `channels.deletedAt` — so the channel row and every + * membership on it survive it, and a join that asks only "is this actor a member" answers yes for + * ever afterwards. This is the path that hands bytes to a MODEL rather than to a browser, so + * without the channel term somebody in a deleted channel could name an id out of their own local + * transcript on a message they compose themselves and have the file read back to them. + */ + const [row] = await database + .select({ + mimeType: attachments.mimeType, + name: attachments.name, + bytes: attachments.bytes, + }) + .from(attachments) + .innerJoin( + channels, + and(eq(channels.id, attachments.channelId), isNull(channels.deletedAt)), + ) + .innerJoin( + channelMemberships, + and( + eq(channelMemberships.channelId, attachments.channelId), + eq(channelMemberships.userId, turn.actorId), + ), + ) + .where( + and( + eq(attachments.id, id), + // The conversation this turn is running in, as a term in the same statement rather than as + // a fact a caller looked up and passed. See {@link inTheTurnsChannel}. + inTheTurnsChannel(database, turn.threadId), + /* + * Sent, or this actor's own. In the same statement as the membership join rather than read + * and then decided on, so a send committing alongside cannot be missed by a check that + * already ran — the same discipline the withdrawal route's WHERE follows. + * + * The uploader's half is not a courtesy: this runs for the message being sent BEFORE + * {@link markAttachmentsSent} stamps it, so without it the ordinary first send of a file + * would refuse to inline the very file it is sending. + */ + or( + isNotNull(attachments.attachedAt), + eq(attachments.uploadedBy, turn.actorId), + ), + ), + ); + return row ?? null; +} + +/** + * The ids a send could not prove it had recorded, carried out of the transaction that has to roll + * back before anybody is told. + * + * A thrown value rather than a returned one because {@link markAttachmentsSent} must undo the stamps + * it DID write when any one of them is missing, and drizzle rolls a transaction back on a throw and + * on nothing else. It never leaves this module: the `.catch` below turns it into the sentence a + * person reads, so no caller has to know this type exists to handle the failure correctly. + */ +class UnrecordedSend extends Error { + constructor(readonly unrecorded: readonly string[]) { + super(`Attachments were not recorded as sent: ${unrecorded.join(", ")}.`); + this.name = "UnrecordedSend"; + } +} + +/** + * Records that these attachments went out in a message, on behalf of the person who sent it. + * + * WRITTEN BY THE SENDER, NOT BY A READER. `attachedAt` means one thing — "this file rode in a + * message somebody actually sent" — and three readers depend on that one meaning: + * `cull-staged-attachments.ts` treats a null as a staged file nobody came back for and eventually + * deletes it; the upload route above treats a null as a slot still occupied against + * `MAX_ATTACHMENTS_PER_MESSAGE`; and `DELETE /api/attachments/:id` treats a non-null as a promise + * to a sent message and refuses to withdraw the file. A column written as a side effect of reading + * cannot carry that meaning, because reading is not sending: history is replayed in full on every + * turn, for whoever happens to be running it, so a read-time stamp says "sent" about every file + * anybody has ever been shown. + * + * SCOPED TO THE UPLOADER, and that is a second, separate reason a read may not write this. Reading + * is scoped to CHANNEL MEMBERSHIP, because members are supposed to see each other's sent files — + * so a stamp on the read path let any member freeze a colleague's still-staged row simply by naming + * its id in a message of their own. That row then answers the colleague's own DELETE with a 409 for + * ever, and the sweeper will not reclaim it either, because both of those read `attachedAt` and + * `attachedAt` now says the file was sent. Only the uploader's own send may stamp the uploader's + * own row, so `uploadedBy` is in the WHERE beside the id. + * + * `isNull(attachments.attachedAt)` keeps the FIRST send, not the most recent one. A message is + * replayed as history on every later turn and a stopped run is retried; neither is a new send, and + * neither should move a timestamp that already means something. + * + * Ids that could not name a row are dropped before the query rather than passed to it. These come + * out of `attachmentIdFor` (attachment-parts.ts), which slices whatever follows + * `/api/attachments/` in a browser-supplied message part, so a query string, a second path segment + * and the empty string all arrive here as "ids" — and comparing text that is not uuid-shaped + * against a `uuid` column raises Postgres `22P02` and throws, exactly as it would in + * {@link loadAttachmentForTurn}. Nothing left to ask about is not a query at all. + * + * AND SCOPED TO THE TURN'S OWN CHANNEL, by the same {@link inTheTurnsChannel} term the reader uses, + * for a harm that was reproduced rather than reasoned about. A person in channels A and B could + * name an id from A on a message they sent in B, and this stamped the row in A. Nothing in A ever + * referred to it, and yet it was now un-withdrawable — the withdrawal route refuses a sent + * attachment with a 409 — and unsweepable, because the culler only reclaims rows with a null + * `attachedAt`. The person was left holding a file they could neither use nor get rid of, in a + * channel that had never seen it. That is the same shape as the freeze the `uploadedBy` term above + * exists to prevent, reached from the other direction. + * + * THE SAME TERM AS THE READER, NOT A STRICTER ONE, and that was a decision. A stricter write — an + * inner join, so an unmapped thread stamps nothing — is unreachable for a hop (a hop's asked + * message is `handoff-delivery.ts`'s synthetic instruction, which names no attachment, so the + * `ids.length === 0` return above fires first) and so looked free. It was rejected because on the + * one surface where it IS reachable, the direct `/bot` chat, it fails in exactly the direction this + * column's whole purpose is to avoid: a file that really was sent silently never gets stamped, and + * the culler deletes it a day later out from under a conversation that shows it. One rule, stated + * once, is also one rule to keep true. + * + * IT RAISES WHEN IT CANNOT PROVE THE STAMP LANDED, AND IT USED TO SWALLOW. The sentence that stood + * here said that a turn is a person waiting for an answer, and that bookkeeping which could not be + * written is not worth failing that answer over. The first half is true. The second rested on a + * premise that is false: that by the time this runs, the answer has been earned. It has not. + * `inlineAttachments` (copilot.ts) calls this BEFORE it hands the history back, and that history is + * what `super.run` / `next.run` is given afterwards — so at this moment no model has been called, no + * token has been spent, and the person's message is still in front of them. Raising here costs a + * retry of something that never started. + * + * Staying silent costs the FILE. The culler reclaims every row whose `attachedAt` is null, so a turn + * that answered happily about an attachment it never stamped leaves a message displaying a file the + * sweeper deletes a day later; the upload cap keeps counting the slot for ever in the meantime. That + * damage is permanent, silent, and lands on somebody who did nothing but send a file. An answer + * somebody can ask for again is the cheaper of the two losses, and this is called at the one point + * in the turn where that trade is still on offer — which is why the third option, refusing before + * the turn is spent, beats both halves of the dilemma rather than splitting it. + * + * Refusing here is also not a new KIND of outcome on this path. `resolvePart`'s `"fail"` mode + * already refuses this same turn at this same moment when the asked message names a file that + * cannot be loaded or cannot be afforded — same reason, same recovery, same sentence-shaped error. + * + * WHAT STILL DOES NOT RAISE, because the old swallow was not protecting nothing: + * + * - AN ID THAT COULD NEVER NAME A ROW, dropped before the query as before. A browser part that is + * not an attachment reference is not a failed send. + * - NOTHING TO RECORD, which is still not a query at all. + * - HISTORY. Only the asked message's ids are passed in, so nothing behind it is stamped or + * checked, and a replayed thread does not acquire new ways to fail. + * - A ROW THAT IS ALREADY SENT. An `attachedAt` that is already set is a SUCCESS here, not a race + * lost: a stopped run retried and a message replayed are both ordinary, and a rule that failed + * them would be failing people for doing nothing wrong. + * + * ZERO UPDATED ROWS IS NOT WHAT IT CHECKS. Postgres reports an UPDATE that matched nothing as a + * successful command (https://www.postgresql.org/docs/current/sql-update.html#SQL-UPDATE-OUTPUTS), + * so the count is silent about the failure that matters — and it is also the wrong question, in both + * directions. It reads zero for a row that was already stamped, which is a success, and zero for a + * colleague's already-sent file named on this message, which the `uploadedBy` term above correctly + * declines to touch. What has to be true is not "this statement changed something" but "this id is, + * NOW, durably recorded as sent in this conversation", so the UPDATE is followed by a SELECT asking + * exactly that, and every id that cannot answer it is named in the refusal. + * + * THAT SELECT DOES NOT REPEAT THE READER'S MEMBERSHIP JOIN, deliberately. Whether this actor may + * see these files was settled by {@link loadAttachmentForTurn} earlier in the same turn, and a + * second, subtly different copy of an access rule is a thing to keep in step rather than a check. + * What is asked here is only what this function is responsible for — the row still exists, it is + * stamped, and it is in this turn's channel. + * + * THE SELECT IS A SECOND STATEMENT, NOT A CTE HANGING OFF THE UPDATE, and that is the whole of the + * idempotence. A data-modifying CTE and the query reading beside it share one snapshot, taken when + * the statement began — so a row a neighbouring session stamped a moment ago is invisible to the + * read, while the UPDATE's own re-check correctly declines to stamp it twice. Two concurrent runs of + * the same message, or a retry overlapping the run it retries, would then refuse each other. Under + * READ COMMITTED a separate statement takes a fresh snapshot and sees the neighbour's commit, which + * is the answer that is actually true. + * + * IN A TRANSACTION, SO A REFUSED SEND LEAVES NO STAMPS BEHIND. A message may name several files and + * only one of them need be missing. Keeping the others' stamps would record a send for a turn that + * never ran, which is the exact freeze the paragraphs above are about: un-withdrawable, unsweepable, + * and referred to by nothing. Throwing inside the transaction rolls them back, so a refusal puts the + * rows back as the turn found them and the person's retry starts from a clean state. + * + * NO ADVISORY LOCK, AND THAT WAS CHECKED RATHER THAN ASSUMED. The upload route holds + * `pg_advisory_xact_lock` because it counts rows and then inserts against that count, which is two + * facts that must not drift apart. There is no count here. The stamp and the two things that can + * take the row out from under it — `DELETE /api/attachments/:id` and + * `server/scripts/cull-staged-attachments.ts`, both of which carry `attached_at is null` in their + * own WHERE — contend for the same ROW, and a row lock already serialises them: whichever commits + * second re-evaluates its own predicate against the row as it then stands. Withdrawal first, and + * this UPDATE matches nothing while the SELECT finds no row, so the send is refused. Stamp first, + * and the withdrawal's `attached_at is null` no longer holds so it deletes nothing, which is the 409 + * pinned by "a send landing mid-request cannot have its file deleted out from under it" in + * attachment-routes.test.ts. They cannot both win. An advisory lock would be a second, weaker + * mechanism laid over the one Postgres already applies to the row itself. + * + * The log names the actor and the ids, because every consequence of a missing stamp — a file the + * sweeper reclaims, a slot that never frees — is about a specific person and a specific row, and a + * line naming neither cannot be acted on. The raised message names the ids as well. There is no + * `app.onError` behind this server, but this refusal never becomes a response status: it leaves + * through the run as an AG-UI error, the road `resolvePart`'s refusals already take, so what the + * composer receives is the sentence rather than a plain-text 500. + */ +export async function markAttachmentsSent( + database: Database, + turn: { actorId: string; threadId: string }, + ids: readonly string[], +): Promise { + const known = ids.filter(isUuidShaped); + if (known.length === 0) return; + + const unrecorded = await database + .transaction(async (transaction) => { + await transaction + .update(attachments) + .set({ attachedAt: new Date() }) + .where( + and( + inArray(attachments.id, known), + eq(attachments.uploadedBy, turn.actorId), + // The channel this send actually happened in. See {@link inTheTurnsChannel}. + inTheTurnsChannel(database, turn.threadId), + isNull(attachments.attachedAt), + ), + ); + + const recorded = await transaction + .select({ id: attachments.id }) + .from(attachments) + .where( + and( + inArray(attachments.id, known), + inTheTurnsChannel(database, turn.threadId), + isNotNull(attachments.attachedAt), + ), + ); + + const durable = new Set(recorded.map((row) => row.id)); + const missing = known.filter((id) => !durable.has(id)); + // Thrown rather than returned, because the rollback is the point: see the paragraph above on + // what keeping a partial set of stamps would leave behind. + if (missing.length > 0) throw new UnrecordedSend(missing); + return []; + }) + .catch((error: unknown) => { + /* + * A failure that is not the verification's own is a failure to reach the database at all — a + * lost connection, an exhausted pool, a `statement_timeout`. Nothing is known to have landed + * and the transaction took back anything that had, so every id is unrecorded. + */ + const unrecorded: readonly string[] = + error instanceof UnrecordedSend ? error.unrecorded : known; + // The ids that are actually unaccounted for, not the whole list that was asked about: a + // message may name four files and have one of them go missing, and it is the one that has to + // be findable from a log line. + console.error( + `Could not record attachments as sent for ${turn.actorId} in ${turn.threadId}: ${unrecorded.join(", ")}.`, + error, + ); + return unrecorded; + }); + + if (unrecorded.length === 0) return; + + const named = unrecorded.map((id) => `"${id}"`).join(", "); + throw new Error( + `This turn was not run, because ${unrecorded.length === 1 ? "an attachment on your message" : "attachments on your message"} could not be recorded as sent (${named}). ` + + "A file withdrawn while the turn was being prepared is the usual cause. Nothing was sent to the Bot — attach the file again and resend.", + ); +} + +/** The columns `POST /:channelId/attachments` hands back on success. */ +type InsertedAttachment = { + id: string; + name: string; + mimeType: string; + sizeBytes: number; +}; + +/** + * The one row the upload's single statement comes back with, whatever it decided. + * + * Nullable columns are the refusal: nothing was inserted, and `waiting`, `held` and `isMember` are + * then the three facts that say which refusal it was — asked in the SAME statement as the insert + * rather than after it, which is the whole point (see the comment on the statement itself). + */ +type UploadAttempt = { + id: string | null; + name: string | null; + mimeType: string | null; + sizeBytes: number | null; + waiting: number; + /** + * Every unsent row this person holds, in every channel and every group — the number + * {@link MAX_STAGED_ATTACHMENTS_PER_UPLOADER} is compared against. + * + * Separate from `waiting` and not derivable from it: `waiting` is one bucket of one composer + * session, this is the whole of what one person has staged, and a refusal has to name whichever + * of the two actually refused. + */ + held: number; + isMember: boolean; +}; + +/** + * The group a client that names none is counted in. + * + * A tab left open across a deploy is running the JavaScript from before it, which sends no + * `uploadGroup` at all. Left ungrouped those rows would have NULL here, `null = ` is never + * true, and the cap would count zero of them and never refuse anything — a hole, not a fallback. So + * every group-less upload from one person in one channel shares this one bucket, which is exactly + * the per-channel counting this server did before the column existed. Old clients keep old + * behaviour; new ones get the per-composer count. + * + * No UUID can collide with it, and `newId()` only mints UUIDs. + */ +const LEGACY_UPLOAD_GROUP = "legacy"; + +/** + * How many unsent attachments ONE PERSON may hold across the whole deployment, whatever they call + * their upload groups and whichever channels they are in. + * + * NOT THE CAP, AND NOT A REPLACEMENT FOR IT. {@link MAX_ATTACHMENTS_PER_MESSAGE} is the number the + * composer knows, shows and refuses on, counted per `upload_group` for the reasons the route's own + * comment gives at length. This is a backstop underneath it, and it exists because that cap counts + * a bucket the CLIENT names: `uploadGroup` arrives as a form field, `uploadGroupOf` deliberately + * does not validate its value, and a caller that mints a fresh one on every request therefore has + * zero prior rows in every bucket it is ever counted against. The cap never fires, and nothing else + * bounded staged `bytea` — the only remaining ceilings were the ~8 MB per-request body limit and + * `scripts/cull-staged-attachments.ts`, whose default window is 24 hours. Reproduced through this + * route before it was closed: 33 uploads, a fresh `newId()`-shaped group on each, 33 rows written. + * + * PER UPLOADER, ACROSS ALL CHANNELS, and both halves of that were forced. + * + * - Per uploader rather than per deployment, because a deployment-wide ceiling is a ceiling one + * person can sit on: fill it and every colleague's upload is refused for a sentence naming + * nothing they did. It also makes every upload contend on one counter. + * - Across all channels rather than per channel, because `POST /api/channels` is open to any + * authenticated user (`channels/routes.ts`). A per-channel backstop is a backstop a client moves + * by creating a channel, which is the same defect as one it moves by choosing a string — a few + * more bytes per bucket, still unbounded. + * + * ROWS RATHER THAN BYTES, which was the harder call. A row here is at most + * `MAX_IMAGE_BYTES` (8 MiB), so counting rows bounds bytes too, at 32 x 8 MiB = 256 MiB of staged + * blobs per person — a real bound, derived rather than declared. A separate `sum(size_bytes)` + * ceiling was written and dropped: it buys a tighter storage number at the cost of a second limit + * to keep honest, a second refusal sentence for a person to make sense of, and a test that has to + * push a quarter of a gigabyte through the route to prove it. If the byte ceiling ever needs to be + * independent of the file ceiling — an attachment kind larger than 8 MiB, say — that is the moment + * to add it, and this comment is where to say so. + * + * FOUR MESSAGES' WORTH, deliberately loose. Ordinary work never comes near it: the composer refuses + * a ninth file per message, so reaching 32 means four full eight-file messages composed and left + * unsent at the same time, inside the sweeper's 24-hour window. Two tabs — the case the per-group + * cap exists to serve — is sixteen. The looseness is the point: this number is not meant to be the + * limit anybody experiences, only the one nobody can walk past. + * + * NO NEW INDEX, AND THAT WAS CHECKED RATHER THAN ASSUMED. The count this drives asks + * `uploaded_by = ? and attached_at is null`, and the obvious worry is that it degrades with an + * uploader's HISTORY — `attachments_uploaded_by_idx` covers every row they ever uploaded, almost + * all of them long since sent. It does not, because a better index already exists for it: + * `attachments_staged_idx` is partial on `attached_at is null`, so it holds only the staged rows in + * the deployment. Measured on this deployment's Postgres with 50,000 SENT rows for one uploader, + * the planner takes that partial index and filters `uploaded_by` inside it — one shared buffer, + * 0.011 ms. The set it scans is the deployment's staged rows, which the sweeper keeps short-lived + * and which THIS CONSTANT now bounds per person, so the query gets cheaper for the same reason it + * exists. A partial index on `(uploaded_by) where attached_at is null` would be tighter still; it + * is not worth a migration until a deployment is seen where it is. + * + * WHAT IT CAN COST AN HONEST PERSON, stated because a backstop that cannot be reached honestly is + * not the same as one that cannot be reached awkwardly. Staged rows in a channel that was + * soft-deleted, or that this person was removed from, cannot be withdrawn (`DELETE + * /api/attachments/:id` joins live channels and membership), so they hold their place until the + * sweeper takes them — which is as long as this deployment's operator has configured, and for ever + * if they have turned the sweep off. Thirty-two of those would refuse the next upload anywhere for + * that whole time. That is the same shape of fault the per-channel cap used to have at EIGHT, which + * is precisely why this sits four times higher, and why the refusal names the deployment rather than + * a deadline it cannot promise. + */ +export const MAX_STAGED_ATTACHMENTS_PER_UPLOADER = + 4 * MAX_ATTACHMENTS_PER_MESSAGE; + +/** + * The longest group this route will count a row under. + * + * A real client sends `newId()`, which is a 36-character UUID, so nothing legitimate comes within + * an order of magnitude of this. What the bound is really for is that a longer one FAILS rather + * than merely wasting space: `upload_group` is the third column of `attachments_upload_group_idx` + * (schema/core.ts), and a btree entry may not exceed about 2704 bytes. An incompressible group of + * 2600 bytes makes the INSERT itself fail — measured against this deployment's Postgres, 2000 bytes + * stored fine and 2600, 2700, 3000 and 8000 all raised. + * + * WHERE THAT FAILURE LANDS, SINCE THIS COMMENT USED TO GET IT WRONG. It claimed the group was also + * interpolated into the `pg_advisory_xact_lock` key, so that an unstorable one failed there instead, + * "one statement earlier, and before the insert is even attempted". That was true while the lock was + * keyed on `(channel, uploader, group)`; the lock was since widened to the uploader alone — the key + * is `attachment-cap-` and carries no group at all — and the sentence was left behind. + * Re-measured: the advisory lock takes an 8000-byte group and a NUL-bearing one without complaint. + * + * The group's first and only appearance is the upload's own CTE statement, which counts and inserts + * together, so an over-long group is refused by the index at the moment the insert runs and there is + * no earlier statement for it to fail in. Nothing is written either way, and the transaction's + * `.catch` turns the raise into the route's 503 rather than into a plain-text 500. + * + * 128 rather than something closer to the index's own ceiling because the bound is not really about + * the index: it is about the difference between a value a composer can plausibly have minted and + * one nothing in this app produces. + */ +const MAX_UPLOAD_GROUP_LENGTH = 128; + +/** + * Which composer session this upload belongs to, as the browser named it. + * + * The VALUE needs no validation and gets none: the column is `text`, it is only ever compared for + * equality against other rows the SAME person staged in the SAME channel, and the worst a person + * can do by choosing their own is give themselves a second bucket of eight — which two tabs already + * do, below. + * + * The SHAPE is a different question, and the comment here used to conflate the two. A group that is + * too long for {@link MAX_UPLOAD_GROUP_LENGTH}, or that carries a U+0000 — which Postgres refuses + * in a `text` value at all, `22021`, before any column is reached — does not give its sender a + * second bucket. It fails the one statement the group ever appears in — the upload's own + * count-and-insert CTE — so nothing is written and the upload is refused outright, as a 503 the + * person can do nothing about. Both were reproduced through this route against the local Postgres. + * + * Neither is refused, though: both fall back to {@link LEGACY_UPLOAD_GROUP}, exactly as a request + * that named no group at all does. A group is a client-side grouping hint, not something anybody + * asked for, so a hint this server cannot store is a hint it can do without — and treating it as a + * hard refusal would turn a stale or third-party client's cosmetic mistake into an upload it can + * never complete, which is a worse answer than counting its files in the same bucket every + * group-less upload already shares. + */ +function uploadGroupOf(formData: FormData): string { + const value = formData.get("uploadGroup"); + if (typeof value !== "string") return LEGACY_UPLOAD_GROUP; + if (value.length === 0 || value.length > MAX_UPLOAD_GROUP_LENGTH) { + return LEGACY_UPLOAD_GROUP; + } + // A NUL is not a length problem and would survive the check above, so it is asked separately: + // Postgres refuses U+0000 in a `text` value outright (`22021`), which takes down the whole + // count-and-insert statement the group is bound into — the only statement it reaches, since the + // advisory lock above it is keyed on the uploader and carries no group. + return value.includes("\u0000") ? LEGACY_UPLOAD_GROUP : value; +} + +/** + * The longest filename this route will store, in UTF-8 bytes. + * + * The fetch route echoes a stored name into `Content-Disposition` TWICE — once quoted, once + * percent-encoded for `filename*` — and percent-encoding can triple a non-ASCII byte, so the header + * runs to roughly four times the name. Nothing capped the name, and a multipart `filename` + * parameter can be as long as the body allows: a 4000-character name was measured through these + * two routes producing a 32 KB `Content-Disposition`. Bun serves that without complaint, but every + * common reverse proxy caps response headers at 4-8 KB, so behind an ingress that attachment is not + * a large download — it is a row nobody can ever fetch again, and no part of this app would say + * why. + * + * 255 bytes because that is the limit almost every filesystem imposes, so it is the number a name + * that came off somebody's disk has already been through. It holds the header under about 1 KB. + * + * TRUNCATED RATHER THAN REFUSED, and the extension is not preserved. A name is a property OF a file + * somebody chose to send; refusing the file over it would be refusing content this app can read for + * a reason that has nothing to do with the content. The stored name is what the 201 hands back, so + * the composer shows what was actually kept rather than what was sent. Rebuilding a `.png` tail + * onto the cut name was considered and dropped: it is more code and more edge cases (no dot, a dot + * at the end, a 300-byte "extension") for a case that only arises with a name no filesystem would + * have held in the first place. + */ +const MAX_FILENAME_BYTES = 255; + +const utf8 = new TextEncoder(); + +/** A filename cut to {@link MAX_FILENAME_BYTES}, never through the middle of a character. */ +function withinFilenameLimit(name: string): string { + if (utf8.encode(name).length <= MAX_FILENAME_BYTES) return name; + let kept = ""; + let bytes = 0; + // Code points, not code units, so the cut cannot split a character into a lone surrogate — and + // never more than 256 iterations, because it stops the moment the budget is spent. + for (const character of name) { + const size = utf8.encode(character).length; + if (bytes + size > MAX_FILENAME_BYTES) break; + bytes += size; + kept += character; + } + return kept; +} + +/** + * The parenthetical that names a refused file's type, or nothing at all when there is nothing to + * name. + * + * WHAT IS ASKED IS `namesNoFormat`, NOT WHETHER THE STRING IS EMPTY, and the difference is the + * whole point. This started as an emptiness test, because `sniffMimeType` used to hand back the + * claim itself when the bytes corroborated nothing — and the claim can be `""`, which this sentence + * interpolated into `'archive' is not a file type this app can read ().` An empty parenthetical is + * strictly worse than the generic sentence it was written to improve on, and the composer shows + * this string verbatim, so it is the only explanation anybody gets. + * + * `sniffMimeType` no longer returns a claim that names nothing — it answers + * `application/octet-stream` instead, which is the honest answer to "what are these bytes" and the + * right contract for a sniffer. That silently defeated an emptiness test: the parenthetical came + * back, now reading `(application/octet-stream)`, which names a string nobody chose and tells the + * reader less than saying nothing would. Both changes are right; only asking the question the + * shared list already answers composes them. A ninth thing that names no format gets added to + * `MIME_NAMES_NOTHING` and this sentence keeps working. + * + * A BLANK CLAIM IS REACHED THROUGH THE FILENAME, not through the request. Measured against this + * deployment's Bun: the multipart parser ignores a part's own `Content-Type` header entirely and + * derives `File.type` from the filename's EXTENSION — `photo.png` declared `text/plain` arrives as + * `image/png`, `drawing.svg` declared `text/plain` arrives as `image/svg+xml`, and a name with no + * extension at all arrives as `""`. So this is not an exotic path: it is any file whose name has no + * dot in it, and the bytes not being valid UTF-8 is the rest of it. + */ +function describeType(mimeType: string): string { + return namesNoFormat(mimeType) ? "" : ` (${mimeType})`; +} + +/** + * A channel's upload door: one file in, one staged row out, or a reason it was refused. + * + * THE TRADE-OFF THE GROUPING MAKES, WRITTEN DOWN BECAUSE IT IS A REAL BEHAVIOUR CHANGE. The cap + * counted every unsent row this person had in this channel; it now counts the ones staged by one + * composer session. Two tabs open on the same channel are two sessions, so they get eight each + * rather than eight between them. That is accepted deliberately: the cap is documented and enforced + * as a PER-MESSAGE limit everywhere else (`MAX_ATTACHMENTS_PER_MESSAGE`, and the composer's own + * screen), two tabs are two messages, and the alternative — the per-channel count — is what made a + * closed tab's leftovers refuse a pick the client had already accepted, naming files nobody could + * see. A cap that occasionally allows a second message's worth is a far smaller fault than one that + * bricks uploads in a channel for 24 hours. + * + * THAT PARAGRAPH DESCRIBED HALF THE PICTURE UNTIL `MAX_STAGED_ATTACHMENTS_PER_UPLOADER` EXISTED. + * "A second bucket of eight" is what a second TAB gets. It is not what a CALLER gets, because the + * bucket is named by a form field this route does not validate: a fresh group on every request has + * nothing in it to count, so the cap never fires, and the number of buckets is however many strings + * the caller cares to type. Nothing else bounded staged `bytea` — the body limit is per request, + * and the culler runs on a 24-hour window — so one authenticated member could stage without limit + * into the only table in this deployment that holds blobs. Reproduced through this route before it + * was closed, not argued from the code. + * + * SO THERE ARE TWO NUMBERS NOW, AND THEY ARE DIFFERENT KINDS OF NUMBER. The cap is the one the + * composer knows, shows and is refused by, counted over a bucket the client names — because the set + * on the screen is the only set the client can reason about. The backstop is counted over every + * unsent row one person holds, in every channel and every group, because that is the only scope a + * client cannot move. It sits four messages higher so that everything the paragraph above accepts — + * two tabs, a closed tab's leftovers, a stopped run — still fits comfortably underneath it. The + * trade survives; what it no longer does is run to infinity. + * + * The refusal is the point of this route as much as the upload is. `classifyAttachment` and + * `sniffMimeType` between them already decide, byte-for-byte, whether a file is something a Bot can + * read; this handler's job is to turn "no" into a sentence a person sees, naming what the file + * actually was rather than "unsupported file type" — the composer surfaces the string verbatim, so + * it is the only explanation anybody gets. + * + * Mounted at `/api/channels` by whoever wires the app together; this file only builds the router. + */ +export function createChannelAttachmentRoutes( + database: Database, + requireUser: MiddlewareHandler<{ Variables: AppVariables }>, +): Hono<{ Variables: AppVariables }> { + const routes = new Hono<{ Variables: AppVariables }>(); + + routes.post("/:channelId/attachments", requireUser, async (context) => { + const actor = context.var.actor; + const channelId = context.req.param("channelId"); + + /* + * The join IS the membership check, not a separate select run after one: a row comes back only + * when this channel exists, is not soft-deleted, and this actor has a membership row on it. No + * row, for any of those three reasons, reads the same from here on: a member of somebody else's + * channel learns nothing about whether it exists. + * + * NOT THE CHECK THE INSERT STANDS ON, though — this one only refuses early, before the file is + * read off the wire. The insert below carries the same join itself, because a check up here and + * an insert down there are two statements with the whole of the upload between them. + */ + const membership = await database + .select({ id: channels.id }) + .from(channels) + .innerJoin( + channelMemberships, + and( + eq(channelMemberships.channelId, channels.id), + eq(channelMemberships.userId, actor.id), + ), + ) + .where(and(eq(channels.id, channelId), isNull(channels.deletedAt))) + // Guarded like every other database call on these routes, and this is the one that runs + // FIRST: without it a database that cannot answer takes the door out before the body is even + // read, as a plain-text 500 the composer can only report as "could not upload". + .catch((error: unknown) => { + console.error( + `Could not check ${actor.id}'s membership of ${channelId} for an upload.`, + error, + ); + return null; + }); + + if (!membership) { + return context.json( + { error: "That file could not be stored just now. Try again." }, + 503, + ); + } + + if (membership.length === 0) { + return context.json( + { error: "You are not a member of this channel." }, + 403, + ); + } + + /* + * Every other body parse in this server is `.catch(() => null)` so a + * malformed body reads as a refusal, not a crash. `formData()` throws + * `ERR_FORMDATA_PARSE_ERROR` on a non-multipart body, and there is no + * `app.onError` to catch it — left unguarded, that throw becomes a + * plain-text 500 instead of the `{ error }` body the client reads off + * every failure. + */ + const formData = await context.req.formData().catch(() => null); + if (!formData) { + return context.json( + { error: 'Send the file as multipart form data under a "file" field.' }, + 400, + ); + } + const file = formData.get("file"); + if (!(file instanceof File)) { + return context.json( + { error: 'Attach a file under the "file" field.' }, + 400, + ); + } + + const bytes = new Uint8Array(await file.arrayBuffer()); + const mimeType = sniffMimeType(bytes, file.type); + const kind = classifyAttachment(mimeType); + + // Bounded once, here, so the same name is what gets stored AND what every refusal below quotes + // back: a sentence naming a file by a name the row does not carry would be its own small lie. + const name = withinFilenameLimit(file.name); + + if (kind === "unsupported-image") { + const error = + mimeType === "image/svg+xml" + ? `'${name}' is an SVG, which can carry scripts and is not accepted.` + : `'${name}' is an ${mimeType} image, which this app cannot read.`; + return context.json({ error }, 415); + } + + if (kind === "unsupported") { + return context.json( + { + error: `'${name}' is not a file type this app can read${describeType(mimeType)}.`, + }, + 415, + ); + } + + if (kind === "image" && bytes.byteLength > MAX_IMAGE_BYTES) { + return context.json( + { + error: `'${name}' is larger than the ${megabytes( + MAX_IMAGE_BYTES, + )} limit for images.`, + }, + 413, + ); + } + + if (kind === "text" && bytes.byteLength > MAX_FILE_BYTES) { + return context.json( + { + error: `'${name}' is larger than the ${megabytes( + MAX_FILE_BYTES, + )} limit for files.`, + }, + 413, + ); + } + + const uploadGroup = uploadGroupOf(formData); + + /* + * THE COUNT AND THE INSERT ARE ONE STATEMENT, UNDER ONE LOCK. + * + * It used to be a count, a comparison, and then an insert. Two uploads in flight at once both + * counted seven and both inserted, and the person held nine — the same time-of-check / + * time-of-use hole `withEnabledCapLock` in routines/store.ts was written for, and this file is + * the more reachable one: dropping eight files on the composer fires eight uploads in parallel + * by design. + * + * BOTH HALVES ARE LOAD-BEARING, and one without the other does not close it. Folding the guard + * into the insert's own `where` means no row can be written by a statement whose count did not + * permit it. But under READ COMMITTED that count still runs against a snapshot taken before the + * other transaction committed, so two such statements can still each see seven. The advisory + * lock is what makes the counts authoritative: it serialises every upload by one uploader, so + * the second one's counts are both taken after the first one's row is committed and visible. + * + * THE KEY IS THE UPLOADER, AND IT USED TO BE (channel, uploader, group). That was the right + * scope while the group cap was the only thing being counted, and it is the wrong scope now: + * `MAX_STAGED_ATTACHMENTS_PER_UPLOADER` is counted over everything one person has staged, so a + * lock keyed on the group serialises the uploads that share a bucket and nothing else — which + * is to say it serialises none of the uploads a client varying its group sends. Two of those + * arriving together would both count 31, both pass, and the person would hold 33: the same + * READ COMMITTED hole the cap itself had to be fixed for, rebuilt one level up. + * + * Widening it costs one person's own parallel uploads their parallelism ACROSS TABS AND + * CHANNELS rather than only within one composer. That is cheap where it lands: the bytes are + * already off the wire, sniffed and classified before this transaction opens, so what takes + * turns is one INSERT each, and eight files dropped on one composer already took turns here. + * Nobody else's uploads wait on this person's. + * + * One lock rather than two — an uploader lock plus the old group lock — because the wider one + * strictly contains the narrower: holding it makes BOTH counts authoritative, and a second lock + * would add a second round trip and an acquisition order to get wrong. + * + * Transaction-scoped (`_xact_`), so the commit or the rollback releases it rather than us + * remembering to. `hashtext` collisions are harmless: two unrelated uploaders sharing a hash + * take turns, which is slower and not wrong. + * + * THE MEMBERSHIP IS IN THE SAME STATEMENT, for the same reason and against a slower race. The + * handler checks membership at the top and used to insert on the strength of what it had read — + * with `await file.arrayBuffer()`, the sniff and the classification in between, so the window is + * as wide as reading an upload off the wire, not as wide as a scheduler tick. A removal landing + * in it put a file into a channel its uploader had just been taken out of. Selecting the row to + * insert FROM the channel-and-membership join closes that: the values are only produced if the + * join still produces a row when the insert runs, so there is no moment at which a non-member's + * file can land. + * + * Zero rows back is now the refusal for either reason — the cap, or no live channel this actor + * is in — so the branch below has to ask which before it can name one. + */ + const outcome = await database + .transaction(async (transaction) => { + await transaction.execute( + sql`select pg_advisory_xact_lock(hashtext(${`attachment-cap-${actor.id}`}))`, + ); + + /* + * THE COUNT THE REFUSAL REPORTS IS THE COUNT THAT REFUSED, because they are the same number + * in the same statement. + * + * This used to be an insert followed, on the refusal path, by a second SELECT that counted + * the staged rows again. Both ran inside this transaction, but READ COMMITTED gives each + * STATEMENT its own snapshot, and the advisory lock serialises other UPLOADS in this group — + * not `DELETE /api/attachments/:id`. A withdrawal committing between the two produced + * "you already have 7 attachments waiting" while the cap is 8, and a composer dropping a + * whole queued message produced 0. The insert was right to refuse both times; only the + * sentence was wrong, and a sentence that contradicts the refusal is worse than no number. + * + * As CTEs there is one snapshot for all three parts: `staged` is literally the number + * `written`'s `where` compared against, so the refusal cannot report a count that would not + * have refused. `membership` is in the same statement for the same reason it was folded into + * the insert in the first place — a check that ran afterwards would be answering a question + * about a later state than the one that decided. + * + * `staged` aggregates without a GROUP BY, so it is always exactly one row, and the LEFT JOIN + * therefore always yields exactly one row whether or not anything was inserted. That is what + * lets one shape carry all three outcomes. + */ + const [attempt] = (await transaction.execute(sql` + with membership as ( + select channels.id + from channels + join channel_memberships + on channel_memberships.channel_id = channels.id + and channel_memberships.user_id = ${actor.id}::text + where channels.id = ${channelId}::text + and channels.deleted_at is null + ), + staged as ( + select count(*)::int as waiting + from attachments + where attachments.channel_id = ${channelId}::text + and attachments.uploaded_by = ${actor.id}::text + and attachments.upload_group = ${uploadGroup}::text + and attachments.attached_at is null + ), + held as ( + select count(*)::int as waiting + from attachments + where attachments.uploaded_by = ${actor.id}::text + and attachments.attached_at is null + ), + written as ( + insert into attachments (channel_id, uploaded_by, upload_group, name, mime_type, size_bytes, bytes) + select + membership.id, + ${actor.id}::text, + ${uploadGroup}::text, + ${name}::text, + ${mimeType}::text, + ${bytes.byteLength}::integer, + ${Buffer.from(bytes)}::bytea + from membership, staged, held + where staged.waiting < ${MAX_ATTACHMENTS_PER_MESSAGE}::integer + and held.waiting < ${MAX_STAGED_ATTACHMENTS_PER_UPLOADER}::integer + returning id, name, mime_type, size_bytes + ) + select + written.id as "id", + written.name as "name", + written.mime_type as "mimeType", + written.size_bytes as "sizeBytes", + staged.waiting as "waiting", + held.waiting as "held", + exists (select 1 from membership) as "isMember" + from staged + cross join held + left join written on true + `)) as unknown as UploadAttempt[]; + + if (attempt.id !== null) { + return { + inserted: { + id: attempt.id, + name: attempt.name as string, + mimeType: attempt.mimeType as string, + sizeBytes: attempt.sizeBytes as number, + } satisfies InsertedAttachment, + }; + } + + /* + * Membership first, because "you are not in this channel" and "you have too many files + * staged" are both reasons for nothing being written and only one of them is true. + * + * Then the per-message cap ahead of the backstop, because where both are true the cap is + * the one that can be acted on: it is the limit the composer already shows, counted over + * the files on the screen in front of the person. Being told about everything they hold + * everywhere, while this composer sits at eight, sends them hunting through other channels + * for a problem that is on this one. The backstop's sentence is only ever the answer when + * the cap would have let this file through. + */ + if (!attempt.isMember) return { forbidden: true } as const; + if (attempt.waiting >= MAX_ATTACHMENTS_PER_MESSAGE) { + return { staged: attempt.waiting }; + } + return { held: attempt.held }; + }) + /* + * EVERY OTHER FAILURE ON THIS ROUTE ANSWERS `{ error }`, AND SO DOES THIS ONE. + * + * The `formData()` call three dozen lines above is wrapped for exactly this reason — there is + * no `app.onError` anywhere behind this router, so an unguarded throw is Hono's default + * plain-text `Internal Server Error` — and then the database calls underneath it were not. + * The composer reads `{ error }` off every failed upload and falls back to a generic + * `Could not upload ""` when the body will not parse as JSON, so a lost connection + * during a rollout, a `statement_timeout`, a lock timeout on the advisory lock, a `53100` + * disk-full on an 8 MiB insert and a serialisation failure were all the same unactionable + * sentence, with nothing written to the log either. + * + * LOGGED WITH WHAT IT WOULD TAKE TO ACT ON IT — who, which channel, and how big the file was + * — because these are operator faults rather than uploader mistakes, and the one refusal + * whose cause lives on this side of the wire is the one nobody could otherwise see. + * + * 503 rather than 500: every failure in that list is a "the store is not able to take this + * right now" and is worth retrying, which is what the sentence tells the person to do. + */ + .catch((error: unknown) => { + console.error( + `Could not store an attachment for ${actor.id} in ${channelId} (${mimeType}, ${bytes.byteLength} bytes).`, + error, + ); + return { unavailable: true } as const; + }); + + if ("unavailable" in outcome) { + return context.json( + { error: "That file could not be stored just now. Try again." }, + 503, + ); + } + + // The same sentence and the same status as the check at the top of the handler, because from + // the uploader's side it is the same refusal — it just became true later than that check ran. + if ("forbidden" in outcome) { + return context.json( + { error: "You are not a member of this channel." }, + 403, + ); + } + + /* + * NOT "IN THIS CHANNEL", WHICH IS THE COUNT THIS SERVER STOPPED COMPUTING. + * + * The cap is counted per composer session now — that is what `upload_group` is for, and the + * header comment above explains why — so a person with a full tab A and an empty tab B was + * being told about a channel total nobody computes, and sent hunting for files that are on + * another screen. That is the exact confusion the grouping was introduced to end. + * + * The limit is named beside the count because the limit is the actionable half: the count says + * what is true now, and `MAX_ATTACHMENTS_PER_MESSAGE` says what to do about it. The count can + * no longer be below the limit — the statement that refused is the statement that counted — so + * the two can never contradict each other in the same sentence. + */ + if ("staged" in outcome) { + return context.json( + { + error: `You can attach ${MAX_ATTACHMENTS_PER_MESSAGE} files to a message, and ${outcome.staged} are already waiting to send.`, + }, + 409, + ); + } + + /* + * THE BACKSTOP'S REFUSAL, AND IT DELIBERATELY DOES NOT SOUND LIKE THE CAP'S. + * + * Nobody reaching this has filled the composer in front of them — the branch above would have + * answered if they had. They are holding four messages' worth of unsent files somewhere, and + * the sentence has to say so, or this is the 409 nobody could act on all over again: "you can + * attach 8 files to a message", said to a composer holding one, is an instruction to go looking + * for seven files that are not there. + * + * NAMES THE SCOPE AND THE WAY OUT. "Across your channels" because the files need not be in this + * one, and sending or removing because those are the two things a person can do to clear a row. + * + * AND IT NAMES THE SWEEP WITHOUT PROMISING A SCHEDULE, WHICH IS A CORRECTION. Some of these rows + * cannot be withdrawn at all — a channel that was soft-deleted, or one this person was removed + * from, still holds their staged rows and `DELETE /api/attachments/:id` will not take them — so a + * sentence offering only "remove some" would be asking for something impossible, and the sentence + * has to say what becomes of those. It used to say they are "cleared within a day", which this + * server is in no position to promise: `attachments.culler.olderThanHours` is the operator's to + * set, and `attachments.culler.enabled: false` is a documented way to keep every staged row for + * ever (charts/openbot/README.md). A deployment that has done either is one where this sentence + * was simply a lie, told to the one person who could not act on it. What is true whatever the + * chart says is that those rows are the deployment's to clear and not this person's, and that is + * what it says now. + * + * The count comes from the statement that refused, exactly as the cap's does, so the number and + * the decision cannot contradict each other. + */ + if ("held" in outcome) { + return context.json( + { + error: `You have ${outcome.held} files waiting to send across your channels, which is as many as one person can hold unsent. Send or remove some before attaching more; any in a channel you can no longer open have to be cleared by whoever runs this deployment.`, + }, + 409, + ); + } + + return context.json(outcome.inserted, 201); + }); + + return routes; +} + +/** + * Replaces every character that cannot legally appear in a header value with `_`. + * + * RFC 9110 allows visible ASCII, SP, HTAB and obs-text (%x80-FF) in a field value and nothing + * else: the C0 controls and DEL are not representable there at all. Two of them, CR and LF, are + * the header-injection pair, and for a long time they were the only two this file took out. The + * rest are just as fatal, only more quietly. Bun's serializer throws on a NUL, and it throws from + * inside `c.body` after the response has already begun, with no `app.onError` behind this router + * to turn that into anything — so one such name is a 500 on EVERY fetch of that attachment rather + * than a served file. The ones Bun does pass through (0x01-0x08, 0x0B, 0x0C, 0x0E-0x1F, 0x7F) are + * still illegal on the wire, arrive mangled at the client, and entitle any proxy in between to + * reject or re-parse the whole header over one byte. + * + * A NUL cannot reach a stored name today — `attachments.name` is a Postgres `text` column and + * Postgres refuses U+0000 in one — but the name is whatever the uploader's browser called the + * file, and header-safety is this function's job to guarantee rather than a column type three + * files away's to imply. + * + * HTAB goes too, and the list above says it is legal. Both are true: a tab is legal in a field + * VALUE and meaningless in a FILENAME, where it would only ever arrive as an accident of whatever + * produced the name and read back as ragged whitespace. The predicate below takes the whole C0 + * range rather than carving one character out of it for no gain. + * + * `_` rather than deletion, matching `foldToLatin1` below: the substitution shows up in the + * downloaded filename, where deletion would silently join whatever sat on either side. + */ +function withoutControlCharacters(name: string): string { + return Array.from(name) + .map((char) => { + const codePoint = char.codePointAt(0) ?? 0; + return codePoint < 0x20 || codePoint === 0x7f ? "_" : char; + }) + .join(""); +} + +/** + * Escapes a stored filename for the quoted `filename` parameter of `Content-Disposition`. + * + * The name came from whatever the uploader's browser called the file, so it can contain a quote + * or a backslash, either of which would end the `filename="..."` value early or splice in + * attacker-controlled header syntax. Neither survives here, and nothing that could not appear in + * a header value at all survives `withoutControlCharacters` above. + */ +function escapeFilename(name: string): string { + return withoutControlCharacters(name) + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"'); +} + +/** + * Folds a filename down to Latin-1 for the quoted `filename` parameter. + * + * Bun's header serializer throws on any value carrying a code unit above U+00FF, and it throws + * from inside `c.body` after headers have already started being written — there is no + * `app.onError` behind this router to turn that into a response, so an uploader who legitimately + * named their file `メモ.txt` or `笔记.md` would 500 the whole handler. Folding those code points + * to `_` here keeps the quoted parameter header-safe for every client; `filename*` below is what + * carries the real name back intact, for the clients that read it. + */ +function foldToLatin1(name: string): string { + return Array.from(name) + .map((char) => { + const codePoint = char.codePointAt(0) ?? 0; + return codePoint > 0xff ? "_" : char; + }) + .join(""); +} + +/** + * Percent-encodes a filename for the RFC 5987 `filename*` parameter (`encodeURIComponent` leaves + * `'`, `(`, `)` and `*` unescaped, none of which are legal in `attr-char`). + */ +function encodeRfc5987ValueChars(value: string): string { + return encodeURIComponent(value).replace( + /['()*]/g, + (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`, + ); +} + +/** + * Builds the `filename="..."; filename*=UTF-8''...` pair for a `Content-Disposition` value: an + * ASCII-folded fallback every client can parse, plus the exact name for the clients that support + * the extended form. + * + * Exported only so a test can reach the NUL case, which no route can: Postgres will not store a + * U+0000 in `attachments.name`, so calling this directly is the one way to show it never hands + * the serializer a byte that would throw. + */ +export function contentDispositionFilename(name: string): string { + // Both parameters are built from the SAME neutralised name, so the fallback and the extended + // form can never disagree about what the file is called. `encodeRfc5987ValueChars` would + // otherwise percent-encode a control character into something a header can legally carry + // (`%0B`) and hand the client back a name it should never have been offered. + const safe = withoutControlCharacters(name); + const quoted = foldToLatin1(escapeFilename(safe)); + const extended = encodeRfc5987ValueChars(safe); + return `filename="${quoted}"; filename*=UTF-8''${extended}`; +} + +/** + * What `GET /api/attachments/:id` says about caching, on the 200 and on the 304 alike so the two + * cannot drift. + * + * `private` because these are somebody's own files: no proxy and no CDN in between may hold a copy + * that a different person could be served. + * + * `no-cache` — WHICH IS NOT `no-store`. The browser still keeps its copy; it just may not use that + * copy without asking here first. This replaced `max-age=3600`, and the hour was not a small + * mistake: deleting an attachment makes this route answer 404, and the browser never asked, so an + * `` already on the page went on painting the file from its own cache at full natural width + * for the rest of the hour and the "this attachment is unavailable" path could not be reached at + * all. The same hour kept bytes readable after a sign-out and after a removal from the channel. + * + * WHAT IS TRADED AWAY IS A ROUND TRIP, and knowingly. Every fetch now costs a request to this + * server even when nothing has changed, where before one in an hour did. The ETag below buys back + * the expensive half — a revalidation that still holds is a 304 with no body — but the request + * itself, and the row lookup behind it, are the price of the property being bought: not freshness + * of CONTENT, which cannot change for a given id, but freshness of EXISTENCE and of entitlement, + * re-decided against this actor's membership on every single fetch. + */ +const ATTACHMENT_CACHE_CONTROL = "private, no-cache"; + +/** + * Whether an `If-None-Match` header says the client already holds this exact representation. + * + * Weak comparison, as RFC 9110 requires for `If-None-Match`: `W/"x"` and `"x"` are a match, and so + * is a list containing either. `*` matches whenever any representation exists at all, which by the + * time this is asked it does. + * + * A header nobody sent is not a match, which is the ordinary first fetch. + */ +function ifNoneMatchHolds(header: string | undefined, etag: string): boolean { + if (!header) return false; + return header + .split(",") + .map((candidate) => candidate.trim()) + .some( + (candidate) => + candidate === "*" || candidate.replace(/^W\//, "") === etag, + ); +} + +/** + * The headers a stored attachment is served under — by `GET` and by `HEAD` alike, from one place, + * because a probe that disagreed with the fetch about the type or the disposition would be worse + * than no probe at all. + */ +function attachmentHeaders( + row: { name: string; mimeType: string }, + etag: string, +): Record { + const kind = classifyAttachment(row.mimeType); + return { + // The type this server sniffed from the bytes at upload time, never the client's original + // claim — that claim is exactly what this header exists to override, so trusting it here + // would undo the sniff. + "Content-Type": row.mimeType, + // Without this, a browser that decides it knows better than the declared type will sniff + // the bytes itself, which is how a "text" file with HTML in it becomes a page rendered on + // this app's own origin instead of the download or plain text it was declared to be. + "X-Content-Type-Options": "nosniff", + // Only an image opens inline. Everything else — including a stray file whose sniffed type + // is not one of the accepted image formats — downloads instead, because inline is exactly + // what would let script-carrying content (the SVG case refused at upload) run if it ever + // reached this endpoint another way. + "Content-Disposition": + kind === "image" + ? "inline" + : `attachment; ${contentDispositionFilename(row.name)}`, + // What may be reused, and on what terms: see {@link ATTACHMENT_CACHE_CONTROL}. The short of + // it is that a stored copy may be kept but not used without asking here again, so a deletion + // or a removal from the channel is seen on the next fetch rather than up to an hour later. + "Cache-Control": ATTACHMENT_CACHE_CONTROL, + // Paired with that: the ask is cheap, because a client that still holds this id's bytes gets + // a 304 instead of them. + ETag: etag, + }; +} + +type AttachmentContext = Context<{ Variables: AppVariables }>; + +/** + * "No such attachment", said the one way, by every read on `GET /api/attachments/:id` and by the + * withdrawal below. + * + * A FUNCTION RATHER THAN FOUR LITERALS, because this sentence is the whole of what an outsider is + * told and its value is that it never varies. `readVisibleAttachment` returns no row for four + * different reasons — no such id, a channel since deleted, not a member of it, a colleague's staged + * draft — and the uuid-shape guards answer a fifth before any query runs. If any one of those ever + * came back phrased differently, or with a different status, the difference would be exactly the bit + * of information the uniform 404 exists to withhold. + */ +function noSuchAttachment(context: AttachmentContext) { + return context.json({ error: "No such attachment." }, 404); +} + +/** + * What `GET /api/attachments/:id` says when the store could not be asked — the fetch, the probe and + * the revalidation alike. + * + * THE PROBE ANSWERS THIS BY CALLING THIS, WHICH IT DID NOT USED TO. `HEAD` had its own + * `context.body(null, 503)` beside a comment claiming its refusals were "the GET's, exactly". They + * were not: Hono answers a HEAD by dispatching the GET handler and re-wrapping the response as + * `new Response(null, )`, so the headers survive even though the body does not — and a + * `context.body(null, ...)` sets no `Content-Type` where `context.json` sets `application/json`. + * Reproduced against an unreachable database: GET gave `503 application/json`, HEAD gave `503` with + * no `Content-Type` at all. That is a probe answering a question the fetch would not, which is the + * one thing the two are not allowed to do. There is now no second spelling to drift from. + */ +function couldNotReadAttachment(context: AttachmentContext) { + return context.json( + { error: "That attachment could not be read just now. Try again." }, + 503, + ); +} + +/** + * Which attachment `GET /api/attachments/:id` may answer about, as a WHERE the fetch, the probe and + * the revalidation all pass the same way. + * + * THE CONDITION IS THE ACCESS CHECK. It holds only when this attachment exists, the channel it was + * uploaded into has not been deleted, and this actor has a membership row on that channel. No row, + * for any of those reasons, is a 404 rather than a 403: a 403 would mean "yes, that id exists, but + * it is not yours", which is a free bit of information for somebody probing ids for attachments they + * cannot see. 404 is the same answer for "no such attachment", "the channel is gone" and "not + * yours", so guessing ids learns nothing either way. + * + * `channels` is inside the `EXISTS` beside the membership, and not only the membership, because + * channels soft-delete: the channel row and every membership on it outlive the deletion, so a + * membership-only test says yes for ever and the bytes stay downloadable — and inlinable — after the + * channel they belong to is gone. + * + * AND A STAGED ROW IS ITS UPLOADER'S ALONE. Membership is what lets people see each other's SENT + * files; a row with no `attachedAt` has been shared with nobody, so a colleague's half-composed + * draft is not a channel's to read. Reaching one needs its v4 uuid, which only the uploader's own + * 201 ever carried, so this is a boundary rather than a leak anybody has — but it is the boundary + * the write side already assumes, and the read side used not to keep. + * + * On the ordinary send that costs nothing: the stamp is written before the message is persisted, so + * by the time another member's browser can name the id the row is no longer staged. What it does + * change is the turns that fail before the stamp — a Bot that is no longer registered, a run that + * throws mid-inline — where the message is persisted anyway and the row stays staged for ever. + * Their attachments now read "unavailable" to everybody but the sender rather than being served out + * of a draft nobody sent. That is the more honest answer of the two, and the underlying leak — a + * persisted message whose rows were never stamped — is a copilot.ts fault worth fixing on its own. + * + * A CONDITION RATHER THAN A WHOLE QUERY, WHICH IS WHAT LETS THE THREE READS DIFFER IN NOTHING BUT + * THEIR COLUMNS. The fetch wants the bytes, the probe wants the size, and a revalidation wants + * nothing at all — and until this was factored out, each of them carried its own copy of the joins + * and the WHERE. That is a standing invitation to drift: a rule added to one and not the others + * turns `HEAD` into a way to learn that an id exists, or that a colleague has a draft, which the + * uniform 404 exists to hide. Stated once, there is no second copy to leave behind. A helper that + * owned the columns too was written first and abandoned: drizzle tracks the legal builder methods in + * the selection's own type, and over a generic selection TypeScript cannot resolve that, so the + * joins would not chain. + * + * AS A CORRELATED `EXISTS` RATHER THAN AS TWO INNER JOINS, which is the shape the withdrawal below + * already uses for the same rule. A condition composes where a join does not — that is the whole + * reason this is reusable — and it cannot multiply the attachment row if a membership is ever + * recorded twice, which a join silently would. + */ +function visibleToActor(database: Database, actorId: string, id: string) { + return and( + eq(attachments.id, id), + exists( + database + .select({ member: sql`1` }) + .from(channelMemberships) + .innerJoin(channels, eq(channels.id, channelMemberships.channelId)) + .where( + and( + eq(channelMemberships.channelId, attachments.channelId), + eq(channelMemberships.userId, actorId), + isNull(channels.deletedAt), + ), + ), + ), + or(isNotNull(attachments.attachedAt), eq(attachments.uploadedBy, actorId)), + ); +} + +/** + * The rows a read on this route came back with, or null when the store could not be asked. + * + * A DATABASE THAT COULD NOT ANSWER IS NOT AN ANSWER OF "NO", and on this route the difference + * matters more than anywhere else in this file. 404 here is load-bearing in the client: the + * transcript treats it as "this attachment is gone" and paints the unavailable tile in its place, + * and {@link ATTACHMENT_CACHE_CONTROL} is `no-cache`, so it asks again on every paint. Answering a + * lost connection with 404 would tell every member that a file which is still there has been + * withdrawn. An empty array is the real "no row", and that alone is the 404. + * + * There is no `app.onError` behind this router, so the alternative to catching here is Hono's + * plain-text default, which the client cannot read as `{ error }` and which leaves no server-side + * trace of a fault that is this side's to fix. One log line for all three reads, naming the row and + * the asker, because that is what it would take to act on it. + */ +async function readOrNull( + query: PromiseLike, + actorId: string, + id: string, +): Promise { + try { + return await query; + } catch (error) { + console.error(`Could not read attachment ${id} for ${actorId}.`, error); + return null; + } +} + +/** + * An attachment's own door: fetch its bytes back, or take it off the shelf before it is sent. + * + * Both routes below start from the same join as `createChannelAttachmentRoutes` above — a channel + * that has not been deleted, and the actor's membership on it — because an attachment is only ever + * visible to the channel it was uploaded into, sender and recipients alike, and only for as long as + * that channel is. + * + * Mounted at `/api/attachments` by whoever wires the app together; this file only builds the + * router. + */ +export function createAttachmentRoutes( + database: Database, + requireUser: MiddlewareHandler<{ Variables: AppVariables }>, +): Hono<{ Variables: AppVariables }> { + const routes = new Hono<{ Variables: AppVariables }>(); + + routes.get("/:id", requireUser, async (context) => { + const actor = context.var.actor; + const id = context.req.param("id"); + + // `attachments.id` is a Postgres `uuid` column: comparing it against arbitrary path text + // raises `22P02` and throws before any row can fail to match. That throw would escape as a + // 500, defeating the "404 hides both reasons" design this route otherwise relies on for + // security — an id that cannot possibly be a uuid gets the same answer as one that just + // isn't a match. + if (!isUuidShaped(id)) { + return noSuchAttachment(context); + } + + /* + * THE VALIDATOR IS KNOWN BEFORE THE ROW IS, AND THAT IS WHAT MAKES A 304 CHEAP. + * + * An attachment's bytes never change once stored — nothing in this file or anywhere else + * updates `attachments.bytes` — so a given id names one exact representation for as long as it + * names anything. A strong ETag of the id therefore needs no digest of the file to compute, and + * no row either: whether the client's `If-None-Match` still holds is decidable HERE, before a + * single column has been chosen. + * + * It used to be asked after the row came back, with `bytes` already in the select list, so every + * revalidation read the whole file out of Postgres in order to send an empty body. The comment + * that stood here called that "the cost of keeping the authorisation and the answer in one + * statement", which was a false trade: the two were never in tension, because the validator + * never needed the row. + * + * NOR IS IT A CORNER. {@link ATTACHMENT_CACHE_CONTROL} is `private, no-cache` deliberately, so + * EVERY image paint in every viewing member's transcript revalidates here — the 304 is the + * common path on this route, not the rare one. Measured through this route against an 8 MiB + * attachment, 12 runs each side: 8,388,608 bytes read and 25.09ms median per 304 before, 0 bytes + * and 0.65ms after. + * + * WHAT IS NOT GIVEN UP IS THE AUTHORISATION. The question a conditional request answers on this + * route is not "have the bytes changed" — they cannot — but "is this still there and still + * yours", so a 304 is still earned by the same channel-and-membership join a 200 is, in the same + * single statement. Somebody removed from the channel gets 404 on their next revalidation, not + * 304. Only the blob left the select list; the decision did not. + * + * AND IT IS ASKED AHEAD OF THE `HEAD` BRANCH, so the probe gets the same saving and the same + * answer from the same lines. A revalidating HEAD and a revalidating GET differ in nothing but + * the body Hono strips, which is what "the probe answers exactly what the fetch would" is + * supposed to mean. + */ + const etag = `"${id}"`; + if (ifNoneMatchHolds(context.req.header("If-None-Match"), etag)) { + // One column, and it is the id the WHERE has already fixed: this statement is asked whether a + // row comes back, never for anything in it. A `select` naming no field at all is not a + // statement Postgres would take, so "nothing" has to be spelled as the cheapest something. + const visible = await readOrNull( + database + .select({ id: attachments.id }) + .from(attachments) + .where(visibleToActor(database, actor.id, id)), + actor.id, + id, + ); + if (!visible) return couldNotReadAttachment(context); + if (!visible[0]) return noSuchAttachment(context); + + return context.body(null, 304, { + ETag: etag, + "Cache-Control": ATTACHMENT_CACHE_CONTROL, + }); + } + + /* + * A PROBE THAT DOES NOT READ THE FILE, AND WHY IT IS A BRANCH RATHER THAN A ROUTE. + * + * Hono intercepts HEAD before routing and re-dispatches it as a GET, returning + * `new Response(null, )` (hono-base.js, `#dispatch`). A + * `routes.on("HEAD", ...)` handler is therefore never reached — that was written first and + * verified not to run — so the only place that can answer a HEAD cheaply is here, in the + * handler Hono actually calls. The REQUEST object is passed through untouched, which is why + * the method is still readable at this point. + * + * It is worth answering cheaply. The transcript probes every document tile it draws with + * `HEAD /api/attachments/`, from every viewing member's browser, and + * {@link ATTACHMENT_CACHE_CONTROL} is `no-cache`, so each probe re-ran a full `bytea` read of a + * file nobody was going to be sent — measured: a HEAD of an 8 MiB attachment came back 200 with + * an empty body, having read all 8 MiB out of Postgres. + * + * Same access decision and the same headers — `size_bytes` in place of `bytes`, and + * `Content-Length` set by hand because a body-less response has nothing to derive it from and + * the size is the thing a probe is usually asking for. The refusals have to be indistinguishable + * from the fetch's, or a probe becomes a way to learn something a fetch would not tell you, so + * they are not repeated here at all: {@link readVisibleAttachment} makes the decision and + * {@link noSuchAttachment} and {@link couldNotReadAttachment} phrase both of its refusals, for + * this branch and the fetch below alike. `attachment-routes.test.ts` still asserts that case by + * case rather than leaving it to this comment. + * + * Revalidation is not handled here either, because the branch above already answered it for both + * methods before this one was reached. + */ + if (context.req.method === "HEAD") { + const probed = await readOrNull( + database + .select({ + name: attachments.name, + mimeType: attachments.mimeType, + sizeBytes: attachments.sizeBytes, + }) + .from(attachments) + .where(visibleToActor(database, actor.id, id)), + actor.id, + id, + ); + + if (!probed) return couldNotReadAttachment(context); + const metadata = probed[0]; + if (!metadata) return noSuchAttachment(context); + + return context.body(null, 200, { + ...attachmentHeaders(metadata, etag), + "Content-Length": String(metadata.sizeBytes), + }); + } + + /* + * The only read on this route that is going to send a body, and so the only one that may name + * `bytes`. + * + * WHAT THAT COLUMN COSTS IS THE POINT, NOT AN ASIDE. `attachments.bytes` is a `bytea` of up to + * {@link MAX_IMAGE_BYTES}, TOASTed out of line, and naming it in a select list is what makes + * Postgres fetch and de-TOAST the whole file into this process. The branch above measured + * 8,388,608 bytes read per 304 while this column was in every read's select list; the rule that + * keeps it out is that only the answer carrying a body may ask for it. + */ + const rows = await readOrNull( + database + .select({ + name: attachments.name, + mimeType: attachments.mimeType, + bytes: attachments.bytes, + }) + .from(attachments) + .where(visibleToActor(database, actor.id, id)), + actor.id, + id, + ); + + if (!rows) { + return couldNotReadAttachment(context); + } + + const row = rows[0]; + if (!row) { + return noSuchAttachment(context); + } + + /* + * A VIEW OVER THE DRIVER'S BUFFER, NEVER `Uint8Array.from` OVER IT. + * + * `row.bytes` is a Node `Buffer` — the driver's mapping for `bytea` — and a `Buffer` is both + * array-like and iterable. `%TypedArray%.from` prefers the ITERATOR, so it walks the file one + * element at a time on the single JS thread: 8.4 million steps for a file at `MAX_IMAGE_BYTES`. + * Measured on this repo's Bun 1.4 over an 8 MiB buffer: `Uint8Array.from` 62-87ms, this view + * 0.0004ms, `new Uint8Array(buffer)` (a copy, no iterator) 0.13ms. End to end, one 8 MiB fetch + * through this route went from 120-140ms to 36-38ms. + * + * That cost lands where it hurts most. {@link ATTACHMENT_CACHE_CONTROL} is `no-cache` + * deliberately, so EVERY image paint revalidates here; a transcript with a handful of large + * images stalls the event loop — for every other person's request on this process too — on + * every scroll-back. It is exactly the CPU the 304 path above was written to avoid paying. + * + * The offset and the length are both passed, rather than `new Uint8Array(row.bytes.buffer)`, + * because a `Buffer` need not own the whole of its `ArrayBuffer`: Node pools small allocations, + * so a short row can arrive as a window into a larger block. Dropping the offset would serve + * whatever else shares that block. The bytes are not copied, which is safe because nothing here + * or downstream writes through this view. + * + * The cast narrows `ArrayBufferLike` to `ArrayBuffer`, which is the only difference between + * what Node's `Buffer` promises and what hono's `Data` accepts: `ArrayBufferLike` admits a + * `SharedArrayBuffer`, and a database driver decoding a `bytea` off a socket does not allocate + * one. It is a type-level narrowing with no run-time step, which is the whole point — the + * expression it replaced was a run-time conversion standing in for a compile-time one. + */ + const bytes = new Uint8Array( + row.bytes.buffer as ArrayBuffer, + row.bytes.byteOffset, + row.bytes.byteLength, + ); + + return context.body(bytes, 200, attachmentHeaders(row, etag)); + }); + + routes.delete("/:id", requireUser, async (context) => { + const actor = context.var.actor; + const id = context.req.param("id"); + + // Same uuid-shape guard, and the same sentence, as the GET route above. + if (!isUuidShaped(id)) { + return noSuchAttachment(context); + } + + /* + * THE WHOLE DECISION IS THE DELETE, and that is what makes the refusal below mean anything. + * + * This route used to read `attachedAt`, decide on what it read, and then delete by id alone. + * Those are two statements with a gap between them, and the send that writes `attachedAt` is + * the third party that fits in it: stamp the row after the read and before the delete, and the + * file goes out from under a message that already claims it, leaving the transcript pointing at + * nothing. The window is not theoretical — it is the sender's own turn racing their own + * composer, which still offers the file for withdrawal until the send is recorded. + * + * Stating the whole rule in the WHERE closes it, because Postgres re-checks that WHERE against + * the row as it stands when the delete actually gets the row: `attachedAt IS NULL` no longer + * holds, nothing is deleted, and the returning list is empty. There is no moment at which a + * sent attachment is deletable. + * + * `uploadedBy` for the reason it was there before. Membership only ever gated visibility, and a + * sent attachment is refused below regardless of who asks; what membership-only scoping still + * allowed was a member deleting a colleague's *unsent* draft, which does real damage — the + * colleague's composer keeps pointing at a now-missing row, and their send fails later when + * nothing can resolve it. A non-uploader gets the same 404 as a non-member. + * + * A LIVE CHANNEL AND MEMBERSHIP ON IT, AS A CORRELATED `EXISTS` RATHER THAN AS A JOIN, because + * the join this route used to lead with lived in a separate SELECT and a delete cannot carry + * one. It is here because "is a member of this live channel" is an access-control boundary, and + * the case dropping it would open — somebody removed from a channel withdrawing their own + * still-unsent draft — is low-harm enough that letting it through would be a boundary loosened + * for no reason at all, inside a change that is nominally about when a column gets written. The + * subquery correlates on `attachments.channelId`, so it is one statement and the whole rule is + * still evaluated at the moment the row is taken. + * + * `channels` is inside that `EXISTS` beside the membership, and not only the membership, + * because channels soft-delete: a membership row outlives its channel's deletion, so a + * membership-only test kept saying yes and this route kept acting inside a channel nobody can + * open. Same scope as the fetch route above and as the upload route, which is what makes "the + * same join" true rather than merely claimed. + */ + const removed = await database + .delete(attachments) + .where( + and( + eq(attachments.id, id), + eq(attachments.uploadedBy, actor.id), + isNull(attachments.attachedAt), + exists( + database + .select({ member: sql`1` }) + .from(channelMemberships) + .innerJoin( + channels, + eq(channels.id, channelMemberships.channelId), + ) + .where( + and( + eq(channelMemberships.channelId, attachments.channelId), + eq(channelMemberships.userId, actor.id), + isNull(channels.deletedAt), + ), + ), + ), + ), + ) + .returning({ id: attachments.id }) + // As on the two routes above, and here the mistaken answer would be the worst of the three: + // an unguarded throw is a plain-text 500, and the composer's only other reading of a failed + // withdrawal is that the file is still staged. Say that the store could not be reached. + .catch((error: unknown) => { + console.error( + `Could not withdraw attachment ${id} for ${actor.id}.`, + error, + ); + return null; + }); + + if (!removed) { + return context.json( + { + error: "That attachment could not be withdrawn just now. Try again.", + }, + 503, + ); + } + + if (removed.length > 0) { + return context.body(null, 204); + } + + /* + * Nothing was withdrawn, and only now is it worth asking why — 404 or 409 is a question about + * how to answer, not about what to do, so it is asked after the act rather than before it. + * + * The same join-is-the-check shape, and the same 404-hides-every-reason answer, as the GET + * route above: no row, whether because there is no such attachment, because its channel has + * been deleted, because the actor is not in that channel, or because somebody else uploaded it, + * reads identically from here. A 403 would mean "yes, that id exists, but it is not yours", + * which is a free bit of information for anybody probing ids. + * + * The channel scope has to be on THIS query as well as on the delete above, or the two would + * disagree: a draft in a deleted channel would refuse to be withdrawn and then be explained + * with a 409 that says it was already sent, which is not what happened. + * + * IT ASKS WHETHER A ROW IS THERE, AND NOTHING ABOUT THE ROW — which is why it no longer selects + * `attachedAt`. It used to, and never read it: the 409 below is unconditional. That was not + * merely a wasted column, it was a claim this query looked like it was checking and was not. + * + * The 409 is unconditional because by this point it is the only answer left, and that follows + * from the two WHEREs rather than from a column. This query repeats every term of the delete's + * except `attachedAt is null` — same id, same uploader, same live channel, same membership — so + * a row coming back means all of those still hold, and the one term the delete had that this one + * does not is therefore the one that refused it. Nothing ever sets `attachedAt` back to null, so + * a row that was stamped when the delete ran is still stamped now. Reading the column could only + * ever confirm what the pair of statements has already established. + */ + const rows = await database + .select({ id: attachments.id }) + .from(attachments) + .innerJoin( + channels, + and(eq(channels.id, attachments.channelId), isNull(channels.deletedAt)), + ) + .innerJoin( + channelMemberships, + and( + eq(channelMemberships.channelId, attachments.channelId), + eq(channelMemberships.userId, actor.id), + ), + ) + .where(and(eq(attachments.id, id), eq(attachments.uploadedBy, actor.id))) + // Only the explanation is left to find, but a failure to find it is still not a 404: the + // delete above already declined to withdraw anything, and answering "no such attachment" + // because the second query failed would tell the composer to drop a row that is still there. + .catch((error: unknown) => { + console.error( + `Could not explain a refused withdrawal of ${id} for ${actor.id}.`, + error, + ); + return null; + }); + + if (!rows) { + return context.json( + { + error: "That attachment could not be withdrawn just now. Try again.", + }, + 503, + ); + } + + const row = rows[0]; + if (!row) { + return noSuchAttachment(context); + } + + // Once an attachment rides in a sent message, it is part of that message's record: pulling it + // out from under a message that already claims it would leave the message pointing at nothing. + // A staged attachment has made no such promise yet, so only that one may still be withdrawn — + // and the delete above is the only thing that decides whether it still is one. + return context.json( + { error: "This attachment is already part of a sent message." }, + 409, + ); + }); + + return routes; +} diff --git a/server/src/copilot.ts b/server/src/copilot.ts index bd0548d4a..2cb9e1411 100644 --- a/server/src/copilot.ts +++ b/server/src/copilot.ts @@ -1,4 +1,4 @@ -import type { BaseEvent, RunAgentInput } from "@ag-ui/client"; +import type { BaseEvent, Message, RunAgentInput } from "@ag-ui/client"; import { AbstractAgent, HttpAgent } from "@ag-ui/client"; import type { BuiltInAgentConfiguration } from "@copilotkit/runtime/v2"; import { @@ -17,6 +17,12 @@ import { import { sanitizeSeededHistory } from "./agents/history-sanitize"; import type { AgentActor } from "./agents/profile-types"; import type { AuditInitiator } from "./audit"; +import { + attachmentIdsIn, + newInlineBudget, + resolveAttachmentParts, + type StoredAttachment, +} from "./channels/attachment-parts"; import type { AgentFetch, StallGuard } from "./channels/stall-guard"; import type { DeploymentConfig } from "./config"; import type { SelectableSkill, Selection } from "./plugins/selection"; @@ -411,6 +417,21 @@ export async function buildAgents( */ loadInstructions?: LoadInstructions, initiator?: AuditInitiator, + /** + * How a message's attached files are put in front of the model. + * + * Appended last on purpose, like the collaborators above it: these are positional, so inserting + * one anywhere else silently shifts every existing call site's arguments by one. + * + * Absent means nothing is inlined and a Bot is shown the URL a file is stored behind rather than + * the file, which is what every deployment did before this existed. + */ + loadAttachment?: LoadAttachment, + /** + * How a run says the files on the message it is answering went out in a send. Appended after + * `loadAttachment` for the positional reason it gives. Absent means nothing is recorded. + */ + markAttachmentsSent?: MarkAttachmentsSent, ): Promise> { let vendors: readonly string[] = []; try { @@ -465,6 +486,8 @@ export async function buildAgents( handoff, instructions ?? null, initiator, + loadAttachment, + markAttachmentsSent, ), ]), ), @@ -480,6 +503,230 @@ export async function buildAgents( */ export type LoadInstructions = () => Promise; +/** + * The bytes behind one stored attachment, fetched when a turn turns out to refer to it. + * + * A closure rather than the rows themselves, for the same reason {@link LoadInstructions} is one: + * which files a turn names is decided by the message, and a request is earlier than a message. It is + * ALSO bound to one person, like {@link LoadInstructions}: the ids reach it out of browser-supplied + * message content, so which rows it may return is decided against the asker's channel memberships. + * Null means this deployment no longer holds it OR it is not this person's to see, and on the + * message the turn is answering `resolveAttachmentParts` turns either into a failed turn rather + * than an answer about a file they never sent. Behind that message it becomes a note that the file + * is gone; see {@link inlineAttachments} for why the two answers differ. + * + * TAKES THE RUN'S THREAD, AND TAKES IT AS A REQUIRED SECOND ARGUMENT. Which files a turn may reach + * is decided by the conversation it is running in, not only by who is asking: an implementation + * resolves the thread to its channel and refuses a file belonging to another one. Required rather + * than optional, and second rather than curried in beside the actor, because those are the two + * shapes that cannot be forgotten — `resolveAttachmentParts` accepts a `(id) => …`, so a + * `LoadAttachment` is deliberately NOT assignable to it, and the binding that adapts one to the + * other is the line where the thread id has to be named. A caller that omits it does not compile. + * Mirrors {@link SignRun}, which takes the thread for the same class of reason. + */ +export type LoadAttachment = ( + id: string, + threadId: string, +) => Promise; + +/** + * Records that these attachments went out in a message somebody sent. + * + * A closure bound to one person, like {@link LoadAttachment}, and for a stricter reason: this one + * WRITES. `attachedAt` is what the sweeper, the upload cap and the withdrawal route all read as + * "this file rode in a message somebody sent", so only the person whose send it was may record it, + * and only for rows they uploaded themselves — see `markAttachmentsSent` in + * channels/attachments.ts, which puts `uploadedBy` in the WHERE for exactly that. + * + * Called with the ids on the message being asked about and nothing else. History is replayed on + * every turn and by whoever runs it, so a message behind the send is not evidence of one. + * + * A FAILURE TO RECORD REFUSES THE TURN, AND THIS PARAGRAPH USED TO PROMISE THE OPPOSITE. It said a + * failure was swallowed and logged, because a turn is somebody waiting for an answer. The waiting + * is real; the conclusion was not. {@link inlineAttachments} calls this BEFORE it returns the + * history, and the history is what the run is given afterwards, so nothing has reached a model when + * this resolves. A rejection here therefore costs a turn that never started, while a silent one + * costs the file itself — `attachedAt` is what stops the culler reclaiming it, so a send recorded + * nowhere is a message that displays a file the sweeper deletes a day later. The long argument, and + * the four things that still do NOT reject, are in `markAttachmentsSent` in channels/attachments.ts. + * + * SO AN IMPLEMENTATION MUST RESOLVE ONLY WHEN THE RECORD IS DURABLE. Resolving because a statement + * was accepted is not enough: an UPDATE that matched no row is a successful command in Postgres, so + * an implementation that cannot distinguish "stamped" from "matched nothing" is reporting a send + * that did not happen. It must also treat a row that is ALREADY sent as recorded, because a replayed + * message and a retried run are both ordinary and neither is an error. + * + * REJECT WITH A SENTENCE, NOT A SYMPTOM. There is no `app.onError` behind this server, and what an + * implementation throws travels out of the run as an AG-UI error and is shown to the person who was + * waiting. It is the same road `resolvePart`'s refusals take, and it wants the same kind of message: + * what happened, and what they can do about it. + * + * Absent still means nothing is recorded, which is what every deployment did before this existed. + */ +export type MarkAttachmentsSent = ( + ids: readonly string[], + /** + * The conversation the send happened in, required for the reason {@link LoadAttachment} takes + * one — and here it is the stronger of the two cases, because this WRITES. A stamp recorded + * against the wrong conversation freezes a row in a channel that never saw the file: it can no + * longer be withdrawn and the culler will no longer reclaim it. + */ + threadId: string, +) => Promise; + +/** + * The same history with every attached file put in front of the model. + * + * USER MESSAGES ONLY. A stored reference gets into a thread by somebody attaching a file to what + * they said; rewriting an assistant or tool message would be rewriting what a model already + * produced. A message that refers to no attachment comes back BY IDENTITY, which is almost all of + * them, so this pass costs nothing on a thread with no files in it. + * + * NOTHING HERE CATCHES, FOR THE MESSAGE BEING ASKED ABOUT. An attachment this deployment cannot load + * fails that turn, deliberately: see `resolvePart` in `channels/attachment-parts.ts` for why a Bot + * answering confidently about an image it never received is the worse of the two outcomes. + * + * THE MESSAGE BEING ASKED ABOUT IS THE LAST USER MESSAGE. Everything after it in `input.messages` is + * the Bot's own work on this turn, and everything before it is a turn already answered; the last + * thing a person said is what the run is a reply to, and so the only message whose attachments the + * answer is going to be about. It is also the only one whose attachments were just uploaded, which + * is what makes failing there recoverable: the person is still there, and can re-attach and re-send. + * + * OLDER MESSAGES DEGRADE INSTEAD. This maps over the WHOLE history, and history is replayed on every + * turn, so one vanished row failing here would fail this channel's every future turn for ever — the + * same geometry as the dangling tool call in `agents/history-sanitize.ts`, which opens by recording + * that exact failure found in production twice: a permanent failure grown out of transient damage, + * and nothing the person did wrong. It is reachable the same way, too: a send whose run is stopped + * before the load never stamps `attachedAt`, and the sweeper deletes the row a day later. So an + * older part whose row is gone becomes text saying so, which keeps the property that matters — the + * model is never left to answer as though a file it cannot see were in front of it — without the + * permanence. + * + * AND IT WALKS BACKWARDS, WHICH IS WHAT MAKES THE BUDGET FAIR. `MAX_INLINED_BYTES_PER_RUN` bounds + * what one turn may inline in total — nothing did, before, and a channel that had seen a few large + * images made every later turn read and base64 all of them again. A budget is only defensible if + * the person's own question is never what it cuts, so the walk starts at the newest message: the + * one being asked about is charged first, and is never cut whatever it costs. What runs out is the + * room left for the history behind it, and that history has already been in front of the model + * once, in the turn it arrived. + * + * NEVER CUT IS NOT THE SAME AS NEVER BOUNDED, and reading the first as the second is what left the + * asked message with no ceiling at all: `resolvePart` stopped spending only under + * `onMissing: "note"`, so a message a browser had just written could name two hundred + * previously-sent files and inline every one of them. It is still never cut — a person is never + * told their own question's attachment was quietly left out — but a message past + * `MAX_INLINED_BYTES_PER_RUN` now fails this turn with a sentence naming the file and the limit. + * Refusing is an answer somebody can act on, since the message is still in front of them; silently + * serving half of it is not. + * + * The walk is sequential for the same reason the parts within a message are: a budget spent by + * whichever database read happened to settle first would cut a different message on each run over + * the same thread. The concurrency given up is one round trip per message that carries a file, + * which is very few messages in very few threads. + */ +async function inlineAttachments( + messages: Message[], + load: LoadAttachment, + /** + * The conversation this run is in, passed to every load and to the stamp. + * + * Taken as a parameter rather than read off anything reachable from here because this function + * is handed a history, not a run — and it is the ONLY place that both knows which message is + * being asked about and is called from every path that can send one. Both call sites below have + * `input.threadId` in hand; neither can supply it by accident, since {@link LoadAttachment} does + * not typecheck without it. + */ + threadId: string, + markSent?: MarkAttachmentsSent, +): Promise { + const asked = messages.reduce( + (latest, message, index) => (message.role === "user" ? index : latest), + -1, + ); + const budget = newInlineBudget(); + const inlined = [...messages]; + /** + * The ids to stamp once the whole walk has come back, or none. + * + * Collected during the walk and spent after it, which is the whole of the fix described at the + * `markSent` call below: the asked message is the FIRST thing this backward loop resolves, so a + * stamp written where it is found is written before any older message has been looked at. + */ + let sentIds: readonly string[] = []; + for (let index = messages.length - 1; index >= 0; index--) { + const message = messages[index]; + if (message.role !== "user") continue; + const content = await resolveAttachmentParts( + message.content, + /* + * The seam narrows to `(id) => …` HERE, and this line is the whole reason the thread id is a + * required parameter rather than something a wiring could forget. `resolveAttachmentParts` + * asks about one id at a time and knows nothing about runs; a `LoadAttachment` is not + * assignable to what it takes, so adapting one to the other cannot be done without naming + * the conversation the ids are being resolved for. + */ + (id) => load(id, threadId), + index === asked ? "fail" : "note", + budget, + ); + /* + * AND THIS IS WHERE `attachedAt` IS WRITTEN, for the asked message alone. + * + * The send that puts a file in front of a Bot goes out through AG-UI, so there is no request + * handler in channels/attachments.ts to hook the write onto; this function is the first place + * in the server that both knows the message and knows it is the one being asked about. Every + * other message here is history, replayed in full on every turn and by whoever is running it, + * which is why the write cannot live in `load`: a read happens for all of them. + * + * AFTER THE WHOLE HISTORY RESOLVES, NOT AFTER THIS MESSAGE DOES, which is why the ids are only + * COLLECTED here and the write happens past the end of the loop. The strict `"fail"` above is + * what refuses a turn that names a file the asker cannot see, and a send is not recorded for a + * turn that never ran — but this loop walks BACKWARDS, so the asked message is the first thing + * it resolves and every older message is still ahead of it. Stamping here meant stamping before + * any of them had been looked at, and a history load that rejects (a pool error, a timeout; + * `"note"` softens a MISSING row, not a failing read) then failed the turn with `attachedAt` + * already written for it. The stamp's entire meaning is "this file reached a message somebody + * actually sent", and three readers act on it — the sweeper's delete, the upload cap, the + * withdrawal route — so a stamp for a turn that never ran is not a cosmetic inaccuracy. + * + * Past the end of the loop is as late as this function can put it, and no later: whether the + * model ever answers is decided by things well downstream of here, and a stamp that waited for + * that would be waiting on something this function does not observe. What it can promise is + * that every message this turn was going to inline was inlined first. + * + * AND IT IS NO LONGER CAUGHT HERE, WHICH IS THE REVERSAL OF WHAT THIS SEAM USED TO DO. A + * `try`/`catch` stood around the call, holding {@link MarkAttachmentsSent}'s old promise that a + * failure to record would never fail an answer. The promise was kept and the outcome was still + * wrong: the run carried on to a model with a file whose `attachedAt` was never written, the + * culler reclaimed the row a day later, and the message went on displaying an attachment that no + * longer existed. Nobody was told, on either side. + * + * THE POINT IS *WHERE* THIS LINE SITS, and it is the reason refusing is affordable at all. + * Everything above has resolved into `inlined`, and `inlined` is RETURNED — the run is handed to + * `super.run` / `next.run` by the caller, after this function comes back. So a rejection on this + * line happens with no model called, no token spent, and the person's message still in front of + * them, which is the same position `resolvePart`'s `"fail"` refusal leaves them in a few lines + * above. The turn is not yet spent, so the trade is not "an answer for a file" — it is a retry + * for a file, and it is only available here. + * + * The implementation logs the actor and the ids; what it throws is a sentence naming the files + * and what to do about them, and that sentence is what leaves through the run as an AG-UI error. + * Wrapping it again here would only put this seam's words in front of the ones that know which + * rows are involved. + */ + if (index === asked && markSent) { + sentIds = attachmentIdsIn(message.content); + } + if (content !== message.content) { + inlined[index] = { ...message, content } as Message; + } + } + if (markSent && sentIds.length > 0) { + await markSent(sentIds, threadId); + } + return inlined; +} + async function buildAgent( agent: RegisteredAgent, model: RuntimeModel, @@ -495,6 +742,10 @@ async function buildAgent( /** Already resolved by {@link buildAgents}, so one roster costs one read. */ standingInstructions: string | null = null, initiator?: AuditInitiator, + /** How this run's attached files are inlined. See {@link buildAgents}. */ + loadAttachment?: LoadAttachment, + /** How this run records that those files were sent. See {@link buildAgents}. */ + markAttachmentsSent?: MarkAttachmentsSent, ): Promise { if (agent.type === "unavailable") { return new UnavailableAgent(agent); @@ -600,6 +851,8 @@ async function buildAgent( signRun, connectedVendors, narrowing ? offeredFor : undefined, + loadAttachment, + markAttachmentsSent, ); } @@ -619,6 +872,8 @@ async function buildAgent( connectedVendors, standingInstructions, ), + loadAttachment, + markAttachmentsSent, ); const whole = withTools(granted); @@ -854,6 +1109,19 @@ function remoteAgentWithStandingRole( * Absent means no narrowing, which is the behaviour every deployment had before this existed. */ narrow?: (input: RunAgentInput) => Promise, + /* + * No `agentFetch` and no `initiator` here any more, and they were not dropped: the transport this + * function used to build itself is now built by `remoteTransport` and handed in as `next`, and + * that is where both went. Passing them again would be two names for one wiring, and the second + * one would be the one nothing reads. + */ + /** + * How this run's attached files are inlined, applied inside the middleware below for the reason + * the sanitiser is: `run` skips `.use()`. Absent means nothing is inlined. + */ + loadAttachment?: LoadAttachment, + /** How this run records that those files were sent. See {@link buildAgents}. */ + markAttachmentsSent?: MarkAttachmentsSent, ) { /* * What this Bot holds, as a second standing message. @@ -933,54 +1201,79 @@ function remoteAgentWithStandingRole( const answeredByResume = new Set( (input.resume ?? []).map((entry) => entry.interruptId), ); - return next.run({ - ...input, - messages: [ - agent.standingMessage, - ...(holdingsMessage ? [holdingsMessage] : []), - ...sanitizeSeededHistory( - input.messages.filter( - (message) => - message.id !== agent.standingMessage.id && - message.id !== holdingsMessage?.id, - ), - answeredByResume, - ), - ], - /* - * The Bot's own grants, added to whatever the surface offered. - * - * Sent on every run rather than configured once on the endpoint, because a grant an - * administrator adds or revokes has to apply to the next run and the endpoint is somebody - * else's process. - */ - tools: [ - ...(input.tools ?? []), - ...tools.map((tool) => ({ - name: tool.name, - description: tool.description, - parameters: z.toJSONSchema(tool.parameters) as Record< - string, - unknown - >, - })), - ], - context: - agent.type === "remote_mastra" - ? [ - ...callerMastraContext(input.context ?? []), - ...mastraOpenBotContext({ - standingMessage: agent.standingMessage, - holdingsMessage, - botId: agent.id, - deploymentTools, - runAssertion, - }), - ] - : input.context, - // Who the Bot is calling back as, so the audit row names it rather than "an agent". - forwardedProps, - } as never); + const history = sanitizeSeededHistory( + input.messages.filter( + (message) => + message.id !== agent.standingMessage.id && + message.id !== holdingsMessage?.id, + ), + answeredByResume, + ); + /* + * And the attached files, inlined here for the same reason that guard is here: this middleware + * is the last thing between the browser's `input.messages` and the endpoint, and `run` skips + * `.use()`. Left alone, a file reaches the endpoint as an `/api/attachments/` URL on a + * server that holds no session here and cannot fetch it, so the Bot answers about a file it + * never received. + * + * ALONGSIDE THE SANITISER, NOT INSTEAD OF IT. One drops what the provider is going to refuse; + * this puts in front of the model what the person actually attached. + */ + return from( + loadAttachment + ? inlineAttachments( + history, + loadAttachment, + // The conversation this run is in, which is what decides whose files it may reach. + input.threadId, + markAttachmentsSent, + ) + : Promise.resolve(history), + ).pipe( + switchMap((messages) => + next.run({ + ...input, + messages: [ + agent.standingMessage, + ...(holdingsMessage ? [holdingsMessage] : []), + ...messages, + ], + /* + * The Bot's own grants, added to whatever the surface offered. + * + * Sent on every run rather than configured once on the endpoint, because a grant an + * administrator adds or revokes has to apply to the next run and the endpoint is somebody + * else's process. + */ + tools: [ + ...(input.tools ?? []), + ...tools.map((tool) => ({ + name: tool.name, + description: tool.description, + parameters: z.toJSONSchema(tool.parameters) as Record< + string, + unknown + >, + })), + ], + context: + agent.type === "remote_mastra" + ? [ + ...callerMastraContext(input.context ?? []), + ...mastraOpenBotContext({ + standingMessage: agent.standingMessage, + holdingsMessage, + botId: agent.id, + deploymentTools, + runAssertion, + }), + ] + : input.context, + // Who the Bot is calling back as, so the audit row names it rather than "an agent". + forwardedProps, + } as never), + ), + ); }; /* @@ -1136,20 +1429,63 @@ class BuiltInAgentWithSaneHistory extends BuiltInAgent { * {@link clone} has to build another one of THIS class rather than of the base. */ private readonly configuration: BuiltInAgentConfiguration; + /** + * How this Bot's attached files are inlined, or absent to leave them alone. + * + * Held for the same reason {@link configuration} is: {@link clone} builds another one of THIS + * class and everything the run depends on has to survive that. + */ + private readonly loadAttachment: LoadAttachment | undefined; + /** + * How this Bot records that the files on the message it is answering were sent, or absent to + * record nothing. Held for the reason {@link loadAttachment} is: {@link clone} builds another one + * of THIS class, and a seam lost in a clone is a seam that never runs. + */ + private readonly markAttachmentsSent: MarkAttachmentsSent | undefined; - constructor(configuration: BuiltInAgentConfiguration) { + constructor( + configuration: BuiltInAgentConfiguration, + loadAttachment?: LoadAttachment, + markAttachmentsSent?: MarkAttachmentsSent, + ) { super(configuration); this.configuration = configuration; + this.loadAttachment = loadAttachment; + this.markAttachmentsSent = markAttachmentsSent; } run(input: RunAgentInput): Observable { const answeredByResume = new Set( (input.resume ?? []).map((entry) => entry.interruptId), ); - return super.run({ - ...input, - messages: sanitizeSeededHistory(input.messages, answeredByResume), - }); + const history = sanitizeSeededHistory(input.messages, answeredByResume); + const load = this.loadAttachment; + /* + * ALONGSIDE THE GUARD ABOVE, NOT INSTEAD OF IT. One drops a conversation the model provider is + * going to refuse; this replaces the stored reference a person's attachment arrives as with the + * bytes themselves, because `BuiltInAgent.run` converts `input.messages` with no seam in + * between and a `/api/attachments/` URL is not something a model provider will go and fetch. + * + * Nothing to load means nothing to inline, and the run goes up exactly as it did before any of + * this existed. + */ + if (!load) return super.run({ ...input, messages: history }); + /* + * Deferred, because `run` has to answer with a stream straight away and reading the bytes is a + * database round trip. `defer` puts that read on the subscription, which is where the run + * actually begins, so nothing is fetched until somebody is listening. + */ + return defer(() => + from( + inlineAttachments( + history, + load, + // As above: the run's own conversation, not the actor's channels at large. + input.threadId, + this.markAttachmentsSent, + ), + ).pipe(switchMap((messages) => super.run({ ...input, messages }))), + ); } /** @@ -1163,7 +1499,11 @@ class BuiltInAgentWithSaneHistory extends BuiltInAgent { * something does, it is not lost in a clone. */ clone(): BuiltInAgentWithSaneHistory { - const cloned = new BuiltInAgentWithSaneHistory(this.configuration); + const cloned = new BuiltInAgentWithSaneHistory( + this.configuration, + this.loadAttachment, + this.markAttachmentsSent, + ); type WithMiddlewares = { middlewares: unknown[] }; (cloned as unknown as WithMiddlewares).middlewares = [ ...(this as unknown as WithMiddlewares).middlewares, @@ -1327,6 +1667,17 @@ export async function resolveRuntimeAgents( loadInstructions?: LoadInstructions, /** Appended after `loadInstructions`, for the positional reason it gives. */ initiator?: AuditInitiator, + /** + * How a message's attached files are put in front of the model. Appended after `initiator`, for + * the same positional reason. Absent means nothing is inlined, which is what every deployment did + * before this existed. + */ + loadAttachment?: LoadAttachment, + /** + * How a send is recorded against the files it carried. Appended after `loadAttachment`, for the + * same positional reason. Absent means nothing is recorded. + */ + markAttachmentsSent?: MarkAttachmentsSent, ): Promise> { const all = await loadAgents(); if (all.length === 0) { @@ -1359,6 +1710,8 @@ export async function resolveRuntimeAgents( handoff, loadInstructions, initiator, + loadAttachment, + markAttachmentsSent, ); } @@ -1440,6 +1793,28 @@ export function createRequestAgents( * so which person it belongs to has to be decided by the session and never by the caller. */ loadInstructionsForActor?: (actorId: string) => LoadInstructions, + /** + * How the files on a person's message are put in front of the model, resolved for whoever is + * asking. + * + * Per actor, and through `identifyActor` rather than anything in the request body, for the reason + * `loadInstructionsForActor` is: the ids arrive inside `input.messages`, which the browser wrote, + * so a turn can name an attachment in a channel the asker was never in. Which rows this may read + * has to be decided by the session, exactly as the fetch route decides it. Appended last for the + * positional reason above. Absent means nothing is inlined, which is what every deployment did + * before this existed. + */ + loadAttachmentForActor?: (actorId: string) => LoadAttachment, + /** + * How a send is recorded against the files it carried, resolved for whoever is asking. + * + * Per actor for a stricter reason than the reader beside it: this one WRITES `attachedAt`, and + * `markAttachmentsSent` will only stamp rows the acting person uploaded themselves. Deciding who + * that is from the session rather than from the request body is what keeps one member from + * recording a send against a colleague's staged file. Appended last, positionally. Absent means + * nothing is recorded. + */ + markAttachmentsSentForActor?: (actorId: string) => MarkAttachmentsSent, ) { return async ({ request }: { request: Request }) => { const actor = await identifyActor(request); @@ -1458,6 +1833,11 @@ export function createRequestAgents( // Every Bot this person can see, so no `onlyBotId` here; the instructions follow it. undefined, loadInstructionsForActor?.(actor.id), + // No initiator: a request is a person asking, which is the default this path has always + // carried. Named only so the attachments after it land in the right position. + undefined, + loadAttachmentForActor?.(actor.id), + markAttachmentsSentForActor?.(actor.id), ); }; } @@ -1588,6 +1968,27 @@ export function mountCopilotRuntime( * into only one of them would be the drift `agentFor` exists to prevent. */ loadInstructionsForActor?: (actorId: string) => LoadInstructions, + /** + * How the files on a message are put in front of the model, resolved per person, on both paths + * below. + * + * Given to the request path and to `agentFor` alike, for the reason `loadInstructionsForActor` is: + * a routine's turn at three in the morning has to inline exactly as a person's chat turn does, and + * a seam wired into only one of them is the drift `agentFor` exists to prevent. Actor-keyed for + * the same reason every other collaborator here is — the ids come out of browser-supplied message + * content, so the person the run belongs to is what decides which attachments it may read. + * Appended last because these are positional. Absent means nothing is inlined. + */ + loadAttachmentForActor?: (actorId: string) => LoadAttachment, + /** + * How a send is recorded against the files it carried, resolved per person, on both paths below. + * + * Given to the request path and to `agentFor` alike, for the reason `loadAttachmentForActor` is: + * a routine's turn at three in the morning sends exactly as a person's chat turn does, and a seam + * wired into only one of them is the drift `agentFor` exists to prevent. Actor-keyed because the + * write is scoped to rows that person uploaded. Appended last. Absent means nothing is recorded. + */ + markAttachmentsSentForActor?: (actorId: string) => MarkAttachmentsSent, ) { const { intelligence } = config.runtime; @@ -1633,6 +2034,8 @@ export function mountCopilotRuntime( input.botId, loadInstructionsForActor?.(actor.id), input.initiator, + loadAttachmentForActor?.(actor.id), + markAttachmentsSentForActor?.(actor.id), ); return agents[input.botId] ?? null; }; @@ -1702,6 +2105,8 @@ export function mountCopilotRuntime( agentFetch, handoffForActor, loadInstructionsForActor, + loadAttachmentForActor, + markAttachmentsSentForActor, ) as never, }); diff --git a/server/src/db/schema/core.ts b/server/src/db/schema/core.ts index 9c58e9e9d..a69dda781 100644 --- a/server/src/db/schema/core.ts +++ b/server/src/db/schema/core.ts @@ -1,6 +1,7 @@ import { sql } from "drizzle-orm"; import { boolean, + customType, index, integer, pgEnum, @@ -15,6 +16,10 @@ import { // JSON string and nothing in this database could be queried by a JSON field. See ./json.ts. import { jsonb } from "./json"; +const bytea = customType<{ data: Buffer; driverData: Buffer }>({ + dataType: () => "bytea", +}); + const createdAt = () => timestamp("created_at", { withTimezone: true }).notNull().defaultNow(); const updatedAt = () => @@ -492,3 +497,97 @@ export const intelligenceChannelMappings = pgTable( uniqueIndex("intelligence_channel_mappings_thread_idx").on(table.threadId), ], ); + +/** + * A file somebody attached to a message in a channel. + * + * The bytes live here rather than on a disk or in a bucket because this deployment is a compose + * file: a volume would split the backup story in two, and an object store would put a bucket + * between a self-hoster and a working install. One `pg_dump` restores a deployment, and that stays + * true. Reads go through one endpoint, so moving the bytes later changes that endpoint and nothing + * else. + */ +export const attachments = pgTable( + "attachments", + { + id: uuid("id").primaryKey().defaultRandom(), + channelId: text("channel_id") + .notNull() + .references(() => channels.id, { onDelete: "cascade" }), + uploadedBy: text("uploaded_by") + .notNull() + .references(() => users.id, { onDelete: "cascade" }), + name: text("name").notNull(), + /** + * What the server decided this is, never what the client claimed. + * + * A browser will happily report `text/plain` for a file it dragged out of another application, + * and a client can send whatever it likes. This column is what the fetch endpoint serves as + * `Content-Type`, so a wrong value here is a security bug rather than a cosmetic one. + */ + mimeType: text("mime_type").notNull(), + sizeBytes: integer("size_bytes").notNull(), + bytes: bytea("bytes").notNull(), + createdAt: createdAt(), + /** + * When this appeared in a sent message. Null means staged. + * + * Attach three files, change your mind and close the tab, and those rows would sit here forever + * with nothing referring to them. The sweeper deletes staged rows past a few hours; a row with a + * date is spoken for and is never swept. + */ + attachedAt: timestamp("attached_at", { withTimezone: true }), + /** + * Which composer session staged this row, so the per-message cap counts the same set the + * composer does. + * + * NOT A DRAFT ID, and the name is the whole of the distinction. Nothing about the message is + * saved here: no text, no ordering, nothing that survives a reload. It is a grouping key over + * rows this table already held, minted fresh by each composer instance and thrown away with it. + * + * The cap it exists for is per message, and the client can only ever see what is on its own + * screen. Counted per channel instead — which is what this server did before this column — a + * closed tab, a stopped run or a removed queued message left staged rows nobody could see, and + * the client would then accept a pick the server refused with a 409 naming files that were on + * nobody's screen. Eight such orphans locked uploads in that channel until the sweeper's + * 24-hour window expired. + * + * NULLABLE, AND DELIBERATELY NOT BACKFILLED. Every row that predates this column has NULL here, + * `null = ` is never true in SQL, and so those rows match no live group and block no + * upload. They are still the sweeper's to reclaim on its own schedule. + * + * `text` rather than `uuid` even though `newId()` mints a UUID: this value arrives as a form + * field the browser chose, and comparing text that is not uuid-shaped against a `uuid` column + * raises `22P02` and throws — the same trap `isUuidShaped` in channels/attachments.ts exists to + * step around. As text, a nonsense group is simply a group with nothing in it. + */ + uploadGroup: text("upload_group"), + }, + (table) => [ + // Postgres does not index foreign key columns on its own. Deleting a + // channel cascades here, and without this index that cascade is a + // sequential scan of the one table in this deployment that holds blobs. + index("attachments_channel_idx").on(table.channelId), + // The same rationale as `attachments_channel_idx` above, for the other + // cascading foreign key on this table. Removing a person deletes their + // `users` row, and that cascade has to find every attachment they ever + // uploaded; unindexed, it is the same sequential scan over the same blob + // table, and it runs on the one operation a deployment cannot retry + // halfway through. + index("attachments_uploaded_by_idx").on(table.uploadedBy), + // The cap's own predicate, and every upload runs it. Partial for the same + // reason `attachments_staged_idx` below is: staged rows are a small, + // short-lived minority, and an index over every row would grow with the + // table for a query that only ever asks about the unstamped ones. + index("attachments_upload_group_idx") + .on(table.channelId, table.uploadedBy, table.uploadGroup) + .where(sql`${table.attachedAt} is null`), + // Partial, on the sweeper's own predicate rather than the whole column. + // The sweeper only ever asks for staged rows (`attached_at is null`), + // which are a small, short-lived minority of the table, so an index + // covering every row would grow with the table for no query that exists. + index("attachments_staged_idx") + .on(table.createdAt) + .where(sql`${table.attachedAt} is null`), + ], +); diff --git a/server/src/index.ts b/server/src/index.ts index 8ac611e66..707c1c29a 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -33,6 +33,10 @@ import { DEV_ACTOR, initializeDevActorUser } from "./auth/dev-actor"; import { createRoleRepository } from "./auth/guards"; import { createIdentityProviderStore } from "./auth/identity-provider-store"; import type { OpenBotRole } from "./auth/roles"; +import { + loadAttachmentForTurn, + markAttachmentsSent, +} from "./channels/attachments"; import { createChannelEventHub, startChannelActivityListener, @@ -544,6 +548,62 @@ const userInstructionsStore = createUserInstructionsStore(database); const loadInstructionsForActor = (actorId: string) => () => userInstructionsStore.read(actorId); +/* + * The file behind an attachment reference, read when a turn turns out to name one. + * + * Read per turn rather than held, for the reason the bytes are in the database at all: a message + * carries a `/api/attachments/` URL, and a model provider is not going to go and fetch it. The + * row is fetched here and the bytes go up inline, so the Bot sees the file the person attached + * instead of a link it cannot follow. + * + * Built per actor and passed to both turn paths — the request path through `mountCopilotRuntime` and + * a routine's turn through `buildAgentFor` — so a routine firing at three in the morning inlines + * exactly as a person's chat turn does, on exactly the same footing. + * + * NARROWED BY ACTOR AND BY CONVERSATION, and not silent. The reference reaches the loader out of + * browser-supplied message content, so a turn can name an attachment in a channel the asker was + * never in — or in one they ARE in but which is not the channel this turn is running in. + * `loadAttachmentForTurn` answers both with the same membership join the fetch route uses plus the + * run's own thread, and null when there is no row this person may see here. + * `resolveAttachmentParts` fails the turn on that null rather than letting a Bot read a file back + * to somebody who cannot open it. + * + * The thread is the CLOSURE'S ARGUMENT rather than something baked in beside the actor, because one + * of these is built per actor per request and then used for however many runs that request makes; + * a thread captured here would be the first run's, silently, for all of them. + * + * A PURE READ. `attachedAt` is written by the send rather than by anything here; see + * `markAttachmentsSentForActor` below. + */ +const loadAttachmentForActor = + (actorId: string) => (id: string, threadId: string) => + loadAttachmentForTurn(database, { actorId, threadId }, id); + +/** + * That the files on a message went out in it, recorded when a turn turns out to be a send. + * + * Bound per actor and handed to the same two turn paths as the reader above, so a routine's send at + * three in the morning is recorded exactly as a person's chat turn is. `inlineAttachments` + * (copilot.ts) calls it with the ids on the message being asked about and no others: history is + * replayed on every turn and by whoever is running it, so nothing behind that message is evidence + * of a send. + * + * NARROWED BY ACTOR, and more strictly than the reader is. Reading is scoped to channel + * membership, because members are meant to see each other's sent files; recording a send is scoped + * to the UPLOADER, because `attachedAt` is what the sweeper, the upload cap and the withdrawal + * route all read as "this file rode in a message somebody sent" — and a member who could write it + * on a colleague's staged row would freeze that colleague's own withdrawal at 409 and leave the row + * unsweepable. See `markAttachmentsSent` in channels/attachments.ts. + * + * AND NARROWED BY CONVERSATION, taking the thread as an argument for the reason the reader does. + * A stamp written against a channel that never saw the file freezes the row the same way, and is + * reached without any colleague being involved: one person, two channels of their own, a file + * named from the wrong one. + */ +const markAttachmentsSentForActor = + (actorId: string) => (ids: readonly string[], threadId: string) => + markAttachmentsSent(database, { actorId, threadId }, ids); + /* * What the deployment tells a remote Bot about the run it is starting. * @@ -700,6 +760,13 @@ const buildAgentFor = async ({ // it is written the way they asked for it to be written, exactly as their chat turn would be. loadInstructionsForActor(actor.id), initiator, + // The same reader the request path gets, bound to the owner the routine runs as, so a file + // attached in a channel reads the same way on a routine's turn as it does on the person's own — + // and is refused the same way when the owner is not in that channel. + loadAttachmentForActor(actor.id), + // And the same recorder, so the files on a routine's own message stop counting as staged the + // moment it sends them, exactly as a person's do. + markAttachmentsSentForActor(actor.id), ); const agent = agents[agentId]; if (!agent) { @@ -862,6 +929,12 @@ const copilotRuntime = mountCopilotRuntime( }, // What this person has told every coworker of theirs, in every channel. See user-instructions.ts. loadInstructionsForActor, + // The files on a message, put in front of the model rather than left as links it cannot follow — + // and only the ones the person whose run this is could open themselves. + loadAttachmentForActor, + // And that those files went out in a send, written by the person who sent them and only for rows + // they uploaded. See markAttachmentsSentForActor. + markAttachmentsSentForActor, ); /** @@ -1146,6 +1219,9 @@ const app = createApp( // The same store every run reads through `loadInstructionsForActor`, so the screen a person edits // and the prompt their coworker is built from can never be two different pieces of text. userInstructionsStore, + // The same database every other store here is built from, so a channel's staged and sent files + // live behind the same connection as the messages that reference them. + database, ); /** diff --git a/server/tests/attachment-mime.test.ts b/server/tests/attachment-mime.test.ts new file mode 100644 index 000000000..af221403f --- /dev/null +++ b/server/tests/attachment-mime.test.ts @@ -0,0 +1,302 @@ +import { describe, expect, test } from "bun:test"; +import { classifyAttachment, namesNoFormat } from "../../shared/attachments"; +import { sniffMimeType } from "../src/channels/attachment-mime"; + +/** + * `claimed` here is `file.type` from the uploading client — a value this + * server later serves back as the `Content-Type` header on its own origin. + * These tests exist to pin down that the byte signature wins over that + * claim, not the other way around. + */ +describe("sniffing a file's real MIME type from its bytes", () => { + test("recognizes a PNG signature", () => { + const bytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]); + expect(sniffMimeType(bytes, "application/octet-stream")).toBe("image/png"); + }); + + test("recognizes a JPEG signature", () => { + const bytes = new Uint8Array([0xff, 0xd8, 0xff, 0xe0]); + expect(sniffMimeType(bytes, "application/octet-stream")).toBe("image/jpeg"); + }); + + test("recognizes a GIF signature", () => { + const bytes = new Uint8Array([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]); + expect(sniffMimeType(bytes, "application/octet-stream")).toBe("image/gif"); + }); + + test("recognizes a WEBP signature (RIFF at 0, WEBP at 8)", () => { + const bytes = new Uint8Array([ + 0x52, + 0x49, + 0x46, + 0x46, // "RIFF" + 0x00, + 0x00, + 0x00, + 0x00, // chunk size, irrelevant here + 0x57, + 0x45, + 0x42, + 0x50, // "WEBP" + ]); + expect(sniffMimeType(bytes, "application/octet-stream")).toBe("image/webp"); + }); + + test("a PNG claiming to be text/plain still sniffs as image/png", () => { + const bytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]); + expect(sniffMimeType(bytes, "text/plain")).toBe("image/png"); + }); + + test("an SVG body claiming image/svg+xml is returned by that name", () => { + // "" decodes as perfectly valid UTF-8, so if the claimed-MIME + // branch didn't win first this would fall through to "text/plain" — + // and the caller's SVG denylist could never see it to refuse it. + const bytes = new TextEncoder().encode(""); + expect(sniffMimeType(bytes, "image/svg+xml")).toBe("image/svg+xml"); + }); + + test("plain text with no recognized claim sniffs as text/plain", () => { + const bytes = new TextEncoder().encode("just some notes"); + expect(sniffMimeType(bytes, "")).toBe("text/plain"); + }); + + test("binary junk with an unknown claim returns the claim", () => { + const bytes = new Uint8Array([0xff, 0xfe, 0x00, 0x01, 0x02, 0xc0]); + expect(sniffMimeType(bytes, "application/x-widget")).toBe( + "application/x-widget", + ); + }); + + test("recognized claims win outright once no image signature matches", () => { + const bytes = new TextEncoder().encode('{"ok":true}'); + expect(sniffMimeType(bytes, "application/json; charset=utf-8")).toBe( + "application/json", + ); + }); + + test("a .txt claiming application/octet-stream sniffs as text/plain", () => { + // "application/octet-stream" names nothing — it's what a browser sends + // for a file it has no idea about — so it must not be preserved over + // the byte sniff the way a specific claim like "image/svg+xml" is. + const bytes = new TextEncoder().encode("just some notes"); + expect(sniffMimeType(bytes, "application/octet-stream")).toBe("text/plain"); + }); + + test("a text/html claim is returned by that name, on purpose", () => { + // "text/html" names a specific format, so it comes back verbatim just + // like "image/svg+xml" does — refusing it is classifyAttachment's job + // downstream, not sniffMimeType's. + const bytes = new TextEncoder().encode(""); + expect(sniffMimeType(bytes, "text/html")).toBe("text/html"); + }); + + test("bytes that are no known image, claimed image/png, are not image/png", () => { + // The claim names a type this app accepts, and nothing in these bytes + // agrees with it. Handing the name back would let any bytes at all be + // stored and served as an accepted image on this app's own origin. + const bytes = new Uint8Array([0x00, 0x01, 0x02, 0xc0, 0xff, 0xfe]); + const sniffed = sniffMimeType(bytes, "image/png"); + expect(sniffed).not.toBe("image/png"); + expect(classifyAttachment(sniffed)).not.toBe("image"); + }); + + test("an SVG body claiming image/png is not accepted as anything", () => { + // The exact laundering the SVG refusal exists to stop: relabel the SVG + // and the caller's denylist never sees the name it refuses by. It must + // not come back as an image, and it must not fall through to + // "text/plain" either — that would still store and serve it. + const bytes = new TextEncoder().encode( + "", + ); + const sniffed = sniffMimeType(bytes, "image/png"); + expect(sniffed).not.toBe("image/png"); + const kind = classifyAttachment(sniffed); + expect(kind).not.toBe("image"); + expect(kind).not.toBe("text"); + }); + + test("a JPEG claiming image/png comes back as what the bytes say", () => { + // Corroboration is per-format, not per-claim: the bytes decide which + // image it is, and a wrong-but-honest claim does not make it unreadable. + const bytes = new Uint8Array([0xff, 0xd8, 0xff, 0xe0]); + expect(sniffMimeType(bytes, "image/png")).toBe("image/jpeg"); + }); + + test("a real PNG claiming image/png is still image/png", () => { + const bytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]); + expect(sniffMimeType(bytes, "image/png")).toBe("image/png"); + }); + + test("an image claim this app does not accept still comes back by name", () => { + // "image/heic" is not corroborated either, but it is refused by name + // downstream, and that name is what makes the refusal say "this app + // cannot read HEIC" instead of "unsupported file". + const bytes = new Uint8Array([0x00, 0x00, 0x00, 0x18, 0x66, 0x74]); + expect(sniffMimeType(bytes, "image/heic")).toBe("image/heic"); + expect(classifyAttachment("image/heic")).toBe("unsupported-image"); + }); +}); + +/** + * The text half of the same rule the image tests above pin down. + * + * A text claim is corroborated, not verified: the bytes can only say whether + * the file is text AT ALL, never which of the four text formats it is. That is + * still worth doing — it is the difference between "the client said `text/plain` + * so it is" and "the client said `text/plain` and the bytes are at least text." + * + * The PNG guard for the other half is `a real PNG claiming image/png is still + * image/png` above; these two are the text-side pair. + */ +describe("a text claim has to be corroborated by the bytes", () => { + test("bytes that are not UTF-8 at all, claimed text/plain, are not text", () => { + // 0xC0 0xC0 is an overlong-prefix pair no UTF-8 decoder accepts, and 0xFF + // never appears in UTF-8 at any position. Nothing here is text, so the + // claim has nothing to stand on. + const bytes = new Uint8Array([0xff, 0xfe, 0x00, 0x01, 0xc0, 0xc0]); + const sniffed = sniffMimeType(bytes, "text/plain"); + expect(sniffed).not.toBe("text/plain"); + expect(classifyAttachment(sniffed)).not.toBe("text"); + }); + + test("a zero-byte file claiming text/plain is not accepted as anything", () => { + // `isValidUtf8` is trivially true for no bytes, so an empty file used to + // walk straight through the claim branch. Nothing corroborates a claim + // about a file that has no contents to corroborate it with. + const sniffed = sniffMimeType(new Uint8Array(0), "text/plain"); + const kind = classifyAttachment(sniffed); + expect(kind).not.toBe("text"); + expect(kind).not.toBe("image"); + }); + + test("a zero-byte file with no claim at all is not accepted either", () => { + // The other way into the same hole: with a blank claim the function falls + // through to the UTF-8 guess, which an empty file also passes trivially. + const sniffed = sniffMimeType(new Uint8Array(0), ""); + const kind = classifyAttachment(sniffed); + expect(kind).not.toBe("text"); + expect(kind).not.toBe("image"); + }); + + test("real UTF-8 text claiming text/plain is still text/plain", () => { + // The guard on the change above: corroboration must not cost the ordinary + // case anything. A .txt file full of text is exactly what this path is for. + const bytes = new TextEncoder().encode("just some notes\n"); + expect(sniffMimeType(bytes, "text/plain")).toBe("text/plain"); + expect(classifyAttachment("text/plain")).toBe("text"); + }); +}); + +/** + * THIS FUNCTION'S ANSWER IS READ ALOUD, SO IT HAS TO BE A NAME. + * + * `attachments.ts` puts the returned string into a refusal a person reads: + * `'x.bin' is not a file type this app can read ().` A blank answer + * renders that sentence with a hole in it — `... can read ().` — and the + * parenthetical names nothing because there was nothing to name. + * + * Every path out of `sniffMimeType` must therefore hand back a media type, + * never the empty string and never a fragment that is not one. The claim is + * not a safe thing to echo at that point: the only way to REACH the last + * line is for `namesNoFormat` to have already said the claim names no + * format, so echoing it is echoing a non-answer by construction. + */ +describe("the answer is always a media type, because a person reads it", () => { + test("binary bytes with a blank claim do not come back blank", () => { + // No image signature, not valid UTF-8, and nothing claimed: the case a + // `.bin` dragged out of a folder by a browser that would not guess hits. + // 0xFF never appears in UTF-8 at any position, and 0xFF 0x00 is not the + // JPEG signature (which needs 0xFF 0xD8 0xFF). + const bytes = new Uint8Array([0xff, 0x00, 0x01, 0xc0]); + const sniffed = sniffMimeType(bytes, ""); + expect(sniffed).not.toBe(""); + expect(sniffed).toBe("application/octet-stream"); + }); + + test("what comes back is a name the refusal knows to say nothing about", () => { + // WHAT THIS PINS, AND WHY IT NO LONGER PINS A SENTENCE. This test used to + // build the refusal here with a template literal and assert it equalled + // the same string spelled out — which could only ever fail if + // `sniffMimeType` changed, and which claimed in its comment to be "the + // exact string `attachments.ts` builds". That claim is false: the route + // asks `describeType`, which suppresses the parenthetical for any type + // `namesNoFormat` recognises, so the real sentence has no parenthetical + // at all. A test asserting a sentence the product does not produce is + // worse than no test, and the sentence itself is pinned where it is + // actually built, in `attachment-routes.test.ts`. + // + // The contract that belongs at THIS layer is the handshake between the + // two files: the sniffer promises never to return a claim that names + // nothing, and to return one the shared list recognises, which is exactly + // what lets the route decide to stay quiet. Both halves are asserted. + const bytes = new Uint8Array([0xff, 0x00, 0x01, 0xc0]); + const sniffed = sniffMimeType(bytes, ""); + expect(classifyAttachment(sniffed)).toBe("unsupported"); + expect(namesNoFormat(sniffed)).toBe(true); + }); + + test("a claim that is not shaped like a MIME type is not echoed back", () => { + // `namesNoFormat` throws this away for having no slash, so by the time + // the last line is reached the claim has already been judged to name + // nothing. Handing "garbage" to the caller would put that word in the + // refusal as though it were a format. + const bytes = new Uint8Array([0xff, 0x00, 0x01, 0xc0]); + expect(sniffMimeType(bytes, "garbage")).toBe("application/octet-stream"); + }); + + test("a generic placeholder claim is normalised to the one generic name", () => { + // `application/unknown` is in `MIME_NAMES_NOTHING` for the same reason + // `application/octet-stream` is. Two spellings of "I don't know" should + // not produce two different refusals for the same file. + const bytes = new Uint8Array([0xff, 0x00, 0x01, 0xc0]); + expect(sniffMimeType(bytes, "application/unknown")).toBe( + "application/octet-stream", + ); + }); + + test("a claim that DOES name a format is still echoed back", () => { + // The guard on the change above. Naming a format the caller will refuse + // is the whole reason `image/svg+xml` and `text/html` survive this + // function, and collapsing them into the generic name would erase the + // signal the caller refuses by. + const bytes = new Uint8Array([0xff, 0x00, 0x01, 0xc0]); + expect(sniffMimeType(bytes, "application/x-widget")).toBe( + "application/x-widget", + ); + }); +}); + +/** + * The claim is normalised by `mediaTypeOf` and by nothing else. + * + * These do not fail against the inline copy this function used to carry — + * that copy was `mediaTypeOf`'s body character for character, so there was no + * behaviour to change and no red to watch. They are here to make the shared + * normalisation load-bearing rather than coincidental: if `mediaTypeOf` grows + * a step (RFC 2045 permits quoted parameters and space before the `;`) these + * follow it, and a future inline copy that did not would fail them. + */ +describe("the claim goes through the shared normalisation", () => { + test("an upper-case claim matches an accepted text type", () => { + const bytes = new TextEncoder().encode("a,b\n1,2\n"); + expect(sniffMimeType(bytes, "TEXT/CSV")).toBe("text/csv"); + }); + + test("case and parameter are stripped together, not one or the other", () => { + const bytes = new TextEncoder().encode("a,b\n1,2\n"); + expect(sniffMimeType(bytes, "Text/CSV; charset=UTF-8")).toBe("text/csv"); + }); + + test("an upper-case accepted image claim is still dropped, not returned", () => { + // The corroboration branch is keyed on the normalised form too. A claim + // of "IMAGE/PNG" over bytes that are not a PNG must not slip past the + // drop by virtue of its spelling. + const bytes = new Uint8Array([0xff, 0x00, 0x01, 0xc0]); + expect(sniffMimeType(bytes, "IMAGE/PNG")).toBe("application/octet-stream"); + }); + + test("an upper-case placeholder claim still names nothing", () => { + const bytes = new TextEncoder().encode("just some notes"); + expect(sniffMimeType(bytes, "Application/Octet-Stream")).toBe("text/plain"); + }); +}); diff --git a/server/tests/attachment-parts.test.ts b/server/tests/attachment-parts.test.ts new file mode 100644 index 000000000..94370def7 --- /dev/null +++ b/server/tests/attachment-parts.test.ts @@ -0,0 +1,930 @@ +import { describe, expect, test } from "bun:test"; +import { MAX_EXTRACTED_CHARACTERS } from "../../shared/attachments"; +import { + newInlineBudget, + resolveAttachmentParts, + type StoredAttachment, +} from "../src/channels/attachment-parts"; + +function loadFrom( + store: Record, +): (id: string) => Promise { + return async (id: string) => store[id] ?? null; +} + +describe("resolving stored attachment references into model-readable content", () => { + test("a url-source image part gets inline data, metadata preserved", async () => { + const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47]); + const metadata = { attachmentId: "img1", filename: "photo.png" }; + const content = [ + { + type: "image", + source: { + type: "url", + value: "/api/attachments/img1", + mimeType: "image/png", + }, + metadata, + }, + ]; + + const result = (await resolveAttachmentParts( + content, + loadFrom({ + img1: { mimeType: "image/png", name: "photo.png", bytes }, + }), + )) as Array>; + + expect(result[0].type).toBe("image"); + expect(result[0].metadata).toBe(metadata); + expect(result[0].source).toEqual({ + type: "data", + value: bytes.toString("base64"), + mimeType: "image/png", + }); + }); + + test("a url-source document part becomes a text part naming the file", async () => { + const bytes = Buffer.from("hello world", "utf8"); + const content = [ + { + type: "document", + source: { type: "url", value: "/api/attachments/doc1" }, + metadata: { attachmentId: "doc1", filename: "notes.txt" }, + }, + ]; + + const result = (await resolveAttachmentParts( + content, + loadFrom({ + doc1: { mimeType: "text/plain", name: "notes.txt", bytes }, + }), + )) as Array>; + + expect(result[0]).toEqual({ + type: "text", + text: 'Attached file "notes.txt":\n\nhello world', + }); + }); + + test("text past MAX_EXTRACTED_CHARACTERS is cut AT that many characters", async () => { + /* + * THE WHOLE PART, NOT `toContain("truncated")`. + * + * The looser assertions this replaces — that the text mentions truncation and is shorter than + * the file — hold for a cut at one character just as well as for a cut at 120,000, so the one + * number the constant exists to set was the one thing not being checked. Written out in full + * so a change to the caption, the blank line before it, or the cut itself is a red test rather + * than a silent change to what every model is shown. + */ + const long = "a".repeat(MAX_EXTRACTED_CHARACTERS + 5000); + const content = [ + { + type: "document", + source: { type: "url", value: "/api/attachments/doc2" }, + }, + ]; + + const result = (await resolveAttachmentParts( + content, + loadFrom({ + doc2: { + mimeType: "text/plain", + name: "big.txt", + bytes: Buffer.from(long, "utf8"), + }, + }), + )) as Array<{ text: string }>; + + expect(result[0].text).toBe( + [ + 'Attached file "big.txt":', + "", + "a".repeat(MAX_EXTRACTED_CHARACTERS), + "", + `[attachment truncated at ${MAX_EXTRACTED_CHARACTERS} characters]`, + ].join("\n"), + ); + }); + + test("text exactly MAX_EXTRACTED_CHARACTERS long is not cut at all", async () => { + // The other side of the same boundary: an off-by-one in `extractDocumentText` shows up here as + // a truncation marker on a file that fitted, and above as a cut in the wrong place. + const exact = "b".repeat(MAX_EXTRACTED_CHARACTERS); + const content = [ + { + type: "document", + source: { type: "url", value: "/api/attachments/doc3" }, + }, + ]; + + const result = (await resolveAttachmentParts( + content, + loadFrom({ + doc3: { + mimeType: "text/plain", + name: "exact.txt", + bytes: Buffer.from(exact, "utf8"), + }, + }), + )) as Array<{ text: string }>; + + expect(result[0].text).toBe(`Attached file "exact.txt":\n\n${exact}`); + }); + + test("a cut that would land inside a character stops one code unit short", async () => { + /* + * `slice` counts UTF-16 code units, so a limit landing between the halves of a surrogate pair + * left a lone high surrogate as the last code unit of the extracted text. That is not a + * character: `JSON.stringify` emits it as a bare `\ud83d` escape, which a provider either + * rejects or silently replaces with U+FFFD — so a file whose 120,000th code unit happens to + * fall inside an emoji damaged a turn for a reason with nothing to do with its contents. + * `withinFilenameLimit` in `channels/attachments.ts` guards the same hazard on the same kind of + * cut, and said so in a comment, while this one — applied to far more bytes, far more often — + * did not. + * + * Built so the pair straddles the limit exactly: MAX-1 filler characters, then one emoji whose + * high half sits at MAX-1 and whose low half sits at MAX, then enough after it to truncate. + */ + const emoji = "😀"; + expect(emoji.length).toBe(2); + const long = `${"a".repeat(MAX_EXTRACTED_CHARACTERS - 1)}${emoji}tail`; + const content = [ + { + type: "document", + source: { type: "url", value: "/api/attachments/pair1" }, + }, + ]; + + const result = (await resolveAttachmentParts( + content, + loadFrom({ + pair1: { + mimeType: "text/plain", + name: "emoji.txt", + bytes: Buffer.from(long, "utf8"), + }, + }), + )) as Array<{ text: string }>; + + // The whole part, for the reason the test above spells out: a looser assertion would hold for + // a cut in the wrong place just as well. + expect(result[0].text).toBe( + [ + 'Attached file "emoji.txt":', + "", + "a".repeat(MAX_EXTRACTED_CHARACTERS - 1), + "", + `[attachment truncated at ${MAX_EXTRACTED_CHARACTERS - 1} characters]`, + ].join("\n"), + ); + // Said directly as well, because it is the property and the line above is one instance of it: + // no lone surrogate survives the cut. + const body = result[0].text.split("\n\n")[1] ?? ""; + const lastUnit = body.charCodeAt(body.length - 1); + expect(lastUnit >= 0xd800 && lastUnit <= 0xdbff).toBe(false); + }); + + test("a cut that lands cleanly still cuts at exactly the limit", async () => { + // The guard must not cost a character on the ordinary file. An emoji ending one unit BEFORE the + // limit is whole inside the cut, so nothing is dropped and the count is the constant. + const long = `${"a".repeat(MAX_EXTRACTED_CHARACTERS - 2)}😀tail`; + const content = [ + { + type: "document", + source: { type: "url", value: "/api/attachments/pair2" }, + }, + ]; + + const result = (await resolveAttachmentParts( + content, + loadFrom({ + pair2: { + mimeType: "text/plain", + name: "emoji2.txt", + bytes: Buffer.from(long, "utf8"), + }, + }), + )) as Array<{ text: string }>; + + expect(result[0].text).toBe( + [ + 'Attached file "emoji2.txt":', + "", + `${"a".repeat(MAX_EXTRACTED_CHARACTERS - 2)}😀`, + "", + `[attachment truncated at ${MAX_EXTRACTED_CHARACTERS} characters]`, + ].join("\n"), + ); + }); + + test("a missing attachment throws, naming the attachment id", async () => { + const content = [ + { + type: "image", + source: { type: "url", value: "/api/attachments/missing1" }, + }, + ]; + + await expect(resolveAttachmentParts(content, loadFrom({}))).rejects.toThrow( + /missing1/, + ); + }); + + test("the refusal names the file the person chose, not only the id they never saw", async () => { + /* + * The uuid is minted by the upload route; what the person picked out of a file dialog is + * `quarterly.png`, and it is the only one of the two they can match against what they did. The + * id stays behind it for whoever is reading a log next to a table. + * + * AND IT DOES NOT COLLAPSE FOUR SITUATIONS INTO ONE CLAIM. `load` answers null for a row the + * sweeper reclaimed, for a file of somebody this asker cannot see, for one in another channel, + * and for an id naming no row at all — different situations with different things to do about + * them, flattened by the `(id) => Promise` seam. Naming one would be a + * guess printed as a fact, so the sentence offers the possibilities instead of picking. + */ + const content = [ + { + type: "image", + source: { type: "url", value: "/api/attachments/missing2" }, + metadata: { attachmentId: "missing2", filename: "quarterly.png" }, + }, + ]; + + const failure = resolveAttachmentParts(content, loadFrom({})); + + await expect(failure).rejects.toThrow(/"quarterly\.png" \(id "missing2"\)/); + await expect(failure).rejects.toThrow(/deleted/); + await expect(failure).rejects.toThrow(/another channel/); + }); + + test("a part with no filename is named once, not twice over", async () => { + // `displayName` falls back to the id, and every part not written by our own composer arrives + // without a filename — so the id would otherwise be printed beside itself. + const content = [ + { + type: "image", + source: { type: "url", value: "/api/attachments/missing3" }, + }, + ]; + + await expect(resolveAttachmentParts(content, loadFrom({}))).rejects.toThrow( + 'Attachment "missing3" could not be loaded', + ); + }); + + test("a missing attachment in history becomes a text part naming the file", async () => { + /* + * "note" is what an older message gets, because history is replayed on + * every turn and a throw there would fail this channel for ever — the + * failure `agents/history-sanitize.ts` records finding in production + * twice. The part still SAYS the file is gone, so nothing answers as + * though it were there. + */ + const content = [ + { + type: "image", + source: { type: "url", value: "/api/attachments/gone1" }, + metadata: { attachmentId: "gone1", filename: "budget.png" }, + }, + ]; + + const result = (await resolveAttachmentParts( + content, + loadFrom({}), + "note", + )) as Array>; + + expect(result[0]).toEqual({ + type: "text", + text: '[attachment "budget.png" is no longer available]', + }); + }); + + test("a missing attachment with no filename is named by its id", async () => { + const content = [ + { + type: "document", + source: { type: "url", value: "/api/attachments/gone2" }, + }, + ]; + + const result = (await resolveAttachmentParts( + content, + loadFrom({}), + "note", + )) as Array>; + + expect(result[0]).toEqual({ + type: "text", + text: '[attachment "gone2" is no longer available]', + }); + }); + + test('an attachment that still loads is inlined under "note" too', async () => { + // "note" changes what a MISSING row does and nothing else: a row that is + // still there resolves exactly as it does for the asked-about message. + const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47]); + const content = [ + { + type: "image", + source: { type: "url", value: "/api/attachments/img3" }, + metadata: { attachmentId: "img3", filename: "kept.png" }, + }, + ]; + + const result = (await resolveAttachmentParts( + content, + loadFrom({ img3: { mimeType: "image/png", name: "kept.png", bytes } }), + "note", + )) as Array>; + + expect(result[0].source).toEqual({ + type: "data", + value: bytes.toString("base64"), + mimeType: "image/png", + }); + }); + + test("a part that already carries a data source is left alone", async () => { + const content = [ + { + type: "image", + source: { type: "data", value: "AAAA", mimeType: "image/png" }, + metadata: { attachmentId: "img2", filename: "already.png" }, + }, + ]; + + const result = await resolveAttachmentParts(content, loadFrom({})); + + expect(result).toBe(content); + }); + + test("string content is returned by identity", async () => { + const content = "just some plain message text"; + + const result = await resolveAttachmentParts(content, loadFrom({})); + + expect(result).toBe(content); + }); +}); + +/** + * WHICH SIDE DECIDES WHAT A FILE IS. + * + * A part's `type` is the browser's claim, fixed from `file.type` before the upload and never + * reconciled with what the upload answered. `mimeType` is `sniffMimeType`'s reading of the actual + * bytes. They disagree in production for an ordinary reason: `sniffMimeType` runs the image + * signatures FIRST, so a PNG whose browser claim was `text/plain` is stored as `image/png` while + * the part that named it still says `document`. + * + * Every test here fails if `resolvePart` goes back to reading `part.type`. + */ +describe("what a stored attachment becomes is decided by its bytes, not by the part", () => { + const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + + function partOf(type: string, id: string, filename?: string) { + return [ + { + type, + source: { type: "url", value: `/api/attachments/${id}` }, + metadata: { attachmentId: id, ...(filename ? { filename } : {}) }, + }, + ]; + } + + test('a "document" part naming a stored image is sent as an image, not as mojibake', async () => { + /* + * The defect in full: `photo.png` renamed and dragged out of an editor arrives claiming + * `text/plain`, the SDK fixes the modality to `document` from that claim, the server sniffs the + * bytes and stores `image/png` — and the old branch ran a PNG through `toString("utf8")` and + * captioned the result `Attached file "photo.png":`. The model was handed a page of noise and + * nothing anywhere said the picture had not been sent. + */ + const result = (await resolveAttachmentParts( + partOf("document", "shot1", "photo.png"), + loadFrom({ + shot1: { mimeType: "image/png", name: "photo.png", bytes: png }, + }), + )) as Array>; + + // `type` is REWRITTEN, not merely left alone: a provider shown a `document` part does not look + // at the picture, so inlining the bytes under the claimed type would fix nothing. + expect(result[0].type).toBe("image"); + expect(result[0].source).toEqual({ + type: "data", + value: png.toString("base64"), + mimeType: "image/png", + }); + }); + + test('an "image" part naming a stored text file is extracted, not base64-ed', async () => { + const result = (await resolveAttachmentParts( + partOf("image", "note1", "notes.txt"), + loadFrom({ + note1: { + mimeType: "text/markdown", + name: "notes.txt", + bytes: Buffer.from("# hello", "utf8"), + }, + }), + )) as Array>; + + expect(result[0]).toEqual({ + type: "text", + text: 'Attached file "notes.txt":\n\n# hello', + }); + }); + + test("a part type AG-UI has and this app never sends still respects MAX_EXTRACTED_CHARACTERS", async () => { + /* + * AG-UI's union is `text | image | audio | video | document | binary`, and message content is + * written by the browser, so all six are constructible by anyone who composes their own request + * — our composer only ever emitting `image`/`document` is not a gate. + * + * Under the old `part.type === "document"` test, `binary` took the else branch: a text file at + * the `MAX_FILE_BYTES` ceiling went to the model as ~1.4 MB of base64 with the extraction cap + * bypassed entirely. The cut is what this asserts, because the cut is what was bypassed. + */ + const long = "c".repeat(MAX_EXTRACTED_CHARACTERS + 1); + + const result = (await resolveAttachmentParts( + partOf("binary", "sneak1", "notes.txt"), + loadFrom({ + sneak1: { + mimeType: "text/plain", + name: "notes.txt", + bytes: Buffer.from(long, "utf8"), + }, + }), + )) as Array<{ type: string; text: string }>; + + expect(result[0].type).toBe("text"); + expect(result[0].text).toBe( + [ + 'Attached file "notes.txt":', + "", + "c".repeat(MAX_EXTRACTED_CHARACTERS), + "", + `[attachment truncated at ${MAX_EXTRACTED_CHARACTERS} characters]`, + ].join("\n"), + ); + }); + + test("a stored type this app cannot read becomes a note naming the type", async () => { + // Not reachable through today's upload route, which runs `classifyAttachment` first. It becomes + // reachable the day a type leaves the accepted lists with rows of it still in the table, and + // the answer must not be mojibake — the model has to be told it cannot see the file. + const result = (await resolveAttachmentParts( + partOf("document", "pdf1", "invoice.pdf"), + loadFrom({ + pdf1: { + mimeType: "application/pdf", + name: "invoice.pdf", + bytes: Buffer.from("%PDF-1.7", "utf8"), + }, + }), + )) as Array>; + + expect(result[0]).toEqual({ + type: "text", + text: '[attachment "invoice.pdf" is a application/pdf file, which cannot be put in front of the model]', + }); + }); + + test("one id named twice in a message is read once and encoded twice", async () => { + // Two parts must not be handed the same object, but they must not cost two reads either. What + // they DO cost twice is the budget, which the next test asserts: two encoded copies, two + // charges, one read. + const reads: string[] = []; + const content = [ + ...partOf("image", "twice1"), + ...partOf("image", "twice1"), + ]; + + const result = (await resolveAttachmentParts(content, async (id) => { + reads.push(id); + return { mimeType: "image/png", name: "photo.png", bytes: png }; + })) as Array>; + + expect(reads).toEqual(["twice1"]); + expect(result[0]).not.toBe(result[1]); + expect(result[0]).toEqual(result[1]); + }); + + /* + * THE OTHER HALF OF THAT SENTENCE, WHICH THE TEST ABOVE DOES NOT REACH: ONE READ, TWO CHARGES. + * + * This test used to assert the opposite — "charged to the budget once", budget 20 minus one + * 8-byte file leaving 12 — and it was wrong in the direction that matters. Deduplicating the + * READ is a saving and stays; deduplicating the CHARGE made every copy after the first free, + * which took the ceiling off the one thing the budget bounds. Two parts naming one file really + * are two base64 strings live at once, so they are two charges. + * + * The two properties are asserted together here precisely because they were once collapsed into + * one claim ("one read per distinct id, and one charge") that read as coherent and was not. + */ + test("one id named twice is read once and charged twice", async () => { + const content = [ + ...partOf("image", "charged1"), + ...partOf("image", "charged1"), + ]; + const eightBytes = Buffer.from("12345678", "utf8"); + const budget = newInlineBudget(20); + const reads: string[] = []; + + const result = (await resolveAttachmentParts( + content, + async (id) => { + reads.push(id); + return { + mimeType: "image/png", + name: "photo.png", + bytes: eightBytes, + }; + }, + "note", + budget, + )) as Array>; + + // One trip to `bytea`: the memo is untouched by the fix. + expect(reads).toEqual(["charged1"]); + // Two copies emitted, so 16 of the 20 bytes are spent, not 8. + expect(budget.remaining).toBe(4); + expect(result).toHaveLength(2); + expect(result[0]).toEqual(result[1]); + }); + + /* + * The consequence, stated separately because it is the one a reader coming from the old behaviour + * would doubt: once the room is gone, a part naming an id that ALREADY FIT is cut like any other. + * + * The rejected argument was that its bytes are in the run already, so the second mention is free + * and cutting it puts one file in front of the model twice over — once as itself, once as a note. + * It is not free. The second mention is a second encoded copy, and exempting it is exactly what + * let a repeated id inline without limit. A note saying one of two mentions was left out is a + * true statement about a turn that ran out of room, and the file is still there on the first. + */ + test("a repeated id is cut like any other once the budget is gone", async () => { + const content = [ + ...partOf("image", "paid"), + ...partOf("image", "big"), + ...partOf("image", "paid"), + ]; + const budget = newInlineBudget(12); + + const result = (await resolveAttachmentParts( + content, + async (id) => ({ + mimeType: "image/png", + name: `${id}.png`, + bytes: Buffer.from(id === "big" ? "1234567890123456" : "12345678"), + }), + "note", + budget, + )) as Array>; + + expect(budget.remaining).toBe(0); + // The first mention fit and was inlined. + expect(result[0]).toMatchObject({ source: { type: "data" } }); + // The oversized one is cut... + expect(JSON.stringify(result[1])).toContain("not included"); + // ...and so is the third, which names a paid id but would cost a second copy of it. + expect(JSON.stringify(result[2])).toContain("not included"); + }); +}); + +/** + * HOW MUCH ONE TURN MAY SPEND ON FILES. + * + * `MAX_IMAGE_BYTES` bounds a file. Nothing bounded a run: history is replayed on every turn, so a + * channel that had seen a few large images re-read and re-base64-ed all of them on every later + * turn, and the way that fails is the pod's heap rather than any refusal a person can read. + * + * The budget is spent NEWEST-FIRST by `inlineAttachments`, so what runs out is the room for + * history. The asked message is the one under `onMissing: "fail"`, and it is never cut — it is + * served whole, or, past the limit, it refuses the turn. Both halves of that sentence are asserted + * below, because for a while only the first was true and the second was not bounded at all. + */ +describe("a run's inlining budget", () => { + const eightBytes = Buffer.from("12345678", "utf8"); + + function imagePart(id: string, filename: string) { + return { + type: "image", + source: { type: "url", value: `/api/attachments/${id}` }, + metadata: { attachmentId: id, filename }, + }; + } + + function pngStore(id: string) { + return { + [id]: { mimeType: "image/png", name: `${id}.png`, bytes: eightBytes }, + }; + } + + test("a history part that does not fit becomes a note that does not claim the file is gone", async () => { + const budget = newInlineBudget(4); + + const result = (await resolveAttachmentParts( + [imagePart("big1", "chart.png")], + loadFrom(pngStore("big1")), + "note", + budget, + )) as Array>; + + /* + * NOT the `is no longer available` wording. The row is still there and a question about it + * makes it the asked message, which the budget is spent on first; telling somebody their file + * was deleted when it was not is a wrong answer that gets acted on. + */ + expect(result[0]).toEqual({ + type: "text", + text: '[attachment "chart.png" from an earlier message was not included in this turn]', + }); + expect(budget.remaining).toBe(0); + }); + + test("a spent budget stops the database read, not just the encoding", async () => { + // The read out of `bytea` is most of what the budget exists to bound, so a part that cannot fit + // must not be fetched to discover that. Once the budget is at zero, later parts cost one + // comparison each. + const reads: string[] = []; + const budget = newInlineBudget(0); + + await resolveAttachmentParts( + [imagePart("skip1", "chart.png")], + async (id) => { + reads.push(id); + return { mimeType: "image/png", name: "chart.png", bytes: eightBytes }; + }, + "note", + budget, + ); + + expect(reads).toEqual([]); + }); + + test("the message being asked about spends the budget and is never cut by it", async () => { + /* + * The property that makes the budget defensible at all. `inlineAttachments` walks backwards, so + * the asked message is charged first; if it were also cuttable, a person attaching one large + * file would be told their own question's attachment was left out of their own turn. + * + * "fail" is what marks that message — the same flag that says an unloadable row there must not + * degrade — so this asserts the two travel together. + * + * Sized to FIT, unlike the version of this test that stood here while the asked message had no + * ceiling at all. That one passed a budget of 4 against an 8-byte file and asserted it was + * inlined anyway, which read as "never cut" but was really "never bounded" — the assertion that + * made the hole below look deliberate. What "never cut" means is asserted here; what happens + * past the limit is the next test, and it is not this. + */ + const budget = newInlineBudget(20); + + const result = (await resolveAttachmentParts( + [imagePart("asked1", "chart.png")], + loadFrom(pngStore("asked1")), + "fail", + budget, + )) as Array>; + + expect(result[0].source).toMatchObject({ type: "data" }); + // Charged, not exempted: it is what leaves less for the history behind it. + expect(budget.remaining).toBe(12); + }); + + /* + * THE HOLE THE BUDGET LEFT OPEN, AND THE SHAPE OF THE ANSWER. + * + * Both places that stopped spending tested `onMissing === "note"`, and the message being asked + * about is resolved under `"fail"`, so neither ever fired for it: nothing capped how many + * attachment parts a browser-written message could carry. Two hundred previously-sent 8 MiB files + * named in one message inlined about 1.6 GiB plus its base64, in one turn — the exact heap + * exhaustion `MAX_INLINED_BYTES_PER_RUN` was written to stop, through the one door it left open. + * + * The answer is NOT to make that message cuttable, which is why these tests assert a rejection + * rather than a note. Cutting it would drop files out of the message somebody is asking a question + * about, silently, which is the failure the strict mode exists to prevent. "Served in full or the + * turn fails loudly" survives; "in full" acquires a ceiling. + */ + test("the message being asked about refuses the turn rather than being cut down to fit", async () => { + const budget = newInlineBudget(4); + + const failure = resolveAttachmentParts( + [imagePart("asked2", "chart.png")], + loadFrom(pngStore("asked2")), + "fail", + budget, + ); + + // The file, so the person knows which one; the limit, so "too big" is a number; and a way out, + // because this is their own most recent action and the message is still in front of them. + await expect(failure).rejects.toThrow(/"chart\.png" \(id "asked2"\)/); + await expect(failure).rejects.toThrow(/more than the 4 bytes/); + await expect(failure).rejects.toThrow(/Send fewer files/); + }); + + test("a refused message is refused, not quietly turned into a note", async () => { + /* + * Stated separately because it is the regression that would be easy to introduce while fixing + * the one above: reusing `notIncludedNote` for the asked message would bound the bytes just as + * well and would be exactly the silent truncation `onMissing: "fail"` exists to rule out. The + * turn must end, not continue with a file missing from the question it is answering. + */ + const budget = newInlineBudget(4); + let resolved: unknown = "never assigned"; + + try { + resolved = await resolveAttachmentParts( + [imagePart("asked3", "chart.png")], + loadFrom(pngStore("asked3")), + "fail", + budget, + ); + } catch { + resolved = "threw"; + } + + expect(resolved).toBe("threw"); + }); + + test("the part past the limit refuses the turn without reading it first", async () => { + /* + * The refusal inherits "cut before the load", because a turn that is going to be refused should + * not pay for the bytes it cannot afford on the way to saying so. Two 8-byte files against a + * budget of 8: the first fits exactly and is read, and the second finds nothing left and must + * never reach the database. + */ + const reads: string[] = []; + const budget = newInlineBudget(8); + + const failure = resolveAttachmentParts( + [imagePart("first", "one.png"), imagePart("second", "two.png")], + async (id) => { + reads.push(id); + return { mimeType: "image/png", name: `${id}.png`, bytes: eightBytes }; + }, + "fail", + budget, + ); + + await expect(failure).rejects.toThrow(/"two\.png"/); + expect(reads).toEqual(["first"]); + }); + + test("a message that fills the budget exactly is served, not refused", async () => { + // The other side of that boundary. An off-by-one turning `>` into `>=` would refuse a message + // that fits, which is a person told their own question is too big when it is not. + const budget = newInlineBudget(8); + + const result = (await resolveAttachmentParts( + [imagePart("exact1", "chart.png")], + loadFrom(pngStore("exact1")), + "fail", + budget, + )) as Array>; + + expect(result[0].source).toMatchObject({ type: "data" }); + expect(budget.remaining).toBe(0); + }); + + test("an unbudgeted caller is still unbounded, refusal or not", async () => { + /* + * The refusal is a property of the BUDGET, not of `onMissing: "fail"`. A single-message caller + * that passes no budget — the documented "absent means unbounded" case, and what every caller + * did before the budget existed — must not start failing because its message is large. + */ + const result = (await resolveAttachmentParts( + [imagePart("free2", "chart.png"), imagePart("free3", "other.png")], + async (id) => ({ + mimeType: "image/png", + name: `${id}.png`, + bytes: Buffer.alloc(64 * 1024 * 1024), + }), + "fail", + )) as Array>; + + expect(result[0].source).toMatchObject({ type: "data" }); + expect(result[1].source).toMatchObject({ type: "data" }); + }); + + /* + * WHAT THE READ MEMO IS SCOPED TO, ASSERTED RATHER THAN ASSUMED. + * + * `resolveAttachmentParts` builds `loadOnce` per call, and it is called once per message, so an + * id quoted in two messages of one thread is read twice. The comment on it once said "this run", + * which was simply false, and the fix was to the comment: per message is the intended scope. A + * run-scoped memo would pin every attachment's buffer live for the whole backward walk — worse + * for the heap this budget exists to protect than the second read is for the clock. + * + * The CHARGE has no scope question left to answer. It runs once per part emitted, here and + * everywhere, so two messages quoting one id pay for it twice for the same reason two parts of + * one message do: two encoded copies reach the run. + */ + test("an id quoted in two messages is read once per message and charged once per part", async () => { + const budget = newInlineBudget(20); + const reads: string[] = []; + const load = async (id: string) => { + reads.push(id); + return { mimeType: "image/png", name: `${id}.png`, bytes: eightBytes }; + }; + + // Two calls, because two messages are two calls: this is the seam the scope question is about. + await resolveAttachmentParts( + [imagePart("quoted", "chart.png")], + load, + "note", + budget, + ); + await resolveAttachmentParts( + [imagePart("quoted", "chart.png")], + load, + "note", + budget, + ); + + expect(reads).toEqual(["quoted", "quoted"]); + expect(budget.remaining).toBe(4); + }); + + test("a history part that fits draws the budget down by the stored bytes", async () => { + const budget = newInlineBudget(20); + + const result = (await resolveAttachmentParts( + [imagePart("fits1", "chart.png")], + loadFrom(pngStore("fits1")), + "note", + budget, + )) as Array>; + + expect(result[0].source).toMatchObject({ type: "data" }); + expect(budget.remaining).toBe(12); + }); + + test("no budget means no ceiling, which is what a single-message caller wants", async () => { + const result = (await resolveAttachmentParts( + [imagePart("free1", "chart.png")], + loadFrom(pngStore("free1")), + "note", + )) as Array>; + + expect(result[0].source).toMatchObject({ type: "data" }); + }); + + /* + * THE BUDGET BOUNDS WHAT COMES OUT, NOT HOW MANY DISTINCT FILES WENT IN. + * + * This is the regression that made the whole number decorative. The charge was deduplicated per + * id, on the reasoning that one id is read once so it should be billed once — but this function + * emits a base64 part for EVERY OCCURRENCE of an id, and every one of those strings is live at the + * same time. Forty parts naming one 1 KiB file against a 1 KiB budget inlined 40 KiB and reported + * the budget spent exactly to zero. At the 8 MiB upload ceiling a hundred references came to about + * 1.04 GiB of base64 against a 32 MiB budget: the heap exhaustion the budget exists to prevent, + * with the counter insisting nothing was wrong. + * + * SO THIS ASSERTS ON DECODED OUTPUT BYTES AND NOT ON `budget.remaining`. The counter is precisely + * what lied: it read zero while forty copies went out. What a run can afford to hold is a fact + * about the parts it returns, so that is the thing measured — sum the bytes behind every `data` + * source that actually left this function. + * + * The read memo is asserted in the same breath, because the fix must not buy the bound back by + * giving up `loadOnce`: one id, one trip to `bytea`, many charges. + */ + test("one id repeated past the budget inlines no more bytes than the budget", async () => { + const kilobyte = Buffer.alloc(1024, 0x41); + const budget = newInlineBudget(1024); + const reads: string[] = []; + const content = Array.from({ length: 40 }, () => + imagePart("repeated", "chart.png"), + ); + + const result = (await resolveAttachmentParts( + content, + async (id) => { + reads.push(id); + return { mimeType: "image/png", name: "chart.png", bytes: kilobyte }; + }, + "note", + budget, + )) as Array>; + + const inlinedBytes = result.reduce((total, part) => { + const source = part.source as + | { type?: unknown; value?: unknown } + | undefined; + if (source?.type !== "data" || typeof source.value !== "string") { + return total; + } + return total + Buffer.from(source.value, "base64").length; + }, 0); + + // One read, because `loadOnce` still memoises: the fix is to the charge, not to the fetch. + expect(reads).toEqual(["repeated"]); + // The bound, measured where it matters: one copy's worth of bytes left this function. + expect(inlinedBytes).toBe(1024); + // Every part is still accounted for — the ones past the bound say so rather than vanishing. + expect(result).toHaveLength(40); + expect(JSON.stringify(result[1])).toContain("not included"); + }); +}); diff --git a/server/tests/attachment-routes.test.ts b/server/tests/attachment-routes.test.ts new file mode 100644 index 000000000..1ffc253d0 --- /dev/null +++ b/server/tests/attachment-routes.test.ts @@ -0,0 +1,3486 @@ +import { afterAll, afterEach, describe, expect, spyOn, test } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { and, eq, isNull, sql } from "drizzle-orm"; +import type { MiddlewareHandler } from "hono"; +import { Hono } from "hono"; +import { + attachmentUrl, + MAX_ATTACHMENTS_PER_MESSAGE, + MAX_FILE_BYTES, + MAX_IMAGE_BYTES, +} from "../../shared/attachments"; +import { createApp, UPLOAD_BODY_LIMIT_BYTES } from "../src/app"; +import type { AppVariables } from "../src/auth/guards"; +import { resolveAttachmentParts } from "../src/channels/attachment-parts"; +import { + contentDispositionFilename, + createAttachmentRoutes, + createChannelAttachmentRoutes, + loadAttachmentForTurn, + markAttachmentsSent, + MAX_STAGED_ATTACHMENTS_PER_UPLOADER, +} from "../src/channels/attachments"; +import { loadConfig } from "../src/config"; +import type { Database } from "../src/db/client"; +import { createDatabase } from "../src/db/client"; +import { + attachments, + channelMemberships, + channels, + intelligenceChannelMappings, + users, +} from "../src/db/schema"; +import { TEST_POOL } from "./support/database"; +import { testEnvironment } from "./support/environment"; + +const databaseUrl = + process.env.DATABASE_URL ?? + "postgres://openbot:openbot@localhost:5432/openbot"; +const database = createDatabase(databaseUrl, TEST_POOL); + +const testPrefix = `attachment-routes-${randomUUID()}`; +const createdChannelIds: string[] = []; +const createdUserIds: string[] = []; + +afterEach(async () => { + // Cascades attachments and memberships tied to the channel. + for (const id of createdChannelIds.splice(0)) { + await database.delete(channels).where(eq(channels.id, id)); + } + for (const id of createdUserIds.splice(0)) { + await database.delete(users).where(eq(users.id, id)); + } +}); + +afterAll(async () => { + await database.$client.close(); + await unreachableDatabase.$client.close(); +}); + +function actorMiddleware( + actorId: string, +): MiddlewareHandler<{ Variables: AppVariables }> { + return async (context, next) => { + context.set("actor", { + id: actorId, + email: `${actorId}@example.test`, + role: "user", + }); + await next(); + }; +} + +/** A member of a channel, a stranger to it, and an app that uploads as the member. */ +async function harness() { + const memberId = `${testPrefix}-member-${randomUUID()}`; + const strangerId = `${testPrefix}-stranger-${randomUUID()}`; + await database.insert(users).values([ + { id: memberId, email: `${memberId}@example.test` }, + { id: strangerId, email: `${strangerId}@example.test` }, + ]); + createdUserIds.push(memberId, strangerId); + + const channelId = `${testPrefix}-channel-${randomUUID()}`; + await database.insert(channels).values({ + id: channelId, + name: "Attachment Route Test Channel", + description: "A channel to upload files into.", + }); + createdChannelIds.push(channelId); + + await database + .insert(channelMemberships) + .values({ channelId, userId: memberId }); + + /* + * The thread this channel's conversation runs in, because in production a channel never exists + * without one: `makeChannel` writes the membership row and the mapping row in a single + * transaction, so membership and mapping are 1:1 and a member with no mapping cannot even see the + * channel (`get` and `list` inner-join it). + * + * It is here rather than in the two tests that care because the turn path is now scoped to the + * conversation as well as to the actor, and a fixture with no mapping would put every test in + * this file on the unmapped branch — the branch that exists for hops and the direct `/bot` chat, + * not for a channel. Tests would then agree with each other and with nothing that ships. + */ + const threadId = `${testPrefix}-thread-${randomUUID()}`; + await database + .insert(intelligenceChannelMappings) + .values({ channelId, userId: memberId, threadId }); + + const member = actorMiddleware(memberId); + const stranger = actorMiddleware(strangerId); + + const app = new Hono<{ Variables: AppVariables }>(); + app.route("/", createChannelAttachmentRoutes(database, member)); + + return { + app, + channelId, + database, + member, + stranger, + memberId, + strangerId, + threadId, + }; +} + +function upload( + app: Hono<{ Variables: AppVariables }>, + channelId: string, + file: File, + uploadGroup?: string, +) { + const formData = new FormData(); + formData.set("file", file); + // Omitted where a test does not care, which is also the shape an older tab still sends: the + // route counts those together in one legacy bucket rather than in nobody's. + if (uploadGroup !== undefined) formData.set("uploadGroup", uploadGroup); + return app.request(`http://test/${channelId}/attachments`, { + method: "POST", + body: formData, + }); +} + +/** An app wired to the `/:id` GET and DELETE routes, acting as the given actor. */ +function attachmentApp( + db: Database, + requireUser: MiddlewareHandler<{ Variables: AppVariables }>, +) { + const app = new Hono<{ Variables: AppVariables }>(); + app.route("/", createAttachmentRoutes(db, requireUser)); + return app; +} + +/** + * A database at an address nothing is listening on. + * + * The failures these routes have to answer for — a connection lost during a rollout, a pool with + * nothing left in it, a `statement_timeout` — are all "the driver could not answer this query", and + * a closed port produces exactly that, from the real driver, without a mock standing in for it. + * Bun's SQL connects lazily, so building this costs nothing and no connection is ever held; the + * query rejects in about 2ms. + */ +const unreachableDatabase = createDatabase( + "postgres://openbot:openbot@127.0.0.1:1/openbot", + { max: 1 }, +); + +/** + * The same database, except that opening a transaction fails. + * + * The unreachable database above cannot reach the transaction at all: the upload route's membership + * check is the first thing to fail, so the deeper guard is never exercised. This one lets every + * real statement through and fails only where the insert happens, which is the line the finding is + * about — a pool exhausted, a lock timeout or a full disk, once the request is already inside. + */ +function databaseWhoseTransactionFails(db: Database): Database { + return new Proxy(db, { + get(target, property) { + if (property === "transaction") { + return () => Promise.reject(new Error("could not open a transaction")); + } + const value = Reflect.get(target, property) as unknown; + return typeof value === "function" ? value.bind(target) : value; + }, + }) as Database; +} + +/** + * The same database, except that a withdrawal lands the moment the upload's own statement has run. + * + * THE RACE THIS PARKS CANNOT BE PARKED WITH A LOCK. The upload holds a `pg_advisory_xact_lock` on + * its uploader, which serialises other UPLOADS and nothing else — a + * `DELETE /api/attachments/:id` takes no such lock and commits straight away. So the only way to + * put a withdrawal exactly between the statement that refuses and any statement that might explain + * the refusal is to hang it off the statements themselves, which is what this does: the first + * `execute` in the transaction is the advisory lock, the second is the upload's own statement, and + * `withdraw` runs the instant that second one comes back. + * + * Under READ COMMITTED every later statement in the same transaction takes a FRESH snapshot, so + * anything the handler asks after this point sees the withdrawn row as gone. That is precisely the + * fault: the count that explains a refusal must come from the statement that refused. + */ +function databaseWithdrawingAfterTheInsert( + db: Database, + withdraw: () => Promise, +): Database { + let statements = 0; + const watch = (transaction: object) => + new Proxy(transaction, { + get(target, property) { + const value = Reflect.get(target, property) as unknown; + if (typeof value !== "function") return value; + if (property !== "execute") return value.bind(target); + return async (...args: unknown[]) => { + const result = await ( + value as (...called: unknown[]) => Promise + ).apply(target, args); + statements += 1; + if (statements === 2) await withdraw(); + return result; + }; + }, + }); + + return new Proxy(db, { + get(target, property) { + const value = Reflect.get(target, property) as unknown; + if (typeof value !== "function") return value; + if (property !== "transaction") return value.bind(target); + return (callback: (transaction: object) => unknown, ...rest: unknown[]) => + (value as (...called: unknown[]) => unknown).call( + target, + (transaction: object) => callback(watch(transaction)), + ...rest, + ); + }, + }) as Database; +} + +/** + * The same database, remembering which columns each `select` asked for. + * + * "The probe does not read the file" is not visible in a HEAD response — the body is empty either + * way, and the only other evidence is a stopwatch, which is a guess about how fast this machine is. + * The column list is the fact itself: `bytes` is either in the statement or it is not. + */ +function databaseRecordingSelections( + db: Database, + selections: string[][], +): Database { + return new Proxy(db, { + get(target, property) { + const value = Reflect.get(target, property) as unknown; + if (typeof value !== "function") return value; + if (property !== "select") return value.bind(target); + return (fields?: Record) => { + selections.push(Object.keys(fields ?? {})); + return (value as (...called: unknown[]) => unknown).call( + target, + fields, + ); + }; + }, + }) as Database; +} + +/** + * Runs `work` with `console.error` captured rather than printed, and hands back what it logged. + * + * Both halves matter. A failure nobody can see from the outside is half the fault these tests pin — + * the finding was that a database error reached the client as an unreadable 500 AND left no trace + * on this side — so the log line is asserted, not merely tolerated. Capturing also keeps a + * deliberate failure from printing a stack trace into a passing suite, where the next person would + * read it as something going wrong. + */ +async function withCapturedErrorLog( + work: () => Promise, +): Promise<{ result: T; logged: string[] }> { + const logged: string[] = []; + const spy = spyOn(console, "error").mockImplementation( + (...args: unknown[]) => { + logged.push(args.map((argument) => String(argument)).join(" ")); + }, + ); + try { + return { result: await work(), logged }; + } finally { + spy.mockRestore(); + } +} + +/** Inserts a staged (or sent, if `attachedAt` is given) attachment row directly, bypassing upload. */ +async function uploadBytes( + db: Database, + options: { + channelId: string; + uploadedBy: string; + name: string; + mimeType: string; + bytes: Uint8Array; + attachedAt?: Date; + }, +): Promise { + const [inserted] = await db + .insert(attachments) + .values({ + channelId: options.channelId, + uploadedBy: options.uploadedBy, + name: options.name, + mimeType: options.mimeType, + sizeBytes: options.bytes.byteLength, + bytes: Buffer.from(options.bytes), + attachedAt: options.attachedAt ?? null, + }) + .returning({ id: attachments.id }); + return inserted.id; +} + +/** Same as `uploadBytes`, for a text body, defaulting to `text/plain`. */ +function uploadText( + db: Database, + options: { + channelId: string; + uploadedBy: string; + name: string; + mimeType?: string; + text: string; + attachedAt?: Date; + }, +): Promise { + return uploadBytes(db, { + channelId: options.channelId, + uploadedBy: options.uploadedBy, + name: options.name, + mimeType: options.mimeType ?? "text/plain", + bytes: new TextEncoder().encode(options.text), + attachedAt: options.attachedAt, + }); +} + +/** + * The `attachedAt` a row currently carries, `null` when it is staged, or `undefined` when there is + * no such row at all. + * + * THREE-VALUED ON PURPOSE, AND THAT IS A TRAP FOR THE CALLER. `expect(...).not.toBeNull()` is + * satisfied by the `undefined` — so a test meaning "the row is still here and stamped" passes just + * as happily when the row was deleted out from under it, which is the failure some of these tests + * exist to catch. Assert `toBeInstanceOf(Date)` for that, never `not.toBeNull()`. + */ +async function attachedAtOf( + db: Database, + id: string, +): Promise { + const [row] = await db + .select({ attachedAt: attachments.attachedAt }) + .from(attachments) + .where(eq(attachments.id, id)); + return row?.attachedAt; +} + +function deferred() { + let resolve!: () => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, reject, resolve }; +} + +/** + * How long {@link waitForBlockedSession} waits before saying so itself. + * + * IT HAS TO BE COMFORTABLY UNDER THE TEST'S OWN TIMEOUT, and it used to be exactly equal to it: the + * deadline was 5s and Bun's default per-test timeout is also 5s, with no override anywhere, so the + * test was always killed by the runner a moment before the helper could raise. The message below — + * the one thing that says WHICH session never blocked — was unreachable, and every failure of these + * races read as a bare "timed out after 5000ms". + * + * So the deadline is a third of the timeout the two callers now declare, which leaves the helper's + * own diagnostic the thing that fires. + */ +const BLOCKED_SESSION_TIMEOUT_MS = 20_000; +const BLOCKED_SESSION_DEADLINE_MS = BLOCKED_SESSION_TIMEOUT_MS / 3; + +/** + * Waits until the named session is actually waiting on somebody else's lock. + * + * A sleep would make the race below a guess about how fast this machine is; `pg_blocking_pids` is + * Postgres saying so itself. `settled` is the other way out: a request that answered without ever + * blocking has nothing left to wait for, and returning false lets the test say which of the two + * happened rather than time out on a question already answered. + * + * The poll has a pause in it, which it did not before. Without one this loop asks + * `pg_stat_activity` as fast as the connection will answer — a view Postgres builds by walking + * every backend — on the one connection the racing request may itself be waiting for, on a database + * other agents' suites are using at the same time. 10ms is far below the window being observed and + * turns thousands of round trips into a handful. + */ +async function waitForBlockedSession( + applicationName: string, + settled: () => boolean, + // Overridden by the test that pins the message below, and by nothing else: waiting the real + // deadline out to watch it fire would put seven seconds on the suite to observe a string. + deadlineMs: number = BLOCKED_SESSION_DEADLINE_MS, +): Promise { + const deadline = Date.now() + deadlineMs; + while (Date.now() < deadline) { + const blocked = await database.execute(sql` + SELECT pid + FROM pg_stat_activity + WHERE application_name = ${applicationName} + AND cardinality(pg_blocking_pids(pid)) > 0 + LIMIT 1 + `); + if (blocked.length > 0) return true; + if (settled()) return false; + await Bun.sleep(10); + } + throw new Error(`Timed out observing blocked session ${applicationName}.`); +} + +/** The transaction handle drizzle hands a `db.transaction` callback, named so a helper can pass it on. */ +type HeldTransaction = Parameters[0]>[0]; + +/** + * Runs `markAttachmentsSent` into a row another transaction is already holding, and lets the caller + * decide what happens to that row while the stamp is stuck waiting for it. + * + * THIS IS THE GAP BETWEEN THE LOAD AND THE STAMP, MADE OBSERVABLE. `inlineAttachments` reads an + * attachment's bytes, then walks the rest of the history, and only then records the send — so + * anything that can take the row commits inside a window that is as long as the history is. A + * `SELECT … FOR UPDATE` holds the row without changing it, which is what puts the stamp in that + * window on purpose rather than hoping the two land in the right order; `pg_blocking_pids` is + * Postgres confirming the stamp really is waiting, so the interleaving is observed rather than + * assumed. Then `whileBlocked` runs in the holding transaction and it commits, and what the stamp + * does when it wakes is the whole of each test below. + * + * A CONNECTION OF ITS OWN, `{ max: 1 }` and named, for the reasons the upload and delete races give + * at length: `pg_blocking_pids` needs a session it can point at, every pool this suite opens is held + * for the whole run, and the stamp issues its statements one after another anyway. + * + * Hands back whatever `markAttachmentsSent` rejected with, or null when it resolved. Returned rather + * than rethrown so the caller can assert on either outcome, and captured in the handler rather than + * left on a floating promise so a rejection is never momentarily unhandled. + */ +async function stampWhileTheRowIsHeld( + db: Database, + turn: { actorId: string; threadId: string }, + id: string, + whileBlocked: (held: HeldTransaction) => Promise, +): Promise { + const applicationName = `attachment_stamp_race_${randomUUID()}`; + const namedUrl = new URL(databaseUrl); + namedUrl.searchParams.set("application_name", applicationName); + const namedDatabase = createDatabase(namedUrl.toString(), { max: 1 }); + + const rowHeld = deferred(); + const release = deferred(); + const holder = db.transaction(async (transaction) => { + await transaction + .select({ id: attachments.id }) + .from(attachments) + .where(eq(attachments.id, id)) + .for("update"); + rowHeld.resolve(); + await release.promise; + await whileBlocked(transaction); + }); + void holder.catch(rowHeld.reject); + + try { + await rowHeld.promise; + let settled = false; + let outcome: unknown = null; + const stamping = markAttachmentsSent(namedDatabase, turn, [id]).then( + () => { + settled = true; + }, + (reason: unknown) => { + settled = true; + outcome = reason; + }, + ); + + expect(await waitForBlockedSession(applicationName, () => settled)).toBe( + true, + ); + release.resolve(); + await holder; + await stamping; + return outcome; + } finally { + release.resolve(); + await holder.catch(() => undefined); + await namedDatabase.$client.close(); + } +} + +/** + * The characters RFC 9110 does not allow in a field value: the C0 controls and DEL. Reported as + * code points rather than as a boolean, so a failure names the byte that got through. + */ +function controlCharactersIn(value: string): string[] { + return Array.from(value) + .filter((char) => { + const codePoint = char.codePointAt(0) ?? 0; + return codePoint < 0x20 || codePoint === 0x7f; + }) + .map((char) => `0x${(char.codePointAt(0) ?? 0).toString(16)}`); +} + +describe("POST /:channelId/attachments", () => { + test("a member uploads a PNG and gets back a staged attachment", async () => { + const { app, channelId } = await harness(); + const bytes = new Uint8Array([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, + ]); + const file = new File([bytes], "photo.png", { type: "image/png" }); + + const response = await upload(app, channelId, file); + + expect(response.status).toBe(201); + const body = (await response.json()) as { + id: string; + name: string; + mimeType: string; + sizeBytes: number; + }; + expect(typeof body.id).toBe("string"); + expect(body.id.length).toBeGreaterThan(0); + expect(body.mimeType).toBe("image/png"); + expect(body.name).toBe("photo.png"); + }); + + test("a non-member uploading to the channel is refused", async () => { + const { database: db, channelId, stranger } = await harness(); + const app = new Hono<{ Variables: AppVariables }>(); + app.route("/", createChannelAttachmentRoutes(db, stranger)); + const file = new File([new Uint8Array([1, 2, 3])], "x.png", { + type: "image/png", + }); + + const response = await upload(app, channelId, file); + + expect(response.status).toBe(403); + }); + + test("an SVG is refused, naming the SVG as the reason", async () => { + const { app, channelId } = await harness(); + const svg = new TextEncoder().encode( + "", + ); + const file = new File([svg], "x.svg", { type: "image/svg+xml" }); + + const response = await upload(app, channelId, file); + + expect(response.status).toBe(415); + const body = (await response.json()) as { error: string }; + expect(body.error).toContain("SVG"); + }); + + test("a text file over the size limit is refused and writes no row", async () => { + const { app, database: db, channelId } = await harness(); + const oversized = new Uint8Array(MAX_FILE_BYTES + 1).fill(0x61); + const file = new File([oversized], "notes.txt", { type: "text/plain" }); + + const response = await upload(app, channelId, file); + + expect(response.status).toBe(413); + const rows = await db + .select({ id: attachments.id }) + .from(attachments) + .where(eq(attachments.channelId, channelId)); + expect(rows.length).toBe(0); + }); + + test("a non-multipart body is refused with a 400 and a reason", async () => { + const { app, channelId } = await harness(); + + const response = await app.request(`http://test/${channelId}/attachments`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ not: "a form" }), + }); + + expect(response.status).toBe(400); + const body = (await response.json()) as { error: string }; + expect(typeof body.error).toBe("string"); + expect(body.error.length).toBeGreaterThan(0); + }); + + test("an unsupported non-image type is refused, naming the type", async () => { + const { app, channelId } = await harness(); + const file = new File([new Uint8Array([0x50, 0x4b, 0x03, 0x04])], "a.zip", { + type: "application/zip", + }); + + const response = await upload(app, channelId, file); + + expect(response.status).toBe(415); + const body = (await response.json()) as { error: string }; + expect(body.error).toContain("application/zip"); + }); + + test("the ninth staged attachment in one upload group is refused", async () => { + const { app, database: db, channelId, memberId } = await harness(); + await db.insert(attachments).values( + Array.from({ length: MAX_ATTACHMENTS_PER_MESSAGE }, (_, index) => ({ + channelId, + uploadedBy: memberId, + uploadGroup: "one-composer", + name: `staged-${index}.txt`, + mimeType: "text/plain", + sizeBytes: 3, + bytes: Buffer.from("abc"), + })), + ); + const file = new File([new TextEncoder().encode("abc")], "one-more.txt", { + type: "text/plain", + }); + + const response = await upload(app, channelId, file, "one-composer"); + + expect(response.status).toBe(409); + const { error } = (await response.json()) as { error: string }; + // The sentence has to name the limit, because the limit is the only actionable half... + expect(error).toContain( + `${MAX_ATTACHMENTS_PER_MESSAGE} files to a message`, + ); + // ...and it may not describe the count as a channel total. It is scoped to one composer + // session, which is the whole point of `upload_group`: a person with a full tab A and an empty + // tab B was being sent hunting for files that are on another screen. + expect(error).not.toContain("in this channel"); + }); + + /** + * THE 409 NOBODY COULD ACT ON, CLOSED. + * + * The cap is per message on the client, which can only count what is on its own screen, and used + * to be per channel here. A closed tab, a stopped run or a removed queued message leaves staged + * rows behind, and eight of them refused every upload a NEW composer made — naming files that + * were on nobody's screen, for the 24 hours until the sweeper's window expired. The orphans are + * still the sweeper's to reclaim; what they must not do is spend somebody else's cap. + */ + test("an orphan from another upload group does not spend a new group's cap", async () => { + const { app, database: db, channelId, memberId } = await harness(); + await db.insert(attachments).values( + Array.from({ length: MAX_ATTACHMENTS_PER_MESSAGE }, (_, index) => ({ + channelId, + uploadedBy: memberId, + // A composer that is gone: its tab was closed with these still staged. + uploadGroup: "a-tab-that-was-closed", + name: `orphan-${index}.txt`, + mimeType: "text/plain", + sizeBytes: 3, + bytes: Buffer.from("abc"), + })), + ); + const file = new File([new TextEncoder().encode("abc")], "first.txt", { + type: "text/plain", + }); + + const response = await upload(app, channelId, file, "a-fresh-composer"); + + expect(response.status).toBe(201); + }); + + /* + * THE HOLE UNDER THE PER-GROUP CAP, DRIVEN THROUGH THE ROUTE THE COMPOSER ACTUALLY POSTS TO. + * + * `uploadGroup` is a multipart form field and `uploadGroupOf` does not validate its value, so the + * cap above counts a bucket the caller names. A caller that names a fresh one every time has zero + * prior rows in every bucket it is ever counted against: before the backstop this loop wrote all + * thirty-three rows, and nothing else in the server bounded staged `bytea` short of the 24-hour + * culler. + * + * SEQUENTIAL, AND THE COUNT OF 201s IS THE ASSERTION. The point is not that some request + * eventually fails; it is that the number of rows this person can stage has a ceiling at all, and + * that the ceiling is the one written down. Whether the refusal arrives on the 33rd request or + * some later one is the difference between a bound and no bound. + */ + test("a fresh upload group on every request does not buy unlimited staging", async () => { + const { app, channelId } = await harness(); + const attempts = MAX_STAGED_ATTACHMENTS_PER_UPLOADER + 1; + const statuses: number[] = []; + let lastError = ""; + + for (let index = 0; index < attempts; index++) { + const file = new File( + [new TextEncoder().encode("abc")], + `evade-${index}.txt`, + { type: "text/plain" }, + ); + // A bucket nothing has ever been staged in, minted the way a real composer mints one. + const response = await upload(app, channelId, file, randomUUID()); + statuses.push(response.status); + if (response.status !== 201) { + lastError = ((await response.json()) as { error: string }).error; + } + } + + expect(statuses.filter((status) => status === 201)).toHaveLength( + MAX_STAGED_ATTACHMENTS_PER_UPLOADER, + ); + expect(statuses.filter((status) => status === 409)).toHaveLength( + attempts - MAX_STAGED_ATTACHMENTS_PER_UPLOADER, + ); + // The two refusals must not be confusable. This one is about everything this person has + // waiting, so it names that number and not the per-message limit — a person told "you can + // attach 8 files to a message" while holding one file in this composer would go looking for + // seven files that are not there. + expect(lastError).toContain(String(MAX_STAGED_ATTACHMENTS_PER_UPLOADER)); + expect(lastError).toContain("waiting to send"); + expect(lastError).not.toContain("files to a message"); + + /* + * AND IT PROMISES NO SCHEDULE, BECAUSE THIS SERVER DOES NOT KNOW ONE. + * + * It used to end "anything still unsent is cleared within a day", which is a promise about + * `attachments.culler.olderThanHours` — an operator's value, defaulting to 24 and documented as + * raisable — made on a deployment that may also have set `attachments.culler.enabled: false`, + * which charts/openbot/README.md offers as the way to "keep every staged row for ever". On such + * a deployment the sentence was a flat lie told to the one person who could not act on it: some + * of these rows are in channels they can no longer open, so withdrawing them is not available + * either. + * + * Asserted as the absence of a deadline rather than as the new wording, so this keeps holding if + * the sentence is rephrased and only stops holding if a duration comes back into it. + */ + expect(lastError).not.toMatch(/within a (day|week|hour)|\d+\s*hours?/i); + }); + + /* + * THE BACKSTOP IS NOT PER CHANNEL, because `POST /api/channels` is open to any authenticated user. + * A ceiling that a new channel resets is a ceiling a client moves at will, which is the same + * defect as one a new group moves. + * + * Seeded rather than uploaded: what is under test is which rows the count includes, and thirty-two + * round trips through the multipart route to establish a precondition would be thirty-two chances + * for this test to be about something else. The upload that decides it is a real one. + */ + test("staged rows in another channel still count against the backstop", async () => { + const { app, database: db, channelId, memberId } = await harness(); + + const elsewhere = `${testPrefix}-channel-${randomUUID()}`; + await db.insert(channels).values({ + id: elsewhere, + name: "Another Channel Entirely", + description: "Where this person has already staged their limit.", + }); + createdChannelIds.push(elsewhere); + await db + .insert(channelMemberships) + .values({ channelId: elsewhere, userId: memberId }); + + await db.insert(attachments).values( + Array.from({ length: MAX_STAGED_ATTACHMENTS_PER_UPLOADER }, (_, i) => ({ + channelId: elsewhere, + uploadedBy: memberId, + // A different group for every one of them, so no per-group count sees more than one. + uploadGroup: randomUUID(), + name: `elsewhere-${i}.txt`, + mimeType: "text/plain", + sizeBytes: 3, + bytes: Buffer.from("abc"), + })), + ); + + const file = new File([new TextEncoder().encode("abc")], "first.txt", { + type: "text/plain", + }); + const response = await upload(app, channelId, file, randomUUID()); + + expect(response.status).toBe(409); + const { error } = (await response.json()) as { error: string }; + expect(error).toContain("waiting to send"); + }); + + /* + * THE BACKSTOP UNDER THE SAME RACE THE CAP HAD TO BE FIXED FOR. + * + * Postgres is READ COMMITTED here, so `insert ... select ... where (select count(*)) < n` is not + * atomic on its own: two statements in flight at once both count against a snapshot taken before + * either committed, and both insert. The per-group cap answers that with a + * `pg_advisory_xact_lock`, and that lock USED to be keyed on (channel, uploader, group) — which is + * exactly the key an evading client varies, so it would have serialised nothing here. The key is + * the uploader now, which is the scope this count is taken over. + * + * One row short of the ceiling, two uploads at once, two different groups, two different sessions + * of the same person: exactly one may be accepted. + */ + test("two uploads racing at the backstop in different groups leave only one accepted", async () => { + const { app, database: db, channelId, memberId } = await harness(); + + await db.insert(attachments).values( + Array.from( + { length: MAX_STAGED_ATTACHMENTS_PER_UPLOADER - 1 }, + (_, i) => ({ + channelId, + uploadedBy: memberId, + uploadGroup: randomUUID(), + name: `already-${i}.txt`, + mimeType: "text/plain", + sizeBytes: 3, + bytes: Buffer.from("abc"), + }), + ), + ); + + const responses = await Promise.all( + [0, 1].map((index) => + upload( + app, + channelId, + new File([new TextEncoder().encode("abc")], `race-${index}.txt`, { + type: "text/plain", + }), + randomUUID(), + ), + ), + ); + const statuses = responses.map((response) => response.status); + + expect(statuses.filter((status) => status === 201)).toHaveLength(1); + expect(statuses.filter((status) => status === 409)).toHaveLength(1); + }); + + /* + * THE TRADE THE GROUPING BOUGHT, STILL BOUGHT. The backstop sits four messages above the cap + * precisely so that the case `upload_group` was introduced for — a second tab, or a closed tab's + * leftovers — keeps working. A person holding a full message's worth of orphans in one composer + * session must still be able to compose the next message; that is the 409 nobody could act on, + * and it stays closed. + */ + test("a second composer session still gets its own eight under the backstop", async () => { + const { app, database: db, channelId, memberId } = await harness(); + + await db.insert(attachments).values( + Array.from({ length: MAX_ATTACHMENTS_PER_MESSAGE }, (_, index) => ({ + channelId, + uploadedBy: memberId, + uploadGroup: "a-tab-that-was-closed", + name: `orphan-${index}.txt`, + mimeType: "text/plain", + sizeBytes: 3, + bytes: Buffer.from("abc"), + })), + ); + + const secondSession = randomUUID(); + const statuses: number[] = []; + for (let index = 0; index < MAX_ATTACHMENTS_PER_MESSAGE; index++) { + const file = new File( + [new TextEncoder().encode("abc")], + `fresh-${index}.txt`, + { type: "text/plain" }, + ); + statuses.push((await upload(app, channelId, file, secondSession)).status); + } + + expect(statuses.every((status) => status === 201)).toBe(true); + }); + + /** + * THE CAP UNDER THE LOAD IT IS ACTUALLY MET WITH. + * + * Dropping nine files on the composer fires nine uploads at once by design, and the guard used to + * be a count, a comparison and then an insert: every one of those requests counted the rows that + * existed before any of them had written, all nine passed, and the person held nine. Sequential + * uploads never showed it. Statuses are counted rather than rows, because what this pins is what + * the door answered, not what the table happens to hold. + */ + test("nine uploads racing in one group still leave only eight accepted", async () => { + const { app, channelId } = await harness(); + const attempts = MAX_ATTACHMENTS_PER_MESSAGE + 1; + + const responses = await Promise.all( + Array.from({ length: attempts }, (_, index) => + upload( + app, + channelId, + new File([new TextEncoder().encode("abc")], `race-${index}.txt`, { + type: "text/plain", + }), + "one-composer-in-a-hurry", + ), + ), + ); + const statuses = responses.map((response) => response.status); + + expect(statuses.filter((status) => status === 201)).toHaveLength( + MAX_ATTACHMENTS_PER_MESSAGE, + ); + expect(statuses.filter((status) => status === 409)).toHaveLength( + attempts - MAX_ATTACHMENTS_PER_MESSAGE, + ); + }); + /* + * THE MEMBERSHIP CHECK AND THE INSERT ARE ONE STATEMENT, for the same reason the count and the + * insert are. + * + * The route read membership at the top of the handler and then inserted on the strength of what + * it had read — with the whole of `await file.arrayBuffer()`, the sniff and the classification in + * between. A removal landing in that gap put a file into a channel its uploader had already been + * taken out of, and the gap is as wide as reading an upload off the wire. + * + * Parked deterministically rather than hoped for. The upload's first act inside its transaction + * is to take its uploader's advisory lock, so holding that exact lock first stops the request + * precisely between its check and its insert. The removal commits, the lock is released, and what + * the insert does when it finally runs is the whole test. + * + * The key is the uploader alone, and was (channel, uploader, group) until the staging backstop + * needed a count no group could move. A group still goes into this upload because the route reads + * one; it just no longer names the lock. + */ + test( + "a membership revoked mid-upload leaves no file in the channel", + async () => { + const { database: db, channelId, memberId, member } = await harness(); + const uploadGroup = randomUUID(); + + /* + * A named connection of its own, so `pg_blocking_pids` can point at the request's session, and + * `{ max: 1 }` rather than `TEST_POOL` for the reason the delete race gives below: every pool + * this suite opens is held for the whole run, and the request issues its statements one after + * another anyway. + */ + const applicationName = `attachment_upload_race_${randomUUID()}`; + const namedUrl = new URL(databaseUrl); + namedUrl.searchParams.set("application_name", applicationName); + const namedDatabase = createDatabase(namedUrl.toString(), { max: 1 }); + + const lockHeld = deferred(); + const releaseLock = deferred(); + const holder = db.transaction(async (transaction) => { + await transaction.execute( + sql`select pg_advisory_xact_lock(hashtext(${`attachment-cap-${memberId}`}))`, + ); + lockHeld.resolve(); + await releaseLock.promise; + }); + void holder.catch(lockHeld.reject); + + let status: number | undefined; + try { + await lockHeld.promise; + const app = new Hono<{ Variables: AppVariables }>(); + app.route("/", createChannelAttachmentRoutes(namedDatabase, member)); + const file = new File( + [new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])], + "photo.png", + { type: "image/png" }, + ); + + let settled = false; + const request = upload(app, channelId, file, uploadGroup).then( + (value) => { + settled = true; + return value; + }, + (reason: unknown) => { + settled = true; + throw reason; + }, + ); + + expect( + await waitForBlockedSession(applicationName, () => settled), + ).toBe(true); + + await db + .delete(channelMemberships) + .where( + and( + eq(channelMemberships.channelId, channelId), + eq(channelMemberships.userId, memberId), + ), + ); + + releaseLock.resolve(); + await holder; + status = (await request).status; + } finally { + releaseLock.resolve(); + await holder.catch(() => undefined); + await namedDatabase.$client.close(); + } + + expect(status).toBe(403); + // The row, because a 403 answered over a file that had already landed would be the same failure + // wearing the right status code. + const rows = await db + .select({ id: attachments.id }) + .from(attachments) + .where(eq(attachments.channelId, channelId)); + expect(rows.length).toBe(0); + // Declared, so the helper's own "which session never blocked" diagnostic is what fires + // when this race does not happen. See BLOCKED_SESSION_TIMEOUT_MS. + }, + BLOCKED_SESSION_TIMEOUT_MS, + ); + /* + * A GROUP THIS SERVER CANNOT STORE IS A GROUP IT DOES WITHOUT. + * + * `upload_group` is the third column of `attachments_upload_group_idx`, and a btree entry may not + * exceed about 2704 bytes, so a long enough group makes the INSERT itself fail. Measured through + * this route against the local Postgres: 2000 bytes stored fine; 2600, 2700, 3000 and 8000 all + * raised, and with no `app.onError` behind this router the person got a plain-text + * `Internal Server Error` — the one shape the composer cannot read a reason out of. + * + * The upload is not refused, though, and that is the behaviour being pinned as much as the + * absence of a 500: a group is a client-side hint nobody asked for, so one this server cannot + * store is counted in the same bucket a request naming no group at all uses. This test proves it + * landed in THAT bucket rather than merely somewhere, by filling the bucket first. + */ + test("an upload group too long for the index is counted with the group-less ones", async () => { + const { app, channelId } = await harness(); + // Incompressible: the index stores what it is given, and 4000 repeated characters would not be + // the same test. + const group = Array.from({ length: 200 }, () => randomUUID()).join(""); + expect(group.length).toBeGreaterThan(2704); + + // Seven with no group at all, which is the bucket a group this server drops falls back into. + for (let index = 0; index < MAX_ATTACHMENTS_PER_MESSAGE - 1; index++) { + const response = await upload( + app, + channelId, + new File([new TextEncoder().encode("abc")], `staged-${index}.txt`, { + type: "text/plain", + }), + ); + expect(response.status).toBe(201); + } + + const eighth = await upload( + app, + channelId, + new File([new TextEncoder().encode("abc")], "eighth.txt", { + type: "text/plain", + }), + group, + ); + const ninth = await upload( + app, + channelId, + new File([new TextEncoder().encode("abc")], "ninth.txt", { + type: "text/plain", + }), + ); + + // Stored, rather than 500ing on the index... + expect(eighth.status).toBe(201); + // ...and stored in the group-less bucket, which the ninth then finds full. + expect(ninth.status).toBe(409); + }); + + /* + * A NUL is not a length problem and would survive a length check. Postgres refuses U+0000 in a + * `text` value outright (`22021`), which takes down the one statement the group is bound into — + * the upload's own count-and-insert CTE — so nothing is written and the upload is refused. + * + * This comment used to say the failure landed "one statement before the insert", on the advisory + * lock. That was true of a lock keyed on `(channel, uploader, group)` and has not been true since + * the key was widened to `attachment-cap-`, which carries no group: re-measured against + * the local Postgres, the lock takes a NUL-bearing group and an 8000-byte one without complaint. + * The fallback below is still right, for the reason it was always really right — an upload with + * nothing wrong with the FILE should not be refused over a cosmetic grouping hint. + */ + test("an upload group carrying a NUL is stored as a group-less upload", async () => { + const { app, channelId, database: db } = await harness(); + + const response = await upload( + app, + channelId, + new File([new TextEncoder().encode("abc")], "notes.txt", { + type: "text/plain", + }), + "composer\u0000session", + ); + + expect(response.status).toBe(201); + const groupless = await upload( + app, + channelId, + new File([new TextEncoder().encode("abc")], "other.txt", { + type: "text/plain", + }), + ); + expect(groupless.status).toBe(201); + + // Compared against what an upload naming no group at all stores, rather than against the + // fallback's spelling, so the test pins the behaviour and not the constant. + const groupOf = async (response: Response) => { + const { id } = (await response.json()) as { id: string }; + const [row] = await db + .select({ uploadGroup: attachments.uploadGroup }) + .from(attachments) + .where(eq(attachments.id, id)); + return row.uploadGroup; + }; + expect(await groupOf(response)).toBe(await groupOf(groupless)); + }); + + /* + * WHAT THE COMPOSER IS OWED WHEN THE STORE IS THE THING THAT FAILED. + * + * `app/src/components/channels/composer/attachments.ts` reads `{ error }` off every failed upload + * and falls back to a generic `Could not upload ""` when the body will not parse as JSON. + * There is no `app.onError` behind this router — the `formData()` guard above says so, and relies + * on it — so an unguarded database call meant a lost connection, a pool with nothing left, a + * `statement_timeout`, a full disk and a lock timeout were all Hono's plain-text + * `Internal Server Error`, and none of them were written down on this side either. + */ + test("a database that cannot be reached refuses the upload as JSON, and says so in the log", async () => { + const { channelId, memberId, member } = await harness(); + const app = new Hono<{ Variables: AppVariables }>(); + app.route("/", createChannelAttachmentRoutes(unreachableDatabase, member)); + + const { result: response, logged } = await withCapturedErrorLog(() => + upload( + app, + channelId, + new File([new TextEncoder().encode("abc")], "notes.txt", { + type: "text/plain", + }), + ), + ); + + expect(response.status).toBe(503); + expect(response.headers.get("Content-Type")).toContain("application/json"); + expect(((await response.json()) as { error: string }).error).toContain( + "Try again", + ); + // Named, because an operator answering "why could nobody upload at 14:05" needs the person and + // the channel, and a line naming neither cannot be acted on. + expect( + logged.some( + (line) => line.includes(memberId) && line.includes(channelId), + ), + ).toBe(true); + }); + + /* + * The same guarantee one statement deeper. The unreachable database above never gets past the + * membership check, so this one lets every real statement through and fails only where the file + * is actually written — which is the line a pool exhaustion, a lock timeout on + * `pg_advisory_xact_lock` or a `53100` full disk would land on. + */ + test("a transaction that cannot be opened refuses the upload as JSON, naming the size", async () => { + const { channelId, member, database: db } = await harness(); + const app = new Hono<{ Variables: AppVariables }>(); + app.route( + "/", + createChannelAttachmentRoutes(databaseWhoseTransactionFails(db), member), + ); + + const { result: response, logged } = await withCapturedErrorLog(() => + upload( + app, + channelId, + new File([new TextEncoder().encode("abcde")], "notes.txt", { + type: "text/plain", + }), + ), + ); + + expect(response.status).toBe(503); + expect(response.headers.get("Content-Type")).toContain("application/json"); + expect(((await response.json()) as { error: string }).error).toContain( + "Try again", + ); + // The sniffed type and the size are in the line because "the disk is full" and "one 8 MiB + // upload is being retried in a loop" look identical without them. + expect(logged.some((line) => line.includes("text/plain"))).toBe(true); + expect(logged.some((line) => line.includes("5 bytes"))).toBe(true); + }); + /* + * A REFUSAL THAT NAMES A NUMBER BELOW THE LIMIT IS A REFUSAL THAT CONTRADICTS ITSELF. + * + * The cap's count and its explanation used to be two statements. Both ran inside the upload's + * transaction, but READ COMMITTED gives each statement its own snapshot and the advisory lock + * does not cover withdrawals, so one `DELETE /api/attachments/:id` committing in between made the + * 409 read "you already have 7 attachments waiting" while the limit is 8 — and a composer + * dropping a whole queued message made it read 0. The upload was still right to refuse; only its + * account of itself was wrong, and "you have 0 attachments, so you may not add one" is a sentence + * no support conversation can recover from. + * + * The assertion is deliberately about NUMBERS rather than about the wording, so it keeps holding + * if the sentence is rewritten again: no number this refusal prints may be below the limit that + * produced it. + */ + test("the cap's refusal never names a count below the limit, even when a withdrawal lands mid-request", async () => { + const { app, channelId, database: db, member } = await harness(); + const uploadGroup = randomUUID(); + + const staged: string[] = []; + for (let index = 0; index < MAX_ATTACHMENTS_PER_MESSAGE; index++) { + const response = await upload( + app, + channelId, + new File([new TextEncoder().encode("abc")], `staged-${index}.txt`, { + type: "text/plain", + }), + uploadGroup, + ); + expect(response.status).toBe(201); + staged.push(((await response.json()) as { id: string }).id); + } + + // A real withdrawal through the real route, on its own connection, committing while the ninth + // upload is between its statements. + const withdrawer = attachmentApp(db, member); + const racing = new Hono<{ Variables: AppVariables }>(); + racing.route( + "/", + createChannelAttachmentRoutes( + databaseWithdrawingAfterTheInsert(db, async () => { + const withdrawn = await withdrawer.request( + `http://test/${staged[0]}`, + { method: "DELETE" }, + ); + expect(withdrawn.status).toBe(204); + }), + member, + ), + ); + + const ninth = await upload( + racing, + channelId, + new File([new TextEncoder().encode("abc")], "ninth.txt", { + type: "text/plain", + }), + uploadGroup, + ); + + expect(ninth.status).toBe(409); + const { error } = (await ninth.json()) as { error: string }; + const counts = [...error.matchAll(/\d+/g)].map((match) => Number(match[0])); + expect(counts.length).toBeGreaterThan(0); + expect( + counts.filter((count) => count < MAX_ATTACHMENTS_PER_MESSAGE), + ).toEqual([]); + // The withdrawal really did land, so the refusal above was explained against a table that had + // already moved on: seven rows are left where the count says eight. + const remaining = await db + .select({ id: attachments.id }) + .from(attachments) + .where( + and( + eq(attachments.channelId, channelId), + eq(attachments.uploadGroup, uploadGroup), + ), + ); + expect(remaining).toHaveLength(MAX_ATTACHMENTS_PER_MESSAGE - 1); + }); + /* + * A NAME THAT CANNOT BE PUT IN A HEADER IS A FILE NOBODY CAN FETCH. + * + * The fetch route echoes the stored name into `Content-Disposition` twice — quoted, and + * percent-encoded for `filename*` — so a non-ASCII name comes back about four times its own + * length. Measured through these two routes before the bound: a 4000-character name uploaded + * fine and then served a 32 KB `Content-Disposition`. Bun serves it; a reverse proxy capping + * response headers at 4-8 KB does not, and behind one that row is unfetchable for ever with + * nothing in the app to say why. + * + * The upload is not refused over it — a name is a property of a file this app CAN read, and the + * 201 hands back what was actually stored — so the assertions are that the file arrives, that the + * name came back cut, and that the header it produces is small enough to survive a proxy. + */ + test("a filename too long for a response header is cut rather than refused", async () => { + const { app, channelId, database: db, member } = await harness(); + // Multi-byte on purpose: it is the percent-encoded copy that blows the header up, and an + // ASCII-only name would not show it. + const name = `${"é".repeat(4000)}.txt`; + + const uploaded = await upload( + app, + channelId, + new File([new TextEncoder().encode("hello")], name, { + type: "text/plain", + }), + ); + + expect(uploaded.status).toBe(201); + const stored = (await uploaded.json()) as { id: string; name: string }; + expect(stored.name.length).toBeLessThan(name.length); + expect(new TextEncoder().encode(stored.name).length).toBeLessThanOrEqual( + 255, + ); + // Never through the middle of a character: a cut that split one would leave a lone surrogate or + // a replacement character rather than the "é" that was there. + expect(stored.name).toBe("é".repeat(stored.name.length)); + + const served = await attachmentApp(db, member).request( + `http://test/${stored.id}`, + ); + const disposition = served.headers.get("Content-Disposition") ?? ""; + expect(served.status).toBe(200); + expect(new TextEncoder().encode(disposition).length).toBeLessThan(2_000); + }); + + /* + * A REFUSAL WITH NOTHING TO NAME MUST NOT PRINT AN EMPTY PAIR OF BRACKETS. + * + * `sniffMimeType` hands the claim back when the bytes corroborate nothing and the claim names no + * format, and the refusal interpolated it: `'archive' is not a file type this app can read ().` + * The composer shows that sentence verbatim, so it is the only explanation anybody gets, and an + * empty parenthetical is worse than the generic sentence it replaced. + * + * REACHED THROUGH THE FILENAME. Measured against this deployment's Bun: the multipart parser + * ignores a part's own `Content-Type` header and derives `File.type` from the filename extension + * (`photo.png` declared `text/plain` arrives as `image/png`), so a name with no extension is how + * a blank claim gets here — the `type` passed to the `File` below is not what the route sees. + */ + test("a refusal with no type to name does not print an empty parenthetical", async () => { + const { app, channelId } = await harness(); + // No extension, and bytes that are neither valid UTF-8 nor any image signature: between them + // the sniffer has nothing at all to report. + const file = new File([new Uint8Array([0xff, 0xfe, 0xff])], "archive", { + type: "application/octet-stream", + }); + + const response = await upload(app, channelId, file); + + expect(response.status).toBe(415); + const { error } = (await response.json()) as { error: string }; + expect(error).toBe("'archive' is not a file type this app can read."); + }); +}); + +describe("GET /:id", () => { + test("a member gets the bytes back byte-for-byte", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const bytes = new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05, 0xff, 0x00]); + const id = await uploadBytes(db, { + channelId, + uploadedBy: memberId, + name: "blob.bin", + mimeType: "application/octet-stream", + bytes, + }); + const app = attachmentApp(db, member); + + const response = await app.request(`http://test/${id}`); + + expect(response.status).toBe(200); + const body = Buffer.from(await response.arrayBuffer()); + expect(Buffer.compare(body, Buffer.from(bytes))).toBe(0); + }); + + /* + * THE ONLY FAULT IN THIS FILE THAT SERVING THE RIGHT BYTES DOES NOT RULE OUT. + * + * `Uint8Array.from(row.bytes)` and a view over the same buffer produce identical responses, so + * every other assertion here passes either way. What separates them is the process: `.from` + * finds `@@iterator` on the `Buffer` and walks the file element by element, 8.4 million steps of + * synchronous work for a file at the ceiling, during which this process answers nobody — + * measured at 62-87ms for the conversion alone and 120-140ms for the whole fetch, against 36-38ms + * once it became a view. + * + * So the assertion is about the EVENT LOOP rather than about the body: a 1ms interval samples + * how long the thread went unavailable while the request ran. The threshold is 40ms against a + * measured 62-87ms fault and a measured sub-millisecond healthy path — roughly half the fault + * and a wide multiple of the healthy case, which is the margin that keeps a loaded machine or a + * garbage collection from failing this while still catching a return to the iterator path. + * + * `MAX_IMAGE_BYTES` and not something smaller, because the cost is linear in the file and the + * ceiling is the size this route is documented to serve. + */ + test("serving an attachment at the size ceiling does not block the event loop", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const bytes = new Uint8Array(MAX_IMAGE_BYTES); + // A real PNG signature, so the route classifies this the way a photo at the ceiling would be. + bytes.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const id = await uploadBytes(db, { + channelId, + uploadedBy: memberId, + name: "huge.png", + mimeType: "image/png", + bytes, + }); + const app = attachmentApp(db, member); + // Warmed, so the first request's own module and pool costs are not counted as a stall. + expect((await app.request(`http://test/${id}`)).status).toBe(200); + + let longestPause = 0; + let previousTick = performance.now(); + const sampler = setInterval(() => { + const now = performance.now(); + longestPause = Math.max(longestPause, now - previousTick); + previousTick = now; + }, 1); + let response: Response; + try { + previousTick = performance.now(); + response = await app.request(`http://test/${id}`); + /* + * ONE TURN OF THE LOOP BEFORE THE SAMPLER IS TAKEN AWAY, and without it this test cannot + * fail. A timer is a macrotask and an awaited promise resumes on the microtask queue, so the + * line above continues BEFORE the interval that would observe the stall gets to run: clearing + * the sampler here measured 2.5ms against a fault that really did block for 62-87ms. The + * gap that spans the blocked stretch is recorded by the first tick after it, so that tick + * has to be allowed to happen. + */ + await new Promise((resume) => setTimeout(resume, 5)); + } finally { + clearInterval(sampler); + } + + expect(response.status).toBe(200); + expect((await response.arrayBuffer()).byteLength).toBe(MAX_IMAGE_BYTES); + // Compared as a sentence rather than with `toBeLessThan`, so a failure names the pause it + // measured: "62ms" against "under 40ms" says which fault came back, where "false is not true" + // would leave the next person to re-measure it. + expect( + longestPause < 40 ? "under 40ms" : `${Math.round(longestPause)}ms`, + ).toBe("under 40ms"); + }, 30_000); + + /* + * A STAGED FILE HAS BEEN SHARED WITH NOBODY. + * + * Membership is what lets people see each other's SENT files. A row with no `attachedAt` is a + * draft somebody has not sent — possibly one they are about to think better of — so serving it to + * a colleague on the strength of the same channel join was the read side quietly disagreeing with + * the write side, which refuses that same colleague any say over the row at all. + * + * Both halves are asserted in one test on purpose: "staged is private" is only correct if "sent + * is shared" still holds, and a test that pinned the first alone could be satisfied by refusing + * everybody everything. + */ + test("a member cannot fetch a colleague's staged draft, and can once it is sent", async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const colleagueId = `${testPrefix}-colleague-${randomUUID()}`; + await db + .insert(users) + .values({ id: colleagueId, email: `${colleagueId}@example.test` }); + createdUserIds.push(colleagueId); + await db + .insert(channelMemberships) + .values({ channelId, userId: colleagueId }); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "draft.txt", + text: "not sent yet", + }); + const colleague = attachmentApp(db, actorMiddleware(colleagueId)); + + expect((await colleague.request(`http://test/${id}`)).status).toBe(404); + + // The uploader is not shut out of their own composer's preview by this. + const uploader = attachmentApp(db, actorMiddleware(memberId)); + expect((await uploader.request(`http://test/${id}`)).status).toBe(200); + + await markAttachmentsSent(db, { actorId: memberId, threadId }, [id]); + + expect((await colleague.request(`http://test/${id}`)).status).toBe(200); + }); + + /* + * The revalidation has to be decided on the same rule as the fetch, or a colleague holding a + * staged id from before the send keeps a 304 that says "your copy is still good" for a file this + * route would no longer hand them. + */ + test("a colleague's staged draft is not revalidated into a 304 either", async () => { + const { database: db, channelId, memberId } = await harness(); + const colleagueId = `${testPrefix}-colleague-${randomUUID()}`; + await db + .insert(users) + .values({ id: colleagueId, email: `${colleagueId}@example.test` }); + createdUserIds.push(colleagueId); + await db + .insert(channelMemberships) + .values({ channelId, userId: colleagueId }); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "draft.txt", + text: "not sent yet", + }); + + const response = await attachmentApp( + db, + actorMiddleware(colleagueId), + ).request(`http://test/${id}`, { + headers: { "If-None-Match": `"${id}"` }, + }); + + expect(response.status).toBe(404); + }); + + test("a non-member is refused with a 404, not a 403", async () => { + const { database: db, channelId, memberId, stranger } = await harness(); + const id = await uploadBytes(db, { + channelId, + uploadedBy: memberId, + name: "secret.txt", + mimeType: "text/plain", + bytes: new TextEncoder().encode("shh"), + }); + const app = attachmentApp(db, stranger); + + const response = await app.request(`http://test/${id}`); + + expect(response.status).toBe(404); + }); + + /* + * Channels soft-delete, so "deleted" is a column and not a missing row, and every read has to + * say so itself. The upload route always did: its join leads with `channels` and + * `deleted_at is null`. This one joined the membership alone — and a membership outlives its + * channel's deletion, so the bytes stayed downloadable, and inlinable, for ever after the channel + * they were uploaded into was deleted. + */ + test("an attachment in a soft-deleted channel is no longer fetchable", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "notes.txt", + text: "hello", + }); + const app = attachmentApp(db, member); + // Fetchable first, so the 404 below is the deletion and not a broken fixture. + expect((await app.request(`http://test/${id}`)).status).toBe(200); + + await db + .update(channels) + .set({ deletedAt: new Date() }) + .where(eq(channels.id, channelId)); + + const response = await app.request(`http://test/${id}`); + + // 404, the same answer as "no such attachment" and as "not yours": whether a channel has been + // deleted is not a bit to be read back off this route either. + expect(response.status).toBe(404); + }); + + test("an image is served with its own Content-Type, nosniff, and inline", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const png = new Uint8Array([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, + ]); + const id = await uploadBytes(db, { + channelId, + uploadedBy: memberId, + name: "photo.png", + mimeType: "image/png", + bytes: png, + }); + const app = attachmentApp(db, member); + + const response = await app.request(`http://test/${id}`); + + expect(response.status).toBe(200); + expect(response.headers.get("Content-Type")).toBe("image/png"); + expect(response.headers.get("X-Content-Type-Options")).toBe("nosniff"); + expect(response.headers.get("Content-Disposition")).toBe("inline"); + expect(response.headers.get("Cache-Control")).toBe("private, no-cache"); + }); + + /* + * A DELETION HAS TO BE VISIBLE, and `private, max-age=3600` made it invisible. + * + * Deleting the row does make this route answer 404 — and the browser never asked. An `` + * already on the page kept painting the file from its own cache, at full natural width, for the + * rest of the hour, so the "this attachment is unavailable" path the transcript has could not be + * reached at all. The same hour kept the bytes readable after a sign-out and after a removal from + * the channel. + * + * `no-cache` is not `no-store`: the copy is still kept, it just may not be used without asking + * first. The bytes behind an id never change, so what is being kept fresh here is EXISTENCE, not + * content — and the ETag below is what keeps the re-ask cheap. + */ + test("the fetch response cannot be reused without revalidating", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "notes.txt", + text: "hello", + }); + const app = attachmentApp(db, member); + + const response = await app.request(`http://test/${id}`); + + expect(response.status).toBe(200); + // `private` still, so no proxy or CDN in between may hold somebody's private file. + expect(response.headers.get("Cache-Control")).toBe("private, no-cache"); + expect(response.headers.get("Cache-Control")).not.toContain("max-age"); + expect(response.headers.get("ETag")).toBe(`"${id}"`); + }); + + /* + * The revalidation itself, on both sides of a deletion. A 304 is what makes `no-cache` affordable + * — the bytes do not go back over the wire — and the 404 after is the point of asking at all: the + * question the conditional request answers is whether the file is still THERE and still THEIRS, + * and it is re-asked against this actor's membership every single time. + */ + test("a revalidation is answered 304 while the row lives and 404 once it is gone", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "notes.txt", + text: "hello", + }); + const app = attachmentApp(db, member); + + const revalidate = () => + app.request(`http://test/${id}`, { + headers: { "If-None-Match": `"${id}"` }, + }); + + const fresh = await revalidate(); + expect(fresh.status).toBe(304); + expect(await fresh.text()).toBe(""); + expect(fresh.headers.get("Cache-Control")).toBe("private, no-cache"); + + await db.delete(attachments).where(eq(attachments.id, id)); + + expect((await revalidate()).status).toBe(404); + }); + + /* + * A 304 SENDS NO BODY, SO IT MUST NOT READ ONE. + * + * `ATTACHMENT_CACHE_CONTROL` is `private, no-cache` deliberately, which means every image paint in + * every viewing member's transcript revalidates through this route — the 304 is the common answer + * here, not the rare one. It was nevertheless served by selecting `bytes` and then discarding + * them: measured through this route against an 8 MiB attachment, 8,388,608 bytes read and 25.09ms + * median per 304. + * + * The column list is the fact, for the same reason the HEAD probe's test gives: an empty body is + * an empty body either way, and a stopwatch is a guess about how fast this machine is. `bytes` is + * either in the statement or it is not. + * + * Both halves in one test, as with the probe: "the revalidation does not read the file" is only + * the right property if "the fetch still does" holds, and a recorder that saw `bytes` nowhere + * would satisfy the first assertion while the route served nothing at all. + */ + test("a revalidation does not read the file it is not going to send", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "notes.txt", + text: "hello", + }); + + const revalidated: string[][] = []; + const revalidating = attachmentApp( + databaseRecordingSelections(db, revalidated), + member, + ); + const notModified = await revalidating.request(`http://test/${id}`, { + headers: { "If-None-Match": `"${id}"` }, + }); + + expect(notModified.status).toBe(304); + expect(await notModified.text()).toBe(""); + expect(revalidated.flat()).not.toContain("bytes"); + + // And the same request without the header does read them, so the assertion above is about this + // handler rather than about how the recorder happens to see drizzle. + const fetched: string[][] = []; + const fetcher = attachmentApp( + databaseRecordingSelections(db, fetched), + member, + ); + expect((await fetcher.request(`http://test/${id}`)).status).toBe(200); + expect(fetched.flat()).toContain("bytes"); + }); + + /* + * The saving above may not be bought with the access check. A revalidation asks "is this still + * there and still mine", so it is the one answer on this route that MUST be re-decided against the + * live channel and this actor's membership every single time — dropping `bytes` from the select + * list must not drop the join that earns the 304. + * + * A stranger revalidating with a valid ETag is the case that would show it: if the ETag alone + * decided, they would be told their copy is still good for a file they may not read. + */ + test("a stranger's revalidation is refused rather than answered from the ETag", async () => { + const { + database: db, + channelId, + memberId, + member, + stranger, + } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "notes.txt", + text: "hello", + attachedAt: new Date(), + }); + const conditional = { headers: { "If-None-Match": `"${id}"` } }; + + // The member holds a good copy, so their revalidation is the 304 this is measured against. + expect( + ( + await attachmentApp(db, member).request( + `http://test/${id}`, + conditional, + ) + ).status, + ).toBe(304); + + expect( + ( + await attachmentApp(db, stranger).request( + `http://test/${id}`, + conditional, + ) + ).status, + ).toBe(404); + + // And once the channel is gone the member's own revalidation goes the same way, which is the + // other half of what the join is there for. + await db + .update(channels) + .set({ deletedAt: new Date() }) + .where(eq(channels.id, channelId)); + expect( + ( + await attachmentApp(db, member).request( + `http://test/${id}`, + conditional, + ) + ).status, + ).toBe(404); + }); + + test("a text file is served attachment, never inline", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "notes.txt", + text: "hello", + }); + const app = attachmentApp(db, member); + + const response = await app.request(`http://test/${id}`); + + expect(response.status).toBe(200); + expect(response.headers.get("Content-Type")).toBe("text/plain"); + const disposition = response.headers.get("Content-Disposition"); + expect(disposition).not.toBe("inline"); + expect(disposition).toContain("attachment"); + expect(disposition).toContain('filename="notes.txt"'); + }); + + test("a non-Latin-1 filename is served instead of 500ing, folded in the quoted param and intact in filename*", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "メモ.txt", + text: "hello", + }); + const app = attachmentApp(db, member); + + const response = await app.request(`http://test/${id}`); + + expect(response.status).toBe(200); + const disposition = response.headers.get("Content-Disposition"); + expect(disposition).toContain("attachment"); + expect(disposition).toContain('filename="__.txt"'); + expect(disposition).toContain("filename*=UTF-8''%E3%83%A1%E3%83%A2.txt"); + }); + + test("a filename carrying C0 control characters is served with none of them in the header", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + // A vertical tab and a DEL. Neither can appear in an HTTP field value, and neither is one of + // the CR/LF pair the escaping used to take out, so between them they stand for the whole + // rest of that class. + name: "report\u000B\u007F.txt", + text: "hello", + }); + const app = attachmentApp(db, member); + + const response = await app.request(`http://test/${id}`); + + expect(response.status).toBe(200); + const disposition = response.headers.get("Content-Disposition") ?? ""; + expect(disposition).toContain("attachment"); + expect(disposition).toContain('filename="report__.txt"'); + // Percent-encoding turns a control character into something a header CAN carry (`%0B`), so the + // extended parameter is checked for the folded name rather than merely for header-safety. + expect(disposition).toContain("filename*=UTF-8''report__.txt"); + expect(controlCharactersIn(disposition)).toEqual([]); + }); + + test("an SVG body uploaded as text/plain is stored, and served so it cannot run", async () => { + // A known and deliberate limit of `sniffMimeType`: an SVG is valid UTF-8, + // so a text claim over SVG bytes is corroborated and accepted. No + // "does this look like XML" sniff is attempted, because that is brittle + // and would refuse legitimate text. What makes it safe is not the sniff + // but this response, so the response is what gets pinned: the stored type + // is served verbatim (`text/plain`, never `image/svg+xml`), `nosniff` + // stops the browser upgrading that guess for itself, and the disposition + // is `attachment`, so nothing renders in a document on this origin. Any + // one of the three would do; all three have to hold. + const { app, channelId, database: db, member } = await harness(); + const svg = + ""; + + const uploaded = await upload( + app, + channelId, + new File([svg], "notes.txt", { type: "text/plain" }), + ); + expect(uploaded.status).toBe(201); + const { id } = (await uploaded.json()) as { id: string }; + + const served = await attachmentApp(db, member).request(`http://test/${id}`); + + expect(served.status).toBe(200); + expect(served.headers.get("Content-Type")).toBe("text/plain"); + expect(served.headers.get("X-Content-Type-Options")).toBe("nosniff"); + expect(served.headers.get("Content-Disposition")).toContain("attachment"); + expect(served.headers.get("Content-Disposition")).not.toBe("inline"); + }); + + test("a non-UUID id is refused with a 404, not a 500", async () => { + const { database: db, member } = await harness(); + const app = attachmentApp(db, member); + + const response = await app.request("http://test/not-a-uuid"); + + expect(response.status).toBe(404); + }); + /* + * WHAT THE TRANSCRIPT'S DOCUMENT PROBE COSTS. + * + * Hono answers HEAD by running the GET handler and dropping the body, and the transcript probes + * every document tile it draws with `HEAD /api/attachments/` from every viewing member's + * browser — with `no-cache`, on every mount. Before the dedicated handler that was a full `bytea` + * read per tile per mount, for bytes nobody would be sent. + * + * Asserted on the SQL rather than on a stopwatch: the fetch asks for `bytes`, and the probe must + * not. + */ + test("a HEAD answers from the metadata alone, without reading the file", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const bytes = new Uint8Array(64 * 1024).fill(7); + const id = await uploadBytes(db, { + channelId, + uploadedBy: memberId, + name: "report.pdf", + mimeType: "text/plain", + bytes, + attachedAt: new Date(), + }); + + const probed: string[][] = []; + const probe = attachmentApp( + databaseRecordingSelections(db, probed), + member, + ); + const head = await probe.request(`http://test/${id}`, { method: "HEAD" }); + + expect(head.status).toBe(200); + expect((await head.arrayBuffer()).byteLength).toBe(0); + // The size a probe is usually asking for, which a body-less response cannot imply. + expect(head.headers.get("Content-Length")).toBe(String(bytes.byteLength)); + expect(probed.flat()).not.toContain("bytes"); + + // And the fetch of the same row does read them, so the assertion above is about this handler + // rather than about how the recorder happens to see drizzle. + const fetched: string[][] = []; + const fetcher = attachmentApp( + databaseRecordingSelections(db, fetched), + member, + ); + expect((await fetcher.request(`http://test/${id}`)).status).toBe(200); + expect(fetched.flat()).toContain("bytes"); + }); + + /* + * A PROBE MAY NOT ANSWER A QUESTION THE FETCH WOULD REFUSE. + * + * The HEAD handler repeats the GET's guards rather than sharing them, so the risk it carries is + * drift: a rule added to one and not the other turns HEAD into a way to learn that an id exists, + * or that a colleague has a draft, which the GET's uniform 404 exists to hide. Every refusal is + * therefore compared against the GET's own answer for the same request. + */ + test("a HEAD answers exactly what a GET would, for every refusal", async () => { + const { + database: db, + channelId, + memberId, + member, + stranger, + } = await harness(); + const colleagueId = `${testPrefix}-colleague-${randomUUID()}`; + await db + .insert(users) + .values({ id: colleagueId, email: `${colleagueId}@example.test` }); + createdUserIds.push(colleagueId); + await db + .insert(channelMemberships) + .values({ channelId, userId: colleagueId }); + + const sent = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "sent.txt", + text: "hello", + attachedAt: new Date(), + }); + const staged = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "draft.txt", + text: "not sent yet", + }); + + const cases: { + name: string; + app: Hono<{ Variables: AppVariables }>; + path: string; + }[] = [ + { + name: "a sent file, to a member", + app: attachmentApp(db, member), + path: sent, + }, + { + name: "a colleague's staged draft", + app: attachmentApp(db, actorMiddleware(colleagueId)), + path: staged, + }, + { + name: "a stranger to the channel", + app: attachmentApp(db, stranger), + path: sent, + }, + { + name: "an id that names no row", + app: attachmentApp(db, member), + path: randomUUID(), + }, + { + name: "an id no uuid column could hold", + app: attachmentApp(db, member), + path: "not-a-uuid", + }, + ]; + + /* + * THE STATUS IS NOT THE WHOLE ANSWER, WHICH IS HOW ONE OF THESE DRIFTED. Hono answers a HEAD by + * dispatching the GET handler and re-wrapping the result as `new Response(null, )`: the body + * is dropped but every header survives. So a refusal built with `context.body(null, status)` + * carries no `Content-Type` where the fetch's `context.json` carries `application/json`, and the + * two are distinguishable to anybody probing — for a 503 that was exactly the case, measured at + * `503 application/json` from the GET against a bare `503` from the HEAD. Comparing the pair + * rather than the number is what would have caught it. + */ + const shapeOf = (response: Response) => ({ + status: response.status, + contentType: response.headers.get("Content-Type"), + }); + + for (const { name, app, path } of cases) { + const fetched = await app.request(`http://test/${path}`); + const probed = await app.request(`http://test/${path}`, { + method: "HEAD", + }); + expect({ [name]: shapeOf(probed) }).toEqual({ + [name]: shapeOf(fetched), + }); + } + }); + + /* + * The refusal the loop above cannot reach, because it needs a database that is not there. + * + * A 503 is the one answer on this route that is about this side rather than about the asker, and + * it was the one the probe spelled differently: `context.body(null, 503)` against the fetch's + * `context.json`. Both now go through the same expression, so there is no second spelling left to + * drift — and the log line is asserted too, because a fault nobody can see from outside is half of + * what the 503 exists to report. + */ + test("a HEAD and a GET refuse an unreachable database in the same shape", async () => { + const memberId = `${testPrefix}-member-${randomUUID()}`; + const app = attachmentApp(unreachableDatabase, actorMiddleware(memberId)); + const id = randomUUID(); + + const { result, logged } = await withCapturedErrorLog(async () => ({ + fetched: await app.request(`http://test/${id}`), + probed: await app.request(`http://test/${id}`, { method: "HEAD" }), + })); + + expect(result.fetched.status).toBe(503); + // Not a 404: the transcript paints "this attachment is gone" on one of those, and a lost + // connection is not a withdrawal. + expect(result.probed.status).toBe(503); + expect(result.probed.headers.get("Content-Type")).toBe( + result.fetched.headers.get("Content-Type"), + ); + expect(result.fetched.headers.get("Content-Type")).toContain( + "application/json", + ); + expect((await result.fetched.json()) as { error: string }).toEqual({ + error: "That attachment could not be read just now. Try again.", + }); + // Both reads are written down, naming the row and the asker, because that is what it would take + // to act on either. + expect(logged.filter((line) => line.includes(id))).toHaveLength(2); + expect(logged.every((line) => line.includes(memberId))).toBe(true); + }); + + /* + * The same question for the answer the probe and the fetch share outright. A revalidation is + * handled before the method is even looked at now, so a HEAD and a GET carrying the same + * `If-None-Match` differ in nothing at all — which is what the two used to only claim. + */ + test("a HEAD and a GET revalidate into the same 304", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "sent.txt", + text: "hello", + attachedAt: new Date(), + }); + const app = attachmentApp(db, member); + const conditional = { headers: { "If-None-Match": `"${id}"` } }; + + const fetched = await app.request(`http://test/${id}`, conditional); + const probed = await app.request(`http://test/${id}`, { + ...conditional, + method: "HEAD", + }); + + for (const response of [fetched, probed]) { + expect(response.status).toBe(304); + expect(response.headers.get("ETag")).toBe(`"${id}"`); + expect(response.headers.get("Cache-Control")).toBe("private, no-cache"); + } + }); + + /* + * The probe revalidates on the same terms as the fetch, or a client holding a stale id would be + * told its copy is still good by one and that the file is gone by the other. + */ + test("a HEAD honours If-None-Match, and stops doing so once the row is gone", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "sent.txt", + text: "hello", + attachedAt: new Date(), + }); + const app = attachmentApp(db, member); + + const held = await app.request(`http://test/${id}`, { + method: "HEAD", + headers: { "If-None-Match": `"${id}"` }, + }); + expect(held.status).toBe(304); + + await db.delete(attachments).where(eq(attachments.id, id)); + + const gone = await app.request(`http://test/${id}`, { + method: "HEAD", + headers: { "If-None-Match": `"${id}"` }, + }); + expect(gone.status).toBe(404); + }); +}); + +describe("DELETE /:id", () => { + test("a staged attachment can be deleted by its uploader, then 404s on GET", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "draft.txt", + text: "not sent yet", + }); + const app = attachmentApp(db, member); + + const deleteResponse = await app.request(`http://test/${id}`, { + method: "DELETE", + }); + expect(deleteResponse.status).toBe(204); + + const getResponse = await app.request(`http://test/${id}`); + expect(getResponse.status).toBe(404); + }); + + test("a sent attachment cannot be deleted", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "sent.txt", + text: "already in a message", + attachedAt: new Date(), + }); + const app = attachmentApp(db, member); + + const response = await app.request(`http://test/${id}`, { + method: "DELETE", + }); + + expect(response.status).toBe(409); + }); + + /* + * THE QUERY THAT EXPLAINS THE REFUSAL ASKS ONLY WHETHER THERE IS A ROW. + * + * It used to select `attachedAt` and never read it: the 409 below it is unconditional. A column + * fetched and dropped is cheap, but a column fetched and dropped in a query that decides between a + * 404 and a 409 reads as a check being made, and no check was. The next person to touch this would + * have had to work out from scratch that the decision comes from the two WHEREs — this query + * repeating every term of the delete's except `attached_at is null`, so a row coming back means + * that term is the one that refused. + * + * Pinned on the column list rather than on the status, because the status was already right. What + * changed is that the statement now says what it is for. + */ + test("the refused withdrawal's explanation reads no column it does not act on", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "sent.txt", + text: "already in a message", + attachedAt: new Date(), + }); + + const selected: string[][] = []; + const app = attachmentApp( + databaseRecordingSelections(db, selected), + member, + ); + const response = await app.request(`http://test/${id}`, { + method: "DELETE", + }); + + expect(response.status).toBe(409); + expect(selected.flat()).not.toContain("attachedAt"); + // The 409 is still the one it was, said the one way it is said. + expect((await response.json()) as { error: string }).toEqual({ + error: "This attachment is already part of a sent message.", + }); + }); + + /* + * The same refusal, when the send lands DURING the request rather than before it. + * + * The route used to read `attachedAt`, decide on what it read, and then delete by id alone. Those + * are two statements, and a send is one more: stamp the row between them and the delete removed a + * file a sent message already pointed at, because its WHERE no longer mentioned the column the + * decision was made on. The window is small and entirely real — the sender's own turn writes that + * stamp while their composer may still be offering the file for withdrawal. + * + * Held open on purpose here rather than hoped for. An uncommitted UPDATE takes the row lock; a + * plain SELECT reads straight past it on the old snapshot, which is exactly why the check saw a + * staged row; and the DELETE has to wait. What it does when it stops waiting is the whole test: + * Postgres re-checks the delete's WHERE against the row as it now stands, so a WHERE that carries + * `attached_at IS NULL` matches nothing and a WHERE that does not still removes the file. + */ + test( + "a send landing mid-request cannot have its file deleted out from under it", + async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "draft.txt", + text: "about to be sent", + }); + + /* + * One more connection, named, so `pg_blocking_pids` can point at the request's own session + * rather than at whatever else is talking to this database. + * + * `{ max: 1 }` rather than `TEST_POOL`, and the difference is not tidiness. Every file that + * opens a pool holds it for the whole run, the suite already sits within sight of PostgreSQL's + * hundred, and a second one here took it over: the failure is `53300 sorry, too many clients + * already` in whichever unrelated file happened to connect next. The request below issues its + * statements one after another, so one connection is all it can use anyway. The blocking + * transaction and the `pg_blocking_pids` poll take the shared pool's two. + */ + const applicationName = `attachment_delete_race_${randomUUID()}`; + const namedUrl = new URL(databaseUrl); + namedUrl.searchParams.set("application_name", applicationName); + const namedDatabase = createDatabase(namedUrl.toString(), { max: 1 }); + + const stampWritten = deferred(); + const releaseSend = deferred(); + const send = db.transaction(async (transaction) => { + await transaction + .update(attachments) + .set({ attachedAt: new Date() }) + .where(eq(attachments.id, id)); + stampWritten.resolve(); + await releaseSend.promise; + }); + void send.catch(stampWritten.reject); + + let status: number | undefined; + try { + await stampWritten.promise; + const app = attachmentApp(namedDatabase, member); + let settled = false; + const withdrawal = app + .request(`http://test/${id}`, { method: "DELETE" }) + .then( + (value) => { + settled = true; + return value; + }, + (reason: unknown) => { + settled = true; + throw reason; + }, + ); + + expect( + await waitForBlockedSession(applicationName, () => settled), + ).toBe(true); + releaseSend.resolve(); + await send; + status = (await withdrawal).status; + } finally { + releaseSend.resolve(); + await send.catch(() => undefined); + await namedDatabase.$client.close(); + } + + expect(status).toBe(409); + // The row itself, because a 409 that answered after the file was already gone would be the same + // failure wearing the right status code. + expect(await attachedAtOf(db, id)).toBeInstanceOf(Date); + // As above: the runner's default 5s timeout used to kill this before the helper could say which + // session it had been watching. See BLOCKED_SESSION_TIMEOUT_MS. + }, + BLOCKED_SESSION_TIMEOUT_MS, + ); + + test("a second channel member cannot delete another member's staged draft", async () => { + const { database: db, channelId, memberId } = await harness(); + const otherMemberId = `${testPrefix}-other-${randomUUID()}`; + await db.insert(users).values({ + id: otherMemberId, + email: `${otherMemberId}@example.test`, + }); + createdUserIds.push(otherMemberId); + await db + .insert(channelMemberships) + .values({ channelId, userId: otherMemberId }); + const otherMember = actorMiddleware(otherMemberId); + + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "draft.txt", + text: "not sent yet", + }); + const app = attachmentApp(db, otherMember); + + const response = await app.request(`http://test/${id}`, { + method: "DELETE", + }); + + expect(response.status).toBe(404); + const rows = await db + .select({ id: attachments.id }) + .from(attachments) + .where(eq(attachments.id, id)); + expect(rows.length).toBe(1); + }); + + /* + * Membership is an access-control boundary, and the withdrawal still stands on it. + * + * The conditional delete carries the whole rule — id, uploader, still-staged AND still a member — + * because a statement that dropped the membership term would be a boundary quietly loosened + * inside a change about when a column is written. The bytes are the person's own and the sweeper + * would reclaim them anyway, so the harm is small; a boundary that erodes one low-harm case at a + * time is the thing being refused here. + * + * 404, not 403, exactly as everywhere else in this router: a former member learns nothing about + * whether the id still names anything. + */ + test("somebody removed from the channel cannot withdraw their own staged draft", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "draft.txt", + text: "not sent yet", + }); + + await db + .delete(channelMemberships) + .where( + and( + eq(channelMemberships.channelId, channelId), + eq(channelMemberships.userId, memberId), + ), + ); + + const app = attachmentApp(db, member); + const response = await app.request(`http://test/${id}`, { + method: "DELETE", + }); + + expect(response.status).toBe(404); + const rows = await db + .select({ id: attachments.id }) + .from(attachments) + .where(eq(attachments.id, id)); + expect(rows.length).toBe(1); + }); + + /* + * The withdrawal stands on the same channel scope as the fetch. A membership row outlives the + * soft deletion of its channel, so a delete scoped by membership alone still acted inside a + * channel nobody can open any more. 404 rather than 204, exactly as everywhere else in this + * router: nothing about the row is visible, including that it is still there. + */ + test("a staged draft in a soft-deleted channel cannot be withdrawn", async () => { + const { database: db, channelId, memberId, member } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "draft.txt", + text: "not sent yet", + }); + await db + .update(channels) + .set({ deletedAt: new Date() }) + .where(eq(channels.id, channelId)); + const app = attachmentApp(db, member); + + const response = await app.request(`http://test/${id}`, { + method: "DELETE", + }); + + expect(response.status).toBe(404); + const rows = await db + .select({ id: attachments.id }) + .from(attachments) + .where(eq(attachments.id, id)); + expect(rows.length).toBe(1); + }); + + test("a non-UUID id is refused with a 404, not a 500", async () => { + const { database: db, member } = await harness(); + const app = attachmentApp(db, member); + + const response = await app.request("http://test/not-a-uuid", { + method: "DELETE", + }); + + expect(response.status).toBe(404); + }); +}); + +describe("attachment route composition", () => { + test("mounts both routers behind createApp authentication with the derived actor", async () => { + const { database: db, channelId, memberId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "notes.txt", + text: "hello", + }); + + let session: { + user: { id: string; email: string; name: string; image: string }; + } | null = null; + const app = createApp( + loadConfig(testEnvironment()), + { + handler: () => new Response(null, { status: 204 }), + api: { getSession: async () => session }, + }, + { rolesForUser: async () => ["user"] }, + // Positions 4-25, ending at userInstructions. `attachmentDatabase` is position 26, the same + // gap channel-routes.test.ts leaves for channelStore at position 11. + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + db, + ); + + const unauthenticated = await app.request( + `http://openbot.test/api/attachments/${id}`, + ); + expect(unauthenticated.status).toBe(401); + + session = { + user: { + id: memberId, + email: `${memberId}@example.test`, + name: "OpenBot Member", + image: "https://example.test/member.png", + }, + }; + + const authenticated = await app.request( + `http://openbot.test/api/attachments/${id}`, + ); + expect(authenticated.status).toBe(200); + + const uploadFormData = new FormData(); + uploadFormData.set( + "file", + // A real PNG signature, not three arbitrary bytes wearing the name: an + // `image/png` claim the bytes do not corroborate is refused, so junk + // here would 415 on the file rather than prove the route is mounted. + new File( + [new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])], + "x.png", + { type: "image/png" }, + ), + ); + const uploadResponse = await app.request( + `http://openbot.test/api/channels/${channelId}/attachments`, + { method: "POST", body: uploadFormData }, + ); + expect(uploadResponse.status).toBe(201); + }); + + test("leaves both routers unmounted when createApp has no database", async () => { + const app = createApp(loadConfig(testEnvironment())); + + const fetchResponse = await app.request( + "http://openbot.test/api/attachments/not-a-uuid", + ); + expect(fetchResponse.status).toBe(404); + + const uploadResponse = await app.request( + "http://openbot.test/api/channels/some-channel/attachments", + { method: "POST" }, + ); + expect(uploadResponse.status).toBe(404); + }); +}); + +/* + * THIS TEST USED TO PIN THE BUG IT WAS MEANT TO GUARD. + * + * It posted `MAX_IMAGE_BYTES + 1` bytes and asserted, POSITIVELY, that the refusal was NOT JSON — + * `expect(() => JSON.parse(text)).toThrow()` — using hono's plain-text `Payload Too Large` as the + * way to tell the door's 413 from the handler's. Both halves of that premise were themselves + * defects, and the door has since been fixed on both counts (`server/src/app.ts`). + * + * The size was wrong because the door measures THE ENVELOPE and every other gate measures THE FILE: + * `bodyLimit` runs before anything has parsed the body, so the multipart boundary, the part headers + * and the `uploadGroup` field (~360 bytes as the composer sends them) counted against the file's own + * ceiling. A file at exactly the published 8MB limit was refused. The door now sits at + * `UPLOAD_BODY_LIMIT_BYTES`, so `MAX_IMAGE_BYTES + 1` sails through it — and the handler is what + * refuses that one, naming the file. This test's old size now reaches `requireUser` and answers 503. + * + * The shape was wrong because the composer reads `{ error }` off every failure + * (`app/src/components/channels/composer/attachments.ts`) and falls back to a generic sentence when + * the body will not parse — so on the one refusal whose reason is both knowable and actionable, the + * person was told nothing. The door now answers JSON, which is what the assertion below requires. + * + * No assertion that lets a file at the ceiling through can also keep the old one: the two sizes + * differ by one byte and the framing is ~360, so whatever the door's number is, `ceiling + 1` is on + * the same side of it as the ceiling. The discriminator has to be something other than the shape of + * the refusal, and it is: a body past the DOOR'S own ceiling can only have been refused by the door. + */ +describe("the upload route's body limit", () => { + /** The app the two tests below share: no auth service, so nothing below the door can answer 2xx. */ + function appWithNoAuth(db: Database) { + return createApp( + loadConfig(testEnvironment()), + // Positions 2-25: no auth needed, since the body limit runs ahead of every route below it, + // including `requireUser`. `attachmentDatabase` is position 26. + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + db, + ); + } + + function postBytes( + app: ReturnType, + channelId: string, + byteLength: number, + ) { + const formData = new FormData(); + formData.set( + "file", + new File([new Uint8Array(byteLength)], "huge.png", { type: "image/png" }), + ); + return app.request( + `http://openbot.test/api/channels/${channelId}/attachments`, + { method: "POST", body: formData }, + ); + } + + test("a POST past the door's own ceiling is refused by the body limit, in a shape the composer can read", async () => { + const { database: db, channelId } = await harness(); + const app = appWithNoAuth(db); + + const response = await postBytes( + app, + channelId, + UPLOAD_BODY_LIMIT_BYTES + 1, + ); + + expect(response.status).toBe(413); + // The refusal has to be `{ error }` or the composer shows "Could not upload ..." and the person + // never learns that the file was simply too big. + const body = (await response.json()) as { error?: string }; + expect(body.error).toContain("8MB"); + // Scoped to this test's own channel: a global row count would collide with the sweeper test, + // which deletes across the whole shared database. + const rows = await db + .select({ id: attachments.id }) + .from(attachments) + .where(eq(attachments.channelId, channelId)); + expect(rows.length).toBe(0); + }); + + /* + * What makes the 413 above the DOOR'S and not somebody else's. + * + * The app is built with no auth service, so every route beneath the middleware answers 503 — which + * means a 413 cannot have come from below it. A small body to the very same URL proves the door is + * not simply refusing everything: it answers 503, so size is the only thing that differs between + * the two requests, and the door is the only thing that measures size before `requireUser`. + */ + test("a small POST to the same URL is not refused by the door, so the 413 above is about size", async () => { + const { database: db, channelId } = await harness(); + const app = appWithNoAuth(db); + + const response = await postBytes(app, channelId, 8); + + expect(response.status).not.toBe(413); + }); +}); + +/** + * A SECOND channel belonging to the same person, with its own conversation. + * + * This is the production shape, not a contrivance: `makeChannel` gives every channel exactly one + * membership row and exactly one mapping row, both for its creator, so one person with two channels + * has two threads and is a member of both. That is all it takes to reach the cross-channel case — + * no second person and no shared channel, which is why it survived three rounds of review looking + * like somebody else's problem. + */ +async function secondChannelFor( + db: Database, + memberId: string, +): Promise<{ channelId: string; threadId: string }> { + const channelId = `${testPrefix}-channel-${randomUUID()}`; + await db.insert(channels).values({ + id: channelId, + name: "The Other Channel", + description: "A second conversation the same person is in.", + }); + createdChannelIds.push(channelId); + await db.insert(channelMemberships).values({ channelId, userId: memberId }); + const threadId = `${testPrefix}-thread-${randomUUID()}`; + await db + .insert(intelligenceChannelMappings) + .values({ channelId, userId: memberId, threadId }); + return { channelId, threadId }; +} + +describe("loadAttachmentForTurn", () => { + /* + * THE CROSS-CHANNEL CASE, which is the one this scope exists for. + * + * The ids reach the loader out of browser-supplied message content, so a person in two channels + * of their own can put an `/api/attachments/` part for a file in A on a message they compose + * in B. Membership alone says yes — they really are a member of A — and before the run's thread + * was passed down, membership alone was the whole of the check. + * + * It is not a confidentiality break: they hold that file either way, and today a channel has + * exactly one human member, so nobody else is even watching. What it leaves behind is a message + * in B whose file belongs to a channel B has nothing to do with — so the day A is deleted, B's + * transcript has an attachment that is permanently broken while the row is still sitting there, + * and a re-run of that turn fails outright rather than degrading, because the message being asked + * about is resolved with `onMissing: "fail"`. + */ + test("a file from another channel of the same person's is not loaded into this turn", async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const other = await secondChannelFor(db, memberId); + + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "budget.txt", + text: "in the first channel", + // Sent, so the staged-row rule is not what refuses it below and the channel term is. + attachedAt: new Date(), + }); + + // Its own conversation still gets it, so the null below is the channel and not a broken fixture. + expect( + (await loadAttachmentForTurn(db, { actorId: memberId, threadId }, id)) + ?.name, + ).toBe("budget.txt"); + + expect( + await loadAttachmentForTurn( + db, + { actorId: memberId, threadId: other.threadId }, + id, + ), + ).toBeNull(); + }); + + /* + * AND THE BRANCH THAT KEEPS HOPS WORKING, which is why the scope is not an inner join. + * + * Three surfaces run turns on threads this deployment deliberately keeps no channel for: a + * forward agent hop (`handoff-delivery.ts` mints a scratch thread of the addressed Bot's own, + * because an Intelligence thread has exactly one agent), the direct `/bot` chat (whose thread + * comes from `POST /api/threads/mint` — "a conversation this deployment keeps no channel for"), + * and a backwards hop relaying into either. + * + * The hop is the one that would have been broken silently. It seeds that scratch thread with the + * ASKING channel's history, attachment parts and all, and history is resolved with + * `onMissing: "note"` — so under an inner join the addressed Bot would have been told + * `[attachment "x" is no longer available]` about files that exist and that the same person may + * read, with nothing raised anywhere. A hop's last user message is the synthetic instruction + * `handoff-delivery.ts` appends, so the "fail" branch that exists to catch a missing file never + * covers a hop's history either. + */ + test("a thread that maps to no channel still loads, so a hop keeps the history it was handed", async () => { + const { database: db, channelId, memberId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "budget.txt", + text: "asked about in the channel, answered on a scratch thread", + attachedAt: new Date(), + }); + + const scratchThread = `${testPrefix}-scratch-${randomUUID()}`; + + expect( + ( + await loadAttachmentForTurn( + db, + { actorId: memberId, threadId: scratchThread }, + id, + ) + )?.name, + ).toBe("budget.txt"); + }); + + /* + * The degrade is about the THREAD having no channel, not about the scope being optional. A + * scratch thread does not become a skeleton key: everything the membership join and the staged + * rule already refused is still refused on one. + */ + test("an unmapped thread is not a way around membership", async () => { + const { database: db, channelId, memberId, strangerId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "budget.txt", + text: "not the stranger's", + attachedAt: new Date(), + }); + + const scratchThread = `${testPrefix}-scratch-${randomUUID()}`; + + expect( + await loadAttachmentForTurn( + db, + { actorId: strangerId, threadId: scratchThread }, + id, + ), + ).toBeNull(); + }); + + test("a member's turn gets the bytes", async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "notes.txt", + text: "hello", + }); + + const loaded = await loadAttachmentForTurn( + db, + { actorId: memberId, threadId }, + id, + ); + + expect(loaded?.name).toBe("notes.txt"); + expect(loaded?.mimeType).toBe("text/plain"); + expect(loaded?.bytes.toString("utf8")).toBe("hello"); + }); + + /* + * The hole this join closes. `input.messages` is the browser's, so a signed-in person can put an + * `/api/attachments/` part for somebody else's channel on a message they compose themselves — + * no uuid guessing needed, because anybody removed from a channel still holds its ids in their + * local transcript. The GET route already refuses this exact id with a 404; the turn path has to + * refuse it too, or the bytes go straight to a model that reads them back to them. + */ + test("a non-member's turn gets nothing, and the turn fails naming the id", async () => { + const { + database: db, + channelId, + memberId, + strangerId, + threadId, + } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "private.txt", + text: "not for you", + }); + + const loaded = await loadAttachmentForTurn( + db, + { actorId: strangerId, threadId }, + id, + ); + expect(loaded).toBeNull(); + + // Null is not a quiet skip: `resolveAttachmentParts` fails the whole turn on it rather than + // letting the Bot answer as if the file were not there. + await expect( + resolveAttachmentParts( + [ + { + type: "image", + source: { type: "url", value: attachmentUrl(id) }, + }, + ], + (partId) => + loadAttachmentForTurn(db, { actorId: strangerId, threadId }, partId), + ), + ).rejects.toThrow(`Attachment "${id}" could not be loaded`); + + // And a refused load stamps nothing: the row is still staged as far as the sweeper is + // concerned, because nobody entitled to it has been shown it. + const [row] = await db + .select({ attachedAt: attachments.attachedAt }) + .from(attachments) + .where(eq(attachments.id, id)); + expect(row?.attachedAt).toBeNull(); + }); + + /* + * A READ IS NOT A SEND, and this is the property the whole meaning of `attachedAt` rests on. + * + * This function is called for the message being asked about AND for every attachment in the + * history behind it, on every turn, by whoever is running that turn. Stamping here therefore said + * "sent" about every file anybody had ever been shown — and three readers take that word + * literally: the sweeper skips the row for ever, the upload cap frees the slot, and the + * withdrawal route answers 409. The send is what writes the column now; see + * `markAttachmentsSent`. + */ + test("a load leaves attachedAt alone, however many turns replay it", async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "draft.txt", + text: "hello", + }); + + // Twice, because history is replayed in full on every turn: the second read is the second turn. + expect( + (await loadAttachmentForTurn(db, { actorId: memberId, threadId }, id)) + ?.name, + ).toBe("draft.txt"); + expect( + (await loadAttachmentForTurn(db, { actorId: memberId, threadId }, id)) + ?.name, + ).toBe("draft.txt"); + + expect(await attachedAtOf(db, id)).toBeNull(); + }); + + /* + * The colleague's staged draft, and why reading it may not write to it. + * + * Reading is scoped to CHANNEL MEMBERSHIP, because members are meant to see each other's sent + * files. So while the read did the stamping, any member could freeze a colleague's still-staged + * row by naming its id in a message of their own — and a frozen row is not a cosmetic problem: + * the colleague's own withdrawal answers 409 for ever, and the sweeper will never reclaim it + * either, because both of them read `attachedAt` and it now says the file was sent. + */ + test("a member who is not the uploader cannot stamp a colleague's staged row", async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const otherMemberId = `${testPrefix}-other-${randomUUID()}`; + await db.insert(users).values({ + id: otherMemberId, + email: `${otherMemberId}@example.test`, + }); + createdUserIds.push(otherMemberId); + await db + .insert(channelMemberships) + .values({ channelId, userId: otherMemberId }); + + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "draft.txt", + text: "not sent yet", + }); + + /* + * Both ways another member's turn touches this row, and both are refused while it is staged. + * The write always was — that is what `uploadedBy` in the update's WHERE is for — and the read + * now is too: a row with no `attachedAt` has been shared with nobody, so a colleague's + * half-composed draft is not a channel's to read back to a model. + */ + expect( + await loadAttachmentForTurn(db, { actorId: otherMemberId, threadId }, id), + ).toBeNull(); + /* + * AND THE WRITE NOW SAYS SO OUT LOUD, where it used to decline quietly. The row is not this + * person's to stamp and it is not recorded as sent by anybody, so the send cannot be recorded — + * which is exactly the state the verification refuses over. The property this test is named for + * is unchanged and is now stronger: not merely "the stamp did not land" but "the stamp did not + * land AND the turn was told". + * + * No real turn reaches this. The loader above returns null for the same row on the same actor, + * so `resolvePart`'s `"fail"` mode refuses that turn several steps earlier; a colleague's staged + * draft never gets as far as being recorded as sent. It is asserted here because the function is + * callable on its own and its answer to an id it cannot account for should not depend on who + * remembered to call the loader first. + */ + const quiet = spyOn(console, "error").mockImplementation(() => {}); + let refused: unknown; + try { + refused = await markAttachmentsSent( + db, + { actorId: otherMemberId, threadId }, + [id], + ).then( + () => null, + (reason: unknown) => reason, + ); + } finally { + quiet.mockRestore(); + } + expect(refused).toBeInstanceOf(Error); + expect((refused as Error).message).toContain(id); + + expect(await attachedAtOf(db, id)).toBeNull(); + + // And the uploader's own send is what the column was always supposed to be about. + await markAttachmentsSent(db, { actorId: memberId, threadId }, [id]); + expect(await attachedAtOf(db, id)).toBeInstanceOf(Date); + + // Once it has been sent it is the conversation's, and the same colleague reads it like any + // other member — which is the half of the rule that must NOT change. + expect( + ( + await loadAttachmentForTurn( + db, + { actorId: otherMemberId, threadId }, + id, + ) + )?.name, + ).toBe("draft.txt"); + }); + + /* + * The turn path carries the channel scope too. This is the one of the three that hands bytes to a + * model rather than to a browser: without it, a member of a deleted channel could name an id out + * of their own local transcript and have the file read back to them for ever. + */ + test("an attachment in a soft-deleted channel is not loaded for a turn", async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "notes.txt", + text: "hello", + }); + // Loadable first, so the null below is the deletion and not a broken fixture. + expect( + await loadAttachmentForTurn(db, { actorId: memberId, threadId }, id), + ).not.toBeNull(); + + await db + .update(channels) + .set({ deletedAt: new Date() }) + .where(eq(channels.id, channelId)); + + expect( + await loadAttachmentForTurn(db, { actorId: memberId, threadId }, id), + ).toBeNull(); + }); + + test("a missing attachment returns null", async () => { + const { database: db, memberId, threadId } = await harness(); + + const loaded = await loadAttachmentForTurn( + db, + { actorId: memberId, threadId }, + randomUUID(), + ); + + expect(loaded).toBeNull(); + }); + + /* + * The id a turn is handed is not a path param that routing shaped. `attachmentIdFor` + * (attachment-parts.ts) slices whatever follows `/api/attachments/` out of a browser-supplied + * message part, so a query string, a second path segment and the empty string all arrive here as + * "ids". Compared against a `uuid` column, each of those raises Postgres `22P02` and THROWS, and a + * throw is not the same answer as a miss: `resolvePart` degrades a NULL into the "no longer + * available" note for an older message, and only fails the turn on the message being asked about. + * A throw skips that degradation entirely, and because history is replayed on every turn, one + * malformed part would fail this channel's every future turn for ever. + */ + test("a malformed id misses instead of throwing, so history still degrades", async () => { + const { database: db, memberId, threadId } = await harness(); + + for (const malformed of [ + `${randomUUID()}?download=1`, + `${randomUUID()}/bytes`, + "not-a-uuid", + "", + ]) { + expect( + await loadAttachmentForTurn( + db, + { actorId: memberId, threadId }, + malformed, + ), + ).toBeNull(); + } + + // The consequence of that difference, at the layer that feels it: the same id on an older + // message becomes a note the model can read, not a turn that can never run again. + const noted = await resolveAttachmentParts( + [ + { + type: "image", + source: { type: "url", value: attachmentUrl("not-a-uuid") }, + metadata: { filename: "chart.png" }, + }, + ], + (partId) => + loadAttachmentForTurn(db, { actorId: memberId, threadId }, partId), + "note", + ); + expect(noted).toEqual([ + { type: "text", text: '[attachment "chart.png" is no longer available]' }, + ]); + }); +}); + +describe("markAttachmentsSent", () => { + /* + * THE HARM THAT WAS REPRODUCED, and the reason the write carries the scope as well as the read. + * + * One person, two channels of their own, no colleague involved. They name a file from A on a + * message they send in B, and this used to stamp the row in A. Nothing in A ever referred to it, + * and the stamp is not cosmetic — three readers treat a non-null `attachedAt` as "this file rode + * in a message somebody sent": the withdrawal route refuses it with a 409, the culler stops + * reclaiming it, and the upload cap stops counting it. So the file became one the person could + * neither send from A, nor withdraw from A, nor wait out. It is the same freeze the `uploadedBy` + * term exists to prevent, reached without anybody else's help. + */ + test("a send in another channel does not stamp a row that lives in this one", async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const other = await secondChannelFor(db, memberId); + + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "budget.txt", + text: "staged in the first channel", + }); + + /* + * The send happens in the OTHER conversation, naming this channel's file — and it is now + * REFUSED rather than quietly declined. The id names a row this conversation cannot account + * for, which is the whole of what the verification asks, and the answer to "I cannot record + * this send" is to say so before the turn is spent rather than to carry on as though it had + * been recorded. + * + * No real turn reaches this either: `loadAttachmentForTurn` carries the same channel term, so + * the bytes are already refused and `resolvePart`'s `"fail"` mode has failed the turn well + * before a stamp is attempted. What this pins is that the row in A is untouched, which was + * always the point and which the rollback below now also guarantees. + */ + const quiet = spyOn(console, "error").mockImplementation(() => {}); + let elsewhere: unknown; + try { + elsewhere = await markAttachmentsSent( + db, + { actorId: memberId, threadId: other.threadId }, + [id], + ).then( + () => null, + (reason: unknown) => reason, + ); + } finally { + quiet.mockRestore(); + } + expect(elsewhere).toBeInstanceOf(Error); + expect(await attachedAtOf(db, id)).toBeNull(); + + // And the row is untouched rather than merely unstamped: still staged, still withdrawable, and + // still stampable by the send it actually belongs to. + await markAttachmentsSent(db, { actorId: memberId, threadId }, [id]); + expect(await attachedAtOf(db, id)).toBeInstanceOf(Date); + }); + + /* + * The same degrade the reader has, and deliberately not a stricter rule. + * + * An inner join here looked free — a hop's asked message is `handoff-delivery.ts`'s synthetic + * instruction, which names no attachment, so nothing would ever reach the statement. It was + * rejected because of the direct `/bot` chat, where an unmapped thread CAN carry a real send: a + * strict write would silently never stamp those rows, and the culler would delete a file out from + * under a conversation that still shows it — failing in the exact direction this column exists to + * prevent. One rule for both is also one rule to keep true. + */ + test("a send on a thread that maps to no channel still records itself", async () => { + const { database: db, channelId, memberId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "budget.txt", + text: "sent from a conversation this deployment keeps no channel for", + }); + + const scratchThread = `${testPrefix}-scratch-${randomUUID()}`; + await markAttachmentsSent( + db, + { actorId: memberId, threadId: scratchThread }, + [id], + ); + + expect(await attachedAtOf(db, id)).toBeInstanceOf(Date); + }); + + test("the uploader's send stamps their staged row, and a replay of it changes nothing", async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "sent.txt", + text: "hello", + }); + + await markAttachmentsSent(db, { actorId: memberId, threadId }, [id]); + const first = await attachedAtOf(db, id); + expect(first).toBeInstanceOf(Date); + + // A stopped run retried, or the same message replayed as history on a later turn. Neither is a + // new send, and neither may move a timestamp that already means something. + await markAttachmentsSent(db, { actorId: memberId, threadId }, [id]); + // Both sides read through `?.`, so a vanished row would have compared `undefined` to + // `undefined` and called the timestamp unmoved. Take the number out of `first` once, above, + // and demand a number on both sides. + const second = await attachedAtOf(db, id); + expect(second).toBeInstanceOf(Date); + expect((second as Date).getTime()).toBe((first as Date).getTime()); + }); + + /* + * These ids are not path params that routing shaped: `attachmentIdFor` slices whatever follows + * `/api/attachments/` out of a browser-supplied message part, so a query string, a second path + * segment and the empty string all arrive as "ids". Compared against a `uuid` column each raises + * Postgres `22P02` and throws — and a throw out of this function now REFUSES THE TURN, so the + * guard matters more than it did when everything here was swallowed: a part that is not an + * attachment reference at all would otherwise fail the send of the real file beside it. + */ + test("ids that could never name a row are dropped rather than asked about", async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "sent.txt", + text: "hello", + }); + + await markAttachmentsSent(db, { actorId: memberId, threadId }, [ + `${randomUUID()}?download=1`, + "not-a-uuid", + "", + id, + ]); + + // The one real id in that list was still recorded, so the guard drops ids rather than sends. + expect(await attachedAtOf(db, id)).toBeInstanceOf(Date); + }); + + /* + * A WRITE THAT NEVER LANDED USED TO REPORT A SEND, and the turn carried on to the model. + * + * The statement was `.catch`ed and logged, so `markAttachmentsSent` resolved whatever the database + * did — and `inlineAttachments` had already read the bytes, so the Bot answered about a file whose + * `attachedAt` stayed null. The culler reclaims exactly those rows, so a day later the message was + * still displaying an attachment that no longer existed. Nobody was told at either end. + * + * Through the real driver rather than a stub that returns a rejected promise: the failures this + * has to answer for are a lost connection, an exhausted pool and a `statement_timeout`, all of + * which arrive as "the driver could not answer this query", which is what a closed port produces. + * + * The unstamped row is asserted on the REAL database, because "it raised" and "it raised and left + * the row alone" are different claims and only the second one makes the retry safe. + */ + test("a write the database refuses is raised rather than reported as a send", async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "receipt.txt", + text: "sent into a database that cannot be reached", + }); + + // Collected into a local rather than read off the spy, because `mockRestore` clears the recorded + // calls and the assertion below would then be made against an empty log whatever happened. + const logged: string[] = []; + const consoleError = spyOn(console, "error").mockImplementation( + (...args: unknown[]) => { + logged.push(args.map(String).join(" ")); + }, + ); + let raised: unknown; + try { + raised = await markAttachmentsSent( + unreachableDatabase, + { actorId: memberId, threadId }, + [id], + ).then( + () => null, + (reason: unknown) => reason, + ); + } finally { + consoleError.mockRestore(); + } + + expect(raised).toBeInstanceOf(Error); + // A sentence somebody can act on, naming the file and saying the turn did not run — this leaves + // through the run as an AG-UI error and is shown to the person who was waiting. + expect((raised as Error).message).toContain(id); + expect((raised as Error).message).toContain("was not run"); + expect((raised as Error).message).toContain("attach the file again"); + // And named for whoever has to act on it from the other side. Every consequence of a missing + // stamp is about a specific person and a specific row. + expect(logged.join(" ")).toContain(memberId); + expect(logged.join(" ")).toContain(id); + // Still staged: still withdrawable, still countable against the cap, still stampable by a retry. + expect(await attachedAtOf(db, id)).toBeNull(); + }); + + /* + * THE ROW DISAPPEARING BETWEEN THE LOAD AND THE STAMP, which is the failure a row count cannot see. + * + * Postgres reports an UPDATE that matched nothing as a successful command, so a withdrawal (or a + * cull) committing in that window left `markAttachmentsSent` resolving happily over a file that + * was no longer there. The window is not theoretical and it is not short: `inlineAttachments` + * reads the asked message's bytes FIRST and then walks the whole history behind it, so it is one + * database round trip per older message wide, and it is the sender's own composer — which goes on + * offering the file for withdrawal until the send is recorded — on the other side of it. + * + * Driven as a real race rather than by deleting the row beforehand, because the claim is about + * what Postgres does when the two statements actually contend: the stamp is blocked on the row, + * `pg_blocking_pids` says so, and only then does the withdrawal commit. The delete carries the + * same WHERE `DELETE /api/attachments/:id` carries, so what is racing the stamp is the withdrawal + * route's own statement and not a convenient approximation of it. + * + * The other direction is pinned by "a send landing mid-request cannot have its file deleted out + * from under it" above: stamp first, and the withdrawal's `attached_at is null` no longer holds so + * it takes nothing and answers 409. Between the two, neither can win twice. + */ + test( + "a withdrawal landing between the load and the stamp refuses the send", + async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "draft.txt", + text: "withdrawn while the turn was being prepared", + }); + + // The bytes really were readable when the turn started, which is what makes this a race rather + // than a send naming a file that was never there. + expect( + await loadAttachmentForTurn(db, { actorId: memberId, threadId }, id), + ).not.toBeNull(); + + const consoleError = spyOn(console, "error").mockImplementation(() => {}); + let refusal: unknown; + try { + refusal = await stampWhileTheRowIsHeld( + db, + { actorId: memberId, threadId }, + id, + async (held) => { + await held + .delete(attachments) + .where( + and( + eq(attachments.id, id), + eq(attachments.uploadedBy, memberId), + isNull(attachments.attachedAt), + ), + ); + }, + ); + } finally { + consoleError.mockRestore(); + } + + expect(refusal).toBeInstanceOf(Error); + expect((refusal as Error).message).toContain(id); + // `toBeUndefined` rather than `not.toBeNull`, which the three-valued helper would satisfy with + // the very absence being asserted. The withdrawal won; the point is that the send says so. + expect(await attachedAtOf(db, id)).toBeUndefined(); + }, + BLOCKED_SESSION_TIMEOUT_MS, + ); + + /* + * AND THE STAMP ANOTHER RUN ALREADY LANDED IS A RECORDED SEND, NOT A LOST RACE. + * + * This is the over-correction the check above has to avoid, and it is the reason the verification + * is a SECOND STATEMENT rather than a CTE reading beside the UPDATE. A data-modifying CTE and the + * query next to it share one snapshot, taken when the statement began, so a row a neighbour + * stamped a moment ago would be invisible to the read while the UPDATE's own re-check correctly + * declined to stamp it twice — and two runs of the same message would refuse each other. Under + * READ COMMITTED a separate statement takes a fresh snapshot and sees the commit. + * + * Reachable without anybody doing anything strange: a stopped run retried, or a second tab. Same + * race as the test above, with the holding transaction stamping instead of withdrawing. + */ + test( + "a stamp another run landed first is a recorded send rather than a refusal", + async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const id = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "retried.txt", + text: "sent twice at once", + }); + + const outcome = await stampWhileTheRowIsHeld( + db, + { actorId: memberId, threadId }, + id, + async (held) => { + await held + .update(attachments) + .set({ attachedAt: new Date() }) + .where(and(eq(attachments.id, id), isNull(attachments.attachedAt))); + }, + ); + + expect(outcome).toBeNull(); + expect(await attachedAtOf(db, id)).toBeInstanceOf(Date); + }, + BLOCKED_SESSION_TIMEOUT_MS, + ); + + /* + * THE OTHER WAY A ROW COUNT LIES: A SENT FILE THIS PERSON DID NOT UPLOAD. + * + * Members are meant to see each other's sent files, so a message may perfectly well name one — and + * `uploadedBy` in the WHERE is what stops a sender freezing a colleague's row, so the UPDATE + * matching it is exactly what must NOT happen. One id in, zero rows out, and nothing wrong. A + * check that compared rows updated against ids requested would refuse this turn, which is why the + * verification asks whether each id IS recorded rather than whether this statement recorded it. + * + * Both ids at once, because the mixed message is the shape that catches a check applied per-list + * instead of per-id. + */ + test("a colleague's already-sent file on this message does not refuse the send", async () => { + const { database: db, channelId, memberId, threadId } = await harness(); + const colleagueId = `${testPrefix}-colleague-${randomUUID()}`; + await db.insert(users).values({ + id: colleagueId, + email: `${colleagueId}@example.test`, + }); + createdUserIds.push(colleagueId); + await db + .insert(channelMemberships) + .values({ channelId, userId: colleagueId }); + + const theirs = await uploadText(db, { + channelId, + uploadedBy: colleagueId, + name: "shared.txt", + text: "sent by somebody else, earlier", + }); + await markAttachmentsSent(db, { actorId: colleagueId, threadId }, [theirs]); + const theirStamp = await attachedAtOf(db, theirs); + expect(theirStamp).toBeInstanceOf(Date); + + const mine = await uploadText(db, { + channelId, + uploadedBy: memberId, + name: "mine.txt", + text: "staged by the person sending this message", + }); + + await expect( + markAttachmentsSent(db, { actorId: memberId, threadId }, [theirs, mine]), + ).resolves.toBeUndefined(); + + expect(await attachedAtOf(db, mine)).toBeInstanceOf(Date); + // Untouched rather than merely unrefused: the colleague's stamp still says when THEY sent it. + expect((await attachedAtOf(db, theirs)) as Date).toEqual( + theirStamp as Date, + ); + }); + + /* + * "NOT A QUERY" IS THE CLAIM, SO A QUERY IS WHAT THIS HAS TO CATCH. + * + * It used to assert `resolves.toBeUndefined()` against a real database, which + * `markAttachmentsSent` satisfied whatever it did: it returns `Promise`, and back when it + * caught its own database failures and only logged them, deleting the guard this claims to pin + * left it green — the ids would have gone to Postgres, raised `22P02` on the `uuid` column, been + * swallowed, and still resolved `undefined`. + * + * A rejection would be visible now that the failure is raised, so that hole has closed on its own. + * The untouchable database stays, because it pins the STRONGER claim the sentence above actually + * makes: not that nothing broke, but that nothing was ASKED. + * + * A database that refuses to be touched is what makes the claim testable: any property this + * function reads off it throws, so the assertion "this resolved quietly" can only be true if + * nothing was asked. + */ + test("nothing to record is not a query", async () => { + const untouchable = new Proxy({} as Database, { + get(_target, property) { + throw new Error( + `markAttachmentsSent reached for database.${String(property)} with nothing to record.`, + ); + }, + }); + + // Every shape `attachmentIdFor` can produce that no `uuid` column could hold. + await expect( + markAttachmentsSent( + untouchable, + { actorId: "nobody", threadId: "nobody" }, + ["not-a-uuid", "", `${randomUUID()}?download=1`], + ), + ).resolves.toBeUndefined(); + + // And an empty list, which is the case the early return is named for. + await expect( + markAttachmentsSent( + untouchable, + { actorId: "nobody", threadId: "nobody" }, + [], + ), + ).resolves.toBeUndefined(); + }); + + /* + * THE HELPER'S OWN DIAGNOSTIC, WHICH USED TO BE UNREACHABLE. + * + * `waitForBlockedSession` gave itself 5s and Bun gives a test 5s by default, with no override + * anywhere, so the runner always won: the message naming the session that never blocked could not + * be printed, and both races failed as a bare "timed out after 5000ms" instead. The constants are + * compared here because that relationship is the fault, and the message is provoked with a short + * deadline because waiting the real one out would cost the suite seven seconds to observe a + * string. + */ + test("the wait for a blocked session says which session it gave up on", async () => { + expect(BLOCKED_SESSION_DEADLINE_MS).toBeLessThan( + BLOCKED_SESSION_TIMEOUT_MS, + ); + + const applicationName = `never_blocks_${randomUUID()}`; + await expect( + waitForBlockedSession(applicationName, () => false, 50), + ).rejects.toThrow(applicationName); + }); +}); + +/** + * `attachments.name` is a Postgres `text` column and Postgres refuses U+0000 in one, so a NUL + * cannot reach the fetch route through the database the way the other control characters can. The + * builder is called directly here because that is the only way left to prove it never hands the + * header serializer a byte it would throw on — and that throw would come from inside `c.body`, + * after the response had already begun, with no `app.onError` behind the router to turn it into + * anything but a 500 on every fetch of the attachment. + */ +describe("contentDispositionFilename", () => { + test("a NUL in the filename never reaches the header value", () => { + const value = `attachment; ${contentDispositionFilename("a\u0000b.txt")}`; + + expect(value).toContain('filename="a_b.txt"'); + expect(value).toContain("filename*=UTF-8''a_b.txt"); + expect(controlCharactersIn(value)).toEqual([]); + expect( + () => new Response(null, { headers: { "Content-Disposition": value } }), + ).not.toThrow(); + }); + + test("a C0 control character in the filename never reaches the header value", () => { + const value = `attachment; ${contentDispositionFilename("a\u0001b\u001Fc.txt")}`; + + expect(value).toContain('filename="a_b_c.txt"'); + expect(value).toContain("filename*=UTF-8''a_b_c.txt"); + expect(controlCharactersIn(value)).toEqual([]); + }); + + test("CR and LF still cannot open a second header line", () => { + const value = contentDispositionFilename("a\r\nX-Injected: 1.txt"); + + expect(value).toContain('filename="a__X-Injected: 1.txt"'); + expect(controlCharactersIn(value)).toEqual([]); + }); + + test("an ordinary name is untouched and a quote or backslash is still escaped", () => { + expect(contentDispositionFilename("notes.txt")).toBe( + `filename="notes.txt"; filename*=UTF-8''notes.txt`, + ); + expect(contentDispositionFilename('a"b\\c.txt')).toContain( + 'filename="a\\"b\\\\c.txt"', + ); + }); +}); diff --git a/server/tests/attachment-store.test.ts b/server/tests/attachment-store.test.ts new file mode 100644 index 000000000..a69b6a930 --- /dev/null +++ b/server/tests/attachment-store.test.ts @@ -0,0 +1,430 @@ +import { afterAll, afterEach, describe, expect, test } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import type { MiddlewareHandler } from "hono"; +import { Hono } from "hono"; +import { MAX_IMAGE_BYTES } from "../../shared/attachments"; +import { createApp, UPLOAD_BODY_LIMIT_BYTES } from "../src/app"; +import type { AppVariables } from "../src/auth/guards"; +import { + createAttachmentRoutes, + createChannelAttachmentRoutes, +} from "../src/channels/attachments"; +import { loadConfig } from "../src/config"; +import { createDatabase } from "../src/db/client"; +import { + attachments, + channelMemberships, + channels, + users, +} from "../src/db/schema"; +import { TEST_POOL } from "./support/database"; +import { testEnvironment } from "./support/environment"; + +const databaseUrl = + process.env.DATABASE_URL ?? + "postgres://openbot:openbot@localhost:5432/openbot"; +const database = createDatabase(databaseUrl, TEST_POOL); + +const testPrefix = `attachment-store-${randomUUID()}`; +const createdAttachmentIds: string[] = []; +const createdChannelIds: string[] = []; +const createdUserIds: string[] = []; + +afterEach(async () => { + for (const id of createdAttachmentIds.splice(0)) { + await database.delete(attachments).where(eq(attachments.id, id)); + } + for (const id of createdChannelIds.splice(0)) { + await database.delete(channels).where(eq(channels.id, id)); + } + for (const id of createdUserIds.splice(0)) { + await database.delete(users).where(eq(users.id, id)); + } +}); + +afterAll(async () => { + await database.$client.close(); +}); + +/** A user and a channel to hang an attachment off of, nothing more. */ +async function seedChannel(db: typeof database) { + const userId = `${testPrefix}-user-${randomUUID()}`; + await db.insert(users).values({ + id: userId, + email: `${userId}@example.test`, + }); + createdUserIds.push(userId); + + const channelId = `${testPrefix}-channel-${randomUUID()}`; + await db.insert(channels).values({ + id: channelId, + name: "Attachment Store Test Channel", + description: "Round-trip test channel.", + }); + createdChannelIds.push(channelId); + + return { userId, channelId }; +} + +describe("an attachment's bytes through the bytea column", () => { + test("survive the round trip unchanged, byte for byte", async () => { + const { userId, channelId } = await seedChannel(database); + // Deliberately includes both a null byte and 0xff: a driver that + // treats bytea as text would mangle one of these on the way through. + const payload = Buffer.from([0x00, 0x01, 0x02, 0xff, 0xfe, 0x00, 0xff]); + + const [inserted] = await database + .insert(attachments) + .values({ + channelId, + uploadedBy: userId, + name: "test.bin", + mimeType: "application/octet-stream", + sizeBytes: payload.byteLength, + bytes: payload, + }) + .returning(); + createdAttachmentIds.push(inserted.id); + + const [read] = await database + .select() + .from(attachments) + .where(eq(attachments.id, inserted.id)); + + expect(Buffer.compare(read.bytes, payload)).toBe(0); + expect(read.attachedAt).toBeNull(); + }); +}); + +/* + * THE ONE TEST THAT CARRIES BYTES THROUGH BOTH DOORS AND COMPARES THEM. + * + * It used to claim to be the only test that used the upload route at all, and that has not been + * true for a while: `attachment-routes.test.ts` uploads through the route and fetches the result + * back in several places — the SVG-body case, the over-long filename case, the cap races. What none + * of those do is compare the BYTES. They assert statuses, headers and stored names, so a POST + * handler that wrote `bytes.slice(0, 1)` or `sizeBytes: 0` would keep every one of them green. + * + * That is what this one is for, and the claim is worth keeping narrow so it stays true. Most + * attachment tests still seed their rows with a direct insert, which leaves the route's own + * handling of the body asserted by nothing; here the bytes are compared against the buffer the + * REQUEST was built from — not against anything read back out of the table — and the headers are + * read off the response rather than derived from the row, so a regression anywhere between the + * multipart body and the served response is a failure here. + */ +describe("a file uploaded through the route and fetched back", () => { + /** Signs every request as one person, which is who the channel membership below is for. */ + function actorMiddleware( + actorId: string, + ): MiddlewareHandler<{ Variables: AppVariables }> { + return async (context, next) => { + context.set("actor", { + id: actorId, + email: `${actorId}@example.test`, + role: "user", + }); + await next(); + }; + } + + test("comes back byte for byte, under the headers it was stored with", async () => { + const { userId, channelId } = await seedChannel(database); + await database.insert(channelMemberships).values({ channelId, userId }); + + /* + * A real PNG signature so the route's sniffer names it `image/png` on the bytes rather than on + * the claim, then a body chosen to break anything that treats these bytes as text: a NUL, a + * 0xff, a lone 0x0d and the 0x0d 0x0a pair a transport that thinks it is handling lines would + * rewrite. + */ + const payload = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0xff, 0xfe, 0x0d, + 0x0d, 0x0a, 0x7f, 0x80, 0x00, 0x01, + ]); + + const actor = actorMiddleware(userId); + const uploads = new Hono<{ Variables: AppVariables }>(); + uploads.route("/", createChannelAttachmentRoutes(database, actor)); + + const body = new FormData(); + body.set( + "file", + new File([payload], "round-trip.png", { type: "image/png" }), + ); + body.set("uploadGroup", randomUUID()); + const uploaded = await uploads.request( + `http://test/${channelId}/attachments`, + { method: "POST", body }, + ); + + expect(uploaded.status).toBe(201); + const created = (await uploaded.json()) as { + id: string; + name: string; + mimeType: string; + sizeBytes: number; + }; + createdAttachmentIds.push(created.id); + expect(created.name).toBe("round-trip.png"); + expect(created.mimeType).toBe("image/png"); + // The length the route recorded, which is the field a truncating upload would have to lie + // about to keep the round trip below looking consistent. + expect(created.sizeBytes).toBe(payload.byteLength); + + const fetches = new Hono<{ Variables: AppVariables }>(); + fetches.route("/", createAttachmentRoutes(database, actor)); + const served = await fetches.request(`http://test/${created.id}`); + + expect(served.status).toBe(200); + const servedBytes = Buffer.from(await served.arrayBuffer()); + // Against the buffer the request was built from. Comparing lengths first only so a failure says + // "truncated" rather than printing two buffers. + expect(servedBytes.byteLength).toBe(payload.byteLength); + expect(Buffer.compare(servedBytes, payload)).toBe(0); + + // The type the server sniffed, never the claim, and the two headers that keep a served + // attachment from being run as a page on this app's origin. + expect(served.headers.get("content-type")).toBe("image/png"); + expect(served.headers.get("x-content-type-options")).toBe("nosniff"); + expect(served.headers.get("content-disposition")).toBe("inline"); + // `no-cache` rather than a max-age, and the ETag beside it, are what let a DELETED attachment + // stop being served: the browser must revalidate every time, and the revalidation is answered + // behind the same channel-and-membership join a 200 is. A max-age here would put the + // "unavailable" path out of reach for the length of the window. + expect(served.headers.get("cache-control")).toBe("private, no-cache"); + expect(served.headers.get("etag")).toBe(`"${created.id}"`); + }); +}); + +/** + * A channel this user is a member of, so an upload through the real app reaches the handler rather + * than its 403. `seedChannel` alone makes a channel nobody belongs to. + */ +async function seedMemberChannel() { + const { userId, channelId } = await seedChannel(database); + await database.insert(channelMemberships).values({ channelId, userId }); + return { userId, channelId }; +} + +/** + * The whole app, wired to this database and signed in as `userId`. + * + * `createApp` positionally, the same as attachment-routes.test.ts does: positions 4-25 are the + * stores this file has nothing to say about, and `attachmentDatabase` is position 26. It has to be + * the whole app rather than `createChannelAttachmentRoutes` on its own, because the body limit + * under test is mounted in app.ts and does not exist on the router by itself. + */ +function appSignedInAs(userId: string) { + return createApp( + loadConfig(testEnvironment()), + { + handler: () => new Response(null, { status: 204 }), + api: { + getSession: async () => ({ + user: { + id: userId, + email: `${userId}@example.test`, + name: "Attachment Store Test User", + image: "https://example.test/avatar.png", + }, + }), + }, + }, + { rolesForUser: async () => ["user"] }, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + database, + ); +} + +/** + * `size` bytes that a mime sniff will call a PNG. The signature is load-bearing: an `image/png` + * claim the bytes do not corroborate comes back 415 on the file, which would pass a status + * assertion about a size limit for entirely the wrong reason. + */ +function pngOfSize(size: number): Uint8Array { + const bytes = new Uint8Array(size); + bytes.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + return bytes; +} + +function uploadRequest( + app: ReturnType, + channelId: string, + file: File, +) { + const formData = new FormData(); + formData.set("file", file); + formData.set("uploadGroup", randomUUID()); + return app.request( + `http://openbot.test/api/channels/${channelId}/attachments`, + { method: "POST", body: formData }, + ); +} + +/** The rows this channel holds, so a refusal can be checked against its own channel and no other. */ +async function rowsIn(channelId: string) { + return await database + .select({ id: attachments.id }) + .from(attachments) + .where(eq(attachments.channelId, channelId)); +} + +describe("the body limit in front of the upload route", () => { + test("accepts an image of exactly MAX_IMAGE_BYTES, the documented ceiling", async () => { + const { userId, channelId } = await seedMemberChannel(); + const app = appSignedInAs(userId); + + const response = await uploadRequest( + app, + channelId, + new File([pngOfSize(MAX_IMAGE_BYTES)], "ceiling.png", { + type: "image/png", + }), + ); + + // RED before the fix: the limit was `MAX_IMAGE_BYTES` measured over the whole multipart + // envelope, so the ~360 bytes of boundary and headers around a file at the ceiling pushed the + // body past it and this came back 413. A file at the number every other gate publishes could + // not be uploaded at all. + expect(response.status).toBe(201); + const created = (await response.json()) as { + id: string; + sizeBytes: number; + }; + createdAttachmentIds.push(created.id); + expect(created.sizeBytes).toBe(MAX_IMAGE_BYTES); + }); + + test("refuses an image one byte past MAX_IMAGE_BYTES with a reason the composer can read", async () => { + const { userId, channelId } = await seedMemberChannel(); + const app = appSignedInAs(userId); + + const response = await uploadRequest( + app, + channelId, + new File([pngOfSize(MAX_IMAGE_BYTES + 1)], "over.png", { + type: "image/png", + }), + ); + + expect(response.status).toBe(413); + const text = await response.text(); + // The composer reads `{ error }` off every failure and falls back to a generic "Could not + // upload" when the body is not JSON, so the shape is the message. + const body = JSON.parse(text) as { error?: string }; + expect(typeof body.error).toBe("string"); + expect(body.error).toContain("8MB"); + expect(await rowsIn(channelId)).toEqual([]); + }); + + test("refuses a body past the door's own ceiling in that same JSON shape", async () => { + const { userId, channelId } = await seedMemberChannel(); + const app = appSignedInAs(userId); + + // Past `UPLOAD_BODY_LIMIT_BYTES`, so the refusal comes from the middleware rather than the + // handler: this is the path that exists to stop an unbounded body being read into memory, and + // it is the one that used to answer in plain text. + const response = await uploadRequest( + app, + channelId, + new File([pngOfSize(UPLOAD_BODY_LIMIT_BYTES + 1)], "enormous.png", { + type: "image/png", + }), + ); + + expect(response.status).toBe(413); + const text = await response.text(); + // RED before the fix: hono's default 413 body is the plain string "Payload Too Large", so this + // parse threw and the person saw the generic "Could not upload" with no reason in it. + const body = JSON.parse(text) as { error?: string }; + expect(typeof body.error).toBe("string"); + expect(body.error).toContain("8MB"); + expect(await rowsIn(channelId)).toEqual([]); + }); +}); + +describe("the attachments table's cascading foreign keys", () => { + test("index uploaded_by, so removing a person is not a sequential scan of the blob table", async () => { + const indexes = (await database.execute(sql` + select indexname, indexdef + from pg_indexes + where schemaname = current_schema() and tablename = 'attachments' + `)) as unknown as { indexname: string; indexdef: string }[]; + + /* + * The LEADING column, not merely a column that appears somewhere. `attachments_upload_group_idx` + * already names `uploaded_by` — as its second key, behind `channel_id`, and behind a partial + * predicate on top — and Postgres cannot drive a lookup on `uploaded_by` alone from it. Matching + * on the definition rather than on an index name keeps this a statement about the property the + * cascade needs, so renaming the index does not fail it and adding an unrelated one does not + * pass it. + */ + const leadsOnUploadedBy = indexes.filter((index) => + /USING btree \(uploaded_by[),]/.test(index.indexdef), + ); + + // RED before the fix: `attachments` had `attachments_channel_idx` for one of its two cascading + // foreign keys and nothing at all for the other, so `delete from users` scanned every row of + // the one table in this deployment that stores blobs. + expect(leadsOnUploadedBy.length).toBeGreaterThan(0); + /* + * AT LEAST ONE OF THEM UNFILTERED, rather than all of them. A cascade has to find every row that + * names the departing user, including the ones already stamped as sent, so a partial index on + * the staged minority cannot serve it and an unfiltered one has to exist. + * + * It may not be the only one, though, and this used to say that it was. The staging backstop + * (`MAX_STAGED_ATTACHMENTS_PER_UPLOADER` in channels/attachments.ts) counts + * `uploaded_by = ? and attached_at is null` on every upload, and the index that would suit it + * best is exactly the partial one the old assertion forbade. Written this way the test still + * pins the property the cascade needs, and stops being the reason a later index cannot be added. + */ + expect( + leadsOnUploadedBy.filter((index) => !index.indexdef.includes("WHERE")), + ).not.toHaveLength(0); + }); + + test("actually cascade a removed person's attachments away", async () => { + const { userId, channelId } = await seedChannel(database); + const [inserted] = await database + .insert(attachments) + .values({ + channelId, + uploadedBy: userId, + name: "theirs.txt", + mimeType: "text/plain", + sizeBytes: 5, + bytes: Buffer.from("hello"), + }) + .returning(); + + await database.delete(users).where(eq(users.id, userId)); + + const survivors = await database + .select({ id: attachments.id }) + .from(attachments) + .where(eq(attachments.id, inserted.id)); + expect(survivors).toEqual([]); + }); +}); diff --git a/server/tests/attachment-sweeper.test.ts b/server/tests/attachment-sweeper.test.ts new file mode 100644 index 000000000..f6526ff82 --- /dev/null +++ b/server/tests/attachment-sweeper.test.ts @@ -0,0 +1,443 @@ +import { afterAll, afterEach, describe, expect, test } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { eq, inArray, notInArray, sql } from "drizzle-orm"; +import { cullStagedAttachments } from "../scripts/cull-staged-attachments"; +import { createDatabase, type Database } from "../src/db/client"; +import { attachments, channels, users } from "../src/db/schema"; +import { TEST_POOL } from "./support/database"; + +const databaseUrl = + process.env.DATABASE_URL ?? + "postgres://openbot:openbot@localhost:5432/openbot"; +const database = createDatabase(databaseUrl, TEST_POOL); + +const testPrefix = `attachment-sweeper-${randomUUID()}`; +const createdAttachmentIds: string[] = []; +const createdChannelIds: string[] = []; +const createdUserIds: string[] = []; + +afterEach(async () => { + for (const id of createdAttachmentIds.splice(0)) { + await database.delete(attachments).where(eq(attachments.id, id)); + } + for (const id of createdChannelIds.splice(0)) { + await database.delete(channels).where(eq(channels.id, id)); + } + for (const id of createdUserIds.splice(0)) { + await database.delete(users).where(eq(users.id, id)); + } +}); + +afterAll(async () => { + await database.$client.close(); +}); + +/** A user and a channel to hang an attachment off of, nothing more. */ +async function seedChannel() { + const userId = `${testPrefix}-user-${randomUUID()}`; + await database.insert(users).values({ + id: userId, + email: `${userId}@example.test`, + }); + createdUserIds.push(userId); + + const channelId = `${testPrefix}-channel-${randomUUID()}`; + await database.insert(channels).values({ + id: channelId, + name: "Attachment Sweeper Test Channel", + description: "Sweep target channel.", + }); + createdChannelIds.push(channelId); + + return { userId, channelId }; +} + +/** + * How far in the past, as an interval Postgres will build from any number. + * + * NOT `make_interval(days => …)`, which is what this used to be. That takes an `int`, so the moment + * a test wants an age of less than a day — which the fractional-window test below does, because a + * half-hour window can only be demonstrated against rows aged in minutes — the helper fails with + * `function make_interval(days => double precision) does not exist` rather than backdating + * anything. Multiplying an interval takes whole numbers and fractions alike, and it is the same + * expression the culler itself now uses to build its cutoff. + */ +function ago(hours: number) { + return sql`now() - ${hours}::float8 * interval '1 hour'`; +} + +/** + * An attachment, backdated by editing `created_at` after the insert. + * + * `attachedAt` stays whatever the caller asks for, including null for a row that was staged and + * never sent. `createdAt` cannot be set through `insert` the way `attachedAt` can — the column + * defaults to `now()` at insert time — so it is pushed into the past with a direct `update` + * afterwards, the same way `page-frame-retention.integration.test.ts` backdates a capture time. + * + * An age is given in days or in hours, whichever reads better at the call site: "a year old" is + * `daysAgo: 365`, and "forty-five minutes ago" is `hoursAgo: 0.75`. + */ +async function stagedAttachment(options: { + channelId: string; + userId: string; + daysAgo?: number; + hoursAgo?: number; + attachedDaysAgo?: number; +}) { + const ageHours = options.hoursAgo ?? (options.daysAgo ?? 0) * 24; + const [inserted] = await database + .insert(attachments) + .values({ + channelId: options.channelId, + uploadedBy: options.userId, + name: "test.bin", + mimeType: "application/octet-stream", + sizeBytes: 3, + bytes: Buffer.from([0x01, 0x02, 0x03]), + attachedAt: + options.attachedDaysAgo === undefined + ? null + : ago(options.attachedDaysAgo * 24), + }) + .returning(); + createdAttachmentIds.push(inserted.id); + + await database + .update(attachments) + .set({ createdAt: ago(ageHours) }) + .where(eq(attachments.id, inserted.id)); + + return inserted.id; +} + +/** Thrown to roll a sweep back; never seen by a test. */ +const ROLLBACK = new Error("attachment sweeper test rollback"); + +type Sweep = { + /** How many rows the sweep deleted, of the rows THIS TEST created and no others. */ + deleted: number; + /** Which of this test's rows the sweep left behind. */ + survivors: Set; + /** + * How many `DELETE` statements the culler issued to do it. + * + * The one observable difference between a sweep that batches and a sweep that does not: both + * delete the same rows and return the same count, and only this says whether one statement held + * a lock on every doomed row at once. The harness's own narrowing delete is not counted — it is + * issued against the transaction directly, and only the culler is handed the counting wrapper. + */ + statements: number; +}; + +/** + * The transaction, with every `delete()` it is asked for counted. + * + * Every member is bound to the real transaction rather than left to be called on the proxy, because + * drizzle's session objects carry state that must be read with `this` pointing at the real object; + * handing them a proxy as `this` is how a wrapper like this one turns into an unrelated failure + * somewhere inside the driver. + */ +function countingDeletes( + transaction: object, + count: { statements: number }, +): Database { + return new Proxy(transaction, { + get(target, property, receiver) { + const value = Reflect.get(target, property, receiver); + if (typeof value !== "function") { + return value; + } + const bound = value.bind(target); + if (property !== "delete") { + return bound; + } + return (...args: unknown[]) => { + count.statements += 1; + return bound(...args); + }; + }, + }) as unknown as Database; +} + +/** + * One sweep, run inside a transaction that is always rolled back, over a table first narrowed to + * the rows this test created. + * + * `cullStagedAttachments` deletes across the whole `attachments` table and returns how many rows it + * removed, so a bare `expect(deleted).toBe(1)` is an assertion about every other row in whatever + * database the suite is pointed at: it holds only while nobody else has a staged attachment past + * the window, and it reads `Expected: 1 Received: 2` the moment somebody does — by which point the + * sweep that produced the 2 has PERMANENTLY DELETED that person's row. Both halves are fixed here, + * not one. + * + * The narrowing delete removes every attachment row this test did not create, so afterwards the only + * rows the culler can possibly find are this test's and the number it returns is exact rather than + * shared: no test ordering, no leftover fixture and no unrelated developer row can move it. EVERY + * row rather than only the staged ones, deliberately — narrowing to what the culler is SUPPOSED to + * match would leave the count global again for exactly the mutations this test exists to catch, and + * `expect(deleted).toBe(1)` would fail with a number that depends on the database rather than on the + * bug. + * + * The transaction is then rolled back, so neither the narrowing delete nor the sweep itself outlives + * the assertion — this test destroys nothing, including the rows it borrowed to narrow. Which is why + * `survivors` is read INSIDE the transaction: after the rollback every row is back, and a read taken + * outside it could only ever say "still there". + */ +async function sweep(options: { + olderThanHours: number; + batchSize?: number; +}): Promise { + if (createdAttachmentIds.length === 0) { + throw new Error("sweep() is meaningless before the test has created a row"); + } + + let outcome: Sweep | undefined; + try { + await database.transaction(async (transaction) => { + await transaction + .delete(attachments) + .where(notInArray(attachments.id, createdAttachmentIds)); + + const count = { statements: 0 }; + const deleted = await cullStagedAttachments( + // A transaction is a `Database` for everything the culler does with one — it selects and + // deletes — but drizzle types the two separately. + countingDeletes(transaction, count), + options, + ); + + const rows = await transaction + .select({ id: attachments.id }) + .from(attachments) + .where(inArray(attachments.id, createdAttachmentIds)); + + outcome = { + deleted, + survivors: new Set(rows.map((row) => row.id)), + statements: count.statements, + }; + throw ROLLBACK; + }); + } catch (error) { + if (error !== ROLLBACK) throw error; + } + + if (!outcome) throw new Error("the sweep transaction never ran its body"); + return outcome; +} + +describe("sweeping staged attachments", () => { + test("a staged attachment older than the window is swept", async () => { + const { userId, channelId } = await seedChannel(); + const id = await stagedAttachment({ channelId, userId, daysAgo: 2 }); + + const { deleted, survivors } = await sweep({ olderThanHours: 6 }); + + expect(deleted).toBe(1); + expect(survivors.has(id)).toBe(false); + }); + + // The one that catches a truthiness bug: `attachedAt` is a real, year-old timestamp here, and a + // check written as `!attachedAt` rather than `IS NULL` would never see it, because a truthy + // check would have to be wrong the other way — treating this row as staged — for it to be swept. + // That is exactly the mistake this predicate must not make. + test("a sent attachment a year old is never swept", async () => { + const { userId, channelId } = await seedChannel(); + const id = await stagedAttachment({ + channelId, + userId, + daysAgo: 365, + attachedDaysAgo: 365, + }); + + const { deleted, survivors } = await sweep({ olderThanHours: 6 }); + + expect(deleted).toBe(0); + expect(survivors.has(id)).toBe(true); + }); + + test("a staged attachment inside the window is left alone", async () => { + const { userId, channelId } = await seedChannel(); + const id = await stagedAttachment({ channelId, userId, daysAgo: 0 }); + + const { deleted, survivors } = await sweep({ olderThanHours: 6 }); + + expect(deleted).toBe(0); + expect(survivors.has(id)).toBe(true); + }); + + // Half an hour is a window an operator can ask for: `attachments.culler.olderThanHours` in the + // chart is handed to this straight, and 0.5 there used to make every hourly sweep die with + // `function make_interval(hours => double precision) does not exist` — the whole sweep wedged, and + // the error naming a Postgres function rather than the value anybody set. It is a window, not an + // integer, so it is checked as one: forty-five minutes is past a half-hour window and fifteen + // minutes is not. + test("a window of half an hour means thirty minutes", async () => { + const { userId, channelId } = await seedChannel(); + const past = await stagedAttachment({ channelId, userId, hoursAgo: 0.75 }); + const inside = await stagedAttachment({ + channelId, + userId, + hoursAgo: 0.25, + }); + + const { deleted, survivors } = await sweep({ olderThanHours: 0.5 }); + + expect(deleted).toBe(1); + expect(survivors.has(past)).toBe(false); + expect(survivors.has(inside)).toBe(true); + }); + + // The sweep must not be one statement over the whole backlog: that holds a row lock on every + // doomed row for the length of the transaction, materialises one returned id per row, and is + // rolled back in its entirety when the CronJob's `activeDeadlineSeconds` kills it — so a + // deployment too far behind to finish inside the ceiling redoes the same doomed work every hour + // and never deletes anything. Three rows and a batch of one: four statements, because the loop + // stops at the first batch that comes back short, and the third full batch cannot be known to be + // the last one until a fourth finds nothing. + test("the delete is issued in batches, not as one statement", async () => { + const { userId, channelId } = await seedChannel(); + const ids = [ + await stagedAttachment({ channelId, userId, daysAgo: 2 }), + await stagedAttachment({ channelId, userId, daysAgo: 3 }), + await stagedAttachment({ channelId, userId, daysAgo: 4 }), + ]; + + const { deleted, survivors, statements } = await sweep({ + olderThanHours: 6, + batchSize: 1, + }); + + expect(deleted).toBe(3); + expect(statements).toBe(4); + for (const id of ids) { + expect(survivors.has(id)).toBe(false); + } + }); +}); + +/** + * A `Database` that fails the instant anything is asked of it. + * + * The batch-size checks below are about what the sweep does BEFORE its first statement, and they + * cannot be written against a real database: the values they pass are the ones that make the loop + * never end, so a test that let one reach Postgres would not fail, it would HANG — holding a + * transaction open and issuing deletes in a tight loop against a database this suite shares with + * every other agent on this machine. A bounded red is the whole point, so nothing here connects. + * + * The message names the property that was touched, so a regression reads as "the sweep reached the + * database (.select)" rather than as an undefined-is-not-a-function from somewhere inside drizzle. + */ +function untouchableDatabase(): Database { + return new Proxy( + {}, + { + get(_target, property) { + throw new Error( + `the sweep reached the database (.${String(property)}) with a batch size it should have refused`, + ); + }, + }, + ) as unknown as Database; +} + +/** + * The batch size, refused when it is one the loop could never come out of. + * + * `batchSize` is optional and defaulted with `??`, which catches an absent value and a null one and + * NOTHING ELSE — zero is a number, so it is taken. Only callers of the exported function can supply + * one; the CLI never does, which is why the CronJob has never hit this and why no test did either. + * + * Each of these was measured against this deployment's Postgres 16 through the same query builder + * the sweep uses, rather than reasoned about: + * + * - `0` builds `limit $1` with `0`, which returns no rows, so the delete removes none and the + * termination test is `0 < 0` — false. The loop issues that pair of statements forever. + * - `0.5` is worse in the way that matters, because it LOOKS like it would at least delete + * something: `LIMIT` takes a bigint, a float8 is rounded to reach one, and 0.5 rounds to zero. + * Same two statements, same endless loop, from a value nobody would read as "none". + * - A negative size does not merely fail to limit: drizzle emits NO `limit` clause at all for one + * (Postgres would itself refuse `LIMIT -1`), so the sweep becomes the single unbounded + * `DELETE ... RETURNING id` over the whole backlog that batching exists to prevent — every doomed + * row locked for the length of one transaction, every id materialised — and THEN loops forever + * too, because no `batch.length` is ever `< -1`. + * - `NaN` takes the same no-`limit` path, for the same reason: drizzle's guard is `>= 0`, and every + * comparison against `NaN` is false. + * - A fraction at or above one terminates, so it is the mild case, and it is still refused. `1.5` + * rounds up to a `LIMIT 2`, so the sweep deletes two rows per statement while testing + * `2 < 1.5` — a full batch that reads as a short one, ending the sweep a batch early every time. + * A ceiling on row locks and WAL per statement is not a number to accept an approximation of. + * + * Asserted as a rejection with a named value rather than as "it did not hang", because a timeout is + * the one thing this must never be: a test that proves the bug by waiting is a test that wedges CI. + */ +describe("a batch size the sweep cannot finish on", () => { + const refused: [string, number][] = [ + ["zero", 0], + ["a half, which Postgres rounds to zero", 0.5], + ["negative, which drops the limit clause entirely", -1], + ["a fraction, which is a limit nobody asked for", 1.5], + ["NaN", Number.NaN], + ["infinite", Number.POSITIVE_INFINITY], + ]; + + for (const [description, batchSize] of refused) { + test(`${description} is refused before a statement is issued`, async () => { + await expect( + cullStagedAttachments(untouchableDatabase(), { + olderThanHours: 6, + batchSize, + }), + ).rejects.toThrow(/batch size.*whole number of at least 1/s); + }); + } + + // The boundary on the other side of the refusal: one row per statement is the smallest sweep that + // can still make progress, and the batching test above depends on it being allowed. + test("a batch size of one is not refused", async () => { + await expect( + cullStagedAttachments(untouchableDatabase(), { + olderThanHours: 6, + batchSize: 1, + }), + ).rejects.toThrow(/the sweep reached the database/); + }); +}); + +/** + * The sweep's documented contract: `DATABASE_URL` and nothing else. + * + * Run as a real process rather than by importing the module, because the thing under test is what + * the script does BEFORE it deletes anything — an in-process call to `cullStagedAttachments` cannot + * fail the way this failed, which is at start-up, in `loadConfig`, with `KEY_ENCRYPTION_KEY must be + * configured`. `docs/deployment.md` tells an operator to run exactly this from an external cron and + * that it needs only the database; the Helm CronJob is not that path and injected five credentials + * this sweep has no use for. So the environment handed to the child is the documented one, built + * from nothing rather than inherited, and the assertion is that the process succeeds. + * + * A WINDOW OF 1,000,000 HOURS, AND THAT NUMBER IS LOAD-BEARING. This is the one test here that runs + * the culler for real rather than inside a rolled-back transaction, against a database other agents + * and other suites are using at the same time. A cutoff in the year 1912 is one no row in any of + * their fixtures can be older than, so the sweep proves it booted and connected while deleting + * nothing whatsoever. Never lower it. + */ +test("the sweep boots with DATABASE_URL and no other configuration", () => { + const result = Bun.spawnSync({ + cmd: [ + process.execPath, + "scripts/cull-staged-attachments.ts", + String(1_000_000), + ], + cwd: join(import.meta.dir, ".."), + env: { PATH: process.env.PATH ?? "", DATABASE_URL: databaseUrl }, + stdout: "pipe", + stderr: "pipe", + }); + + const stderr = result.stderr.toString(); + const stdout = result.stdout.toString(); + expect(stderr).not.toContain("KEY_ENCRYPTION_KEY"); + expect({ code: result.exitCode, stderr }).toEqual({ code: 0, stderr: "" }); + expect(JSON.parse(stdout)).toEqual({ type: "attachment-cull", deleted: 0 }); +}); diff --git a/server/tests/bunfig-preload.test.ts b/server/tests/bunfig-preload.test.ts new file mode 100644 index 000000000..9b81765b9 --- /dev/null +++ b/server/tests/bunfig-preload.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "bun:test"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +/** + * The two bunfig.toml files, held against each other. + * + * Bun reads bunfig.toml from the current working directory and nowhere else, so the root one does + * not cover `bun test` run from inside `server`, and there has to be a second one here. A second + * file is a second thing to forget: a preload added to one and not the other reintroduces exactly + * the failure the preload exists to prevent, and reintroduces it silently, because a test file that + * throws while being imported reports nothing at all. + * + * So the drift is asserted away rather than documented away. Both lists are resolved against their + * own file's directory and compared as absolute paths, which is the comparison that matters: the two + * are meant to name the same scripts, not to contain the same strings. + */ + +const repositoryRoot = resolve(import.meta.dir, "..", ".."); + +function preloadedScripts(bunfigPath: string): string[] { + const parsed = Bun.TOML.parse(readFileSync(bunfigPath, "utf8")) as { + test?: { preload?: string[] }; + }; + const declared = parsed.test?.preload ?? []; + return declared.map((entry) => resolve(dirname(bunfigPath), entry)).sort(); +} + +describe("bunfig preload", () => { + const rootBunfig = resolve(repositoryRoot, "bunfig.toml"); + const serverBunfig = resolve(repositoryRoot, "server", "bunfig.toml"); + + test("the root and server configs preload the same scripts", () => { + expect(preloadedScripts(serverBunfig)).toEqual( + preloadedScripts(rootBunfig), + ); + }); + + test("every preloaded script exists", () => { + const scripts = preloadedScripts(rootBunfig); + + expect(scripts.length).toBeGreaterThan(0); + for (const script of scripts) { + expect(existsSync(script)).toBe(true); + } + }); +}); diff --git a/server/tests/copilot.test.ts b/server/tests/copilot.test.ts index 99085d1b3..8cf7f4fb0 100644 --- a/server/tests/copilot.test.ts +++ b/server/tests/copilot.test.ts @@ -1,13 +1,15 @@ import { describe, expect, spyOn, test } from "bun:test"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import type { RunAgentInput } from "@ag-ui/client"; +import type { AbstractAgent, RunAgentInput } from "@ag-ui/client"; import { HttpAgent } from "@ag-ui/client"; import { LLMock } from "@copilotkit/aimock"; import { BuiltInAgent } from "@copilotkit/runtime/v2"; import { EMPTY } from "rxjs"; import { PROVENANCE_GUIDANCE } from "../../shared/bot-prompt"; +import { MAX_INLINED_BYTES_PER_RUN } from "../src/channels/attachment-parts"; import { loadConfig } from "../src/config"; +import type { LoadAttachment } from "../src/copilot"; import { buildAgents, builtInAgentConfiguration, @@ -22,6 +24,30 @@ import { grantedToolGuidance } from "../src/plugins/tools"; import { loadTenantPackage } from "../src/tenant-package"; import { testEnvironment } from "./support/environment"; +/** + * The Bot under test, or a failure that says it was never built. + * + * `agents["general-assistant"]?.run(...)` on an undefined agent never subscribes, so the promise + * around that subscribe never settles and the test hangs to the suite's timeout with nothing in + * the output naming the cause. A `buildAgents` that stopped returning this Bot is a failure to + * report, not a five second wait. The optional-chaining form fails just as quietly without a + * subscribe in play: `agent?.setMessages(...)` on an undefined agent is a no-op, and the assertions + * after it then describe a Bot that was never run. + * + * AT MODULE SCOPE, not inside the describe that first needed it. Two describes reach for this — the + * attachment one and the refused-conversation one — and while it lived in the first, the second + * kept the `agent?.` shape it was written with, which is the whole failure above. A guard that has + * to be copied to be used is a guard the next block will not have. + */ +function built( + agents: Record, + id: string, +): AbstractAgent { + const agent = agents[id]; + if (!agent) throw new Error(`buildAgents returned no "${id}" to run`); + return agent; +} + // Every agent row now joins its profile, so the row a coworker is built from always names it. const assistantRow = { id: "general-assistant", @@ -582,15 +608,38 @@ describe("standing agent roles", () => { const agent = agents.agent_expense; agent?.setMessages([userMessage("Sort these.")]); - await agent?.runAgent(); + const result = await agent?.runAgent(); - const sent = endpoint.requests.at(-1); - expect(JSON.stringify(sent?.forwardedProps ?? {})).not.toContain( + /* + * That a request was sent AT ALL is the first assertion, and it is the one that makes the rest + * mean anything. `requests.at(-1)` on an empty log is `undefined`, and `JSON.stringify(undefined + * ?? {})` is `"{}"`, which contains no "standing-role" and never will: every line below passed + * with the agent key misspelled and no run performed. + */ + expect(endpoint.requests).toHaveLength(1); + expect(result?.newMessages?.at(-1)?.content).toBe("Categorized."); + + const [sent] = endpoint.requests; + // And the standing role really did travel, so "not in forwardedProps, not in state" is an + // assertion about WHERE it went rather than about whether it exists. + expect(JSON.stringify(sent.messages)).toContain( + "standing-role:agent_expense", + ); + expect(JSON.stringify(sent.forwardedProps ?? {})).not.toContain( "standing-role", ); - expect(JSON.stringify(sent?.state ?? {})).not.toContain("standing-role"); + expect(JSON.stringify(sent.state ?? {})).not.toContain("standing-role"); }); + /* + * Main's clone-preserving form of this test, kept, with the built-in probe this branch added. + * + * `fetch` alone does not cover the failure that branch was written against: if the + * `type === "unavailable"` branch in `buildRegisteredAgent` stops catching this row, the tombstone + * falls through to the BUILT-IN path, which answers from a model rather than by dialling an + * endpoint. `BuiltInAgent.prototype.run` is the only place that shows up, so it is spied on + * alongside the network. + */ test("preserves the deleted coworker refusal through runtime clones without network calls", async () => { const reason = "Expense Manager has been deleted and can no longer run. Its conversations remain readable."; @@ -598,6 +647,9 @@ describe("standing agent roles", () => { const network = spyOn(globalThis, "fetch").mockImplementation(() => { throw new Error("An unavailable agent must not make network calls"); }); + const builtInRun = spyOn(BuiltInAgent.prototype, "run").mockImplementation( + () => EMPTY, + ); const consoleError = spyOn(console, "error").mockImplementation(() => {}); try { const agents = await resolveRuntimeAgents( @@ -643,8 +695,10 @@ describe("standing agent roles", () => { } expect(modelKeyRequests).toBe(0); expect(network).not.toHaveBeenCalled(); + expect(builtInRun).not.toHaveBeenCalled(); } finally { consoleError.mockRestore(); + builtInRun.mockRestore(); network.mockRestore(); } }); @@ -673,12 +727,24 @@ describe("standing agent roles", () => { }); test("rebuilds each agent from the loader so an edited role applies to the next run", async () => { + /* + * READ OFF THE SECOND AGENT'S OWN RUN, not recomputed from the test's local. + * + * This closed with `expect(standingRoleMessage({ ...profile, roleDescription }).content)`, + * which calls the same pure function the assertion is about with the same argument and asserts + * it agrees with itself. It holds whatever `createRequestAgents` did with the roster, so the + * one claim in the test's name — that the SECOND build carries the edited role — was never + * made. A memoised roster behind a per-request rebuild passes it: two distinct agent objects, + * both still saying "Review receipts." + * + * So the edited role is asserted where a person would meet it, on the wire out of the rebuilt + * agent, which is also the only place a remote Bot ever hears its role at all. + */ + await using endpoint = fakeAgUiEndpoint(); let roleDescription = "Review receipts."; const factory = createRequestAgents( async () => ({ id: "user-7", role: "user" as const }), - async () => [ - remoteAgent("http://coworker.internal/ag-ui", { roleDescription }), - ], + async () => [remoteAgent(endpoint.url, { roleDescription })], { provider: "openai", defaultModel: "gpt-5.6-terra" }, async () => null, ); @@ -689,7 +755,17 @@ describe("standing agent roles", () => { const after = await factory({ request }); expect(before.agent_expense).not.toBe(after.agent_expense); - expect(standingRoleMessage({ ...profile, roleDescription }).content).toBe( + + const rebuilt = after.agent_expense; + if (!rebuilt) + throw new Error("createRequestAgents returned no agent_expense"); + rebuilt.setMessages([userMessage("Sort these.")]); + await rebuilt.runAgent(); + + const sent = endpoint.requests.at(-1) as + | { messages?: { content?: string }[] } + | undefined; + expect(sent?.messages?.[0]?.content).toBe( [ "You are Expense Manager, Finance Operations.", "Reconcile corporate card statements.", @@ -1414,9 +1490,19 @@ describe("a chat turn is not sent a conversation the model API refuses", () => { { id: "m3", role: "user", content: "Did that work?" }, ]; + /** + * The Bot under test, or a failure that says it was never built. + * + * Returned non-optional on purpose: `agent?.run(...)` on an undefined agent runs nothing, and what + * the callers below then assert against is an empty `seen`, which reads as "the guard dropped + * everything" rather than as "there was no agent". The subscribing caller has it worse and hangs + * to the suite's timeout. + */ async function builtIn() { const agents = await buildAgents([assistant], model, "openai-secret"); - return agents["general-assistant"]; + const agent = agents["general-assistant"]; + if (!agent) throw new Error('buildAgents returned no "general-assistant"'); + return agent; } test("the unanswerable call is gone from what the run converts", async () => { @@ -1424,7 +1510,7 @@ describe("a chat turn is not sent a conversation the model API refuses", () => { const { seen, restore } = captureRuns(); try { - agent?.run(input(danglingCall)); + agent.run(input(danglingCall)); } finally { restore(); } @@ -1441,11 +1527,11 @@ describe("a chat turn is not sent a conversation the model API refuses", () => { test("the clone the runtime runs guards it too", async () => { // `agents[agentId].clone()` happens before every single run, and the base class's clone builds a // plain `BuiltInAgent`. Inherited unchanged, the guard would never once be reached in production. - const agent = (await builtIn())?.clone(); + const agent = (await builtIn()).clone(); const { seen, restore } = captureRuns(); try { - agent?.run(input(danglingCall)); + agent.run(input(danglingCall)); } finally { restore(); } @@ -1465,7 +1551,7 @@ describe("a chat turn is not sent a conversation the model API refuses", () => { const { seen, restore } = captureRuns(); try { - agent?.run( + agent.run( input(danglingCall, [ { interruptId: "chatcmpl-tool-8dd56dc7497c5ea9", status: "resolved" }, ]), @@ -1501,10 +1587,17 @@ describe("a chat turn is not sent a conversation the model API refuses", () => { null, ); - const agent = agents.risk; - agent?.setMessages(danglingCall as never[]); - await agent?.runAgent(); + // `built`, not `agents.risk` with an optional chain. On an undefined agent the chained form + // makes `setMessages` and `runAgent` no-ops, and the test then fails — if it fails at all — on + // `expect(endpoint.requests).toHaveLength(1)`, which says nothing was sent rather than that + // there was nothing to send it with. + const agent = built(agents, "risk"); + agent.setMessages(danglingCall as never[]); + await agent.runAgent(); + // Not vacuous without this — `sent` would be undefined and `sent.map` would throw — but it + // throws saying "undefined is not an object" rather than "nothing was ever sent". + expect(endpoint.requests).toHaveLength(1); const sent = endpoint.requests.at(-1)?.messages as { id: string; toolCalls?: unknown[]; @@ -1548,20 +1641,805 @@ describe("a chat turn is not sent a conversation the model API refuses", () => { floor: 0, }, ); + const agent = agents["general-assistant"]; + if (!agent) throw new Error('buildAgents returned no "general-assistant"'); const { seen, restore } = captureRuns(); + // Kept rather than discarded: `error: () => resolve()` here turned a run that failed outright + // into a passing test, and the same handler is what swallows anything thrown inside the + // narrowing callbacks this build is wired with. + const failed: Error[] = []; try { // Subscribed, because the narrowing wrapper builds the inner agent lazily on subscription. await new Promise((resolve) => { - agents["general-assistant"] - ?.run(input(danglingCall)) - .subscribe({ complete: resolve, error: () => resolve() }); + agent.run(input(danglingCall)).subscribe({ + complete: resolve, + error: (error: Error) => { + failed.push(error); + resolve(); + }, + }); }); } finally { restore(); } + expect(failed).toEqual([]); expect(seen[0]?.messages).toHaveLength(3); expect(seen[0]?.messages?.[1]).not.toHaveProperty("toolCalls"); }); }); + +/** + * The two places `resolveAttachmentParts` is called, and the one place it deliberately is not. + * + * `server/tests/attachment-parts.test.ts` covers the pure resolver, and covers it well, but nothing + * anywhere pins that the resolver is actually reached from a run. Delete either call below and that + * whole suite stays green, because it never builds an agent. A loader that returns null makes + * `resolveAttachmentParts` throw, naming the id; nothing here catches, so the throw is the proof the + * call happened at all. + * + * `RunBuiltAgent.run` (built when narrowing or handoff is active) is asserted by exclusion: the loader + * must be called exactly once for a one-attachment message, because `RunBuiltAgent.run` delegates to + * an inner `BuiltInAgentWithSaneHistory` whose own `run` is the one that inlines the attachment. A + * second call anywhere in that path would make it two. That loader has to RESOLVE for the count to + * mean anything; see the test itself. + * + * And the last two tests are the other half of the throw: it belongs to the message being asked + * about, which is the last user message, and NOT to the history behind it, where a vanished row + * would otherwise kill the channel permanently. + */ +describe("where an attachment reaches the model, and where it deliberately does not", () => { + const assistant = { + id: "general-assistant", + name: "General Assistant", + type: "built_in" as const, + systemPrompt: "Be helpful.", + }; + const model = { provider: "openai" as const, defaultModel: "gpt-5.6-terra" }; + + function input(messages: unknown[]): RunAgentInput { + return { + threadId: "thread_1", + runId: "run_1", + messages: messages as RunAgentInput["messages"], + tools: [], + context: [], + forwardedProps: {}, + state: {}, + }; + } + + /** + * Runs to its first error, with the model held off. + * + * THE SPY IS NOT A CONVENIENCE. Every caller here is asserting that a run REFUSES over an + * attachment it could not load, and the way that assertion regresses is the refusal disappearing — + * at which point the run carries on into `BuiltInAgent.run` and a real model call against + * whatever key the environment happens to hold. `EMPTY` completes at once instead, and completion + * is what this returns as the failure, so the regression is a fast red test rather than a live + * request. + */ + async function runToError( + agent: AbstractAgent, + runInput: RunAgentInput, + ): Promise { + const spy = spyOn(BuiltInAgent.prototype, "run").mockImplementation( + () => EMPTY, + ); + try { + return await new Promise((resolve) => { + agent.run(runInput).subscribe({ + error: resolve, + complete: () => resolve(new Error("expected the run to error")), + }); + }); + } finally { + spy.mockRestore(); + } + } + + /** + * Runs to completion with the model held off, handing back whatever the run failed with. + * + * RETURNED RATHER THAN SWALLOWED. `error: () => resolve()` is what these subscribes used to say, + * which quietly turns a failed run — and any assertion thrown inside a loader the run calls — into + * a passing test. A caller that expects the run to succeed asserts on an empty array and finds out + * either way. + * + * `onRun` is handed the input `BuiltInAgent.run` was called with, which is where a caller checks + * what the model would have been sent. + */ + async function runToCompletion( + agent: AbstractAgent, + runInput: RunAgentInput, + onRun: (received: RunAgentInput) => void = () => {}, + ): Promise { + const spy = spyOn(BuiltInAgent.prototype, "run").mockImplementation( + (received: RunAgentInput) => { + onRun(received); + return EMPTY; + }, + ); + const failed: Error[] = []; + try { + await new Promise((resolve) => { + agent.run(runInput).subscribe({ + complete: resolve, + error: (error: Error) => { + failed.push(error); + resolve(); + }, + }); + }); + } finally { + spy.mockRestore(); + } + return failed; + } + + /** One user message, one attachment, pointing at a file this deployment cannot load. */ + const attachmentMessage = [ + { + id: "m1", + role: "user" as const, + content: [ + { + type: "image", + source: { type: "url", value: "/api/attachments/abc" }, + metadata: { attachmentId: "abc" }, + }, + ], + }, + ]; + + test("a built-in Bot's run fails naming the attachment it could not load", async () => { + // Protects `BuiltInAgentWithSaneHistory.run` (copilot.ts:~924). Drop the + // `inlineAttachments` call there and this run completes instead of erroring. + const agents = await buildAgents( + [assistant], + model, + "openai-secret", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + async () => null, + ); + const error = await runToError( + built(agents, "general-assistant"), + input(attachmentMessage), + ); + + expect(error.message).toContain('"abc"'); + }); + + test("a remote Bot's run fails naming the attachment it could not load", async () => { + // Protects the remote `.use()` middleware's `runWith` (copilot.ts:~777). + // Drop the `inlineAttachments` call there and this rejection never fires. + const agents = await buildAgents( + [ + { + id: "risk", + name: "Risk", + type: "remote_ag_ui" as const, + endpoint: "http://risk.internal/ag-ui", + standingMessage: standingRoleMessage(riskRow), + }, + ], + model, + null, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + async () => null, + ); + // Through `built`, not `agents.risk?.`, for the reason `built` was written: on an optional + // chain a `buildAgents` that stopped returning this Bot makes `setMessages` a silent no-op and + // `expect(undefined).rejects` a type complaint, neither of which names the actual failure. + const agent = built(agents, "risk"); + agent.setMessages(attachmentMessage as never[]); + + const consoleError = spyOn(console, "error").mockImplementation(() => {}); + try { + await expect(agent.runAgent()).rejects.toThrow( + 'Attachment "abc" could not be loaded', + ); + } finally { + consoleError.mockRestore(); + } + }); + + test("the narrowed built-in path reads the attachment once, not twice", async () => { + /* + * Protects the exclusion: `RunBuiltAgent.run` (copilot.ts:~989) deliberately does not call + * `resolveAttachmentParts` itself. It only delegates to the built-in agent its `build()` + * produces, and that agent's own `run` is what inlines the attachment. Re-adding the call at + * `RunBuiltAgent.run` would read the same attachment a second time, which is what turns this + * count from one into two. + */ + /* + * A loader that RESOLVES, which is what makes the count mean anything. With one that returned + * null the first resolution threw, the second never ran, and the count was one whether or not + * `RunBuiltAgent.run` inlined as well — this test passed with the very double call it exists to + * forbid. Verified by adding that call back: with a real row here it fails at 2. + * + * WHAT IT WAS ASKED FOR IS RECORDED, NOT ASSERTED HERE. This loader runs inside the subscribe + * below, whose `error` handler resolves the promise rather than rethrowing, and bun does not + * fail a test on an `expect` whose throw was caught by something: `expect(id).toBe("WRONG-ID")` + * in this position was green. The recorded ids are asserted after the run, where a failure is + * the test's own. + */ + const loaded: string[] = []; + const loadAttachment = async (id: string) => { + loaded.push(id); + return { + mimeType: "image/png", + name: "abc.png", + bytes: Buffer.from([0x89, 0x50, 0x4e, 0x47]), + }; + }; + const granted = Array.from({ length: 3 }, (_, index) => ({ + ref: `drive/tool_${index}`, + name: `mcp__drive__tool_${index}`, + description: `drive tool ${index}`, + })) as never[]; + + // Narrowing active, same fixtures as "the narrowed path is guarded" above, so this Bot is + // built as a `RunBuiltAgent` rather than a plain `BuiltInAgentWithSaneHistory`. + const agents = await buildAgents( + [assistant], + model, + "openai-secret", + undefined, + async () => granted, + undefined, + undefined, + undefined, + { + loadSkills: async () => [ + { + slug: "drive-audit", + title: "Drive audit", + summary: "Read documents out of Google Drive.", + tools: ["drive/tool_0"], + }, + ], + choose: async () => JSON.stringify({ skills: ["drive-audit"] }), + floor: 0, + }, + undefined, + undefined, + undefined, + undefined, + loadAttachment, + ); + const failed = await runToCompletion( + built(agents, "general-assistant"), + input(attachmentMessage), + ); + + expect(failed).toEqual([]); + // One read, of the attachment this message actually names. Two is the double call this test + // forbids; a different id is a read of something nobody asked for. + expect(loaded).toEqual(["abc"]); + }); + + /** A user message carrying one attachment, named so a note about it can be recognised. */ + function attached(id: string, filename: string, messageId: string) { + return { + id: messageId, + role: "user" as const, + content: [ + { + type: "image", + source: { type: "url", value: `/api/attachments/${id}` }, + metadata: { attachmentId: id, filename }, + }, + ], + }; + } + + /** Somebody attached a file a while ago, said something else since, and is asking again now. */ + const twoTurns = [ + attached("old", "budget.png", "m1"), + { id: "m2", role: "assistant" as const, content: "Looks fine." }, + attached("abc", "photo.png", "m3"), + ]; + + const stored = { + mimeType: "image/png", + name: "photo.png", + bytes: Buffer.from([0x89, 0x50, 0x4e, 0x47]), + }; + + async function builtInWith(loadAttachment: LoadAttachment) { + const agents = await buildAgents( + [assistant], + model, + "openai-secret", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + loadAttachment, + ); + return built(agents, "general-assistant"); + } + + test("an attachment that vanished from an older message becomes a note", async () => { + /* + * The permanence half of the same argument. `inlineAttachments` maps over the WHOLE history and + * history is replayed every turn, so a row deleted by the sweeper after an interrupted send + * would otherwise fail this channel's every future turn for ever, exactly as the dangling call + * in `agents/history-sanitize.ts` did in production. The old part says the file is gone; the + * one the person is actually asking about still arrives as bytes. + */ + const agent = await builtInWith(async (id) => + id === "abc" ? stored : null, + ); + + const seen: RunAgentInput[] = []; + const failed = await runToCompletion(agent, input(twoTurns), (received) => { + seen.push(received); + }); + + // The thread still runs. That is the whole point: one dead file, not a dead channel. + expect(failed).toEqual([]); + const messages = seen[0]?.messages ?? []; + expect(messages.map((message) => message.id)).toEqual(["m1", "m2", "m3"]); + expect((messages[0] as { content?: unknown }).content).toEqual([ + { + type: "text", + text: '[attachment "budget.png" is no longer available]', + }, + ]); + expect( + (messages[2] as { content?: { source?: unknown }[] }).content?.[0] + ?.source, + ).toMatchObject({ type: "data" }); + }); + + test("the message being asked about still fails, history behind it or not", async () => { + // The strictness that matters is unchanged: the file THIS turn names is unloadable, and no Bot + // is going to answer about it. Only the messages behind it are allowed to degrade. + const agent = await builtInWith(async (id) => + id === "old" ? { ...stored, name: "budget.png" } : null, + ); + + const error = await runToError(agent, input(twoTurns)); + + expect(error.message).toContain('"abc"'); + }); + + /* + * Which message is a SEND, and therefore which attachments `attachedAt` may be written for. + * + * Only the last user message is: everything behind it is history, replayed in full on every turn + * and by whoever happens to be running that turn. A stamp written where the file is READ cannot + * tell those apart, so it says "sent" about every file anybody has ever been shown — which is the + * one thing `attachedAt` must never mean, because the sweeper, the upload cap and the withdrawal + * route all read it as "this rode in a message somebody sent". + * + * Delete the mark from `inlineAttachments` and `marked` stays empty; move it out of the + * `index === asked` branch and `old` joins it. Both are the failure this pins. + */ + test("the message being asked about is marked as sent, and the history behind it is not", async () => { + const marked: string[][] = []; + const agents = await buildAgents( + [assistant], + model, + "openai-secret", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + // Everything resolves, so nothing throws and every message in `twoTurns` is inlined — which + // is exactly the condition under which a read-time stamp would have marked both of them. + async () => stored, + async (ids: readonly string[]) => { + marked.push([...ids]); + }, + ); + const failed = await runToCompletion( + built(agents, "general-assistant"), + input(twoTurns), + ); + + expect(failed).toEqual([]); + expect(marked).toEqual([["abc"]]); + }); + + /* + * AND IT IS STAMPED ONLY IF THE WHOLE WALK CAME BACK, which is a question about WHEN rather than + * about which message. + * + * `inlineAttachments` walks backwards, so the message being asked about is the FIRST thing it + * resolves and every older message is still ahead of it. The stamp was written the moment that + * message resolved, so a history load that rejected afterwards failed the turn with `attachedAt` + * already recorded for it — a stamp for a turn that never ran. `"note"` does not cover this: it + * softens a row that is MISSING, not a read that fails, so a pool error or a timeout on any older + * message still propagates and still fails the run. + * + * That is not a cosmetic inaccuracy. The stamp's entire meaning is "this file reached a message + * somebody actually sent", and three readers act on it — the sweeper's delete, the upload cap, the + * withdrawal route. Moving the write past the end of the loop is what this pins: with it inside, + * `marked` holds `["abc"]` here, and the file is treated for ever as having been sent by a turn + * that errored. + */ + test("a history load that fails leaves nothing stamped as sent", async () => { + const marked: string[][] = []; + const agents = await buildAgents( + [assistant], + model, + "openai-secret", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + // The message being asked about resolves cleanly — that is the point. It is the OLDER one, + // reached after the stamp used to be written, whose read falls over. + async (id: string) => { + if (id === "old") { + throw new Error("connection terminated unexpectedly"); + } + return stored; + }, + async (ids: readonly string[]) => { + marked.push([...ids]); + }, + ); + + const failed = await runToCompletion( + built(agents, "general-assistant"), + input(twoTurns), + ); + + // The turn really did fail, so the assertion below is about a turn that never ran rather than + // about a run that quietly succeeded. + expect(failed.map((error) => error.message)).toEqual([ + "connection terminated unexpectedly", + ]); + expect(marked).toEqual([]); + }); + + test("a message naming more bytes than a turn may inline refuses the turn", async () => { + /* + * `MAX_INLINED_BYTES_PER_RUN` bounded only the half of a run that could degrade: both places + * that stopped spending tested `onMissing === "note"`, and the message being asked about is + * resolved under `"fail"`, so nothing capped it at all. A member naming two hundred + * previously-sent 8 MiB attachments in one message inlined about 1.6 GiB, plus its base64, in a + * single turn — the heap exhaustion this budget exists to prevent, through the one door it left + * open. End to end rather than in `attachment-parts.test.ts` alone, because what was wrong was + * the pairing of the budget with the strict mode, and only this file wires the two together. + * + * Sized off the constant rather than off a literal, so raising the budget moves this test with + * it instead of quietly making it assert nothing. Five quarters of the budget on ONE message: + * four fit exactly, and the fifth is what there is no room for. + */ + const quarter = MAX_INLINED_BYTES_PER_RUN / 4; + const big = { ...stored, bytes: Buffer.alloc(quarter) }; + const names = ["one.png", "two.png", "three.png", "four.png", "five.png"]; + const asking = { + id: "m1", + role: "user" as const, + content: names.map((filename, index) => ({ + type: "image", + source: { type: "url", value: `/api/attachments/a${index}` }, + metadata: { attachmentId: `a${index}`, filename }, + })), + }; + + const loaded: string[] = []; + const agent = await builtInWith(async (id) => { + loaded.push(id); + return big; + }); + + const error = await runToError(agent, input([asking])); + + // A sentence naming the problem, not a truncated turn: the file, the limit, and something to do + // about it, because the message is still in front of the person who wrote it. + expect(error.message).toContain('"five.png"'); + expect(error.message).toContain("could not be included"); + expect(error.message).toContain(String(MAX_INLINED_BYTES_PER_RUN)); + expect(error.message).toContain("Send fewer files"); + + // And the part it refused over was never read. A turn about to be refused should not pay for + // the bytes it cannot afford on the way to saying so. + expect(loaded).toEqual(["a0", "a1", "a2", "a3"]); + }); + + /* + * THE RUN'S OWN CONVERSATION REACHES BOTH SEAMS, which is the half of the channel scope that + * lives in this file and cannot be tested from the other side. + * + * `loadAttachmentForTurn` and `markAttachmentsSent` refuse a file belonging to a different + * channel by resolving the thread they are given to its channel. That is worth nothing if the + * thread they are given is not the thread the run is in — and a fix spanning three files can + * half-land and still look green, because every test on the database side passes whatever thread + * it likes and every test on this side used to ignore the argument entirely. This is the seam + * where the two halves meet: `inlineAttachments` takes the thread from `input`, and a wiring that + * passed a constant, a stale capture, or the run id would satisfy the types and break the scope + * in the direction that fails open. + * + * Both seams, because they are wired separately: the loader through `resolveAttachmentParts` + * (which narrows to `(id) => …`, so the binding is hand-written) and the stamp directly. + */ + test("the run's own thread is what both attachment seams are asked about", async () => { + const loadedOn: string[] = []; + const markedOn: string[] = []; + const agents = await buildAgents( + [assistant], + model, + "openai-secret", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + async (_id: string, threadId: string) => { + loadedOn.push(threadId); + return stored; + }, + async (_ids: readonly string[], threadId: string) => { + markedOn.push(threadId); + }, + ); + + const failed = await runToCompletion( + built(agents, "general-assistant"), + input(twoTurns), + ); + expect(failed).toEqual([]); + + // `input()` runs on "thread_1". Both messages in `twoTurns` carry a file, so the loader is + // asked twice, and the stamp once — for the message being asked about. + expect(loadedOn).toEqual(["thread_1", "thread_1"]); + expect(markedOn).toEqual(["thread_1"]); + }); + + test("a send that could not be recorded refuses the turn before the model sees it", async () => { + /* + * THIS TEST USED TO ASSERT THE OPPOSITE, and the assertion it made was the bug. + * + * `MarkAttachmentsSent` promised that a failure to record was swallowed, because "a turn is + * somebody waiting for an answer". The waiting is real. What the sentence quietly assumed is + * that by the time the stamp runs, the answer has been earned — and it has not. `markSent` is + * the last thing `inlineAttachments` does BEFORE returning the history, and the history is what + * the run is given afterwards. So the swallow bought an answer at the price of the FILE: + * `attachedAt` stayed null, the culler reclaimed the row a day later, and the message went on + * displaying an attachment that no longer existed. + * + * Refusing instead costs a turn that never started. The two assertions below are that trade, + * stated as facts rather than as an argument: the run errors, and `BuiltInAgent.run` was never + * reached — so no model was called, no token was spent, and the person's message is still in + * front of them to send again. `runToCompletion`'s spy standing in for the model is what makes + * the second one observable; `seen` staying empty is the whole claim about WHEN this happens. + * + * A readable sentence, because there is no `app.onError` behind this server: what a rejected run + * carries is what the composer shows, so an implementation's message has to name the file and + * say what to do. This one is the test's own, since the production wording lives in + * channels/attachments.ts and is pinned there. + */ + const agents = await buildAgents( + [assistant], + model, + "openai-secret", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + async () => stored, + async (ids: readonly string[]) => { + throw new Error( + `This turn was not run, because an attachment on your message could not be recorded as sent ("${ids.join('", "')}").`, + ); + }, + ); + + const seen: RunAgentInput[] = []; + const failed = await runToCompletion( + built(agents, "general-assistant"), + input(twoTurns), + (received) => { + seen.push(received); + }, + ); + + expect(failed.map((error) => error.message)).toEqual([ + 'This turn was not run, because an attachment on your message could not be recorded as sent ("abc").', + ]); + // The turn was not yet spent, which is the whole reason refusing here is the cheaper loss. Put + // the `markSent` call after the run instead of before it and this is the assertion that goes red. + expect(seen).toEqual([]); + }); + + /* + * AND THE SEAM DOES NOT DRESS THE REFUSAL UP, which is what the `try`/`catch` that used to stand + * around this call did to a synchronous throw as much as to a rejection. + * + * `MarkAttachmentsSent` is an optional parameter four wirings expose — `buildAgents`, + * `resolveRuntimeAgents`, `createRequestAgents` and `mountCopilotRuntime` — so an implementation + * that throws before it ever returns a promise is a shape this seam has to carry, and it has to + * carry it WITHOUT replacing the message: the words that reach the person are the ones from the + * implementation that knows which rows are involved. + */ + test("an implementation that throws synchronously still refuses with its own words", async () => { + const agents = await buildAgents( + [assistant], + model, + "openai-secret", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + async () => stored, + // Not `async`: this throws on the call itself rather than returning a rejected promise, which + // is the case a `.catch()` on the result would never have seen at all. + (): Promise => { + throw new Error("the pool had nothing left to give"); + }, + ); + + const seen: RunAgentInput[] = []; + const failed = await runToCompletion( + built(agents, "general-assistant"), + input(twoTurns), + (received) => { + seen.push(received); + }, + ); + + expect(failed.map((error) => error.message)).toEqual([ + "the pool had nothing left to give", + ]); + expect(seen).toEqual([]); + }); + + test("a stored image on a document part reaches the model as an image", async () => { + /* + * End to end, because the unit test for this can only prove `resolvePart` does the right thing + * with arguments a test chose. What matters is that the run hands the provider an `image` part: + * `photo.png` renamed and dragged out of an editor claims `text/plain`, the SDK fixes the + * modality to `document` from that claim before the upload, and the server sniffs the bytes and + * stores `image/png`. Reading `part.type` here ran a PNG through `toString("utf8")` and + * captioned the noise `Attached file "photo.png":`. + */ + const agent = await builtInWith(async () => stored); + + const seen: RunAgentInput[] = []; + const failed = await runToCompletion( + agent, + input([ + { + id: "m1", + role: "user" as const, + content: [ + { + type: "document", + source: { type: "url", value: "/api/attachments/abc" }, + metadata: { attachmentId: "abc", filename: "photo.png" }, + }, + ], + }, + ]), + (received) => { + seen.push(received); + }, + ); + + expect(failed).toEqual([]); + const part = ( + seen[0]?.messages?.[0] as { + content?: { type?: string; source?: { mimeType?: string } }[]; + } + )?.content?.[0]; + expect(part?.type).toBe("image"); + expect(part?.source?.mimeType).toBe("image/png"); + }); + + test("the run's byte budget is spent newest-first, so it is the oldest history that is cut", async () => { + /* + * Nothing bounded a turn before this. `MAX_IMAGE_BYTES` bounds one file, but history is + * replayed on every turn, so a channel that had seen a few large images read and base64-ed all + * of them again on every later turn — and that failure arrives as the pod's heap, taking every + * other person's in-flight run with it, rather than as anything a person can read. + * + * Sized off `MAX_INLINED_BYTES_PER_RUN` rather than off a literal, so raising the budget moves + * this test with it instead of quietly making it assert nothing. Five messages of a quarter of + * the budget each: the newest four fit, and the oldest is what runs out. + */ + const quarter = MAX_INLINED_BYTES_PER_RUN / 4; + const big = { ...stored, bytes: Buffer.alloc(quarter) }; + const history = [ + attached("h1", "one.png", "m1"), + attached("h2", "two.png", "m2"), + attached("h3", "three.png", "m3"), + attached("h4", "four.png", "m4"), + attached("h5", "five.png", "m5"), + ]; + + const loaded: string[] = []; + const agent = await builtInWith(async (id) => { + loaded.push(id); + return big; + }); + + const seen: RunAgentInput[] = []; + const failed = await runToCompletion(agent, input(history), (received) => { + seen.push(received); + }); + + expect(failed).toEqual([]); + const messages = seen[0]?.messages ?? []; + + // The oldest is a note that does NOT say the file is gone — it is still there, and asking about + // it directly would make it the message being asked about, which is charged first. + expect((messages[0] as { content?: unknown }).content).toEqual([ + { + type: "text", + text: '[attachment "one.png" from an earlier message was not included in this turn]', + }, + ]); + // And it was never read: the point of the budget is the round trip it does not make, not just + // the base64 it does not build. + expect(loaded).not.toContain("h1"); + expect(loaded.length).toBe(4); + + // The message being asked about is whole, which is the property that makes a budget defensible + // at all — a person is never told their own question's attachment was left out of their turn. + expect( + (messages[4] as { content?: { source?: unknown }[] }).content?.[0] + ?.source, + ).toMatchObject({ type: "data" }); + }); +}); diff --git a/server/tests/handoff-caps-defaults.test.ts b/server/tests/handoff-caps-defaults.test.ts index a2f6efac7..63843ad17 100644 --- a/server/tests/handoff-caps-defaults.test.ts +++ b/server/tests/handoff-caps-defaults.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "bun:test"; +import { resolve } from "node:path"; import { parse } from "yaml"; import { loadConfig } from "../src/config"; import { testEnvironment } from "./support/environment"; @@ -17,11 +18,22 @@ import { testEnvironment } from "./support/environment"; * operator debugging a refusal against a number their deployment never had. */ -const chart = parse(await Bun.file("charts/openbot/values.yaml").text()) as { +/* + * Resolved from this file rather than from the working directory. Both reads are at the top level, + * so a relative path that misses takes the whole file's tests with it and reports nothing: the suite + * gets smaller and stays green. That is what `bun test` from inside `server/` used to do. + */ +const repositoryRoot = resolve(import.meta.dir, "..", ".."); + +const chart = parse( + await Bun.file(resolve(repositoryRoot, "charts/openbot/values.yaml")).text(), +) as { config?: { handoff?: { maxDepth?: number; maxPerRun?: number } }; }; -const docs = await Bun.file("docs/configuration.md").text(); +const docs = await Bun.file( + resolve(repositoryRoot, "docs/configuration.md"), +).text(); /** What `handoffCaps` falls back to with nothing in the environment. */ const code = loadConfig(testEnvironment()).handoff; diff --git a/shared/attachments.test.ts b/shared/attachments.test.ts new file mode 100644 index 000000000..9bcd72649 --- /dev/null +++ b/shared/attachments.test.ts @@ -0,0 +1,310 @@ +import { describe, expect, test } from "bun:test"; +import { + classifyAttachment, + MAX_ATTACHMENTS_PER_MESSAGE, + MAX_EXTRACTED_CHARACTERS, + MAX_FILE_BYTES, + mediaTypeOf, + namesNoFormat, + shouldClaimPaste, +} from "./attachments"; + +describe("classifyAttachment", () => { + test("names the four accepted image types", () => { + for (const mimeType of [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ]) { + expect(classifyAttachment(mimeType)).toBe("image"); + } + }); + + test("an SVG is not an image we will serve", () => { + expect(classifyAttachment("image/svg+xml")).toBe("unsupported-image"); + }); + + test("a HEIC photo is refused as an image, not as an unknown file", () => { + expect(classifyAttachment("image/heic")).toBe("unsupported-image"); + }); + + test("text-ish files are text", () => { + expect(classifyAttachment("text/markdown")).toBe("text"); + expect(classifyAttachment("application/json")).toBe("text"); + }); + + test("anything else is unsupported", () => { + expect(classifyAttachment("application/zip")).toBe("unsupported"); + }); + + test("a charset parameter does not defeat the match", () => { + expect(classifyAttachment("text/plain;charset=utf-8")).toBe("text"); + expect(classifyAttachment("application/json;charset=utf-8")).toBe("text"); + }); + + test("a parameter with a space and mixed case still matches", () => { + expect(classifyAttachment("text/plain; charset=UTF-8")).toBe("text"); + }); + + test("a trailing space does not misfire as an unsupported image", () => { + expect(classifyAttachment("image/png ")).toBe("image"); + }); +}); + +describe("shouldClaimPaste", () => { + test("a spreadsheet cell pastes its text, and does not attach a screenshot of itself", () => { + // The flavour Chrome really produces for a copied cell: an `image/png` file AND the text. An + // earlier version of this test used `kinds: ["text"]`, which is a text FILE — a shape Excel + // never puts on the clipboard — so it passed while the real paste was broken. + expect(shouldClaimPaste({ kinds: ["image"], plainText: "a\tb" })).toBe( + false, + ); + }); + + test("a screenshot is claimed, because it arrives with no text at all", () => { + expect(shouldClaimPaste({ kinds: ["image"], plainText: "" })).toBe(true); + }); + + test("an unsupported image with no text is claimed, so it can be refused out loud", () => { + expect( + shouldClaimPaste({ kinds: ["unsupported-image"], plainText: "" }), + ).toBe(true); + }); + + test("an unsupported file with no text is claimed, so it can be refused out loud", () => { + // The sibling of the `unsupported-image` case above, and the branch that + // matters for a pasted `.zip`. Not claiming it would let the paste fall + // through to the browser's default, which does nothing visible at all — + // the file would simply not appear, with no reason given. + expect(shouldClaimPaste({ kinds: ["unsupported"], plainText: "" })).toBe( + true, + ); + }); + + test("text wins over a plain file too", () => { + expect(shouldClaimPaste({ kinds: ["text"], plainText: "a\tb" })).toBe( + false, + ); + }); + + test("a file with no text alongside it is claimed", () => { + expect(shouldClaimPaste({ kinds: ["text"], plainText: "" })).toBe(true); + }); + + test("an empty clipboard is not claimed", () => { + expect(shouldClaimPaste({ kinds: [], plainText: "" })).toBe(false); + }); +}); + +test("the per-message cap is eight", () => { + expect(MAX_ATTACHMENTS_PER_MESSAGE).toBe(8); +}); + +/** + * The two text limits, pinned against each other rather than each alone. + * + * They bound different things — bytes uploaded versus characters the model + * reads — and they are far apart, so a text file can be accepted whole and + * still reach the model as a fraction of itself. That is deliberate and the + * constants now explain it, but it is the kind of relationship that gets + * broken by an innocent-looking edit to one number. This is here so the edit + * lands on a failing test with the reasoning attached, rather than silently + * changing what a person gets an answer about. + */ +describe("what is uploaded and what the model reads are different limits", () => { + test("a text file may be accepted far larger than the model will read", () => { + // If these ever converge, the truncation warning the composer owes the + // sender stops being needed — and if they invert, `MAX_EXTRACTED_CHARACTERS` + // stops doing anything at all, since no accepted file could reach it. + expect(MAX_FILE_BYTES).toBeGreaterThan(MAX_EXTRACTED_CHARACTERS); + }); + + test("a file at the byte ceiling reaches the model as about an eighth of itself", () => { + // Stated as the ratio rather than as the two numbers, because the ratio is + // the thing a person would be surprised by. ASCII text, where a byte is a + // character; anything multi-byte loses proportionally less. + const readable = MAX_EXTRACTED_CHARACTERS / MAX_FILE_BYTES; + expect(readable).toBeLessThan(0.125); + expect(readable).toBeGreaterThan(0.1); + }); + + test("the byte size is a sound one-sided test for whether text will be cut", () => { + // The predicate the composer can screen with: UTF-8 spends at least one + // byte per code point, so a file of N bytes never decodes to more than N + // characters. A file at or under the character ceiling therefore CANNOT be + // truncated, which is what makes `file.size > MAX_EXTRACTED_CHARACTERS` a + // warning that never fires on a file that arrives whole. + const mayBeTruncated = (byteLength: number) => + byteLength > MAX_EXTRACTED_CHARACTERS; + + expect(mayBeTruncated(MAX_EXTRACTED_CHARACTERS)).toBe(false); + expect(mayBeTruncated(MAX_EXTRACTED_CHARACTERS + 1)).toBe(true); + // The case the gap is about: an accepted upload that will still be cut. + expect(mayBeTruncated(MAX_FILE_BYTES)).toBe(true); + }); +}); + +/** + * The two functions the "the browser told us nothing, so the server gets to + * look" design rests on, neither of which was tested anywhere in the repo. + * + * That is a gap worth closing rather than a style point. `namesNoFormat` is + * the ONE copy of the list both halves screen against, and its whole reason + * for living in this file is that the composer and the server drifted apart on + * it once already: the server threw an unnamed claim away and read the bytes, + * while the composer refused the same file at pick time on the claim alone. + * Nothing failed while they disagreed, because nothing asked either of them + * anything. `mediaTypeOf` is the normalisation every comparison in this file + * and in `attachment-mime.ts` runs first, so a change to it moves every gate + * at once. + */ +describe("mediaTypeOf", () => { + test("a bare media type is handed back unchanged", () => { + expect(mediaTypeOf("text/plain")).toBe("text/plain"); + }); + + test("the charset Bun's File constructor appends is dropped", () => { + // Not hypothetical: `new File(["x"], "a.txt", { type: "text/plain" })` + // reports `text/plain;charset=utf-8`, so test fixtures and clipboard + // entries in this app routinely arrive with a parameter attached. + expect(mediaTypeOf("text/plain;charset=utf-8")).toBe("text/plain"); + expect(mediaTypeOf("application/json;charset=utf-8")).toBe( + "application/json", + ); + }); + + test("a space after the semicolon does not survive into the type", () => { + expect(mediaTypeOf("text/plain; charset=UTF-8")).toBe("text/plain"); + }); + + test("case is folded, because RFC 2045 makes the type case-insensitive", () => { + // A `File` this app never built — one off a drop or a clipboard — carries + // whatever its source wrote, and `new File(...)` only lower-cases what it + // is handed itself. + expect(mediaTypeOf("IMAGE/PNG")).toBe("image/png"); + expect(mediaTypeOf("Text/Markdown")).toBe("text/markdown"); + }); + + test("surrounding whitespace is trimmed", () => { + expect(mediaTypeOf(" image/png ")).toBe("image/png"); + expect(mediaTypeOf("image/png ")).toBe("image/png"); + }); + + test("case, parameter and whitespace are all handled at once", () => { + // The combination is the realistic one; handling each alone is not enough. + expect(mediaTypeOf(" TEXT/CSV ; charset=UTF-8 ")).toBe("text/csv"); + }); + + test("more than one parameter still leaves just the type", () => { + expect(mediaTypeOf("text/plain; charset=utf-8; boundary=xyz")).toBe( + "text/plain", + ); + }); + + test("a blank claim normalises to a blank string, not to a guess", () => { + // This function normalises; it does not invent. `namesNoFormat` is what + // turns the blank into a decision. + expect(mediaTypeOf("")).toBe(""); + expect(mediaTypeOf(" ")).toBe(""); + }); +}); + +describe("namesNoFormat", () => { + test("every generic placeholder a browser sends names nothing", () => { + // All four members of MIME_NAMES_NOTHING. Only the first had coverage + // anywhere in the repo, and this list is precisely what the two sides + // drifted on, so it is pinned member by member rather than sampled. + for (const claim of [ + "application/octet-stream", + "binary/octet-stream", + "application/unknown", + "application/force-download", + ]) { + expect(namesNoFormat(claim)).toBe(true); + } + }); + + test("a blank claim names nothing", () => { + // The commonest case of all: a browser with no mapping for an extension + // sets `file.type` to the empty string rather than to a placeholder. + expect(namesNoFormat("")).toBe(true); + }); + + test("anything not shaped like a MIME type names nothing", () => { + // The `!mediaType.includes("/")` half of the rule, which nothing covered. + expect(namesNoFormat("garbage")).toBe(true); + expect(namesNoFormat("text")).toBe(true); + expect(namesNoFormat("plain-text-please")).toBe(true); + }); + + test("a placeholder still names nothing through case and parameters", () => { + // The list is matched against the NORMALISED form. Matching the raw claim + // would let `APPLICATION/OCTET-STREAM` pose as a named format and be + // refused at pick time — the exact drift this list exists to prevent. + expect(namesNoFormat("APPLICATION/OCTET-STREAM")).toBe(true); + expect(namesNoFormat("Application/Octet-Stream; charset=binary")).toBe( + true, + ); + expect(namesNoFormat(" binary/octet-stream ")).toBe(true); + }); + + test("a claim that names a real format names a format", () => { + for (const claim of [ + "text/plain", + "text/csv", + "image/png", + "image/heic", + "image/svg+xml", + "text/html", + "application/zip", + "application/pdf", + ]) { + expect(namesNoFormat(claim)).toBe(false); + } + }); + + test("a near-miss of a placeholder is not a placeholder", () => { + // Set membership on the whole media type, not a prefix or a substring, so + // a real format whose name merely starts the same way is not swept up. + expect(namesNoFormat("application/octet-stream-plus")).toBe(false); + expect(namesNoFormat("application/unknown-format")).toBe(false); + }); +}); + +/** + * The pairing that actually decides what happens to an unnamed file. + * + * The composer defers to the server only when BOTH are true of a pick: + * `classifyAttachment` says `unsupported` AND `namesNoFormat` says the claim + * named nothing (`picked-files.ts`, the `unnamed` branch). Testing the two + * functions apart would not catch a change that broke the conjunction — if a + * placeholder ever started classifying as something other than `unsupported`, + * that branch would stop firing and the round trip that lets the server read + * the bytes would quietly disappear, with both functions still passing their + * own tests. + */ +describe("an unnamed claim is unsupported AND unnamed, which is what defers to the server", () => { + test("each placeholder is refused by kind and excused by name", () => { + for (const claim of [ + "application/octet-stream", + "binary/octet-stream", + "application/unknown", + "application/force-download", + "", + ]) { + expect(classifyAttachment(claim)).toBe("unsupported"); + expect(namesNoFormat(claim)).toBe(true); + } + }); + + test("a named refusal is refused by kind and NOT excused by name", () => { + // The other half of the branch: the two sides already agree about a claim + // that names a format, so the composer refuses it itself rather than + // spending a round trip to be told the same thing. + for (const claim of ["application/zip", "text/html"]) { + expect(classifyAttachment(claim)).toBe("unsupported"); + expect(namesNoFormat(claim)).toBe(false); + } + }); +}); diff --git a/shared/attachments.ts b/shared/attachments.ts new file mode 100644 index 000000000..3ce58ccee --- /dev/null +++ b/shared/attachments.ts @@ -0,0 +1,274 @@ +/** + * What may be attached to a message, and how much of it. + * + * One declaration read from both sides, the same as `routine-firing.ts`: the composer refuses a + * file before uploading it and the server refuses it again on arrival, and those two refusals have + * to agree. A limit written twice is a limit that drifts, and the drift shows up as a file the + * composer accepted and the server threw away with no explanation. + */ + +export const MAX_ATTACHMENTS_PER_MESSAGE = 8; + +/** + * A hard ceiling, not a downscaling threshold: nothing in this path resizes + * or re-encodes an image, so a file under this limit is stored and sent + * whole. `resolvePart` (`server/src/channels/attachment-parts.ts`) then + * base64-encodes those bytes into a single content part, which runs about + * 4/3 the byte size — a file at this ceiling becomes a part of roughly + * 10.7 MB. + * + * This deployment only ever targets `openai` (`tenant-package.ts` refuses + * to load a package whose `model.provider` is anything else), and OpenAI's + * vision input limit is 20 MB per image. 8 MB was chosen, not the 15 MB + * that would sit exactly at that line, because it is the encoded form that + * has to fit under the provider's number, not the stored one, and because + * it is not certain from that number alone which side of the encoding it + * was measured on — so this sits at roughly half of it either way. + * + * Downscaling is deliberately not built yet: a file over this ceiling is + * refused at pick time rather than shrunk, and building the shrink step is + * a known follow-up, not something this constant can stand in for. + */ +export const MAX_IMAGE_BYTES = 8 * 1024 * 1024; + +/** + * How large a text file may be UPLOADED. Not how much of it the model reads. + * + * Text files are not downscaled, so this is a hard refusal like `MAX_IMAGE_BYTES`. + * + * THIS IS NOT THE LIMIT THAT DECIDES WHAT REACHES THE MODEL, AND THE TWO NUMBERS ARE FAR APART. + * `MAX_EXTRACTED_CHARACTERS` below is 120,000, and this is 1,048,576. A file accepted at this + * ceiling is stored whole and then handed to the model as its first 120,000 characters — for + * ASCII-ish text, where a byte is a character, roughly its first EIGHTH, with about 89% cut. Text + * in a script that costs several bytes per character loses proportionally less, because the + * ceilings are counted in different units, but a file anywhere near this size is cut. + * + * The two are deliberately different numbers because they bound different things, and neither can + * stand in for the other: + * + * - this one bounds what is UPLOADED, STORED and SERVED — bandwidth, disk, and the size of the + * response `/api/attachments/:id` has to produce; + * - `MAX_EXTRACTED_CHARACTERS` bounds what one attachment may spend of a shared CONTEXT WINDOW, + * which is a budget split with the conversation and with up to seven other attachments. + * + * REJECTED: lowering this to 120,000 so the two agree. It would refuse files the app can already + * do something useful with — the first 120,000 characters of a large CSV usually answers the + * question that was asked of it — and would trade a partial read for no read at all. + * + * REJECTED: raising `MAX_EXTRACTED_CHARACTERS` to match this. A megabyte of text is a few hundred + * thousand tokens, which is the context window this deployment targets spent entirely on one + * attachment. That is the outcome that constant exists to prevent. + * + * So the gap stays, and what it costs is stated here rather than left for somebody to derive by + * dividing one constant by the other. What is NOT yet resolved is that the person who uploads the + * file is not told — see `MAX_EXTRACTED_CHARACTERS`. + */ +export const MAX_FILE_BYTES = 1024 * 1024; + +/** + * How much extracted text may reach the model from one file. + * + * The real limit on a text attachment is tokens, not bytes: a one-megabyte file is a few hundred + * thousand tokens and would fill the window on its own. 120,000 characters is roughly 30,000 + * tokens, which every model this deployment targets can hold alongside a conversation. Text past + * this point is cut and the part says so, rather than being silently dropped. + * + * THE PART SAYS SO TO THE MODEL. NOBODY SAYS SO TO THE PERSON WHO ATTACHED THE FILE. + * + * `extractDocumentText` (`server/src/channels/attachment-parts.ts`) appends + * `[attachment truncated at 120000 characters]` to what it sends, so the model is never left + * answering questions about a file it read only part of without knowing. That is half the promise. + * The other half is not kept: this ceiling is 120,000 while `MAX_FILE_BYTES` above is 1,048,576, so + * a 1 MB CSV is accepted at pick time, uploaded whole, shown as a `1.0MB` tile — and then read by + * the model as roughly its first eighth, with nothing anywhere in the UI saying so. Somebody who + * asks "how many rows have status=failed" gets a confident answer about the part that fit. + * + * That is a real gap and it is recorded rather than fixed here, because the fix is a composer + * change and this file is only where the number lives. What the composer needs is already exported: + * a file whose SIZE IN BYTES exceeds this constant may be truncated, and one at or under it cannot + * be. That direction is exact rather than approximate — UTF-8 spends at least one byte per code + * point, so a file of N bytes can never decode to more than N characters — which makes + * `file.size > MAX_EXTRACTED_CHARACTERS` a warning that never fires on a file that will arrive + * whole. It is deliberately a "may", since the same byte count is fewer characters in any script + * that costs more than a byte each. + */ +export const MAX_EXTRACTED_CHARACTERS = 120_000; + +/** + * `image/svg+xml` is deliberately absent. + * + * An SVG is an image and can also carry script. Served inline from this app's own origin, one + * pasted into a channel is stored XSS against everybody in it. Nobody pastes an SVG expecting a + * model to read it, so it is refused at the door rather than sanitised. + */ +export const ACCEPTED_IMAGE_MIME = [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", +] as const; + +export const ACCEPTED_TEXT_MIME = [ + "text/plain", + "text/markdown", + "text/csv", + "application/json", +] as const; + +/** + * THE MEDIA TYPE ON ITS OWN: LOWER CASE, PARAMETERS GONE — AND THE ONLY FORM ANYTHING HERE COMPARES. + * + * A `File`'s type is whatever produced it. Bun's constructor appends `;charset=utf-8` to + * `text/plain` and `application/json`, and a browser does the same for some clipboard entries; RFC + * 2045 makes the type case-insensitive besides, and while `new File(...)` ASCII-lower-cases what it + * is given, a `File` this app never built — one off a drop or a clipboard — carries whatever its + * source wrote. `sniffMimeType` already normalises the same two ways before handing its answer to + * `classifyAttachment`. + * + * Exported because the composer needs the identical normalisation for a different reason: the SDK's + * own `accept` check is a case-sensitive `file.type === filter`, so anything handed to it has to + * have been through here first or the two gates disagree about the same file. + */ +export function mediaTypeOf(mimeType: string): string { + return mimeType.toLowerCase().split(";")[0].trim(); +} + +/** + * CLAIMS THAT NAME NO FORMAT AT ALL, AND THE ONE COPY OF THAT LIST. + * + * A browser sends one of these for a file it has no mapping for — a `.txt` dragged out of an + * editor, anything with an unfamiliar extension. Both sides have to treat them the same way and for + * a while did not: `sniffMimeType` discards such a claim and reads the bytes, so the server accepts + * the text file behind it, while the composer screened the claim alone and refused the same file at + * pick time. That is the drift the note at the top of this file exists to prevent, so the list lives + * here and `attachment-mime.ts` reads it from here rather than keeping its own. + * + * A blank claim, and anything not shaped like a MIME type, names nothing by the same reasoning. + */ +const MIME_NAMES_NOTHING: ReadonlySet = new Set([ + "application/octet-stream", + "binary/octet-stream", + "application/unknown", + "application/force-download", +]); + +export function namesNoFormat(mimeType: string): boolean { + const mediaType = mediaTypeOf(mimeType); + return !mediaType.includes("/") || MIME_NAMES_NOTHING.has(mediaType); +} + +export type AttachmentKind = + | "image" + | "text" + | "unsupported-image" + | "unsupported"; + +/** + * `unsupported-image` is a separate answer from `unsupported` so the refusal can name the real + * problem. "That is not a supported image type" tells somebody with a HEIC photo what to do; + * "unsupported file" leaves them guessing whether images work at all. + * + * THE MIME TYPE AND NOTHING ELSE, WHICH IS WHY THE FILENAME IS NO LONGER ASKED FOR. This took + * `{ name, mimeType }` and read only the second, and the dead argument was not merely untidy: it + * read as a promise that a file called `notes.txt` would be given the benefit of the doubt, which + * is exactly the doubt the composer's callers had. It must not be kept, either. The server calls + * this with the type `sniffMimeType` earned from the BYTES, and letting an extension override that + * answer is how a binary blob named `.txt` — or an empty file, which sniffs to + * `application/octet-stream` on purpose — would be stored and served from this origin as text. + * Where a name does deserve the benefit of the doubt the door is `namesNoFormat`, and it is the + * client's alone: it is asked of a claim nobody has corroborated yet, not of an answer the bytes + * have already given. + */ +export function classifyAttachment(mimeType: string): AttachmentKind { + // The media type is what decides here — see `mediaTypeOf`. Without this the composer refused a + // file the server then accepted, which is the drift the note at the top of this file exists to + // prevent. + const mediaType = mediaTypeOf(mimeType); + if ((ACCEPTED_IMAGE_MIME as readonly string[]).includes(mediaType)) { + return "image"; + } + if (mediaType.startsWith("image/")) return "unsupported-image"; + if ((ACCEPTED_TEXT_MIME as readonly string[]).includes(mediaType)) { + return "text"; + } + return "unsupported"; +} + +/** + * Whether a paste is an attachment or an ordinary text paste. + * + * Text wins whenever the clipboard carries any. A file is ours only when there is no text to + * prefer. + * + * This is the one rule here that is NOT T3 Code's. Theirs claims an image even when text came with + * it, on the reasoning that an image is unambiguously what was meant. It is not: copying a cell + * from a spreadsheet, or a block from a word processor, puts an `image/png` rendering of the + * selection on the clipboard ALONGSIDE the text. Under "an image always wins" that paste attached a + * screenshot of the cell and typed nothing — verified in Chrome against the real app, not + * theorised. + * + * A screenshot carries no text at all, so the case this rule exists for is untouched. + * + * DECLINING A PASTE IS NOT THE SAME AS THE TEXT BEING TYPED, AND FOR SOME SOURCES IT IS NOT WHAT + * HAPPENS. This function only says "not ours". What the paste then does is PromptArea's rule, and + * that rule is: if the clipboard carries an image AND the `text/html` is Microsoft Office markup — + * it looks for `urn:schemas-microsoft-com:office`, a `ProgId` name, a `Mso` class, or an `mso-*` + * property — the text is inserted and the image ignored. Anything else with an image on the + * clipboard calls `onImagePaste` and RETURNS, having already called `preventDefault` and inserted + * nothing. + * + * So the two halves of the spreadsheet case land differently, and it is worth being exact about + * which is which: + * + * - Word and Excel put Office markup in the `text/html`. Their text is typed. Handled. + * - Google Sheets, Numbers, and every other source that behaves like Office without being it, + * take the second branch. Their IMAGE is attached and THEIR TEXT IS NOT TYPED — the same + * outcome "an image always wins" gave, arrived at by a different route. + * + * That second bullet is a known defect and is being kept for now rather than fixed, so this comment + * says so instead of implying the spreadsheet case is covered. Fixing it means teaching this rule + * or PromptArea's the difference between an image that IS the selection and an image that merely + * accompanies it, which no clipboard flag reports. + * + * The other trade-off, stated because it is also real: copying an image from a web page can put the + * image's URL in `text/plain`, and that pastes the URL rather than attaching the image. Drag or the + * `+` button still attach it, and a URL landing in the box is visible and undoable — where a + * swallowed paste is neither. + */ +export function shouldClaimPaste(input: { + kinds: readonly AttachmentKind[]; + plainText: string; +}): boolean { + if (input.plainText.length > 0) return false; + return input.kinds.length > 0; +} + +/** + * What a sent message carries instead of the bytes. + * + * This is an AG-UI `image`/`document` part with a URL source, NOT a part type of our own. AG-UI has + * no reference member: `RunAgentInputSchema.parse` rejects `{type:"attachment"}` outright and the + * runtime answers 400, verified against the installed 0.0.59 schema. The union is + * `text | image | audio | video | document | binary`, and a source is `{type:"data"|"url"}` — there + * is no `base64` literal and no `media_type` key anywhere. + * + * The URL is relative and is never fetched by a provider. `copilot.ts` swaps the source for + * `{type:"data", value:}` as the run is built, which is what keeps the bytes out of the + * stored thread while still putting the image in front of the model. `metadata` carries the id + * because it is `z.unknown().optional()` and survives the parse; a sibling key would be silently + * stripped. + */ +export type AttachmentSource = + | { type: "data"; value: string; mimeType: string } + | { type: "url"; value: string; mimeType?: string }; + +export type AttachmentPart = { + type: "image" | "document"; + source: AttachmentSource; + metadata?: { attachmentId: string; filename?: string }; +}; + +/** The relative URL form stored in a sent message. Resolved server-side, never by a provider. */ +export function attachmentUrl(attachmentId: string): string { + return `/api/attachments/${attachmentId}`; +}