diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d7d421f8..0d8eeb23a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### The Bot in the box and the LangGraph Bot read a message that has a file attached + +A message with a file attached reached both Bots as `[object Object],[object Object]`, in place of +what the person typed and the file both: they read a message as a string, and one carrying a file is +a list of parts. Each part now reaches the model as what it is — the words, the text of an attached +file, an attached image — and a part neither can read is named rather than dropped. + ### A fractional or out-of-range computer setting takes the fallback instead of breaking the boot `numberFromEnv` accepted anything `Number` called finite and positive, so `PORT=80.5` bound diff --git a/agent-bot/src/history.ts b/agent-bot/src/history.ts index 881debfc4..670cf1b98 100644 --- a/agent-bot/src/history.ts +++ b/agent-bot/src/history.ts @@ -8,6 +8,7 @@ import type { RunAgentInput } from "@ag-ui/core"; import type OpenAI from "openai"; import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt"; +import { userContent } from "../../shared/user-content"; export { NO_ANSWER_CAME }; @@ -58,7 +59,7 @@ export function toProviderMessages( // Placed with the call they answer, below, rather than wherever they arrived. if (message.role === "tool") continue; if (message.role === "user") { - messages.push({ role: "user", content: String(message.content ?? "") }); + messages.push({ role: "user", content: userContent(message.content) }); continue; } if (message.role === "system" || message.role === "developer") { diff --git a/agent-bot/tests/history.test.ts b/agent-bot/tests/history.test.ts index f8c1854d6..6fcb03a09 100644 --- a/agent-bot/tests/history.test.ts +++ b/agent-bot/tests/history.test.ts @@ -279,3 +279,66 @@ describe("a tool call restored from the thread store", () => { expect(fn?.arguments).toBe('{"url":"https://news.ycombinator.com"}'); }); }); + +/** + * A message somebody attached a file to. + * + * The composer sends it as a list of parts rather than a string: what the person typed, then the + * file. `copilot.ts` resolves the file before the run leaves the server, so a text file arrives here + * as a text part and an image as an `image` part carrying its bytes. `String()` of that list is + * `[object Object],[object Object]`, and that is what the model was sent in place of the question and + * the file both. + */ +describe("a message with a file attached", () => { + const typed = { type: "text", text: "How many rows say failed?" }; + const csv = 'Attached file "runs.csv":\n\nid,status\n1,failed\n2,ok'; + + function userContent(content: unknown) { + const [user] = withoutGuidance( + toProviderMessages( + input([{ id: "1", role: "user", content } as unknown as Message]), + ), + ); + return user?.content; + } + + test("keeps what the person typed and the text of the file", () => { + expect(userContent([typed, { type: "text", text: csv }])).toEqual([ + { type: "text", text: "How many rows say failed?" }, + { type: "text", text: csv }, + ]); + }); + + test("puts an attached image in front of the model", () => { + const image = { + type: "image", + source: { type: "data", value: "iVBORw0KGgo=", mimeType: "image/png" }, + metadata: { attachmentId: "a1", filename: "chart.png" }, + }; + expect(userContent([typed, image])).toEqual([ + { type: "text", text: "How many rows say failed?" }, + { + type: "image_url", + image_url: { url: "data:image/png;base64,iVBORw0KGgo=" }, + }, + ]); + }); + + test("names a part it cannot read rather than dropping it", () => { + // A model told "[audio]" can say something was attached that it cannot hear. A model handed + // nothing answers as though nothing was attached. + const audio = { + type: "audio", + source: { type: "data", value: "UklGRg==", mimeType: "audio/wav" }, + }; + expect(userContent([typed, audio])).toEqual([ + { type: "text", text: "How many rows say failed?" }, + { type: "text", text: "[audio]" }, + ]); + }); + + test("sends a message that is only text exactly as it was typed", () => { + // Nearly every message. Unchanged by this, and pinned so it stays that way. + expect(userContent("What is 17 times 3?")).toBe("What is 17 times 3?"); + }); +}); diff --git a/agent-langgraph/src/history.ts b/agent-langgraph/src/history.ts index 3b9db862b..4e124df9e 100644 --- a/agent-langgraph/src/history.ts +++ b/agent-langgraph/src/history.ts @@ -14,6 +14,7 @@ import { ToolMessage, } from "@langchain/core/messages"; import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt"; +import { userContent } from "../../shared/user-content"; /* * Re-exported so this module's own tests and callers keep reading it from here, while the wording @@ -57,7 +58,9 @@ export function toLangChainMessages(input: RunAgentInput): BaseMessage[] { for (const message of input.messages) { if (message.role === "user") { - messages.push(new HumanMessage(String(message.content ?? ""))); + messages.push( + new HumanMessage({ content: userContent(message.content) }), + ); continue; } if (message.role === "system" || message.role === "developer") { diff --git a/agent-langgraph/tests/history.test.ts b/agent-langgraph/tests/history.test.ts index f078fb4ca..0e8fe7649 100644 --- a/agent-langgraph/tests/history.test.ts +++ b/agent-langgraph/tests/history.test.ts @@ -139,3 +139,62 @@ describe("history with a tool call nobody answered", () => { expect(byId.get("orphan")).toBe(NO_ANSWER_CAME); }); }); + +/** + * A message somebody attached a file to. + * + * The composer sends it as a list of parts rather than a string: what the person typed, then the + * file. `copilot.ts` resolves the file before the run leaves the server, so a text file arrives here + * as a text part and an image as an `image` part carrying its bytes. `String()` of that list is + * `[object Object],[object Object]`, and that is what the model was sent in place of the question and + * the file both. + */ +describe("a message with a file attached", () => { + const typed = { type: "text", text: "How many rows say failed?" }; + const csv = 'Attached file "runs.csv":\n\nid,status\n1,failed\n2,ok'; + + function userContent(content: unknown) { + return toLangChainMessages(input([{ role: "user", content }])).at(-1) + ?.content; + } + + test("keeps what the person typed and the text of the file", () => { + expect(userContent([typed, { type: "text", text: csv }])).toEqual([ + { type: "text", text: "How many rows say failed?" }, + { type: "text", text: csv }, + ]); + }); + + test("puts an attached image in front of the model", () => { + const image = { + type: "image", + source: { type: "data", value: "iVBORw0KGgo=", mimeType: "image/png" }, + metadata: { attachmentId: "a1", filename: "chart.png" }, + }; + expect(userContent([typed, image])).toEqual([ + { type: "text", text: "How many rows say failed?" }, + { + type: "image_url", + image_url: { url: "data:image/png;base64,iVBORw0KGgo=" }, + }, + ]); + }); + + test("names a part it cannot read rather than dropping it", () => { + // A model told "[audio]" can say something was attached that it cannot hear. A model handed + // nothing answers as though nothing was attached. + const audio = { + type: "audio", + source: { type: "data", value: "UklGRg==", mimeType: "audio/wav" }, + }; + expect(userContent([typed, audio])).toEqual([ + { type: "text", text: "How many rows say failed?" }, + { type: "text", text: "[audio]" }, + ]); + }); + + test("sends a message that is only text exactly as it was typed", () => { + // Nearly every message. Unchanged by this, and pinned so it stays that way. + expect(userContent("What is 17 times 3?")).toBe("What is 17 times 3?"); + }); +}); diff --git a/shared/user-content.ts b/shared/user-content.ts new file mode 100644 index 000000000..0859ac9cc --- /dev/null +++ b/shared/user-content.ts @@ -0,0 +1,51 @@ +/** + * What a person said, in the shape a chat model reads. Read by both Bots. + * + * A message is a string until somebody attaches a file. Then the composer sends a list of parts — + * what the person typed, then the file — and `copilot.ts` resolves each file before the run leaves + * the server: a text file becomes a text part carrying its contents, an image an `image` part + * carrying its bytes. Both Bots read `content` with `String()`, which for that list is + * `[object Object],[object Object]`, so that is what the model was sent in place of the question and + * the file both. + * + * ONE DECLARATION FOR BOTH BOTS, for the reason `bot-prompt.ts` gives for `NO_ANSWER_CAME`. The two + * shapes returned are the ones OpenAI's chat completions take, and LangChain's OpenAI, Anthropic and + * Google converters read the same two blocks, so one answer serves `agent-bot` and `agent-langgraph` + * alike and the two cannot drift. + * + * A part neither shape can carry is NAMED, not dropped, which is the rule `resultText` in + * `server/src/plugins/mcp.ts` follows for a tool result: a model told "[audio]" can say something was + * attached that it cannot hear, and a model handed nothing answers as though nothing was attached. + */ +export type UserContentPart = + | { type: "text"; text: string } + | { type: "image_url"; image_url: { url: string } }; + +export function userContent(content: unknown): string | UserContentPart[] { + if (!Array.isArray(content)) return String(content ?? ""); + return content.map((part): UserContentPart => { + const item = (part ?? {}) as { + type?: unknown; + text?: unknown; + source?: { type?: unknown; value?: unknown; mimeType?: unknown } | null; + }; + if (item.type === "text" && typeof item.text === "string") { + return { type: "text", text: item.text }; + } + const source = item.source; + if ( + item.type === "image" && + source?.type === "data" && + typeof source.value === "string" && + typeof source.mimeType === "string" + ) { + return { + type: "image_url", + image_url: { url: `data:${source.mimeType};base64,${source.value}` }, + }; + } + const name = + typeof item.type === "string" && item.type ? item.type : "unknown"; + return { type: "text", text: `[${name}]` }; + }); +}