Read a message with a file attached as its words and the file, not as [object Object]

CHANGELOG.md

@@ -8,6 +8,13 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
 
 ## Unreleased
 
+### The Bot in the box and the LangGraph Bot read a message that has a file attached
+
+A message with a file attached reached both Bots as `[object Object],[object Object]`, in place of
+what the person typed and the file both: they read a message as a string, and one carrying a file is
+a list of parts. Each part now reaches the model as what it is — the words, the text of an attached
+file, an attached image — and a part neither can read is named rather than dropped.
+
 ### A fractional or out-of-range computer setting takes the fallback instead of breaking the boot
 
 `numberFromEnv` accepted anything `Number` called finite and positive, so `PORT=80.5` bound

agent-bot/src/history.ts

@@ -8,6 +8,7 @@
 import type { RunAgentInput } from "@ag-ui/core";
 import type OpenAI from "openai";
 import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt";
+import { userContent } from "../../shared/user-content";
 
 export { NO_ANSWER_CAME };
 
@@ -58,7 +59,7 @@ export function toProviderMessages(
     // Placed with the call they answer, below, rather than wherever they arrived.
     if (message.role === "tool") continue;
     if (message.role === "user") {
-      messages.push({ role: "user", content: String(message.content ?? "") });
+      messages.push({ role: "user", content: userContent(message.content) });
       continue;
     }
     if (message.role === "system" || message.role === "developer") {

agent-bot/tests/history.test.ts

@@ -279,3 +279,66 @@ describe("a tool call restored from the thread store", () => {
     expect(fn?.arguments).toBe('{"url":"https://news.ycombinator.com"}');
   });
 });
+
+/**
+ * A message somebody attached a file to.
+ *
+ * The composer sends it as a list of parts rather than a string: what the person typed, then the
+ * file. `copilot.ts` resolves the file before the run leaves the server, so a text file arrives here
+ * as a text part and an image as an `image` part carrying its bytes. `String()` of that list is
+ * `[object Object],[object Object]`, and that is what the model was sent in place of the question and
+ * the file both.
+ */
+describe("a message with a file attached", () => {
+  const typed = { type: "text", text: "How many rows say failed?" };
+  const csv = 'Attached file "runs.csv":\n\nid,status\n1,failed\n2,ok';
+
+  function userContent(content: unknown) {
+    const [user] = withoutGuidance(
+      toProviderMessages(
+        input([{ id: "1", role: "user", content } as unknown as Message]),
+      ),
+    );
+    return user?.content;
+  }
+
+  test("keeps what the person typed and the text of the file", () => {
+    expect(userContent([typed, { type: "text", text: csv }])).toEqual([
+      { type: "text", text: "How many rows say failed?" },
+      { type: "text", text: csv },
+    ]);
+  });
+
+  test("puts an attached image in front of the model", () => {
+    const image = {
+      type: "image",
+      source: { type: "data", value: "iVBORw0KGgo=", mimeType: "image/png" },
+      metadata: { attachmentId: "a1", filename: "chart.png" },
+    };
+    expect(userContent([typed, image])).toEqual([
+      { type: "text", text: "How many rows say failed?" },
+      {
+        type: "image_url",
+        image_url: { url: "data:image/png;base64,iVBORw0KGgo=" },
+      },
+    ]);
+  });
+
+  test("names a part it cannot read rather than dropping it", () => {
+    // A model told "[audio]" can say something was attached that it cannot hear. A model handed
+    // nothing answers as though nothing was attached.
+    const audio = {
+      type: "audio",
+      source: { type: "data", value: "UklGRg==", mimeType: "audio/wav" },
+    };
+    expect(userContent([typed, audio])).toEqual([
+      { type: "text", text: "How many rows say failed?" },
+      { type: "text", text: "[audio]" },
+    ]);
+  });
+
+  test("sends a message that is only text exactly as it was typed", () => {
+    // Nearly every message. Unchanged by this, and pinned so it stays that way.
+    expect(userContent("What is 17 times 3?")).toBe("What is 17 times 3?");
+  });
+});

agent-langgraph/src/history.ts

@@ -14,6 +14,7 @@ import {
   ToolMessage,
 } from "@langchain/core/messages";
 import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt";
+import { userContent } from "../../shared/user-content";
 
 /*
  * Re-exported so this module's own tests and callers keep reading it from here, while the wording
@@ -57,7 +58,9 @@ export function toLangChainMessages(input: RunAgentInput): BaseMessage[] {
 
   for (const message of input.messages) {
     if (message.role === "user") {
-      messages.push(new HumanMessage(String(message.content ?? "")));
+      messages.push(
+        new HumanMessage({ content: userContent(message.content) }),
+      );
       continue;
     }
     if (message.role === "system" || message.role === "developer") {

agent-langgraph/tests/history.test.ts

@@ -139,3 +139,62 @@ describe("history with a tool call nobody answered", () => {
     expect(byId.get("orphan")).toBe(NO_ANSWER_CAME);
   });
 });
+
+/**
+ * A message somebody attached a file to.
+ *
+ * The composer sends it as a list of parts rather than a string: what the person typed, then the
+ * file. `copilot.ts` resolves the file before the run leaves the server, so a text file arrives here
+ * as a text part and an image as an `image` part carrying its bytes. `String()` of that list is
+ * `[object Object],[object Object]`, and that is what the model was sent in place of the question and
+ * the file both.
+ */
+describe("a message with a file attached", () => {
+  const typed = { type: "text", text: "How many rows say failed?" };
+  const csv = 'Attached file "runs.csv":\n\nid,status\n1,failed\n2,ok';
+
+  function userContent(content: unknown) {
+    return toLangChainMessages(input([{ role: "user", content }])).at(-1)
+      ?.content;
+  }
+
+  test("keeps what the person typed and the text of the file", () => {
+    expect(userContent([typed, { type: "text", text: csv }])).toEqual([
+      { type: "text", text: "How many rows say failed?" },
+      { type: "text", text: csv },
+    ]);
+  });
+
+  test("puts an attached image in front of the model", () => {
+    const image = {
+      type: "image",
+      source: { type: "data", value: "iVBORw0KGgo=", mimeType: "image/png" },
+      metadata: { attachmentId: "a1", filename: "chart.png" },
+    };
+    expect(userContent([typed, image])).toEqual([
+      { type: "text", text: "How many rows say failed?" },
+      {
+        type: "image_url",
+        image_url: { url: "data:image/png;base64,iVBORw0KGgo=" },
+      },
+    ]);
+  });
+
+  test("names a part it cannot read rather than dropping it", () => {
+    // A model told "[audio]" can say something was attached that it cannot hear. A model handed
+    // nothing answers as though nothing was attached.
+    const audio = {
+      type: "audio",
+      source: { type: "data", value: "UklGRg==", mimeType: "audio/wav" },
+    };
+    expect(userContent([typed, audio])).toEqual([
+      { type: "text", text: "How many rows say failed?" },
+      { type: "text", text: "[audio]" },
+    ]);
+  });
+
+  test("sends a message that is only text exactly as it was typed", () => {
+    // Nearly every message. Unchanged by this, and pinned so it stays that way.
+    expect(userContent("What is 17 times 3?")).toBe("What is 17 times 3?");
+  });
+});

shared/user-content.ts

@@ -0,0 +1,51 @@
+/**
+ * What a person said, in the shape a chat model reads. Read by both Bots.
+ *
+ * A message is a string until somebody attaches a file. Then the composer sends a list of parts —
+ * what the person typed, then the file — and `copilot.ts` resolves each file before the run leaves
+ * the server: a text file becomes a text part carrying its contents, an image an `image` part
+ * carrying its bytes. Both Bots read `content` with `String()`, which for that list is
+ * `[object Object],[object Object]`, so that is what the model was sent in place of the question and
+ * the file both.
+ *
+ * ONE DECLARATION FOR BOTH BOTS, for the reason `bot-prompt.ts` gives for `NO_ANSWER_CAME`. The two
+ * shapes returned are the ones OpenAI's chat completions take, and LangChain's OpenAI, Anthropic and
+ * Google converters read the same two blocks, so one answer serves `agent-bot` and `agent-langgraph`
+ * alike and the two cannot drift.
+ *
+ * A part neither shape can carry is NAMED, not dropped, which is the rule `resultText` in
+ * `server/src/plugins/mcp.ts` follows for a tool result: a model told "[audio]" can say something was
+ * attached that it cannot hear, and a model handed nothing answers as though nothing was attached.
+ */
+export type UserContentPart =
+  | { type: "text"; text: string }
+  | { type: "image_url"; image_url: { url: string } };
+
+export function userContent(content: unknown): string | UserContentPart[] {
+  if (!Array.isArray(content)) return String(content ?? "");
+  return content.map((part): UserContentPart => {
+    const item = (part ?? {}) as {
+      type?: unknown;
+      text?: unknown;
+      source?: { type?: unknown; value?: unknown; mimeType?: unknown } | null;
+    };
+    if (item.type === "text" && typeof item.text === "string") {
+      return { type: "text", text: item.text };
+    }
+    const source = item.source;
+    if (
+      item.type === "image" &&
+      source?.type === "data" &&
+      typeof source.value === "string" &&
+      typeof source.mimeType === "string"
+    ) {
+      return {
+        type: "image_url",
+        image_url: { url: `data:${source.mimeType};base64,${source.value}` },
+      };
+    }
+    const name =
+      typeof item.type === "string" && item.type ? item.type : "unknown";
+    return { type: "text", text: `[${name}]` };
+  });
+}