From 6de3efd93e5aeb1e7782a649c2580902da052407 Mon Sep 17 00:00:00 2001 From: Wout Stiens <71498452+StiensWout@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:57:04 +0200 Subject: [PATCH 01/10] feat(chat): render math across web and mobile --- .../modules/t3-markdown-text/package.json | 4 +- .../src/NativeMarkdownSelectableText.tsx | 5 + .../t3-markdown-text/src/NativeMathText.tsx | 172 +++++++++++++ .../src/SelectableMarkdownText.tsx | 9 +- .../src/nativeMarkdownMath.ts | 51 ++++ .../src/nativeMarkdownText.ts | 16 +- .../src/nativeMathDocument.ts | 34 +++ .../t3-markdown-text/src/nativeMathHtml.ts | 102 ++++++++ apps/mobile/package.json | 3 + .../mobile/src/lib/nativeMarkdownMath.test.ts | 72 ++++++ .../mobile/src/lib/nativeMathDocument.test.ts | 94 ++++++++ .../mobile/src/lib/wideMarkdownBlocks.test.ts | 6 + apps/mobile/src/lib/wideMarkdownBlocks.ts | 5 +- apps/web/package.json | 4 + apps/web/src/components/ChatMarkdown.tsx | 13 + apps/web/src/components/MarkdownMath.tsx | 61 +++++ apps/web/src/components/MathTypeset.tsx | 12 + apps/web/src/index.css | 51 ++++ apps/web/src/markdown-clipboard.ts | 12 +- apps/web/src/mathClipboard.test.ts | 51 ++++ apps/web/src/mathRendering.test.ts | 36 +++ apps/web/src/mathRendering.ts | 29 +++ docs/user/composer.md | 8 + packages/client-runtime/package.json | 5 + .../client-runtime/src/markdownMath.test.ts | 62 +++++ packages/client-runtime/src/markdownMath.ts | 227 ++++++++++++++++++ 26 files changed, 1133 insertions(+), 11 deletions(-) create mode 100644 apps/mobile/modules/t3-markdown-text/src/NativeMathText.tsx create mode 100644 apps/mobile/modules/t3-markdown-text/src/nativeMarkdownMath.ts create mode 100644 apps/mobile/modules/t3-markdown-text/src/nativeMathDocument.ts create mode 100644 apps/mobile/modules/t3-markdown-text/src/nativeMathHtml.ts create mode 100644 apps/mobile/src/lib/nativeMarkdownMath.test.ts create mode 100644 apps/mobile/src/lib/nativeMathDocument.test.ts create mode 100644 apps/web/src/components/MarkdownMath.tsx create mode 100644 apps/web/src/components/MathTypeset.tsx create mode 100644 apps/web/src/mathClipboard.test.ts create mode 100644 apps/web/src/mathRendering.test.ts create mode 100644 apps/web/src/mathRendering.ts create mode 100644 packages/client-runtime/src/markdownMath.test.ts create mode 100644 packages/client-runtime/src/markdownMath.ts diff --git a/apps/mobile/modules/t3-markdown-text/package.json b/apps/mobile/modules/t3-markdown-text/package.json index 376befbeb152..85f34bcc3051 100644 --- a/apps/mobile/modules/t3-markdown-text/package.json +++ b/apps/mobile/modules/t3-markdown-text/package.json @@ -36,9 +36,11 @@ "expo-clipboard": "*", "expo-haptics": "*", "expo-symbols": "*", + "mathjax-full": "*", "react": "*", "react-native": "*", - "react-native-nitro-markdown": "*" + "react-native-nitro-markdown": "*", + "react-native-webview": "*" }, "codegenConfig": { "name": "T3MarkdownTextSpec", diff --git a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.tsx b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.tsx index 50a381bae288..9b5ec79a2a93 100644 --- a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.tsx +++ b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.tsx @@ -1,5 +1,6 @@ import { createContext, useCallback, useContext, useMemo } from "react"; import { decodeComposerContextFragment } from "@t3tools/shared/composerContextClipboard"; +import { NativeMathText } from "./NativeMathText"; import { findNodeHandle, Image, @@ -326,6 +327,10 @@ export function NativeMarkdownSelectableText(props: { props.textStyle.contextChipBorderColor, ].join(":"); + if (props.runs.some((run) => run.mathSource !== undefined)) { + return ; + } + return ( | undefined; +function loadRenderer() { + return (rendererPromise ??= import("./nativeMathHtml").catch((error) => { + rendererPromise = undefined; + throw error; + })); +} + +/** One view per math-containing text chunk preserves inline layout and selection on both OSes. */ +export const NativeMathText = memo(function NativeMathText(props: { + readonly runs: ReadonlyArray; + readonly textStyle: NativeMarkdownTextStyle; + readonly onLinkPress?: (href: string) => void; + readonly fileContextMenu?: (href: string) => MarkdownFileContextMenu | undefined; + readonly onFileContextMenuAction?: (href: string, actionId: string) => void; +}) { + const webView = useRef(null); + const ready = useRef(false); + const [height, setHeight] = useState(props.textStyle.lineHeight); + const [failed, setFailed] = useState(false); + const [renderer, setRenderer] = useState(); + const { fontScale } = useWindowDimensions(); + useEffect(() => { + let active = true; + void loadRenderer().then( + (module) => { + if (active) setRenderer(module); + }, + () => { + if (active) setFailed(true); + }, + ); + return () => { + active = false; + }; + }, []); + const fileContextMenu = props.fileContextMenu; + const update = useMemo(() => { + if (!renderer) return null; + const style = { + ...props.textStyle, + fontSize: props.textStyle.fontSize * fontScale, + lineHeight: props.textStyle.lineHeight * fontScale, + headingFontSizes: props.textStyle.headingFontSizes?.map((size) => size * fontScale), + }; + return { + runs: props.runs.map((run) => + renderer.nativeMathRunHtml(run, style, run.href ? fileContextMenu?.(run.href) : undefined), + ), + color: style.color, + fontSize: style.fontSize, + lineHeight: style.lineHeight, + }; + }, [renderer, props.runs, props.textStyle, fileContextMenu, fontScale]); + const revision = useRef(0); + const latest = useRef(update); + useEffect(() => { + latest.current = update; + if (ready.current && update) { + revision.current += 1; + webView.current?.injectJavaScript( + `window.updateMath(${JSON.stringify({ ...update, revision: revision.current })});true;`, + ); + } + }, [update]); + + if (failed || !renderer) + return ( + + {props.runs.map((run) => run.text).join("")} + + ); + return ( + request.url === "about:blank"} + scrollEnabled={false} + bounces={false} + showsVerticalScrollIndicator={false} + dataDetectorTypes="none" + allowFileAccess={false} + setSupportMultipleWindows={false} + style={{ height, backgroundColor: "transparent", flex: 0 }} + onError={() => setFailed(true)} + onContentProcessDidTerminate={() => setFailed(true)} + onRenderProcessGone={() => setFailed(true)} + onMessage={(event) => { + let message: unknown; + try { + message = JSON.parse(event.nativeEvent.data); + } catch { + return; + } + if (!message || typeof message !== "object" || !("type" in message)) return; + if (message.type === "ready") { + ready.current = true; + if (latest.current) { + revision.current += 1; + webView.current?.injectJavaScript( + `window.updateMath(${JSON.stringify({ ...latest.current, revision: revision.current })});true;`, + ); + } + return; + } + if (!("revision" in message) || message.revision !== revision.current) return; + if ( + message.type === "height" && + "height" in message && + typeof message.height === "number" && + Number.isFinite(message.height) && + message.height > 0 + ) + setHeight(message.height); + if (message.type === "copy" && "text" in message && typeof message.text === "string") { + const text = message.text; + const copyRevision = revision.current; + void setStringAsync(text).then( + () => { + if (revision.current === copyRevision) + webView.current?.injectJavaScript( + `window.mathCopyResult(${JSON.stringify(text)});true;`, + ); + }, + () => Alert.alert("Could not copy text"), + ); + } + if ( + message.type === "file-action" && + "href" in message && + typeof message.href === "string" && + "action" in message && + typeof message.action === "string" + ) { + const href = message.href; + const actionId = message.action; + if ( + props.runs.some((run) => run.href === href) && + props + .fileContextMenu?.(href) + ?.actions.some((action) => action.id === actionId && !action.disabled) + ) + props.onFileContextMenuAction?.(href, actionId); + } + if (message.type === "link" && "href" in message && typeof message.href === "string") { + if (props.onLinkPress) props.onLinkPress(message.href); + else if (/^https?:\/\//i.test(message.href)) void Linking.openURL(message.href); + } + }} + /> + ); +}); diff --git a/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.tsx b/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.tsx index 8e59256b12b8..8053088c140e 100644 --- a/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.tsx +++ b/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.tsx @@ -1,3 +1,4 @@ +import { parseNativeMarkdownMath } from "./nativeMarkdownMath"; import { useMemo } from "react"; import { View } from "react-native"; import { parseMarkdownWithOptions } from "react-native-nitro-markdown/headless"; @@ -51,11 +52,9 @@ export function SelectableMarkdownText({ marginBottom = 0, }: SelectableMarkdownTextProps) { const chunks = useMemo(() => { - const parsedDocument = parseMarkdownWithOptions(markdown, { - gfm: true, - html: true, - math: false, - }); + const parsedDocument = parseNativeMarkdownMath(markdown, (source) => + parseMarkdownWithOptions(source, { gfm: true, html: true, math: false }), + ); const document = preserveSoftBreaks ? nativeMarkdownWithPreservedSoftBreaks(parsedDocument) : parsedDocument; diff --git a/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownMath.ts b/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownMath.ts new file mode 100644 index 000000000000..c6a6a0a36110 --- /dev/null +++ b/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownMath.ts @@ -0,0 +1,51 @@ +import { markdownMathRanges } from "@t3tools/client-runtime/markdown-math"; +import type { MarkdownNode } from "react-native-nitro-markdown/headless"; + +/** Shield math from the native parser, then restore typed nodes using the shared grammar. */ +export function parseNativeMarkdownMath( + source: string, + parse: (markdown: string) => MarkdownNode, +): MarkdownNode { + const ranges = markdownMathRanges(source); + if (ranges.length === 0) return parse(source); + let prefix = ":t3-math-"; + while (source.includes(prefix)) prefix += "-"; + const mathByMarker = new Map(ranges.map((math, index) => [`${prefix}${index}:`, math])); + // Text markers cannot merge with neighboring Markdown code-span delimiters. + let cursor = 0; + let protectedSource = ""; + for (const [marker, math] of mathByMarker) { + protectedSource += source.slice(cursor, math.start) + marker; + cursor = math.end; + } + protectedSource += source.slice(cursor); + const markerPattern = new RegExp(`${prefix}\\d+:`, "g"); + function restore(node: MarkdownNode): MarkdownNode { + if (!node.children) return node; + return { + ...node, + children: node.children.flatMap((child) => { + if (child.type !== "text" || !child.content) return [restore(child)]; + const restored: MarkdownNode[] = []; + let offset = 0; + for (const match of child.content.matchAll(markerPattern)) { + const math = mathByMarker.get(match[0]); + if (!math) continue; + if (match.index > offset) + restored.push({ ...child, content: child.content.slice(offset, match.index) }); + restored.push({ + type: math.math ? (math.math.display ? "math_block" : "math_inline") : "text", + content: math.source, + beg: math.start, + end: math.end, + }); + offset = match.index + match[0].length; + } + if (offset < child.content.length) + restored.push({ ...child, content: child.content.slice(offset) }); + return restored; + }), + }; + } + return restore(parse(protectedSource)); +} diff --git a/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts b/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts index ec8cf74fee2b..dc1556c6fe85 100644 --- a/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts +++ b/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts @@ -131,6 +131,7 @@ import { export interface NativeMarkdownTextRun { readonly text: string; + readonly mathSource?: string; readonly bold?: boolean; readonly italic?: boolean; readonly strikethrough?: boolean; @@ -259,6 +260,8 @@ function inlineHtmlText(value: string): string { function sameRunStyle(left: NativeMarkdownTextRun, right: NativeMarkdownTextRun): boolean { return ( + left.mathSource === undefined && + right.mathSource === undefined && left.bold === right.bold && left.italic === right.italic && left.strikethrough === right.strikethrough && @@ -344,7 +347,7 @@ function decorateSkillRuns( const decorated: NativeMarkdownTextRun[] = []; for (const run of runs) { - if (run.code || run.href || run.fileIcon || run.role === "code-block") { + if (run.mathSource || run.code || run.href || run.fileIcon || run.role === "code-block") { decorated.push(run); continue; } @@ -436,8 +439,15 @@ function appendNode( context: RunContext, ): NativeMarkdownTextRun[] { switch (node.type) { - case "text": case "math_inline": + case "math_block": + runs.push({ + text: nodeTextContent(node), + mathSource: nodeTextContent(node), + ...(context.href ? { href: context.href } : {}), + }); + return runs; + case "text": return appendRun(runs, textNodeContent(nodeTextContent(node)), context); case "html_inline": return appendRun(runs, inlineHtmlText(nodeTextContent(node)), context); @@ -839,7 +849,7 @@ function appendDocumentBlock( }); return appendBlockTerminator(runs, { ...EMPTY_CONTEXT, role: "body", depth }); case "math_block": - appendRun(runs, nodeTextContent(node), { ...EMPTY_CONTEXT, role: "body", depth }); + appendNode(runs, node, { ...EMPTY_CONTEXT, role: "body", depth }); return appendBlockTerminator(runs, { ...EMPTY_CONTEXT, role: "body", depth }); default: appendInlineChildren(runs, node, { ...EMPTY_CONTEXT, role: "body", depth }); diff --git a/apps/mobile/modules/t3-markdown-text/src/nativeMathDocument.ts b/apps/mobile/modules/t3-markdown-text/src/nativeMathDocument.ts new file mode 100644 index 000000000000..2799f20aaff9 --- /dev/null +++ b/apps/mobile/modules/t3-markdown-text/src/nativeMathDocument.ts @@ -0,0 +1,34 @@ +// This document is installed once. Updates reconcile runs so streaming does not +// reload the WebView or recreate completed equations and open source disclosures. +export const NATIVE_MATH_DOCUMENT = `
`; diff --git a/apps/mobile/modules/t3-markdown-text/src/nativeMathHtml.ts b/apps/mobile/modules/t3-markdown-text/src/nativeMathHtml.ts new file mode 100644 index 000000000000..68e9988620f2 --- /dev/null +++ b/apps/mobile/modules/t3-markdown-text/src/nativeMathHtml.ts @@ -0,0 +1,102 @@ +import { markdownMath } from "@t3tools/client-runtime/markdown-math"; +import { mathjax } from "mathjax-full/js/mathjax.js"; +import { TeX } from "mathjax-full/js/input/tex.js"; +import { SVG } from "mathjax-full/js/output/svg.js"; +import { liteAdaptor } from "mathjax-full/js/adaptors/liteAdaptor.js"; +import { AssistiveMmlHandler } from "mathjax-full/js/a11y/assistive-mml.js"; +import { RegisterHTMLHandler } from "mathjax-full/js/handlers/html.js"; +import "mathjax-full/js/input/tex/ams/AmsConfiguration.js"; +import type { NativeMarkdownTextRun } from "./nativeMarkdownText"; +import type { + MarkdownFileContextMenu, + NativeMarkdownTextStyle, +} from "./SelectableMarkdownText.types"; + +const adaptor = liteAdaptor(); +AssistiveMmlHandler(RegisterHTMLHandler(adaptor)); +const tex = new TeX({ packages: ["base", "ams"], maxBuffer: 16_384, maxMacros: 1000 }); +const renderer = mathjax.document("", { InputJax: tex, OutputJax: new SVG({ fontCache: "none" }) }); +const cache = new Map(); +let cacheSize = 0; + +function escapeHtml(text: string): string { + return text.replace( + /[&<>"']/g, + (character) => + ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[character]!, + ); +} + +/** Self-contained SVG needs no font downloads, browser typesetter or network access. */ +export function nativeMathSvg(source: string): string | null { + if (cache.has(source)) { + const cached = cache.get(source) ?? null; + cache.delete(source); + cache.set(source, cached); + return cached; + } + const math = markdownMath(source); + if (!math) return null; + let svg: string | null = null; + try { + renderer.reset(); + tex.reset(); + const node = renderer.convert(math.tex, { display: math.display }); + const output = adaptor.outerHTML(node); + if (!output.includes('data-mml-node="merror"')) svg = output; + } catch { + // Leave unsupported or malformed math as its original source. + } + const size = source.length + (svg?.length ?? 0); + if (size <= 1_000_000) { + while (cache.size >= 128 || cacheSize + size > 1_000_000) { + const oldest = cache.entries().next().value; + if (!oldest) break; + cacheSize -= oldest[0].length + (oldest[1]?.length ?? 0); + cache.delete(oldest[0]); + } + cache.set(source, svg); + cacheSize += size; + } + return svg; +} + +/** Render typed native text runs, never raw Markdown HTML, inside the math text view. */ +export function nativeMathRunHtml( + run: NativeMarkdownTextRun, + style: NativeMarkdownTextStyle, + menu?: MarkdownFileContextMenu, +): string { + if (run.mathSource) { + const math = markdownMath(run.mathSource); + const source = escapeHtml(run.mathSource); + const svg = nativeMathSvg(run.mathSource); + const equation = `${svg ?? source}`; + return math?.display + ? `${equation}` + : run.href + ? `${equation}` + : equation; + } + const heading = run.role === "heading"; + const fontSize = heading + ? (style.headingFontSizes?.[(run.headingLevel ?? 1) - 1] ?? style.fontSize * 1.3) + : style.fontSize; + const color = run.href + ? style.linkColor + : run.code + ? style.inlineCodeColor + : heading || run.bold + ? style.strongColor + : style.color; + const css = `color:${color};font-size:${fontSize}px;font-weight:${heading || run.bold ? 700 : 400};font-style:${run.italic ? "italic" : "normal"};font-family:${run.code ? "monospace" : "inherit"};text-decoration:${run.strikethrough ? "line-through" : "none"}`; + const content = escapeHtml(run.text); + if (run.href) { + const href = escapeHtml(run.href); + const actions = menu + ? `` + : ""; + return `${content}${actions}`; + } + return `${content}`; +} diff --git a/apps/mobile/package.json b/apps/mobile/package.json index 0fd35faf7528..31284b0804b4 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -107,6 +107,7 @@ "expo-video": "~57.0.3", "expo-web-browser": "~57.0.2", "expo-widgets": "~57.0.15", + "mathjax-full": "^3.2.2", "react": "19.2.3", "react-dom": "19.2.3", "react-native": "0.86.3", @@ -129,9 +130,11 @@ "devDependencies": { "@effect/vitest": "catalog:", "@pierre/trees": "1.0.0-beta.4", + "@types/jsdom": "^30.0.0", "@types/react": "~19.2.0", "@types/react-dom": "~19.2.3", "babel-preset-expo": "~57.0.9", + "jsdom": "^30.0.1", "tailwindcss": "^4.0.0", "typescript": "catalog:" }, diff --git a/apps/mobile/src/lib/nativeMarkdownMath.test.ts b/apps/mobile/src/lib/nativeMarkdownMath.test.ts new file mode 100644 index 000000000000..1147373e5a31 --- /dev/null +++ b/apps/mobile/src/lib/nativeMarkdownMath.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vite-plus/test"; +import { parseNativeMarkdownMath } from "../../modules/t3-markdown-text/src/nativeMarkdownMath"; +import { + nativeMathSvg, + nativeMathRunHtml, +} from "../../modules/t3-markdown-text/src/nativeMathHtml"; +import { nativeMarkdownDocumentRuns } from "../../modules/t3-markdown-text/src/nativeMarkdownText"; + +describe("native math", () => { + it("uses shared math recognition and keeps code and currency literal", () => { + let input = ""; + const source = String.raw`Use \(x_i\), keep ` + "`$code$` and pay $20 or $30."; + const tree = parseNativeMarkdownMath(source, (markdown) => { + input = markdown; + return { type: "paragraph", children: [{ type: "text", content: ":t3-math-0:" }] }; + }); + expect(input).toContain(":t3-math-0:"); + expect(input).toContain("`$code$`"); + expect(input).toContain("$20 or $30"); + expect(nativeMarkdownDocumentRuns(tree)).toEqual([ + { text: String.raw`\(x_i\)`, mathSource: String.raw`\(x_i\)` }, + ]); + }); + + it.each([ + String.raw`$x_i^2+\alpha$`, + String.raw`\(\frac{a}{b}\)`, + String.raw`$$\begin{pmatrix}1&2\\3&4\end{pmatrix}$$`, + String.raw`\[\begin{aligned}x&=1\\y&=2\end{aligned}\]`, + ])("produces a self-contained equation for %s", (source) => { + const svg = nativeMathSvg(source); + expect(svg).toContain(" { + expect(nativeMathSvg(String.raw`$\frac{$`)).toBeNull(); + nativeMathSvg(String.raw`$\gdef\privateMacro{x}\privateMacro$`); + expect(nativeMathSvg(String.raw`$\privateMacro$`)).toBeNull(); + }); + + it("keeps original TeX in the selection-copy contract", () => { + const source = String.raw`\[x < y\]`; + const html = nativeMathRunHtml( + { text: source, mathSource: source }, + { + color: "#fff", + strongColor: "#fff", + mutedColor: "#aaa", + linkColor: "#acf", + inlineCodeColor: "#fff", + codeColor: "#fff", + codeBackgroundColor: "#111", + codeBlockBackgroundColor: "#111", + fileTextColor: "#fff", + skillTextColor: "#fff", + quoteMarkerColor: "#888", + dividerColor: "#444", + fontSize: 15, + lineHeight: 22, + fontFamily: "system-ui", + headingFontFamily: "system-ui", + boldFontFamily: "system-ui", + }, + ); + expect(html).toContain('data-source="\\[x < y\\]"'); + expect(html).toContain('data-copy="\\[x < y\\]"'); + }); +}); diff --git a/apps/mobile/src/lib/nativeMathDocument.test.ts b/apps/mobile/src/lib/nativeMathDocument.test.ts new file mode 100644 index 000000000000..9ed741498c66 --- /dev/null +++ b/apps/mobile/src/lib/nativeMathDocument.test.ts @@ -0,0 +1,94 @@ +import { JSDOM } from "jsdom"; +import { describe, expect, it } from "vite-plus/test"; +import { NATIVE_MATH_DOCUMENT } from "../../modules/t3-markdown-text/src/nativeMathDocument"; + +function fixture() { + const messages: unknown[] = []; + const dom = new JSDOM(NATIVE_MATH_DOCUMENT, { + runScripts: "dangerously", + beforeParse(window) { + Object.defineProperty(window, "ReactNativeWebView", { + value: { postMessage: (message: string) => messages.push(JSON.parse(message)) }, + }); + Object.defineProperty(window, "ResizeObserver", { + value: class { + observe() {} + }, + }); + }, + }); + const update = (runs: string[], revision: number) => + dom.window.eval( + `window.updateMath(${JSON.stringify({ runs, revision, color: "white", fontSize: 15, lineHeight: 22 })})`, + ); + return { dom, messages, update }; +} + +describe("native math text bridge", () => { + it("keeps a completed equation and its open source while adjacent text streams", () => { + const { dom, update, messages } = fixture(); + try { + const equation = + 'x'; + update([equation, "Starting"], 1); + const original = dom.window.document.querySelector(".equation"); + dom.window.document.querySelector("button")!.click(); + update([equation, "Starting a longer answer"], 2); + expect(dom.window.document.querySelector(".equation")).toBe(original); + expect(dom.window.document.querySelector(".source")!.hidden).toBe(false); + expect(messages).toContainEqual({ type: "ready", revision: 0 }); + expect(messages).toContainEqual({ type: "height", revision: 2, height: 0 }); + } finally { + dom.window.close(); + } + }); + + it("lets file menus close without taking an action", () => { + const { dom, update, messages } = fixture(); + try { + update( + [ + ``, + ], + 1, + ); + const { document } = dom.window; + const button = document.querySelector("button")!; + button.click(); + expect(document.querySelector('[role="menu"]')).not.toBeNull(); + button.click(); + expect(document.querySelector('[role="menu"]')).toBeNull(); + button.click(); + document.dispatchEvent(new dom.window.KeyboardEvent("keydown", { key: "Escape" })); + expect(document.querySelector('[role="menu"]')).toBeNull(); + button.click(); + document.body.click(); + expect(document.querySelector('[role="menu"]')).toBeNull(); + expect(messages).not.toContainEqual(expect.objectContaining({ type: "file-action" })); + } finally { + dom.window.close(); + } + }); + + it("copies a selected equation as TeX and forwards links to the app", () => { + const { dom, update, messages } = fixture(); + try { + update( + [ + 'Before x after reference', + ], + 1, + ); + const { document } = dom.window; + const range = document.createRange(); + range.selectNodeContents(document.querySelector(".equation span")!); + dom.window.getSelection()!.addRange(range); + document.dispatchEvent(new dom.window.Event("copy", { cancelable: true })); + expect(messages).toContainEqual({ type: "copy", revision: 1, text: String.raw`\(x\)` }); + document.querySelector("a")!.click(); + expect(messages).toContainEqual({ type: "link", revision: 1, href: "https://example.com" }); + } finally { + dom.window.close(); + } + }); +}); diff --git a/apps/mobile/src/lib/wideMarkdownBlocks.test.ts b/apps/mobile/src/lib/wideMarkdownBlocks.test.ts index 4f745aed8851..ecd99083afa2 100644 --- a/apps/mobile/src/lib/wideMarkdownBlocks.test.ts +++ b/apps/mobile/src/lib/wideMarkdownBlocks.test.ts @@ -3,6 +3,12 @@ import { describe, expect, it } from "vite-plus/test"; import { hasWideMarkdownBlock } from "./wideMarkdownBlocks"; describe("hasWideMarkdownBlock", () => { + it("gives math a definite width without expanding currency or literal code", () => { + expect(hasWideMarkdownBlock("An equation $x_i$ here")).toBe(true); + expect(hasWideMarkdownBlock(String.raw`\[x+y\]`)).toBe(true); + expect(hasWideMarkdownBlock("Costs $20 or $30")).toBe(false); + expect(hasWideMarkdownBlock("`$x$`")).toBe(false); + }); it("ignores prose, inline code, and emphasis", () => { expect(hasWideMarkdownBlock("just a message")).toBe(false); expect(hasWideMarkdownBlock("I found it in `secteurs_intervention` earlier")).toBe(false); diff --git a/apps/mobile/src/lib/wideMarkdownBlocks.ts b/apps/mobile/src/lib/wideMarkdownBlocks.ts index 054c60be087e..1eca81cf86d1 100644 --- a/apps/mobile/src/lib/wideMarkdownBlocks.ts +++ b/apps/mobile/src/lib/wideMarkdownBlocks.ts @@ -1,6 +1,8 @@ +import { markdownMathRanges } from "@t3tools/client-runtime/markdown-math"; + /** * Detects markdown that the renderer draws as a block requiring a definite - * user-bubble width: fenced and indented code blocks, GFM tables, ordered + * user-bubble width: math-containing text, fenced and indented code blocks, GFM tables, ordered * lists, and blockquotes when requested by the caller. * * Fenced code blocks and tables report an intrinsic width equal to their @@ -121,6 +123,7 @@ export function hasWideMarkdownBlock( if (options.includeOrderedLists !== false && hasOrderedListItem(text)) { return true; } + if (markdownMathRanges(text).some((range) => range.math !== null)) return true; if (!text.includes("|")) { return false; } diff --git a/apps/web/package.json b/apps/web/package.json index 16579e730f1b..3e6d6b1bcd07 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -40,6 +40,7 @@ "jose": "catalog:", "jsonc-parser": "3.3.1", "jszip": "3.10.1", + "katex": "^0.16.22", "lexical": "^0.41.0", "lucide-react": "^0.564.0", "react": "19.2.6", @@ -62,6 +63,8 @@ "@types/compression": "^1.8.1", "@types/culori": "^4.0.1", "@types/mdast": "^4.0.4", + "@types/jsdom": "^30.0.0", + "@types/katex": "^0.16.7", "@types/react": "~19.2.14", "@types/react-dom": "~19.2.3", "@types/react-test-renderer": "19.1.0", @@ -69,6 +72,7 @@ "@vitejs/plugin-react": "^6.0.0", "babel-plugin-react-compiler": "1.0.0", "compression": "^1.8.1", + "jsdom": "^30.0.1", "react-test-renderer": "19.2.6", "tailwindcss": "^4.0.0", "unified": "^11.0.5", diff --git a/apps/web/src/components/ChatMarkdown.tsx b/apps/web/src/components/ChatMarkdown.tsx index 3aeaaa5b8443..2277af4d0edd 100644 --- a/apps/web/src/components/ChatMarkdown.tsx +++ b/apps/web/src/components/ChatMarkdown.tsx @@ -1,4 +1,6 @@ import { usePullRequestLinking } from "~/hooks/usePullRequestLinking"; +import { remarkMath } from "@t3tools/client-runtime/markdown-math"; +import { MarkdownMath } from "./MarkdownMath"; import { useAtomValue } from "@effect/atom-react"; import { COMPOSER_CONTEXT_CLIPBOARD_MIME, @@ -462,6 +464,7 @@ const CHAT_MARKDOWN_SANITIZE_SCHEMA = { attributes: { ...defaultSchema.attributes, "*": (defaultSchema.attributes?.["*"] ?? []).filter((attribute) => attribute !== "title"), + span: [...(defaultSchema.attributes?.span ?? []), "dataMathSource"], code: [...(defaultSchema.attributes?.code ?? []), "dataCodeMeta", "dataInlineCode"], blockquote: [...(defaultSchema.attributes?.blockquote ?? []), "dataAlert"], div: [...(defaultSchema.attributes?.div ?? []), ...CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES], @@ -482,6 +485,7 @@ const CHAT_MARKDOWN_SANITIZE_SCHEMA = { const CHAT_MARKDOWN_REMARK_PLUGINS = [ remarkGfm, + remarkMath, remarkGithubAlerts, remarkNormalizeListItemIndentation, remarkCodexDirectives, @@ -491,6 +495,7 @@ const CHAT_MARKDOWN_REMARK_PLUGINS = [ const CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS = [ remarkGfm, + remarkMath, remarkGithubAlerts, remarkNormalizeListItemIndentation, remarkCodexDirectives, @@ -2717,6 +2722,14 @@ const CHAT_MARKDOWN_COMPONENTS = { h4: markdownHeadingRenderer(4), h5: markdownHeadingRenderer(5), h6: markdownHeadingRenderer(6), + span: ({ node, children, ...props }) => { + const source = node?.properties.dataMathSource; + return typeof source === "string" ? ( + + ) : ( + {children} + ); + }, div: function MarkdownDiv({ node, children, ...props }) { const { onUseArtifactTemplate } = use(ChatMarkdownRendererContext); const artifactTemplate = artifactTemplateFromHastProperties(node?.properties); diff --git a/apps/web/src/components/MarkdownMath.tsx b/apps/web/src/components/MarkdownMath.tsx new file mode 100644 index 000000000000..3247d2d666f3 --- /dev/null +++ b/apps/web/src/components/MarkdownMath.tsx @@ -0,0 +1,61 @@ +import { lazy, memo, Suspense, useState } from "react"; +import { markdownMath } from "@t3tools/client-runtime/markdown-math"; +import { RenderErrorBoundary } from "./RenderErrorBoundary"; +import { toastManager } from "./ui/toast"; + +// The renderer and its fonts are loaded only when a message contains math. +const MathTypeset = lazy(() => import("./MathTypeset")); + +export const MarkdownMath = memo(function MarkdownMath({ source }: { source: string }) { + const math = markdownMath(source); + const [showSource, setShowSource] = useState(false); + if (!math) return <>{source}; + const fallback = {source}; + return ( + + {math.display ? ( + + + + + ) : null} + + + + + + + + {showSource && math.display ? {source} : null} + + ); +}); diff --git a/apps/web/src/components/MathTypeset.tsx b/apps/web/src/components/MathTypeset.tsx new file mode 100644 index 000000000000..2698dde28f5f --- /dev/null +++ b/apps/web/src/components/MathTypeset.tsx @@ -0,0 +1,12 @@ +import { memo } from "react"; +import { renderMathHtml } from "../mathRendering"; +import "katex/dist/katex.min.css"; + +export default memo(function MathTypeset({ source }: { source: string }) { + const html = renderMathHtml(source); + return html === null ? ( + {source} + ) : ( + + ); +}); diff --git a/apps/web/src/index.css b/apps/web/src/index.css index a7f6d91f2e40..a618255d51cc 100644 --- a/apps/web/src/index.css +++ b/apps/web/src/index.css @@ -2104,3 +2104,54 @@ code { visibility: visible; } } + +/* Math shares the prose layout; only a wide equation owns horizontal overflow. */ +.markdown-math { + color: inherit; +} +.markdown-math-display { + display: block; + position: relative; + margin-block: 1.25em; + min-width: 0; +} +.markdown-math-display .markdown-math-viewport { + display: block; + overflow-x: auto; + padding-block: 0.5em; +} +.markdown-math .katex-display { + margin: 0; +} +.markdown-math .katex-display > .katex { + display: block; + width: max-content; + min-width: 100%; +} +.markdown-math-actions { + display: flex; + justify-content: flex-end; + gap: 0.9em; + font-size: 0.75rem; + opacity: 0; +} +.markdown-math-actions button { + cursor: pointer; +} +.markdown-math:hover .markdown-math-actions, +.markdown-math:focus-within .markdown-math-actions { + opacity: 1; +} +.markdown-math-source { + display: block; + white-space: pre-wrap; + overflow-wrap: anywhere; + font-family: var(--font-mono); + font-size: 0.75rem; + margin-top: 0.5em; +} +@media (hover: none) { + .markdown-math-actions { + opacity: 1; + } +} diff --git a/apps/web/src/markdown-clipboard.ts b/apps/web/src/markdown-clipboard.ts index 4a96c8b31d13..2ec17e0d5b2e 100644 --- a/apps/web/src/markdown-clipboard.ts +++ b/apps/web/src/markdown-clipboard.ts @@ -384,8 +384,18 @@ export function chatMarkdownClipboardPayload( const texts: string[] = []; const htmls: string[] = []; for (let index = 0; index < selection.rangeCount; index += 1) { - const range = selection.getRangeAt(index); + const range = selection.getRangeAt(index).cloneRange(); if (range.collapsed) continue; + // A fraction may put each endpoint several spans inside the rendered math. + // Copy the equation as one source token, never as flattened glyphs. + const mathAt = (node: Node) => + (node.nodeType === Node.ELEMENT_NODE ? (node as Element) : node.parentElement)?.closest( + "[data-markdown-math]", + ); + const startMath = mathAt(range.startContainer); + const endMath = mathAt(range.endContainer); + if (startMath) range.setStartBefore(startMath); + if (endMath) range.setEndAfter(endMath); const container = document.createElement("div"); container.appendChild(range.cloneContents()); const ancestor = range.commonAncestorContainer; diff --git a/apps/web/src/mathClipboard.test.ts b/apps/web/src/mathClipboard.test.ts new file mode 100644 index 000000000000..39fa9936ce1d --- /dev/null +++ b/apps/web/src/mathClipboard.test.ts @@ -0,0 +1,51 @@ +import { JSDOM } from "jsdom"; +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; +import { chatMarkdownClipboardPayload } from "./markdown-clipboard"; +import { renderMathHtml } from "./mathRendering"; + +const windows: JSDOM[] = []; +afterEach(() => { + for (const dom of windows.splice(0)) dom.window.close(); + vi.unstubAllGlobals(); +}); + +function fixture() { + const dom = new JSDOM("

Before after.

"); + windows.push(dom); + const { document } = dom.window; + vi.stubGlobal("document", document); + vi.stubGlobal("Node", dom.window.Node); + const math = document.querySelector("[data-markdown-math]")!; + math.setAttribute("data-markdown-copy", String.raw`\(\frac{x}{y}\)`); + math.innerHTML = renderMathHtml(String.raw`\(\frac{x}{y}\)`) ?? ""; + return { document, math, selection: dom.window.getSelection()! }; +} + +describe("copying rendered equations", () => { + it("copies the original TeX when selection is inside a fraction", () => { + const { document, math, selection } = fixture(); + const numerator = [...math.querySelectorAll(".mord")].find((node) => node.textContent === "x")!; + const range = document.createRange(); + range.selectNodeContents(numerator); + selection.addRange(range); + expect(chatMarkdownClipboardPayload(selection)?.text).toBe(String.raw`\(\frac{x}{y}\)`); + }); + + it("preserves surrounding prose without duplicating accessible math", () => { + const { document, selection } = fixture(); + const range = document.createRange(); + range.selectNodeContents(document.querySelector("p")!); + selection.addRange(range); + expect(chatMarkdownClipboardPayload(selection)?.text).toBe( + String.raw`Before \(\frac{x}{y}\) after.`, + ); + }); + + it("does not copy a collapsed caret inside an equation", () => { + const { document, math, selection } = fixture(); + const range = document.createRange(); + range.setStart(math, 0); + selection.addRange(range); + expect(chatMarkdownClipboardPayload(selection)).toBeNull(); + }); +}); diff --git a/apps/web/src/mathRendering.test.ts b/apps/web/src/mathRendering.test.ts new file mode 100644 index 000000000000..c247f1c5f942 --- /dev/null +++ b/apps/web/src/mathRendering.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vite-plus/test"; +import { renderMathHtml } from "./mathRendering"; + +describe("math rendering", () => { + it.each([ + String.raw`$x_i^2 + \alpha$`, + String.raw`\(\frac{a}{b}\)`, + String.raw`\[\sum_{i=1}^n x_i\]`, + String.raw`$$\begin{pmatrix}1 & 2 \\ 3 & 4\end{pmatrix}$$`, + String.raw`$$\begin{aligned}x &= 1 \\ y &= 2\end{aligned}$$`, + ])("renders accessible math for %s", (source) => { + const html = renderMathHtml(source); + expect(html).toContain(' { + expect(renderMathHtml(source)).toBeNull(); + expect(renderMathHtml("$x$")).toContain(" { + const html = renderMathHtml(String.raw`$\href{javascript:alert(1)}{click}$`); + expect(html ?? "").not.toContain("href="); + expect(renderMathHtml(String.raw`$\htmlClass{bad}{x}$`)).toBeNull(); + }); + + it("does not share macro definitions between equations", () => { + renderMathHtml(String.raw`$\gdef\privateMacro{x}\privateMacro$`); + expect(renderMathHtml(String.raw`$\privateMacro$`)).toBeNull(); + }); +}); diff --git a/apps/web/src/mathRendering.ts b/apps/web/src/mathRendering.ts new file mode 100644 index 000000000000..72a4c70ef73c --- /dev/null +++ b/apps/web/src/mathRendering.ts @@ -0,0 +1,29 @@ +import katex from "katex"; +import { markdownMath } from "@t3tools/client-runtime/markdown-math"; +import { LRUCache } from "./lib/lruCache"; + +const cache = new LRUCache<{ html: string | null }>(128, 2 * 1024 * 1024); + +/** Only generated KaTeX markup reaches innerHTML. Authored HTML never enters this path. */ +export function renderMathHtml(source: string): string | null { + const cached = cache.get(source); + if (cached) return cached.html; + const math = markdownMath(source); + if (!math) return null; + let html: string | null = null; + try { + html = katex.renderToString(math.tex, { + displayMode: math.display, + output: "htmlAndMathml", + throwOnError: true, + strict: "error", + trust: false, + maxExpand: 1000, + maxSize: 20, + }); + } catch { + // Malformed and unsupported expressions remain readable and copyable TeX. + } + cache.set(source, { html }, (source.length + (html?.length ?? 0)) * 2); + return html; +} diff --git a/docs/user/composer.md b/docs/user/composer.md index 4da452215893..601df430d9aa 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -210,3 +210,11 @@ automatically. HTML previews cannot access your T3 Code session. On mobile, select a PDF attachment or link to open it. iOS uses the native viewer; Android opens a compatible installed file viewer. + + +## Math in messages + +Inline math uses `$...$` or `\(...\)`. Display equations use `$$...$$` or `\[...\]`. +Web, desktop, iOS, and Android render supported TeX, including fractions, matrices, and aligned equations. Wide equations scroll horizontally. + +Use **Copy TeX** on a display equation to reuse its original source. Selecting math or copying the whole response also preserves the TeX. Code stays literal, and invalid or unfinished expressions remain readable as source. diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index b4a3d5da118d..d3d581a72d0f 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -254,6 +254,10 @@ "./file-preview": { "types": "./src/filePreview.ts", "default": "./src/filePreview.ts" + }, + "./markdown-math": { + "types": "./src/markdownMath.ts", + "default": "./src/markdownMath.ts" } }, "scripts": { @@ -272,6 +276,7 @@ }, "devDependencies": { "@effect/vitest": "catalog:", + "@types/mdast": "^4.0.4", "micromark-util-types": "^2.0.2", "vite-plus": "catalog:" } diff --git a/packages/client-runtime/src/markdownMath.test.ts b/packages/client-runtime/src/markdownMath.test.ts new file mode 100644 index 000000000000..ddc08c085d20 --- /dev/null +++ b/packages/client-runtime/src/markdownMath.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vite-plus/test"; +import remarkParse from "remark-parse"; +import { unified } from "unified"; +import { markdownMathRanges, remarkMath } from "./markdownMath.ts"; + +const parser = unified().use(remarkParse).use(remarkMath); + +describe("Markdown math", () => { + it.each([ + ["$x_i^2$", "x_i^2", false], + ["$$x_i^2$$", "x_i^2", true], + [String.raw`\(\frac{a}{b}\)`, String.raw`\frac{a}{b}`, false], + [String.raw`\[\sum_i x_i\]`, String.raw`\sum_i x_i`, true], + ["$$\nx_i + y\n$$", "x_i + y", true], + [ + "\\[\n\\begin{aligned}\nx &= 1 \\\\\ny &= 2\n\\end{aligned}\n\\]", + "\\begin{aligned}\nx &= 1 \\\\\ny &= 2\n\\end{aligned}", + true, + ], + ])("recognizes %s", (source, tex, display) => { + expect(markdownMathRanges(source)).toEqual([ + { source, math: { source, tex, display }, start: 0, end: source.length }, + ]); + }); + + it.each([ + "Costs $20 today and $30 tomorrow.", + "$20.00, $30.00, and $40.00", + "`$x$` and `\\(x\\)`", + "```tex\n$$x$$\n```", + " \\[x\\]", + String.raw`\$x\$ and \\(x\\)`, + "[link](https://example.com/$x$)", + "$$\nunfinished\n\nNext paragraph $5", + "$ incomplete$ and $incomplete $", + "$$$x$$$", + ])("leaves literal content unchanged: %s", (source) => { + expect(markdownMathRanges(source)).toEqual([]); + }); + + it("keeps exact source offsets alongside emoji, lists and emphasis", () => { + const source = "🙂 **Fit** $x_i$\n\n- [ ] Verify \\(y\\)"; + const ranges = markdownMathRanges(source); + expect(ranges.map((range) => source.slice(range.start, range.end))).toEqual([ + "$x_i$", + "\\(y\\)", + ]); + expect(ranges.map((range) => range.math?.tex)).toEqual(["x_i", "y"]); + }); + + it("preserves incomplete backslash openers during streaming", () => { + const paragraph = parser.parse(String.raw`Use \(x`).children[0]; + expect( + paragraph?.type === "paragraph" && + paragraph.children.map((node) => ("value" in node ? node.value : "")).join(""), + ).toBe(String.raw`Use \(x`); + }); + + it("does not interpret escaped dollar signs inside an expression as its end", () => { + expect(markdownMathRanges(String.raw`$x + \$5$`)[0]?.math?.tex).toBe(String.raw`x + \$5`); + }); +}); diff --git a/packages/client-runtime/src/markdownMath.ts b/packages/client-runtime/src/markdownMath.ts new file mode 100644 index 000000000000..f4cb347f63da --- /dev/null +++ b/packages/client-runtime/src/markdownMath.ts @@ -0,0 +1,227 @@ +import type { Literal, Root } from "mdast"; +import { markdownLineEnding, markdownSpace } from "micromark-util-character"; +import type { Code, Construct, State, Tokenizer } from "micromark-util-types"; +import remarkParse from "remark-parse"; +import { unified, type Processor } from "unified"; + +export interface MarkdownMath { + readonly source: string; + readonly tex: string; + readonly display: boolean; +} + +interface MathNode extends Literal { + type: "inlineMath"; + data: { + math: MarkdownMath | null; + hName: "span"; + hProperties: { dataMathSource: string }; + }; +} + +declare module "mdast" { + interface RootContentMap { + inlineMath: MathNode; + } + interface PhrasingContentMap { + inlineMath: MathNode; + } +} + +declare module "micromark-util-types" { + interface TokenTypeMap { + t3Math: "t3Math"; + t3MathData: "t3MathData"; + } +} + +// Bound both incomplete-delimiter lookahead and work handed to a TeX renderer. +export const MAX_MATH_LENGTH = 16_384; + +export function markdownMath(source: string): MarkdownMath | null { + const opener = source.startsWith("$$") ? "$$" : source.slice(0, 2); + const delimiter = opener === "\\(" || opener === "\\[" || opener === "$$" ? opener : "$"; + const close = delimiter === "\\(" ? "\\)" : delimiter === "\\[" ? "\\]" : delimiter; + if ( + !source.startsWith(delimiter) || + !source.endsWith(close) || + source.length <= delimiter.length + close.length || + source.length > MAX_MATH_LENGTH + ) + return null; + const tex = source.slice(delimiter.length, -close.length).trim(); + return tex ? { source, tex, display: delimiter === "$$" || delimiter === "\\[" } : null; +} + +const tokenizeMath: Tokenizer = function (effects, ok, nok) { + let opener: "$" | "$$" | "\\(" | "\\[" = "$"; + let count = 0; + let previous: Code = null; + let closingCount = 0; + let hasContent = false; + let lineStart = false; + let dataOpen = false; + const closeData = () => { + if (dataOpen) effects.exit("t3MathData"); + dataOpen = false; + }; + const consume = (code: Code) => { + if (!dataOpen) effects.enter("t3MathData"); + dataOpen = true; + effects.consume(code); + count += 1; + previous = code; + }; + return start; + + function unfinished(code: Code): State | undefined { + if (opener === "$" || opener === "$$") return nok(code); + closeData(); + effects.exit("t3Math"); + return ok(code); + } + function start(code: Code): State | undefined { + effects.enter("t3Math"); + consume(code); + return code === 92 ? backslashOpen : dollarOpen; + } + function backslashOpen(code: Code): State | undefined { + if (code !== 40 && code !== 91) return nok(code); + opener = code === 40 ? "\\(" : "\\["; + consume(code); + return body; + } + function dollarOpen(code: Code): State | undefined { + if (code === 36) { + opener = "$$"; + consume(code); + return displayStart; + } + if (code === null || markdownLineEnding(code) || markdownSpace(code)) return nok(code); + return body(code); + } + function displayStart(code: Code): State | undefined { + return code === 36 ? nok(code) : body(code); + } + function body(code: Code): State | undefined { + if (code === null || count >= MAX_MATH_LENGTH) return unfinished(code); + if (markdownLineEnding(code)) { + // A blank line ends a candidate. Never swallow later paragraphs while streaming. + if (lineStart || opener === "$") return unfinished(code); + lineStart = true; + closeData(); + effects.enter("lineEnding"); + effects.consume(code); + effects.exit("lineEnding"); + count += 1; + previous = code; + return body; + } + if (!markdownSpace(code)) lineStart = false; + if (code === 92) { + consume(code); + return escaped; + } + if (code === 36 && (opener === "$" || opener === "$$")) { + if ( + !hasContent || + (opener === "$" && (markdownSpace(previous) || markdownLineEnding(previous))) + ) + return nok(code); + closingCount = 1; + consume(code); + return closeDollar; + } + if (!markdownSpace(code)) hasContent = true; + consume(code); + return body; + } + function escaped(code: Code): State | undefined { + if ((opener === "\\(" && code === 41) || (opener === "\\[" && code === 93)) { + consume(code); + closeData(); + effects.exit("t3Math"); + return ok; + } + if (code === null || markdownLineEnding(code)) return body(code); + hasContent = true; + consume(code); + return body; + } + function closeDollar(code: Code): State | undefined { + if (code === 36) { + closingCount += 1; + consume(code); + return closeDollar; + } + if (closingCount !== opener.length) return nok(code); + // Pandoc-style dollar boundaries keep "$20 and $30" as ordinary prose. + if (opener === "$" && code !== null && code >= 48 && code <= 57) return nok(code); + closeData(); + effects.exit("t3Math"); + return ok(code); + } +}; + +/** Recognize math in Markdown's text grammar, so code, links and escapes keep their semantics. */ +function attachMath(this: Processor) { + const data = this.data(); + const construct: Construct = { tokenize: tokenizeMath }; + (data.micromarkExtensions ??= []).push({ + text: { 36: construct, 92: construct }, + }); + (data.fromMarkdownExtensions ??= []).push({ + enter: { + t3Math(token) { + const source = this.sliceSerialize(token); + const math = markdownMath(source); + this.enter( + { + type: "inlineMath", + value: math?.tex ?? source, + data: { + math, + hName: "span", + hProperties: { dataMathSource: source }, + }, + }, + token, + ); + }, + }, + exit: { + t3Math(token) { + this.exit(token); + }, + }, + }); +} + +export const remarkMath = attachMath; + +const parser = unified().use(remarkParse).use(remarkMath).freeze(); + +/** Source ranges let native Markdown use the same grammar without a second delimiter scanner. */ +export function markdownMathRanges(source: string) { + const matches: Array<{ source: string; math: MarkdownMath | null; start: number; end: number }> = + []; + if (!source.includes("$") && !source.includes("\\(") && !source.includes("\\[")) return matches; + const tree = parser.parse(source); + function visit(node: Root | Root["children"][number] | MathNode): void { + if (node.type === "inlineMath") { + const start = node.position?.start.offset; + const end = node.position?.end.offset; + if (start !== undefined && end !== undefined) + matches.push({ + source: node.data.hProperties.dataMathSource, + math: node.data.math, + start, + end, + }); + } else if ("children" in node) { + for (const child of node.children) visit(child); + } + } + visit(tree); + return matches; +} From 3dd13b5ef6970e95a5f8232b46ef78e62030f070 Mon Sep 17 00:00:00 2001 From: Wout Stiens <71498452+StiensWout@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:05:42 +0200 Subject: [PATCH 02/10] fix(web): use bundled KaTeX types --- apps/web/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/web/package.json b/apps/web/package.json index 3e6d6b1bcd07..c276dcb501aa 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -40,7 +40,7 @@ "jose": "catalog:", "jsonc-parser": "3.3.1", "jszip": "3.10.1", - "katex": "^0.16.22", + "katex": "^0.16.47", "lexical": "^0.41.0", "lucide-react": "^0.564.0", "react": "19.2.6", @@ -64,7 +64,6 @@ "@types/culori": "^4.0.1", "@types/mdast": "^4.0.4", "@types/jsdom": "^30.0.0", - "@types/katex": "^0.16.7", "@types/react": "~19.2.14", "@types/react-dom": "~19.2.3", "@types/react-test-renderer": "19.1.0", From 2291e187b6bcd0f4887dbdfa84eb0ff8275e4362 Mon Sep 17 00:00:00 2001 From: Wout Stiens <71498452+StiensWout@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:24:09 +0200 Subject: [PATCH 03/10] fix(mobile): preserve formatting around equations --- .../modules/t3-markdown-text/package.json | 1 + .../src/NativeMarkdownSelectableText.tsx | 1 - .../t3-markdown-text/src/NativeMathText.tsx | 28 ++++- .../src/nativeMarkdownRunStyle.ts | 109 ++++++++++++++++++ .../src/nativeMarkdownText.ts | 20 +++- .../t3-markdown-text/src/nativeMathAssets.ts | 49 ++++++++ .../src/nativeMathDocument.ts | 6 +- .../t3-markdown-text/src/nativeMathHtml.ts | 43 ++++--- .../mobile/src/lib/nativeMarkdownMath.test.ts | 93 +++++++++++---- .../mobile/src/lib/nativeMathDocument.test.ts | 14 +++ .../client-runtime/src/markdownMath.test.ts | 9 ++ packages/client-runtime/src/markdownMath.ts | 2 +- 12 files changed, 318 insertions(+), 57 deletions(-) create mode 100644 apps/mobile/modules/t3-markdown-text/src/nativeMarkdownRunStyle.ts create mode 100644 apps/mobile/modules/t3-markdown-text/src/nativeMathAssets.ts diff --git a/apps/mobile/modules/t3-markdown-text/package.json b/apps/mobile/modules/t3-markdown-text/package.json index 85f34bcc3051..ec5ced0e4ee3 100644 --- a/apps/mobile/modules/t3-markdown-text/package.json +++ b/apps/mobile/modules/t3-markdown-text/package.json @@ -34,6 +34,7 @@ "expo": "*", "expo-asset": "*", "expo-clipboard": "*", + "expo-file-system": "*", "expo-haptics": "*", "expo-symbols": "*", "mathjax-full": "*", diff --git a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.tsx b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.tsx index 9b5ec79a2a93..7efe18815130 100644 --- a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.tsx +++ b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.tsx @@ -8,7 +8,6 @@ import { Platform, StyleSheet, Text as RNText, - type TextStyle, useColorScheme, View, } from "react-native"; diff --git a/apps/mobile/modules/t3-markdown-text/src/NativeMathText.tsx b/apps/mobile/modules/t3-markdown-text/src/NativeMathText.tsx index c607d7b3ba67..687a70be14e3 100644 --- a/apps/mobile/modules/t3-markdown-text/src/NativeMathText.tsx +++ b/apps/mobile/modules/t3-markdown-text/src/NativeMathText.tsx @@ -7,6 +7,7 @@ import type { MarkdownFileContextMenu, NativeMarkdownTextStyle, } from "./SelectableMarkdownText.types"; +import { loadNativeMathIcons, nativeMathIconKey } from "./nativeMathAssets"; import { NATIVE_MATH_DOCUMENT } from "./nativeMathDocument"; const documentSource = { html: NATIVE_MATH_DOCUMENT }; @@ -31,6 +32,22 @@ export const NativeMathText = memo(function NativeMathText(props: { const [height, setHeight] = useState(props.textStyle.lineHeight); const [failed, setFailed] = useState(false); const [renderer, setRenderer] = useState(); + const [icons, setIcons] = useState>({}); + useEffect(() => { + let active = true; + void loadNativeMathIcons(props.runs).then((loaded) => { + if (active) + setIcons((current) => + Object.keys(current).length === Object.keys(loaded).length && + Object.entries(loaded).every(([key, value]) => current[key] === value) + ? current + : loaded, + ); + }); + return () => { + active = false; + }; + }, [props.runs]); const { fontScale } = useWindowDimensions(); useEffect(() => { let active = true; @@ -57,13 +74,18 @@ export const NativeMathText = memo(function NativeMathText(props: { }; return { runs: props.runs.map((run) => - renderer.nativeMathRunHtml(run, style, run.href ? fileContextMenu?.(run.href) : undefined), + renderer.nativeMathRunHtml( + run, + style, + run.href ? fileContextMenu?.(run.href) : undefined, + icons[nativeMathIconKey(run) ?? ""], + ), ), color: style.color, fontSize: style.fontSize, lineHeight: style.lineHeight, }; - }, [renderer, props.runs, props.textStyle, fileContextMenu, fontScale]); + }, [renderer, props.runs, props.textStyle, fileContextMenu, fontScale, icons]); const revision = useRef(0); const latest = useRef(update); useEffect(() => { @@ -164,7 +186,7 @@ export const NativeMathText = memo(function NativeMathText(props: { } if (message.type === "link" && "href" in message && typeof message.href === "string") { if (props.onLinkPress) props.onLinkPress(message.href); - else if (/^https?:\/\//i.test(message.href)) void Linking.openURL(message.href); + else void Linking.openURL(message.href); } }} /> diff --git a/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownRunStyle.ts b/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownRunStyle.ts new file mode 100644 index 000000000000..22721ac6e65d --- /dev/null +++ b/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownRunStyle.ts @@ -0,0 +1,109 @@ +import type { TextStyle } from "react-native"; +import type { NativeMarkdownTextRun } from "./nativeMarkdownText"; +import type { NativeMarkdownTextStyle } from "./SelectableMarkdownText.types"; + +const PARAGRAPH_STYLE_ENCODING_OFFSET = 1000; +const DEFAULT_BODY_FONT_SIZE = 15; +const DEFAULT_HEADING_FONT_SIZES = [22, 19, 17, 16, 15, 15] as const; + +function resolveHeadingFontSize(textStyle: NativeMarkdownTextStyle, headingLevel: number): number { + const index = Math.max(0, Math.min(5, headingLevel - 1)); + const configured = textStyle.headingFontSizes?.[index]; + if (typeof configured === "number" && Number.isFinite(configured)) { + return configured; + } + + const scale = textStyle.fontSize / DEFAULT_BODY_FONT_SIZE; + return Math.max(12, Math.round(DEFAULT_HEADING_FONT_SIZES[index] * scale)); +} + +export function nativeMarkdownRunStyle( + run: NativeMarkdownTextRun, + textStyle: NativeMarkdownTextStyle, + monoFontFamily: string, +) { + const isFile = run.fileIcon != null; + const isSkill = run.skillName != null; + const headingLevel = Math.max(1, Math.min(6, run.headingLevel ?? 1)); + const headingFontSize = resolveHeadingFontSize(textStyle, headingLevel); + const isHeading = run.role === "heading"; + const isCodeBlock = run.role === "code-block" || run.role === "code-language"; + const hasParagraphStyle = run.headIndent !== undefined; + const textDecorationLine = run.strikethrough + ? "line-through" + : run.href && !isFile + ? "underline" + : "none"; + + return { + color: isFile + ? textStyle.fileTextColor + : isSkill + ? textStyle.skillTextColor + : run.href + ? textStyle.linkColor + : isHeading + ? textStyle.strongColor + : run.role === "quote-marker" + ? textStyle.quoteMarkerColor + : run.role === "divider" + ? textStyle.dividerColor + : run.role === "code-language" + ? textStyle.mutedColor + : run.role === "list-marker" + ? textStyle.mutedColor + : isCodeBlock + ? textStyle.codeColor + : run.code + ? textStyle.inlineCodeColor + : run.bold + ? textStyle.strongColor + : textStyle.color, + fontFamily: + isFile || isSkill + ? textStyle.boldFontFamily + : run.code || isCodeBlock + ? monoFontFamily + : isHeading + ? textStyle.headingFontFamily + : run.bold + ? textStyle.boldFontFamily + : textStyle.fontFamily, + fontSize: + run.role === "spacer" + ? (run.spacing ?? 10) + : run.role === "list-break" + ? textStyle.fontSize + : isHeading + ? headingFontSize + : run.role === "code-language" + ? Math.max(10, Math.round(textStyle.fontSize * 0.73)) + : run.code || isCodeBlock + ? Math.max(12, textStyle.fontSize - 2) + : textStyle.fontSize, + lineHeight: + run.role === "spacer" + ? (run.spacing ?? 10) + : run.role === "list-break" + ? textStyle.lineHeight + (run.spacing ?? 0) + : isHeading + ? Math.max(headingFontSize + 6, textStyle.lineHeight + 2) + : isCodeBlock + ? Math.max(16, textStyle.lineHeight - 2) + : textStyle.lineHeight, + fontStyle: run.italic ? "italic" : "normal", + fontWeight: isHeading || run.bold || isFile || isSkill ? "700" : "400", + textDecorationLine, + backgroundColor: isCodeBlock ? textStyle.codeBlockBackgroundColor : undefined, + ...(hasParagraphStyle + ? { + shadowColor: "transparent", + shadowOffset: { + width: run.firstLineHeadIndent ?? 0, + height: run.headIndent, + }, + shadowRadius: PARAGRAPH_STYLE_ENCODING_OFFSET + (run.paragraphSpacing ?? 0), + } + : {}), + } satisfies TextStyle; +} diff --git a/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts b/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts index dc1556c6fe85..19ca768b5b84 100644 --- a/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts +++ b/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts @@ -285,6 +285,7 @@ function appendRun( runs: NativeMarkdownTextRun[], text: string, context: RunContext, + mathSource?: string, ): NativeMarkdownTextRun[] { if (text.length === 0) { return runs; @@ -292,6 +293,7 @@ function appendRun( const run: NativeMarkdownTextRun = { text, + ...(mathSource ? { mathSource } : {}), ...(context.bold ? { bold: true } : {}), ...(context.italic ? { italic: true } : {}), ...(context.strikethrough ? { strikethrough: true } : {}), @@ -441,12 +443,7 @@ function appendNode( switch (node.type) { case "math_inline": case "math_block": - runs.push({ - text: nodeTextContent(node), - mathSource: nodeTextContent(node), - ...(context.href ? { href: context.href } : {}), - }); - return runs; + return appendRun(runs, nodeTextContent(node), context, nodeTextContent(node)); case "text": return appendRun(runs, textNodeContent(nodeTextContent(node)), context); case "html_inline": @@ -857,7 +854,18 @@ function appendDocumentBlock( } } +function containsMath(node: MarkdownNode): boolean { + return ( + node.type === "math_inline" || + node.type === "math_block" || + (node.children ?? []).some(containsMath) + ); +} + function containsRichBlock(node: MarkdownNode): boolean { + // NativeList already owns nested indentation and hanging markers. Keep that + // layout when a list item needs a math text view. + if (node.type === "list" && containsMath(node)) return true; if ( node.type === "code_block" || node.type === "blockquote" || diff --git a/apps/mobile/modules/t3-markdown-text/src/nativeMathAssets.ts b/apps/mobile/modules/t3-markdown-text/src/nativeMathAssets.ts new file mode 100644 index 000000000000..f53459cc44f2 --- /dev/null +++ b/apps/mobile/modules/t3-markdown-text/src/nativeMathAssets.ts @@ -0,0 +1,49 @@ +import { Asset } from "expo-asset"; +import { File } from "expo-file-system"; +import { markdownFileIconSource } from "./markdownFileIcons"; +import { markdownLinkIconSource } from "./markdownLinkIcons"; +import { resolveMarkdownLinkIcon } from "./markdownLinks"; +import type { NativeMarkdownTextRun } from "./nativeMarkdownText"; + +// These keys come only from the finite set of bundled Markdown icon assets. +const icons = new Map>(); + +export function nativeMathIconKey(run: NativeMarkdownTextRun): string | undefined { + if (run.fileIcon) return `file:${run.fileIcon}`; + const linkIcon = run.externalHost && resolveMarkdownLinkIcon(run.externalHost); + return linkIcon ? `link:${linkIcon}` : undefined; +} + +/** Embed bundled icons so the WebView never needs file access or remote image permissions. */ +export async function loadNativeMathIcons(runs: ReadonlyArray) { + const entries = new Map>(); + for (const run of runs) { + const key = nativeMathIconKey(run); + if (!key || entries.has(key)) continue; + let icon = icons.get(key); + if (!icon) { + const linkIcon = run.externalHost && resolveMarkdownLinkIcon(run.externalHost); + const source = run.fileIcon + ? markdownFileIconSource(run.fileIcon) + : linkIcon + ? markdownLinkIconSource(linkIcon) + : undefined; + if (typeof source !== "number") continue; + icon = Asset.fromModule(source) + .downloadAsync() + .then(async (asset) => { + if (!asset.localUri) return ""; + return `data:image/png;base64,${await new File(asset.localUri).base64()}`; + }) + .catch(() => { + icons.delete(key); + return ""; + }); + icons.set(key, icon); + } + entries.set(key, icon); + } + return Object.fromEntries( + await Promise.all([...entries].map(async ([key, icon]) => [key, await icon])), + ); +} diff --git a/apps/mobile/modules/t3-markdown-text/src/nativeMathDocument.ts b/apps/mobile/modules/t3-markdown-text/src/nativeMathDocument.ts index 2799f20aaff9..c80c7a74c626 100644 --- a/apps/mobile/modules/t3-markdown-text/src/nativeMathDocument.ts +++ b/apps/mobile/modules/t3-markdown-text/src/nativeMathDocument.ts @@ -1,7 +1,7 @@ // This document is installed once. Updates reconcile runs so streaming does not // reload the WebView or recreate completed equations and open source disclosures. export const NATIVE_MATH_DOCUMENT = `