feat(chat): render math across web and mobile

apps/mobile/modules/t3-markdown-text/package.json

@@ -34,11 +34,14 @@
     "expo": "*",
     "expo-asset": "*",
     "expo-clipboard": "*",
+    "expo-file-system": "*",
     "expo-haptics": "*",
     "expo-symbols": "*",
+    "mathjax-full": "*",
     "react": "*",
     "react-native": "*",
-    "react-native-nitro-markdown": "*"
+    "react-native-nitro-markdown": "*",
+    "react-native-webview": "*"
   },
   "codegenConfig": {
     "name": "T3MarkdownTextSpec",

apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.tsx

@@ -1,13 +1,15 @@
 import { createContext, useCallback, useContext, useMemo } from "react";
 import { decodeComposerContextFragment } from "@t3tools/shared/composerContextClipboard";
+import { shouldTypesetNativeMath } from "./nativeMarkdownMath";
+import { NativeMathText } from "./NativeMathText";
+import { nativeMarkdownRunStyle } from "./nativeMarkdownRunStyle";
 import {
   findNodeHandle,
   Image,
   Linking,
   Platform,
   StyleSheet,
   Text as RNText,
-  type TextStyle,
   useColorScheme,
   View,
 } from "react-native";
@@ -49,7 +51,6 @@ const SKILL_ICON_PLACEHOLDER = "\uFFFC";
 // height. Any other single character keeps the run's attributes; the native side swaps it
 // for the chip attachment either way.
 const IOS_CHIP_PLACEHOLDER = "\u200B";
-const PARAGRAPH_STYLE_ENCODING_OFFSET = 1000;
 const MONO_FONT_FAMILY = Platform.select({
   ios: "ui-monospace",
   android: "monospace",
@@ -87,111 +88,6 @@ function runKeySignature(run: NativeMarkdownTextRun): string {
   ].join(":");
 }
 
-const DEFAULT_BODY_FONT_SIZE = 15;
-const DEFAULT_HEADING_FONT_SIZES = [22, 19, 17, 16, 15, 15] as const;
-
-function resolveHeadingFontSize(textStyle: NativeMarkdownTextStyle, headingLevel: number): number {
-  const index = Math.max(0, Math.min(5, headingLevel - 1));
-  const configured = textStyle.headingFontSizes?.[index];
-  if (typeof configured === "number" && Number.isFinite(configured)) {
-    return configured;
-  }
-
-  const scale = textStyle.fontSize / DEFAULT_BODY_FONT_SIZE;
-  return Math.max(12, Math.round((DEFAULT_HEADING_FONT_SIZES[index] ?? 15) * scale));
-}
-
-function runStyle(run: NativeMarkdownTextRun, textStyle: NativeMarkdownTextStyle): TextStyle {
-  const isFile = run.fileIcon != null;
-  const isSkill = run.skillName != null;
-  const headingLevel = Math.max(1, Math.min(6, run.headingLevel ?? 1));
-  const headingFontSize = resolveHeadingFontSize(textStyle, headingLevel);
-  const isHeading = run.role === "heading";
-  const isCodeBlock = run.role === "code-block" || run.role === "code-language";
-  const hasParagraphStyle = run.headIndent !== undefined;
-  const textDecorationLine = run.strikethrough
-    ? "line-through"
-    : run.href && !isFile
-      ? "underline"
-      : "none";
-
-  return {
-    color: isFile
-      ? textStyle.fileTextColor
-      : isSkill
-        ? textStyle.skillTextColor
-        : run.href
-          ? textStyle.linkColor
-          : isHeading
-            ? textStyle.strongColor
-            : run.role === "quote-marker"
-              ? textStyle.quoteMarkerColor
-              : run.role === "divider"
-                ? textStyle.dividerColor
-                : run.role === "code-language"
-                  ? textStyle.mutedColor
-                  : run.role === "list-marker"
-                    ? textStyle.mutedColor
-                    : isCodeBlock
-                      ? textStyle.codeColor
-                      : run.code
-                        ? textStyle.inlineCodeColor
-                        : run.bold
-                          ? textStyle.strongColor
-                          : textStyle.color,
-    fontFamily:
-      isFile || isSkill
-        ? textStyle.boldFontFamily
-        : run.code || isCodeBlock
-          ? MONO_FONT_FAMILY
-          : isHeading
-            ? textStyle.headingFontFamily
-            : run.bold
-              ? textStyle.boldFontFamily
-              : textStyle.fontFamily,
-    fontSize:
-      run.role === "spacer"
-        ? (run.spacing ?? 10)
-        : run.role === "list-break"
-          ? textStyle.fontSize
-          : isHeading
-            ? headingFontSize
-            : run.role === "code-language"
-              ? Math.max(10, Math.round(textStyle.fontSize * 0.73))
-              : run.code || isCodeBlock
-                ? Math.max(12, textStyle.fontSize - 2)
-                : textStyle.fontSize,
-    lineHeight:
-      run.role === "spacer"
-        ? (run.spacing ?? 10)
-        : run.role === "list-break"
-          ? textStyle.lineHeight + (run.spacing ?? 0)
-          : isHeading
-            ? Math.max(headingFontSize + 6, textStyle.lineHeight + 2)
-            : isCodeBlock
-              ? Math.max(16, textStyle.lineHeight - 2)
-              : textStyle.lineHeight,
-    fontStyle: run.italic ? "italic" : "normal",
-    fontWeight: isHeading || run.bold || isFile || isSkill ? "700" : "400",
-    textDecorationLine,
-    backgroundColor: isCodeBlock
-      ? textStyle.codeBlockBackgroundColor
-      : parseComposerContextHref(run.href ?? "")
-        ? textStyle.codeBackgroundColor
-        : undefined,
-    ...(hasParagraphStyle
-      ? {
-          shadowColor: "transparent",
-          shadowOffset: {
-            width: run.firstLineHeadIndent ?? 0,
-            height: run.headIndent,
-          },
-          shadowRadius: PARAGRAPH_STYLE_ENCODING_OFFSET + (run.paragraphSpacing ?? 0),
-        }
-      : {}),
-  };
-}
-
 export function NativeMarkdownSelectableText(props: {
   readonly runs: ReadonlyArray<NativeMarkdownTextRun>;
   readonly textStyle: NativeMarkdownTextStyle;
@@ -326,7 +222,7 @@ export function NativeMarkdownSelectableText(props: {
     props.textStyle.contextChipBorderColor,
   ].join(":");
 
-  return (
+  const nativeText = (
     <MarkdownTextPrimitive
       key={appearanceKey}
       nativeTextRef={attachAndroidText}
@@ -375,7 +271,7 @@ export function NativeMarkdownSelectableText(props: {
             }
             contextMenuConfig={contextMenu ? JSON.stringify(contextMenu) : undefined}
             style={[
-              runStyle(run, props.textStyle),
+              nativeMarkdownRunStyle(run, props.textStyle, MONO_FONT_FAMILY ?? "monospace"),
               chip ? { backgroundColor: "transparent" } : undefined,
             ]}
             onPress={onPress}
@@ -433,4 +329,9 @@ export function NativeMarkdownSelectableText(props: {
       })}
     </MarkdownTextPrimitive>
   );
+  return shouldTypesetNativeMath(props.runs) ? (
+    <NativeMathText {...props} {...(menu ?? {})} fallback={nativeText} />
+  ) : (
+    nativeText
+  );
 }

apps/mobile/modules/t3-markdown-text/src/NativeMathText.tsx

@@ -0,0 +1,187 @@
+import { memo, useEffect, useMemo, useRef, useState, type ReactNode } from "react";
+import { Alert, Linking, useWindowDimensions } from "react-native";
+import { setStringAsync } from "expo-clipboard";
+import { WebView } from "react-native-webview";
+import type { NativeMarkdownTextRun } from "./nativeMarkdownText";
+import type {
+  MarkdownFileContextMenu,
+  NativeMarkdownTextStyle,
+} from "./SelectableMarkdownText.types";
+import { loadNativeMathIcons, nativeMathIconKey } from "./nativeMathAssets";
+import { NATIVE_MATH_DOCUMENT } from "./nativeMathDocument";
+
+const documentSource = { html: NATIVE_MATH_DOCUMENT };
+let rendererPromise: Promise<typeof import("./nativeMathHtml")> | undefined;
+function loadRenderer() {
+  return (rendererPromise ??= import("./nativeMathHtml").catch((error) => {
+    rendererPromise = undefined;
+    throw error;
+  }));
+}
+
+/** One view per math-containing text chunk preserves inline layout and selection on both OSes. */
+export const NativeMathText = memo(function NativeMathText(props: {
+  readonly runs: ReadonlyArray<NativeMarkdownTextRun>;
+  readonly fallback: ReactNode;
+  readonly textStyle: NativeMarkdownTextStyle;
+  readonly onLinkPress?: (href: string) => void;
+  readonly fileContextMenu?: (href: string) => MarkdownFileContextMenu | undefined;
+  readonly onFileContextMenuAction?: (href: string, actionId: string) => void;
+}) {
+  const webView = useRef<WebView>(null);
+  const ready = useRef(false);
+  const [height, setHeight] = useState(props.textStyle.lineHeight);
+  const [failed, setFailed] = useState(false);
+  const [renderer, setRenderer] = useState<typeof import("./nativeMathHtml")>();
+  const [icons, setIcons] = useState<Record<string, string>>({});
+  useEffect(() => {
+    let active = true;
+    void loadNativeMathIcons(props.runs).then((loaded) => {
+      if (active)
+        setIcons((current) =>
+          Object.keys(current).length === Object.keys(loaded).length &&
+          Object.entries(loaded).every(([key, value]) => current[key] === value)
+            ? current
+            : loaded,
+        );
+    });
+    return () => {
+      active = false;
+    };
+  }, [props.runs]);
+  const { fontScale } = useWindowDimensions();
+  useEffect(() => {
+    let active = true;
+    void loadRenderer().then(
+      (module) => {
+        if (active) setRenderer(module);
+      },
+      () => {
+        if (active) setFailed(true);
+      },
+    );
+    return () => {
+      active = false;
+    };
+  }, []);
+  const fileContextMenu = props.fileContextMenu;
+  const update = useMemo(() => {
+    if (!renderer) return null;
+    const style = {
+      ...props.textStyle,
+      fontSize: props.textStyle.fontSize * fontScale,
+      lineHeight: props.textStyle.lineHeight * fontScale,
+      headingFontSizes: props.textStyle.headingFontSizes?.map((size) => size * fontScale),
+    };
+    const prefixedLinks = new Set<string>();
+    return {
+      runs: props.runs.map((run) => {
+        const showExternalIcon = !run.href || !prefixedLinks.has(run.href);
+        if (run.externalHost && run.href) prefixedLinks.add(run.href);
+        return renderer.nativeMathRunHtml(
+          run,
+          style,
+          run.fileIcon && run.href ? fileContextMenu?.(run.href) : undefined,
+          icons[nativeMathIconKey(run) ?? ""],
+          showExternalIcon,
+        );
+      }),
+      color: style.color,
+      fontSize: style.fontSize,
+      lineHeight: style.lineHeight,
+    };
+  }, [renderer, props.runs, props.textStyle, fileContextMenu, fontScale, icons]);
+  const revision = useRef(0);
+  const latest = useRef(update);
+  useEffect(() => {
+    latest.current = update;
+    if (ready.current && update) {
+      revision.current += 1;
+      webView.current?.injectJavaScript(
+        `window.updateMath(${JSON.stringify({ ...update, revision: revision.current })});true;`,
+      );
+    }
+  }, [update]);
+
+  if (failed || !renderer) return props.fallback;
+  return (
+    <WebView
+      ref={webView}
+      source={documentSource}
+      originWhitelist={["*"]}
+      onShouldStartLoadWithRequest={(request) => request.url === "about:blank"}
+      scrollEnabled={false}
+      bounces={false}
+      showsVerticalScrollIndicator={false}
+      dataDetectorTypes="none"
+      allowFileAccess={false}
+      setSupportMultipleWindows={false}
+      style={{ height, backgroundColor: "transparent", flex: 0 }}
+      onError={() => setFailed(true)}
+      onContentProcessDidTerminate={() => setFailed(true)}
+      onRenderProcessGone={() => setFailed(true)}
+      onMessage={(event) => {
+        let message: unknown;
+        try {
+          message = JSON.parse(event.nativeEvent.data);
+        } catch {
+          return;
+        }
+        if (!message || typeof message !== "object" || !("type" in message)) return;
+        if (message.type === "ready") {
+          ready.current = true;
+          if (latest.current) {
+            revision.current += 1;
+            webView.current?.injectJavaScript(
+              `window.updateMath(${JSON.stringify({ ...latest.current, revision: revision.current })});true;`,
+            );
+          }
+          return;
+        }
+        if (!("revision" in message) || message.revision !== revision.current) return;
+        if (
+          message.type === "height" &&
+          "height" in message &&
+          typeof message.height === "number" &&
+          Number.isFinite(message.height) &&
+          message.height > 0
+        )
+          setHeight(message.height);
+        if (message.type === "copy" && "text" in message && typeof message.text === "string") {
+          const text = message.text;
+          const copyRevision = revision.current;
+          void setStringAsync(text).then(
+            () => {
+              if (revision.current === copyRevision)
+                webView.current?.injectJavaScript(
+                  `window.mathCopyResult(${JSON.stringify(text)});true;`,
+                );
+            },
+            () => Alert.alert("Could not copy text"),
+          );
+        }
+        if (
+          message.type === "file-action" &&
+          "href" in message &&
+          typeof message.href === "string" &&
+          "action" in message &&
+          typeof message.action === "string"
+        ) {
+          const href = message.href;
+          const actionId = message.action;
+          if (
+            props.runs.some((run) => run.fileIcon && run.href === href) &&
+            props
+              .fileContextMenu?.(href)
+              ?.actions.some((action) => action.id === actionId && !action.disabled)
+          )
+            props.onFileContextMenuAction?.(href, actionId);
+        }
+        if (message.type === "link" && "href" in message && typeof message.href === "string") {
+          if (props.onLinkPress) props.onLinkPress(message.href);
+          else void Linking.openURL(message.href);
+        }
+      }}
+    />
+  );
+});

apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.tsx

@@ -1,3 +1,4 @@
+import { parseNativeMarkdownMath } from "./nativeMarkdownMath";
 import { useMemo } from "react";
 import { View } from "react-native";
 import { parseMarkdownWithOptions } from "react-native-nitro-markdown/headless";
@@ -51,11 +52,9 @@ export function SelectableMarkdownText({
   marginBottom = 0,
 }: SelectableMarkdownTextProps) {
   const chunks = useMemo(() => {
-    const parsedDocument = parseMarkdownWithOptions(markdown, {
-      gfm: true,
-      html: true,
-      math: false,
-    });
+    const parsedDocument = parseNativeMarkdownMath(markdown, (source) =>
+      parseMarkdownWithOptions(source, { gfm: true, html: true, math: false }),
+    );
     const document = preserveSoftBreaks
       ? nativeMarkdownWithPreservedSoftBreaks(parsedDocument)
       : parsedDocument;