feat(voice): let agents reply with a voice recording

README.md

@@ -45,6 +45,7 @@ This is a personal fork of [pingdotgg/t3code](https://github.com/pingdotgg/t3cod
 
 - **Voice dictation** — ElevenLabs-powered voice transcription in the composer, including mobile.
 - **Message listening** — optional spoken versions of assistant messages with playback controls; per-message summaries and speech artifacts are persisted, with mobile playback support and server-side ElevenLabs model and voice overrides.
+- **Agent voice replies** — agents get a `voice_reply` tool that turns a script they write for the ear into an ElevenLabs recording, attached to their final message when the turn completes. Web and mobile then lead with the player and fold the written reply behind a "Show written reply" toggle; a turn that ends with no written message publishes the transcript as the message text. Needs `ELEVENLABS_API_KEY` on the server; the switch in Settings → Extras (on by default) withholds the tool from newly started agent sessions.
 
 ### Agents & skills
 

apps/mobile/src/features/threads/ThreadFeed.tsx

@@ -1231,38 +1231,55 @@ function renderFeedEntry(
       );
     }
 
-    // Skip empty assistant messages (no text, no attachments) — they would
-    // render as an orphaned timestamp and break adjacent activity-group merging.
-    if (message.text.trim().length === 0 && attachments.length === 0) {
+    const agentVoiceReply = message.speech?.origin === "agent" ? message.speech : null;
+    // Skip empty assistant messages (no text, no attachments, no voice
+    // reply) — they would render as an orphaned timestamp and break adjacent
+    // activity-group merging.
+    if (message.text.trim().length === 0 && attachments.length === 0 && agentVoiceReply === null) {
       return null;
     }
 
     const enterAnimated = isFreshTimestamp(message.createdAt);
+    const writtenReply =
+      message.text.trim().length > 0 ? (
+        hasNativeSelectableMarkdownText() ? (
+          <SelectableMarkdownText
+            markdown={message.text}
+            skills={props.skills}
+            textStyle={styles.nativeTextStyle}
+            onLinkPress={props.onMarkdownLinkPress}
+            renderImage={props.renderMarkdownImage}
+          />
+        ) : (
+          <Markdown
+            options={{ gfm: true }}
+            renderers={styles.renderers}
+            styles={styles.styles}
+            theme={styles.theme}
+          >
+            {message.text}
+          </Markdown>
+        )
+      ) : null;
     return (
       <Animated.View
         className={cn(showAssistantMeta ? "mb-5 px-1" : "mb-2 px-1")}
         {...(enterAnimated ? { entering: FadeIn.duration(220) } : {})}
       >
-        {message.text.trim().length > 0 ? (
-          hasNativeSelectableMarkdownText() ? (
-            <SelectableMarkdownText
-              markdown={message.text}
-              skills={props.skills}
-              textStyle={styles.nativeTextStyle}
-              onLinkPress={props.onMarkdownLinkPress}
-              renderImage={props.renderMarkdownImage}
-            />
-          ) : (
-            <Markdown
-              options={{ gfm: true }}
-              renderers={styles.renderers}
-              styles={styles.styles}
-              theme={styles.theme}
-            >
-              {message.text}
-            </Markdown>
-          )
-        ) : null}
+        {agentVoiceReply !== null ? (
+          <AssistantAgentVoiceReply
+            environmentId={props.environmentId}
+            speech={agentVoiceReply}
+            iconSubtleColor={iconSubtleColor}
+            writtenReplyDuplicatesTranscript={
+              message.text.trim() === agentVoiceReply.transcript.trim()
+            }
+          >
+            {writtenReply}
+          </AssistantAgentVoiceReply>
+        ) : (
+          writtenReply
+        )}
         {attachments.map((attachment) => {
           return (
             <MessageAttachmentImage
@@ -1312,6 +1329,68 @@ function formatPlaybackTime(seconds: number): string {
   return `${Math.floor(wholeSeconds / 60)}:${String(wholeSeconds % 60).padStart(2, "0")}`;
 }
 
+/**
+ * An agent-staged voice recording rendered as the message's main content:
+ * player first, the written reply collapsed behind a toggle.
+ */
+function AssistantAgentVoiceReply(props: {
+  readonly environmentId: EnvironmentId;
+  readonly speech: MessageSpeechSynthesisResult;
+  readonly iconSubtleColor: ColorValue;
+  /**
+   * A voice-only turn's text is the transcript itself; the toggle is skipped
+   * for it (the player's "View transcript" already covers it), but a dead
+   * recording still forces the text visible — it is the message then.
+   */
+  readonly writtenReplyDuplicatesTranscript: boolean;
+  readonly children: ReactNode;
+}) {
+  const [transcriptExpanded, setTranscriptExpanded] = useState(false);
+  const [writtenReplyExpanded, setWrittenReplyExpanded] = useState(false);
+  // When the recording's file is gone, the written reply becomes the message
+  // content and is forced visible instead of hiding behind the toggle.
+  const [audioUnavailable, setAudioUnavailable] = useState(false);
+  const onAudioUnavailable = useCallback(() => setAudioUnavailable(true), []);
+  const hasWrittenReply = props.children !== null;
+
+  return (
+    <View>
+      <AssistantSpeechPlayer
+        environmentId={props.environmentId}
+        speech={props.speech}
+        iconSubtleColor={props.iconSubtleColor}
+        transcriptExpanded={transcriptExpanded}
+        onToggleTranscript={() => setTranscriptExpanded((current) => !current)}
+        onRetry={null}
+        onAudioUnavailable={onAudioUnavailable}
+        primary
+      />
+      {hasWrittenReply && !props.writtenReplyDuplicatesTranscript && !audioUnavailable ? (
+        <Pressable
+          accessibilityRole="button"
+          accessibilityState={{ expanded: writtenReplyExpanded }}
+          className="min-h-8 flex-row items-center gap-1 px-0.5"
+          onPress={() => setWrittenReplyExpanded((current) => !current)}
+        >
+          <Text className="font-t3-medium text-xs text-foreground-muted">
+            {writtenReplyExpanded ? "Hide written reply" : "Show written reply"}
+          </Text>
+          <SymbolView
+            name={writtenReplyExpanded ? "chevron.up" : "chevron.down"}
+            size={13}
+            tintColor={props.iconSubtleColor}
+            type="monochrome"
+          />
+        </Pressable>
+      ) : null}
+      {hasWrittenReply &&
+      (audioUnavailable || (writtenReplyExpanded && !props.writtenReplyDuplicatesTranscript))
+        ? props.children
+        : null}
+    </View>
+  );
+}
+
 function AssistantMessageMetaAndArtifacts(props: {
   readonly environmentId: EnvironmentId;
   readonly messageId: MessageId;
@@ -1349,6 +1428,10 @@ function AssistantMessageMetaAndArtifacts(props: {
   );
   const speech = sessionArtifacts.speech ?? props.persistedSpeech;
   const summary = sessionArtifacts.summary ?? props.persistedSummary;
+  // Agent voice replies render their own player above the message; the meta
+  // row must not offer a second one (or a regeneration that would replace the
+  // agent's recording with a synthesized listening version).
+  const isAgentVoiceReply = speech !== null && speech.origin === "agent";
 
   const prepareSpeech = useCallback(async () => {
     if (preparing) return;
@@ -1453,7 +1536,7 @@ function AssistantMessageMetaAndArtifacts(props: {
             )}
           </Pressable>
         ) : null}
-        {props.textToSpeechAvailable || speech !== null ? (
+        {(props.textToSpeechAvailable || speech !== null) && !isAgentVoiceReply ? (
           <Pressable
             accessibilityRole="button"
             accessibilityLabel={
@@ -1515,7 +1598,7 @@ function AssistantMessageMetaAndArtifacts(props: {
           )}
         </View>
       ) : null}
-      {speech !== null && expanded ? (
+      {speech !== null && expanded && !isAgentVoiceReply ? (
         <AssistantSpeechPlayer
           environmentId={props.environmentId}
           speech={speech}
@@ -1535,7 +1618,12 @@ function AssistantSpeechPlayer(props: {
   readonly iconSubtleColor: ColorValue;
   readonly transcriptExpanded: boolean;
   readonly onToggleTranscript: () => void;
-  readonly onRetry: () => void;
+  /** null hides the regenerate action (agent recordings cannot be re-made client-side). */
+  readonly onRetry: (() => void) | null;
+  /** Lets the row fall back to the written reply when the audio is gone. */
+  readonly onAudioUnavailable?: () => void;
+  /** Agent voice replies render the player as the message's main content. */
+  readonly primary?: boolean;
 }) {
   const { blocked, speed } = useListeningPlaybackSnapshot();
   // The transport sits on `bg-foreground`, so its glyph has to come from the
@@ -1549,8 +1637,19 @@ function AssistantSpeechPlayer(props: {
     attachmentId: props.speech.speechId,
   });
   const audioUrl = audioUrlState._tag === "Success" ? audioUrlState.url : null;
-  const player = useAudioPlayer(audioUrl, { updateInterval: 250 });
+  // Primary players mount for every voice-reply row in the feed, so they must
+  // not fetch their MP3 until the user asks to play — a thread can hold many
+  // recordings and the app may be on a remote or cellular link. Secondary
+  // players only mount after an explicit expand, which is consent enough.
+  const [activated, setActivated] = useState(props.primary !== true);
+  const [pendingPlay, setPendingPlay] = useState(false);
+  const player = useAudioPlayer(activated ? audioUrl : null, { updateInterval: 250 });
   const status = useAudioPlayerStatus(player);
+  const audioUnavailable = audioUrlState._tag === "Failure";
+  const onAudioUnavailableProp = props.onAudioUnavailable;
+  useEffect(() => {
+    if (audioUnavailable) onAudioUnavailableProp?.();
+  }, [audioUnavailable, onAudioUnavailableProp]);
   const progress = status.duration > 0 ? Math.min(1, status.currentTime / status.duration) : 0;
 
   const pausePlayer = useCallback(() => {
@@ -1582,54 +1681,81 @@ function AssistantSpeechPlayer(props: {
     [pausePlayer, props.speech.speechId],
   );
 
+  const startPlayback = useCallback(
+    () =>
+      startListeningPlayback({
+        id: props.speech.speechId,
+        pause: pausePlayer,
+        restartFromBeginning: status.duration > 0 && status.currentTime >= status.duration - 0.1,
+        seekToBeginning: () => player.seekTo(0),
+        prepareAudioMode: () =>
+          setAudioModeAsync({ allowsRecording: false, playsInSilentMode: true }),
+        applyPlaybackRate,
+        play: () => player.play(),
+      }),
+    [
+      applyPlaybackRate,
+      pausePlayer,
+      player,
+      props.speech.speechId,
+      status.currentTime,
+      status.duration,
+    ],
+  );
+
   const onTogglePlayback = useCallback(async () => {
     if (status.playing) {
       pausePlayer();
       return;
     }
     if (blocked) return;
-    await startListeningPlayback({
-      id: props.speech.speechId,
-      pause: pausePlayer,
-      restartFromBeginning: status.duration > 0 && status.currentTime >= status.duration - 0.1,
-      seekToBeginning: () => player.seekTo(0),
-      prepareAudioMode: () =>
-        setAudioModeAsync({ allowsRecording: false, playsInSilentMode: true }),
-      applyPlaybackRate,
-      play: () => player.play(),
-    });
-  }, [
-    applyPlaybackRate,
-    blocked,
-    pausePlayer,
-    player,
-    props.speech.speechId,
-    status.currentTime,
-    status.duration,
-    status.playing,
-  ]);
+    if (!activated) {
+      setActivated(true);
+      setPendingPlay(true);
+      return;
+    }
+    await startPlayback();
+  }, [activated, blocked, pausePlayer, startPlayback, status.playing]);
+
+  // First tap on a deferred player: wait for the source to attach, then play.
+  useEffect(() => {
+    if (!pendingPlay || !activated || audioUrl === null) return;
+    setPendingPlay(false);
+    void startPlayback();
+  }, [activated, audioUrl, pendingPlay, startPlayback]);
 
   return (
-    <View className="mt-2 gap-2 rounded-2xl border border-border bg-subtle p-3">
+    <View
+      className={cn(
+        "gap-2 rounded-2xl border border-border bg-subtle p-3",
+        props.primary ? "mb-1" : "mt-2",
+      )}
+    >
       <View className="flex-row items-center gap-2">
         <SymbolView
           name="headphones"
           size={14}
           tintColor={props.iconSubtleColor}
           type="monochrome"
         />
-        <Text className="font-t3-bold text-xs text-foreground">Listening version</Text>
+        <Text className="font-t3-bold text-xs text-foreground">
+          {props.primary ? "Voice reply" : "Listening version"}
+        </Text>
       </View>
       {audioUrlState._tag === "Failure" ? (
         <View className="gap-2 py-1">
           <Text className="text-xs text-foreground-muted">
-            The audio file is unavailable. Regenerate it to listen again.
+            {props.onRetry === null
+              ? "The audio file is unavailable."
+              : "The audio file is unavailable. Regenerate it to listen again."}
           </Text>
-          <Pressable accessibilityRole="button" className="min-h-8" onPress={props.onRetry}>
-            <Text className="font-t3-medium text-xs text-foreground">Regenerate</Text>
-          </Pressable>
+          {props.onRetry !== null ? (
+            <Pressable accessibilityRole="button" className="min-h-8" onPress={props.onRetry}>
+              <Text className="font-t3-medium text-xs text-foreground">Regenerate</Text>
+            </Pressable>
+          ) : null}
         </View>
-      ) : audioUrl === null ? (
+      ) : audioUrl === null && activated ? (
         <View className="flex-row items-center gap-2 py-1">
           <ActivityIndicator size="small" color={props.iconSubtleColor} />
           <Text className="text-xs text-foreground-muted">Loading audio…</Text>
@@ -1641,10 +1767,10 @@ function AssistantSpeechPlayer(props: {
               accessibilityRole="button"
               accessibilityLabel={
                 blocked
-                  ? "Play listening version unavailable while recording"
+                  ? `Play ${props.primary ? "voice reply" : "listening version"} unavailable while recording`
                   : status.playing
-                    ? "Pause listening version"
-                    : "Play listening version"
+                    ? `Pause ${props.primary ? "voice reply" : "listening version"}`
+                    : `Play ${props.primary ? "voice reply" : "listening version"}`
               }
               accessibilityState={{ disabled: blocked }}
               className={cn(
@@ -1689,7 +1815,7 @@ function AssistantSpeechPlayer(props: {
         onPress={props.onToggleTranscript}
       >
         <Text className="font-t3-medium text-xs text-foreground-muted">
-          View listening transcript
+          {props.primary ? "View transcript" : "View listening transcript"}
         </Text>
         <SymbolView
           name={props.transcriptExpanded ? "chevron.up" : "chevron.down"}

apps/server/integration/OrchestrationEngineHarness.integration.ts

@@ -87,6 +87,7 @@ import { VcsStatusBroadcaster } from "../src/vcs/VcsStatusBroadcaster.ts";
 import { GitWorkflowService } from "../src/git/GitWorkflowService.ts";
 import * as VcsProcess from "../src/vcs/VcsProcess.ts";
 import * as AgentAwarenessRelay from "../src/relay/AgentAwarenessRelay.ts";
+import * as AgentVoiceReply from "../src/voice/AgentVoiceReply.ts";
 
 const decodeCodexSettings = Schema.decodeEffect(CodexSettings);
 
@@ -406,6 +407,7 @@ export const makeOrchestrationIntegrationHarness = (
       Layer.provideMerge(ServerSettingsService.layerTest()),
       Layer.provideMerge(ServerConfig.layerTest(workspaceDir, rootDir)),
       Layer.provideMerge(ProviderInstanceHealthLive),
+      Layer.provideMerge(AgentVoiceReply.layerNoop),
       Layer.provideMerge(NodeServices.layer),
     );