- View listening transcript
+ {primary ? "View transcript" : "View listening transcript"}
{speech.transcript}
diff --git a/apps/web/src/components/settings/ExtrasSettingsPanel.tsx b/apps/web/src/components/settings/ExtrasSettingsPanel.tsx
index ca52861fe47e..a591c8371bdf 100644
--- a/apps/web/src/components/settings/ExtrasSettingsPanel.tsx
+++ b/apps/web/src/components/settings/ExtrasSettingsPanel.tsx
@@ -941,10 +941,40 @@ function VoiceExtrasSection() {
}
/>
+
+ updateSettings({
+ voice: {
+ enableAgentVoiceReplies: DEFAULT_UNIFIED_SETTINGS.voice.enableAgentVoiceReplies,
+ },
+ })
+ }
+ />
+ ) : null
+ }
+ control={
+
+ updateSettings({ voice: { enableAgentVoiceReplies: Boolean(checked) } })
+ }
+ aria-label="Allow agent voice replies"
+ />
+ }
+ />
+
- Speech playback needs ELEVENLABS_API_KEY in the server's environment. When set,
- these fields override the server's ELEVENLABS_TTS_MODEL and{" "}
- ELEVENLABS_TTS_VOICE_ID environment variables.
+ Speech playback and agent voice replies need ELEVENLABS_API_KEY in the server's
+ environment. When set, these fields override the server's ELEVENLABS_TTS_MODEL{" "}
+ and ELEVENLABS_TTS_VOICE_ID environment variables.
);
diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts
index 83e985a2e960..7a898ddea480 100644
--- a/apps/web/src/components/settings/settingsSearch.ts
+++ b/apps/web/src/components/settings/settingsSearch.ts
@@ -297,6 +297,11 @@ export const SETTINGS_SEARCH_ITEMS = [
title: "Voice & Listening",
to: "/settings/extras",
},
+ {
+ id: "agent-voice-replies",
+ title: "Agent voice replies",
+ to: "/settings/extras",
+ },
{
id: "archive",
title: "Archived threads",
diff --git a/packages/client-runtime/src/state/threadReducer.test.ts b/packages/client-runtime/src/state/threadReducer.test.ts
index 985b6f0aca37..8647a2575a22 100644
--- a/packages/client-runtime/src/state/threadReducer.test.ts
+++ b/packages/client-runtime/src/state/threadReducer.test.ts
@@ -619,6 +619,62 @@ describe("applyThreadDetailEvent", () => {
}
});
+ it("maps agent voice-reply speech from a completion onto the existing message", () => {
+ const threadWithMessage: OrchestrationThread = {
+ ...baseThread,
+ messages: [
+ {
+ id: MessageId.make("msg-voice"),
+ role: "assistant",
+ text: "Written reply",
+ turnId: TurnId.make("turn-1"),
+ streaming: false,
+ createdAt: "2026-04-01T06:00:00.000Z",
+ updatedAt: "2026-04-01T06:00:00.000Z",
+ },
+ ],
+ };
+
+ const result = applyThreadDetailEvent(threadWithMessage, {
+ ...baseEventFields,
+ sequence: 7,
+ occurredAt: "2026-04-01T06:01:00.000Z",
+ aggregateKind: "thread",
+ aggregateId: ThreadId.make("thread-1"),
+ type: "thread.message-sent",
+ payload: {
+ threadId: ThreadId.make("thread-1"),
+ messageId: MessageId.make("msg-voice"),
+ role: "assistant",
+ text: "",
+ speech: {
+ speechId: "speech-1",
+ transcript: "Spoken version",
+ mimeType: "audio/mpeg",
+ sizeBytes: 4321,
+ sourceTextHash: "hash",
+ voiceId: "voice-1",
+ ttsModel: "eleven_flash_v2_5",
+ origin: "agent",
+ createdAt: "2026-04-01T06:01:00.000Z",
+ },
+ turnId: TurnId.make("turn-1"),
+ streaming: false,
+ createdAt: "2026-04-01T06:01:00.000Z",
+ updatedAt: "2026-04-01T06:01:00.000Z",
+ },
+ } as Parameters[1]);
+
+ expect(result.kind).toBe("updated");
+ if (result.kind === "updated") {
+ const updated = result.thread.messages[0];
+ expect(updated?.text).toBe("Written reply");
+ expect(updated?.speech?.origin).toBe("agent");
+ expect(updated?.speech?.speechId).toBe("speech-1");
+ expect(updated?.speech?.transcript).toBe("Spoken version");
+ }
+ });
+
it("updates latestTurn for assistant messages with a turn", () => {
const result = applyThreadDetailEvent(baseThread, {
...baseEventFields,
diff --git a/packages/client-runtime/src/state/threadReducer.ts b/packages/client-runtime/src/state/threadReducer.ts
index b393ae508f7c..d168a5b78714 100644
--- a/packages/client-runtime/src/state/threadReducer.ts
+++ b/packages/client-runtime/src/state/threadReducer.ts
@@ -489,6 +489,21 @@ function applyThreadDetailEventUnretained(
}
case "thread.message-sent": {
+ // An agent-staged voice recording arrives as speech metadata on the
+ // completion event; map it onto the message so the live timeline shows
+ // the player without waiting for a snapshot refresh.
+ const speech: OrchestrationMessage["speech"] =
+ event.payload.speech !== undefined
+ ? {
+ messageId: event.payload.messageId,
+ speechId: event.payload.speech.speechId,
+ transcript: event.payload.speech.transcript,
+ mimeType: event.payload.speech.mimeType,
+ sizeBytes: event.payload.speech.sizeBytes,
+ origin: event.payload.speech.origin,
+ createdAt: event.payload.speech.createdAt,
+ }
+ : undefined;
const message: OrchestrationMessage = {
id: event.payload.messageId,
role: event.payload.role,
@@ -496,6 +511,7 @@ function applyThreadDetailEventUnretained(
...(event.payload.attachments !== undefined
? { attachments: event.payload.attachments }
: {}),
+ ...(speech !== undefined ? { speech } : {}),
turnId: event.payload.turnId,
streaming: event.payload.streaming,
createdAt: event.payload.createdAt,
@@ -520,6 +536,7 @@ function applyThreadDetailEventUnretained(
...(message.attachments !== undefined
? { attachments: message.attachments }
: {}),
+ ...(speech !== undefined ? { speech } : {}),
},
)
: Arr.append(thread.messages, message);
diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts
index 0162d7c25dc9..084e53f25e3c 100644
--- a/packages/contracts/src/orchestration.ts
+++ b/packages/contracts/src/orchestration.ts
@@ -23,7 +23,11 @@ import {
TurnId,
} from "./baseSchemas.ts";
import { ProviderDriverKind, ProviderInstanceId } from "./providerInstance.ts";
-import { MessageSpeechSynthesisResult, MessageSummaryResult } from "./voice.ts";
+import {
+ MessageSpeechAttachment,
+ MessageSpeechSynthesisResult,
+ MessageSummaryResult,
+} from "./voice.ts";
export const ORCHESTRATION_WS_METHODS = {
dispatchCommand: "orchestration.dispatchCommand",
@@ -1128,6 +1132,13 @@ const ThreadMessageAssistantCompleteCommand = Schema.Struct({
threadId: ThreadId,
messageId: MessageId,
turnId: Schema.optional(TurnId),
+ // An agent-staged voice recording to attach to the completed message. The
+ // MP3 already sits in the attachments directory; this is metadata only.
+ speech: Schema.optional(MessageSpeechAttachment),
+ // Text the decider publishes only when the target message does not exist
+ // (or has no text of its own), so a voice reply for a turn that produced
+ // no written message still lands as one atomic command.
+ fallbackText: Schema.optional(Schema.String),
createdAt: IsoDateTime,
});
@@ -1403,6 +1414,7 @@ export const ThreadMessageSentPayload = Schema.Struct({
text: Schema.String,
attachments: Schema.optional(Schema.Array(ChatAttachment)),
inputOrigin: Schema.optional(MessageInputOrigin),
+ speech: Schema.optional(MessageSpeechAttachment),
turnId: Schema.NullOr(TurnId),
streaming: Schema.Boolean,
createdAt: IsoDateTime,
diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts
index e33615fa4c05..5b2d028c78f7 100644
--- a/packages/contracts/src/previewAutomation.ts
+++ b/packages/contracts/src/previewAutomation.ts
@@ -634,7 +634,7 @@ export type PreviewAutomationResponse = typeof PreviewAutomationResponse.Type;
export class PreviewAutomationUnavailableError extends Schema.TaggedErrorClass()(
"PreviewAutomationUnavailableError",
{
- capability: Schema.Literal("preview"),
+ capability: Schema.Literals(["preview", "voice"]),
environmentId: EnvironmentId,
threadId: ThreadId,
providerSessionId: TrimmedNonEmptyString,
diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts
index dadb7e53805f..3404e4b9d9c1 100644
--- a/packages/contracts/src/settings.test.ts
+++ b/packages/contracts/src/settings.test.ts
@@ -361,19 +361,38 @@ describe("ClientSettings extras", () => {
describe("ServerSettings.voice", () => {
it("defaults to unset, so the server keeps its env/default resolution", () => {
- expect(DEFAULT_SERVER_SETTINGS.voice).toEqual({ ttsModelId: "", ttsVoiceId: "" });
- expect(decodeServerSettings({}).voice).toEqual({ ttsModelId: "", ttsVoiceId: "" });
+ expect(DEFAULT_SERVER_SETTINGS.voice).toEqual({
+ ttsModelId: "",
+ ttsVoiceId: "",
+ enableAgentVoiceReplies: true,
+ });
+ expect(decodeServerSettings({}).voice).toEqual({
+ ttsModelId: "",
+ ttsVoiceId: "",
+ enableAgentVoiceReplies: true,
+ });
});
it("trims values in both the settings and the patch", () => {
expect(decodeServerSettings({ voice: { ttsModelId: " eleven_v3 " } }).voice).toEqual({
ttsModelId: "eleven_v3",
ttsVoiceId: "",
+ enableAgentVoiceReplies: true,
});
expect(decodeServerSettingsPatch({ voice: { ttsVoiceId: " abc " } })).toEqual({
voice: { ttsVoiceId: "abc" },
});
});
+
+ it("round-trips the agent voice replies toggle through the patch", () => {
+ expect(
+ decodeServerSettings({ voice: { enableAgentVoiceReplies: false } }).voice
+ .enableAgentVoiceReplies,
+ ).toBe(false);
+ expect(decodeServerSettingsPatch({ voice: { enableAgentVoiceReplies: false } })).toEqual({
+ voice: { enableAgentVoiceReplies: false },
+ });
+ });
});
describe("ServerSettings.providerInstances (slice-2 invariant)", () => {
diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts
index 29946425c2b6..68a7c3ab5b26 100644
--- a/packages/contracts/src/settings.ts
+++ b/packages/contracts/src/settings.ts
@@ -858,6 +858,10 @@ export type ObservabilitySettings = typeof ObservabilitySettings.Type;
export const VoiceSettings = Schema.Struct({
ttsModelId: TrimmedString.pipe(Schema.withDecodingDefault(Effect.succeed(""))),
ttsVoiceId: TrimmedString.pipe(Schema.withDecodingDefault(Effect.succeed(""))),
+ // Exposes the voice_reply MCP tool to agent sessions so they can answer
+ // with a staged recording. Only effective while the server has an
+ // ELEVENLABS_API_KEY; defaults to on so setting the key is enough.
+ enableAgentVoiceReplies: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
});
export type VoiceSettings = typeof VoiceSettings.Type;
@@ -1244,6 +1248,7 @@ export const ServerSettingsPatch = Schema.Struct({
Schema.Struct({
ttsModelId: Schema.optionalKey(TrimmedString),
ttsVoiceId: Schema.optionalKey(TrimmedString),
+ enableAgentVoiceReplies: Schema.optionalKey(Schema.Boolean),
}),
),
providers: Schema.optionalKey(
diff --git a/packages/contracts/src/voice.ts b/packages/contracts/src/voice.ts
index 093698317609..f7107544802a 100644
--- a/packages/contracts/src/voice.ts
+++ b/packages/contracts/src/voice.ts
@@ -38,6 +38,15 @@ export const MESSAGE_SPEECH_MAX_SCRIPT_CHARS = 40_000;
export const MESSAGE_SUMMARY_MAX_SOURCE_CHARS = 120_000;
export const MESSAGE_SUMMARY_MAX_TEXT_CHARS = 12_000;
+/**
+ * Who produced a message's speech artifact. "user" is the on-demand listening
+ * version a client requested; "agent" is a recording the agent staged itself
+ * through the voice_reply MCP tool. Agent recordings are presented as the
+ * primary form of the message; user ones stay an opt-in secondary artifact.
+ */
+export const MessageSpeechOrigin = Schema.Literals(["user", "agent"]);
+export type MessageSpeechOrigin = typeof MessageSpeechOrigin.Type;
+
export const MessageSpeechSynthesisRequest = Schema.Struct({
messageId: MessageId,
});
@@ -49,10 +58,63 @@ export const MessageSpeechSynthesisResult = Schema.Struct({
transcript: TrimmedNonEmptyString.check(Schema.isMaxLength(MESSAGE_SPEECH_MAX_SCRIPT_CHARS)),
mimeType: Schema.Literal("audio/mpeg"),
sizeBytes: NonNegativeInt,
+ // Optional so payloads persisted before agent voice replies still decode;
+ // absent means "user".
+ origin: Schema.optional(MessageSpeechOrigin),
createdAt: IsoDateTime,
});
export type MessageSpeechSynthesisResult = typeof MessageSpeechSynthesisResult.Type;
+export const AGENT_VOICE_REPLY_MAX_SCRIPT_CHARS = 10_000;
+
+/**
+ * Speech metadata carried on an assistant-message completion. The audio bytes
+ * live in the server attachments directory under `.mp3` (written
+ * before the command is dispatched, mirroring how user image attachments are
+ * persisted by the normalizer); the event stream only ever sees metadata.
+ */
+export const MessageSpeechAttachment = Schema.Struct({
+ speechId: TrimmedNonEmptyString,
+ transcript: TrimmedNonEmptyString.check(Schema.isMaxLength(MESSAGE_SPEECH_MAX_SCRIPT_CHARS)),
+ mimeType: Schema.Literal("audio/mpeg"),
+ sizeBytes: NonNegativeInt,
+ sourceTextHash: TrimmedNonEmptyString,
+ voiceId: TrimmedNonEmptyString,
+ ttsModel: TrimmedNonEmptyString,
+ origin: MessageSpeechOrigin,
+ createdAt: IsoDateTime,
+});
+export type MessageSpeechAttachment = typeof MessageSpeechAttachment.Type;
+
+export const AgentVoiceReplyInput = Schema.Struct({
+ script: TrimmedNonEmptyString.check(Schema.isMaxLength(AGENT_VOICE_REPLY_MAX_SCRIPT_CHARS)),
+});
+export type AgentVoiceReplyInput = typeof AgentVoiceReplyInput.Type;
+
+export const AgentVoiceReplyResult = Schema.Struct({
+ status: Schema.Literal("staged"),
+ transcriptChars: NonNegativeInt,
+ audioSizeBytes: NonNegativeInt,
+});
+export type AgentVoiceReplyResult = typeof AgentVoiceReplyResult.Type;
+
+export class AgentVoiceReplyError extends Schema.TaggedErrorClass()(
+ "AgentVoiceReplyError",
+ {
+ // turn_unavailable: the thread has no identifiable active turn, or the
+ // active turn changed while the recording was being synthesized (the turn
+ // was steered or aborted), so the recording has no turn to attach to.
+ reason: Schema.Literals([
+ "unavailable",
+ "empty_script",
+ "script_too_long",
+ "turn_unavailable",
+ "provider_failed",
+ "storage_failed",
+ ]),
+ },
+) {}
+
export const MessageSummaryRequest = Schema.Struct({
messageId: MessageId,
});
diff --git a/packages/shared/src/serverSettings.test.ts b/packages/shared/src/serverSettings.test.ts
index a2898de52c84..a40bd0daf9ca 100644
--- a/packages/shared/src/serverSettings.test.ts
+++ b/packages/shared/src/serverSettings.test.ts
@@ -66,18 +66,27 @@ describe("serverSettings helpers", () => {
it("merges the voice group one field at a time", () => {
const current = {
...DEFAULT_SERVER_SETTINGS,
- voice: { ttsModelId: "eleven_v3", ttsVoiceId: "voice-a" },
+ voice: { ttsModelId: "eleven_v3", ttsVoiceId: "voice-a", enableAgentVoiceReplies: true },
};
- // A patch touching one field leaves the other alone...
+ // A patch touching one field leaves the others alone...
expect(applyServerSettingsPatch(current, { voice: { ttsVoiceId: "voice-b" } }).voice).toEqual({
ttsModelId: "eleven_v3",
ttsVoiceId: "voice-b",
+ enableAgentVoiceReplies: true,
});
// ...and an empty string clears a field back to "unset".
expect(applyServerSettingsPatch(current, { voice: { ttsModelId: "" } }).voice).toEqual({
ttsModelId: "",
ttsVoiceId: "voice-a",
+ enableAgentVoiceReplies: true,
+ });
+ expect(
+ applyServerSettingsPatch(current, { voice: { enableAgentVoiceReplies: false } }).voice,
+ ).toEqual({
+ ttsModelId: "eleven_v3",
+ ttsVoiceId: "voice-a",
+ enableAgentVoiceReplies: false,
});
expect(applyServerSettingsPatch(current, {}).voice).toEqual(current.voice);
});