diff --git a/.fern/metadata.json b/.fern/metadata.json index f4f7a0d..62289a3 100644 --- a/.fern/metadata.json +++ b/.fern/metadata.json @@ -14,5 +14,5 @@ }, "exclude_types_from_init_exports": true }, - "originGitCommit": "3fdb45ab45802742325afca544fdc22f5ae58db8" + "originGitCommit": "26314a9d3199f5e135302c3c3645b6bbb8f556d4" } \ No newline at end of file diff --git a/.fern/replay.lock b/.fern/replay.lock index a1ce605..699a092 100644 --- a/.fern/replay.lock +++ b/.fern/replay.lock @@ -66,14 +66,20 @@ generations: cli_version: unknown generator_versions: fernapi/fern-python-sdk: 4.37.0 -current_generation: 3885ea61224cbf0671c305350b8d1cddef509443 + - commit_sha: f7e9b5d993665a16b43eb1d7d442b118fbeffa4b + tree_hash: 68519b3d82b64ac128a76d6f5a86b1d4e5661410 + timestamp: 2026-09-16T11:18:46.374Z + cli_version: unknown + generator_versions: + fernapi/fern-python-sdk: 4.37.0 +current_generation: f7e9b5d993665a16b43eb1d7d442b118fbeffa4b patches: - id: patch-13d8e068 content_hash: sha256:97d879ab169016b9abaf1866427a8ebc45af608f69e063a07e5126c9613c329b original_commit: 13d8e0683ffa08bcfc9a381eb9604adf5abdbe0b original_message: "chore: ignore local venv and planning docs" original_author: plutoless - base_generation: 3885ea61224cbf0671c305350b8d1cddef509443 + base_generation: f7e9b5d993665a16b43eb1d7d442b118fbeffa4b files: - .gitignore patch_content: | @@ -102,7 +108,7 @@ patches: original_commit: a065088b4c45e2ca7e1bb91b1a899b8444800121 original_message: "fix(vendors): send Speechmatics STT key instead of api_key" original_author: digitallysavvy - base_generation: 3885ea61224cbf0671c305350b8d1cddef509443 + base_generation: f7e9b5d993665a16b43eb1d7d442b118fbeffa4b files: - src/agora_agent/types/speechmatics_asr_params.py patch_content: | @@ -240,3 +246,69 @@ patches: smart_union = True extra = pydantic.Extra.allow user_owned: true + - id: patch-33a436b3 + content_hash: sha256:04db41b4eefe647eab420cbfbf8b86e78a2f42fb3c72122e535b149b89bc3459 + original_commit: 33a436b3bdb6f2a6fe4d8987b6536c2f0ac2862e + original_message: "fix: address v2.8.0 release review findings" + original_author: digitallysavvy + base_generation: f7e9b5d993665a16b43eb1d7d442b118fbeffa4b + files: + - src/agora_agent/agents/types/start_agents_request_properties_advanced_features.py + patch_content: | + diff --git a/src/agora_agent/agents/types/start_agents_request_properties_advanced_features.py b/src/agora_agent/agents/types/start_agents_request_properties_advanced_features.py + index e75f098..667e20e 100644 + --- a/src/agora_agent/agents/types/start_agents_request_properties_advanced_features.py + +++ b/src/agora_agent/agents/types/start_agents_request_properties_advanced_features.py + @@ -29,7 +29,7 @@ class StartAgentsRequestPropertiesAdvancedFeatures(UncheckedBaseModel): + + enable_tools: typing.Optional[bool] = pydantic.Field(default=None) + """ + - Enable tool invocation. When enabled, the agent can invoke tools provided by the MCP server to implement advanced functionality. + + Enable invocation for MCP servers and inline REST tools. + """ + + if IS_PYDANTIC_V2: + theirs_snapshot: + src/agora_agent/agents/types/start_agents_request_properties_advanced_features.py: | + # This file was auto-generated by Fern from our API Definition. + + import typing + + import pydantic + from ...core.pydantic_utilities import IS_PYDANTIC_V2 + from ...core.unchecked_base_model import UncheckedBaseModel + + + class StartAgentsRequestPropertiesAdvancedFeatures(UncheckedBaseModel): + """ + Advanced features configuration. + """ + + enable_mllm: typing.Optional[bool] = pydantic.Field(default=None) + """ + Use `mllm.enable` instead. Enable Multimodal Large Language Model for voice-to-voice processing. Enabling MLLM automatically disables ASR, LLM, and TTS since the MLLM handles end-to-end voice processing directly. See `turn_detection.type` for turn detection options available with MLLM. + """ + + enable_rtm: typing.Optional[bool] = pydantic.Field(default=None) + """ + Whether to enable the Signaling (RTM) service. When enabled, the agent can combine the capabilities provided by Signaling to implement advanced functions, such as delivering custom information. Before enabling the Signaling service, make sure the token includes both RTC and RTM privileges. + """ + + enable_sal: typing.Optional[bool] = pydantic.Field(default=None) + """ + Enable Selective Attention Locking (SAL). When enabled, configure the `sal` field to set up speaker recognition or locking modes. + """ + + enable_tools: typing.Optional[bool] = pydantic.Field(default=None) + """ + Enable invocation for MCP servers and inline REST tools. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/changelog.md b/changelog.md index c1c0996..6edd7b1 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/). +## [v2.10.0] — 2026-09-16 + +### Changed + +- **Gemini Live production routing and compatibility** — Gemini 3.8 sessions now use the normal production gateway without a preview feature header. Existing imports from `agora_agent.agentkit.preview` remain compatible and resolve to the production `GeminiLive` implementation and model constants. + ## [v2.9.0] — 2026-09-15 ### Added @@ -19,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). ### Changed +- **OpenAI GPT Live production routing** — moved `OpenAIGPTLive` to the production MLLM vendors and removed its preview registration. Existing imports from `agora_agent.agentkit.preview` remain compatible and now route to Production automatically. - **OpenAI GPT Live defaults** — `OpenAIGPTLive` now defaults to the shortened `gpt-live-1` model name and omits the alpha selector. Explicit `model` and `alpha_selector` options remain available for future preview contracts. ## [v2.8.0] — 2026-09-10 diff --git a/compat/agora-agent-server-sdk/pyproject.toml b/compat/agora-agent-server-sdk/pyproject.toml index a8eea54..7618bee 100644 --- a/compat/agora-agent-server-sdk/pyproject.toml +++ b/compat/agora-agent-server-sdk/pyproject.toml @@ -3,7 +3,7 @@ name = "agora-agent-server-sdk" [tool.poetry] name = "agora-agent-server-sdk" -version = "v2.9.0" +version = "v2.10.0" description = "Compatibility shim for the renamed agora-agents package." readme = "README.md" authors = [] @@ -35,7 +35,7 @@ Repository = 'https://github.com/AgoraIO/agora-agents-python' [tool.poetry.dependencies] python = "^3.8" -agora-agents = ">=2.9.0,<3.0.0" +agora-agents = ">=2.10.0,<3.0.0" [build-system] requires = ["poetry-core"] diff --git a/docs/concepts/vendors.md b/docs/concepts/vendors.md index 3ba71f9..13f6cd5 100644 --- a/docs/concepts/vendors.md +++ b/docs/concepts/vendors.md @@ -158,9 +158,9 @@ Used with `agent.with_mllm()` for the [MLLM flow](../guides/mllm-flow.md). These | Class | Provider | Area | Required Parameters | |---|---|---|---| | `OpenAIRealtime` | OpenAI Realtime | Global | `api_key`; optional `turn_detection` | -| `OpenAIGPTLive` (preview) | OpenAI GPT Live | Global | `api_key`; optional `greeting` | +| `OpenAIGPTLive` | OpenAI GPT Live | Global | `api_key`; optional `greeting` | | `AzureOpenAIRealtime` | Azure OpenAI Realtime | Global | `api_key`, `url`, `turn_detection`; optional `max_history` | -| `GeminiLive` | Google Gemini Live API | Global | `api_key`; `model` defaults to `models/gemini-3.8-live`. The two 3.8 IDs use preview routing; older IDs use production. | +| `GeminiLive` | Google Gemini Live API | Global | `api_key`; `model` defaults to `models/gemini-3.8-live`. All model IDs use production routing. | | `VertexAI` | Vertex AI (Gemini Live) | Global | `model`, `project_id`, `location`, `adc_credentials_string`; optional `turn_detection` | | `XaiGrok` | xAI Grok (`mllm.vendor`: `xai`) | Global | `api_key`; optional `voice`, `language`, `sample_rate`, `turn_detection` | | `QwenOmni` | Alibaba Cloud Qwen Omni Realtime | CN | `api_key`, `url`; optional `turn_detection` | diff --git a/docs/guides/mllm-flow.md b/docs/guides/mllm-flow.md index 4f4414a..44135f2 100644 --- a/docs/guides/mllm-flow.md +++ b/docs/guides/mllm-flow.md @@ -95,7 +95,7 @@ asyncio.run(main()) ## Gemini Live -Use `GeminiLive` for the existing Gemini Live models and both Gemini 3.8 models. The 3.8 IDs select the preview route automatically; Extended Thinking also accepts `thinking_level`. See the [Preview Endpoint guide](./preview-endpoint.md). +Use `GeminiLive` for the existing Gemini Live models and both Gemini 3.8 models. All use the production route; Extended Thinking also accepts `thinking_level`. Gemini Live uses a Google AI API key: diff --git a/docs/guides/openai-gpt-live-v3.md b/docs/guides/openai-gpt-live-v3.md index 6b5e001..f8d0c56 100644 --- a/docs/guides/openai-gpt-live-v3.md +++ b/docs/guides/openai-gpt-live-v3.md @@ -1,6 +1,6 @@ -# GPT Live v3 preview +# GPT Live v3 -This preview targets `gpt-live-1` on `/v1/live/sessions`. Use an alpha-enabled OpenAI key. Do not use this alpha for production traffic. +GPT Live targets `gpt-live-1` on `/v1/live/sessions` through the production gateway. Existing imports from `agora_agent.agentkit.preview` remain supported as compatibility aliases. ```python @@ -51,15 +51,15 @@ GPT Live places MCP at `properties.mllm.mcp_servers`, the tool gate at `properti } ``` -This fragment omits the normal name, channel, token and UID fields populated by the SDK session. The preview route and `agora-feature: live-models` gate are selected automatically from the vendor. +This fragment omits the normal name, channel, token and UID fields populated by the SDK session. The production route is selected automatically from the vendor. The SDK omits `alpha_selector` by default. Set it only when a future preview contract requires an `OpenAI-Alpha` selector. ## Silence and backend rollout -Keep silence settings in the existing agent parameters builder, never in vendor params. The public API spelling is `silence_config`, with `{timeout_ms, action, content}`. The supplied extension contract describes internal `parameters.main.silence` and supports `action: "think"`; the public documentation currently says `silence_config` does not apply to MLLM. Serialization is covered by tests, but the public documentation does not establish that the preview allocator maps it to GPT Live's internal MAIN setting. Confirm that backend mapping before relying on silence nudges. The SDK does not invent a new public `main` field. +Keep silence settings in the existing agent parameters builder, never in vendor params. The public API spelling is `silence_config`, with `{timeout_ms, action, content}`. The supplied extension contract describes internal `parameters.main.silence` and supports `action: "think"`; the public documentation currently says `silence_config` does not apply to MLLM. Serialization is covered by tests, but the public documentation does not establish that the allocator maps it to GPT Live's internal MAIN setting. Confirm that backend mapping before relying on silence nudges. The SDK does not invent a new public `main` field. -## Preview-only options +## Compatibility options The supplied backend contract marks custom voice objects, `responses_params`, and first-class context management as pending PR #1522. The SDK does not expose typed options for those fields. Use raw params only once your target backend supports that PR. Until then, context management is reachable via `session_params.context_management`. diff --git a/docs/guides/preview-endpoint.md b/docs/guides/preview-endpoint.md index 39148ea..3596932 100644 --- a/docs/guides/preview-endpoint.md +++ b/docs/guides/preview-endpoint.md @@ -10,10 +10,8 @@ Some providers may be released through a preview gateway before their production and `AsyncAgentSession` detect registered preview providers from the resolved start request and route the entire session automatically. -OpenAI GPT Live uses the `live-models` feature. Gemini 3.8 MLLMs use `gemini-live`. -Gemini STT has graduated to -production and uses the normal regional endpoint. Existing imports of `GeminiSTT` and `GeminiSTTModels` from -`agora_agent.agentkit.preview` remain supported as compatibility aliases. +OpenAI GPT Live, Gemini STT, and Gemini Live have graduated to the production gateway and use the normal regional +endpoint. Existing imports from `agora_agent.agentkit.preview` remain supported as compatibility aliases. ```python from agora_agent import Agent, OpenAIGPTLive @@ -26,16 +24,13 @@ session = ( agent_id = session.start() ``` -This session uses the preview base URL and sends `agora-feature: live-models`. A session using `GeminiSTT` uses the -client's normal GA regional endpoint without that header. +Preview providers use the preview base URL and `agora-feature` gate. Sessions using `GeminiSTT`, `GeminiLive`, or GPT +Live use the client's normal production regional endpoint without that header. -Use the single `GeminiLive(api_key=..., model=...)` class with `with_mllm`. -The model IDs are `models/gemini-3.8-live` and -`models/gemini-3.8-live-extended-thinking`; the low-latency ID is the default. -Set `thinking_level="medium"` for extended thinking. `GeminiLive` sends it -only for the extended-thinking ID. The Gemini +Use the single production `GeminiLive(api_key=..., model=...)` class with `with_mllm`. The model IDs are +`models/gemini-3.8-live` and `models/gemini-3.8-live-extended-thinking`; the low-latency ID is the default. Set +`thinking_level="medium"` for extended thinking. `GeminiLive` sends it only for the extended-thinking ID. The Gemini credential is sent once as `mllm.api_key`, never as `mllm.params.api_key`. -Gemini sessions send `agora-feature: gemini-live`, while GPT Live retains `live-models`. ## Session-scoped routing @@ -61,7 +56,7 @@ The registry is keyed first by request category and then by the serialized vendo ```python _PREVIEW_FEATURES_BY_CATEGORY = { "asr": {"new_vendor": "new-vendor-feature"}, - "mllm": {"openai_gpt_live": "live-models"}, + "mllm": {}, } ``` diff --git a/docs/reference/agent.md b/docs/reference/agent.md index 50e1915..68e46f1 100644 --- a/docs/reference/agent.md +++ b/docs/reference/agent.md @@ -228,7 +228,7 @@ filler_words = FillerWordsConfig( agent = agent.with_filler_words(filler_words) ``` -`generated_config` is optional. Its `llm_provider`, `prompt`, and `fallback_strategy` fields are optional as well; when omitted, the service uses its defaults. +`generated_config` is optional. Its `llm_provider`, `prompt`, and `fallback_strategy` fields are optional as well; when omitted, the service uses its defaults. Set `context_message_limit` to cap the number of recent messages and `history_character_limit` to cap the number of conversation-history characters passed to the filler-word model. ## `create_session()` diff --git a/docs/reference/vendors.md b/docs/reference/vendors.md index d22d705..959690b 100644 --- a/docs/reference/vendors.md +++ b/docs/reference/vendors.md @@ -449,6 +449,27 @@ AgentKit serializes `credential_mode` at the top level of the Rime TTS configura | `sample_rate` | `int` | No | `None` | Audio sample rate | | `skip_patterns` | `List[int]` | No | `None` | Skip patterns | +### `SmallestAITTS` + +Global-only Smallest AI text-to-speech provider (`tts.vendor: "smallestai"`). + +| Parameter | Type | Required | Default | Description | +|---|---|---|---|---| +| `api_key` | `str` | Yes | — | Smallest AI API key | +| `url` | `str` | No | `None` | Streaming HTTP endpoint | +| `model` | `str` | No | `None` | TTS model name | +| `voice_id` | `str` | No | `None` | Voice identifier | +| `sample_rate` | `int` | No | `None` | Output audio sample rate in Hz | +| `speed` | `float` | No | `None` | Speech rate multiplier | +| `language` | `str` | No | `None` | Synthesis language code | +| `number_pronunciation_language` | `str` | No | `None` | Language used to pronounce numbers | +| `math_notation` | `bool` | No | `None` | Verbalize mathematical notation | +| `pronunciation_dicts` | `List[str]` | No | `None` | Pronunciation dictionaries | +| `session_id` | `str` | No | `None` | Client-provided session identifier | +| `request_id` | `str` | No | `None` | Client-provided request identifier | +| `additional_params` | `Dict[str, Any]` | No | `None` | Additional Smallest AI parameters | +| `skip_patterns` | `List[int]` | No | `None` | Skip patterns | + ### `GenericTTS` `GenericTTS` currently supports HTTP and HTTPS endpoints. WebSocket endpoints are rejected until a WebSocket-backed generic TTS implementation is available. AgentKit serializes the current HTTP implementation with the internal vendor value `generic_http`. @@ -615,6 +636,37 @@ stt = GeminiSTT( | `language` | `str` | No | `None` | Language code for speech recognition | | `additional_params` | `Dict[str, Any]` | No | `None` | Additional xAI STT parameters | +### `SmallestAISTT` + +Global-only Smallest AI speech-to-text provider (`asr.vendor: "smallestai"`). + +| Parameter | Type | Required | Default | Description | +|---|---|---|---|---| +| `api_key` | `str` | Yes | — | Smallest AI API key | +| `language` | `str` | No | `None` | Recognition language code | +| `url` | `str` | No | `None` | Streaming WebSocket endpoint | +| `sample_rate` | `int` | No | `None` | Input audio sample rate in Hz | +| `encoding` | `str` | No | `None` | Input audio encoding | +| `word_timestamps` | `bool` | No | `None` | Include word timestamps | +| `sentence_timestamps` | `bool` | No | `None` | Include sentence timestamps | +| `diarize` | `bool` | No | `None` | Enable speaker diarization | +| `vad_events` | `bool` | No | `None` | Return voice activity events | +| `endpointing` | `bool` | No | `None` | Enable endpoint detection | +| `eou_timeout_ms` | `int` | No | `None` | End-of-utterance timeout in milliseconds | +| `format` | `bool` | No | `None` | Format the transcript | +| `finalize_on_words` | `bool` | No | `None` | Finalize results based on recognized words | +| `max_words` | `str` | No | `None` | Maximum words per result | +| `punctuate` | `bool` | No | `None` | Add punctuation | +| `capitalize` | `bool` | No | `None` | Capitalize transcript text | +| `itn_normalize` | `bool` | No | `None` | Enable inverse text normalization | +| `full_transcript` | `bool` | No | `None` | Return the full transcript | +| `keywords` | `str` | No | `None` | Comma-separated `keyword:weight` boosts | +| `redact_pii` | `bool` | No | `None` | Redact personally identifiable information | +| `redact_pci` | `bool` | No | `None` | Redact payment-card information | +| `additional_params` | `Dict[str, Any]` | No | `None` | Additional Smallest AI parameters | + +Boolean options are exposed as Python `bool` values and serialized to the API's string representation (`"true"` or `"false"`). + --- ## CN Vendors @@ -625,10 +677,10 @@ All CN LLM helpers reuse the `OpenAI`-compatible shape and set a different vendo | Class | Key parameters | |---|---| -| `AliyunLLM` | `base_url`, `model`, `api_key?`, `system_messages?`, `greeting_message?`, `failure_message?`, `max_history?`, `params?`, `headers?` | -| `BytedanceLLM` | `base_url`, `model`, `api_key?`, `system_messages?`, `greeting_message?`, `failure_message?`, `max_history?`, `params?`, `headers?` | -| `DeepSeekLLM` | `base_url`, `model`, `api_key?`, `system_messages?`, `greeting_message?`, `failure_message?`, `max_history?`, `params?`, `headers?` | -| `TencentLLM` | `base_url`, `model`, `api_key?`, `system_messages?`, `greeting_message?`, `failure_message?`, `max_history?`, `params?`, `headers?` | +| `AliyunLLM` | `base_url`, `model`, `api_key?`, `system_messages?`, `greeting_message?`, `failure_message?`, `max_history?`, `params?`, `headers?`, `tools?`, `mcp_servers?` | +| `BytedanceLLM` | `base_url`, `model`, `api_key?`, `system_messages?`, `greeting_message?`, `failure_message?`, `max_history?`, `params?`, `headers?`, `tools?`, `mcp_servers?` | +| `DeepSeekLLM` | `base_url`, `model`, `api_key?`, `system_messages?`, `greeting_message?`, `failure_message?`, `max_history?`, `params?`, `headers?`, `tools?`, `mcp_servers?` | +| `TencentLLM` | `base_url`, `model`, `api_key?`, `system_messages?`, `greeting_message?`, `failure_message?`, `max_history?`, `params?`, `headers?`, `tools?`, `mcp_servers?` | ### CN TTS Vendors @@ -872,9 +924,11 @@ agent = Agent(client=client).with_llm(llm).with_tools() ``` `server.body` is only valid for `POST`. Template values may use `{{args.name}}` in URLs and bodies, and `{{template_variables.name}}` or `{{tool_call_id}}` in URLs, headers, and bodies. `execution.mode` currently supports only `sync`; `timeout_ms` must be between `1000` and `100000`. -### OpenAIGPTLive (preview) -GPT Live v3 uses `mllm.vendor: "openai_gpt_live"`, model `gpt-live-1`, and `wss://api.openai.com/v1/live/sessions`. Sessions route through the preview gateway automatically. This alpha must not carry production traffic. +The same `tools` and `mcp_servers` fields are available on all AgentKit MLLM vendors. They serialize at the top level of `mllm`, not inside `mllm.params`. Use `LlmToolConfig` and `McpServerConfig` for typed configuration; dictionaries remain supported for backward compatibility. MCP transport defaults to `streamable_http` when omitted. Tool execution still requires `Agent.with_tools()`. +### OpenAIGPTLive + +GPT Live v3 uses `mllm.vendor: "openai_gpt_live"`, model `gpt-live-1`, and `wss://api.openai.com/v1/live/sessions`. Sessions use the production gateway. The historical import from `agora_agent.agentkit.preview` remains a compatibility alias. The SDK omits `params.alpha_selector` by default. Set `alpha_selector` only when a future preview contract requires an `OpenAI-Alpha` selector. Other tuning defaults remain owned by the provider. Explicit options override entries in `params`. Zero and false values are preserved. @@ -904,6 +958,7 @@ The SDK omits `params.alpha_selector` by default. Set `alpha_selector` only when | `greeting` | string | `mllm.greeting_message`; v3 may reword this request. | | `messages` | list | `mllm.messages`; prior conversation seeded by Agora. | | `mcp_servers` | list | `mllm.mcp_servers`; MCP servers exposed to GPT Live. Requires `Agent.with_tools()`. | +| `tools` | list | `mllm.tools`; inline REST tools exposed to GPT Live. Requires `Agent.with_tools()`. | | `failure_message` | string | `mllm.failure_message` | | `input_modalities / output_modalities` | string lists | Agora outer `mllm.input_modalities` / `mllm.output_modalities` | | `params` | object | Additional snake_case provider parameters. | @@ -938,6 +993,8 @@ Global Azure OpenAI Realtime vendor (`mllm.vendor`: `"azure"`). CN Alibaba Cloud Qwen Omni Realtime vendor (`mllm.vendor`: `"qwen_omni"`). Import it with `from agora_agent.cn import QwenOmni`. +`QwenOmni` accepts both `tools` and `mcp_servers`. Both fields serialize at the top level of `mllm`; typed `LlmToolConfig`/`McpServerConfig` values and dictionaries are supported. Enable execution with `Agent.with_tools()`. + | Parameter | Type | Required | Default | Description | |---|---|---|---|---| | `api_key` | `str` | Yes | — | Alibaba Cloud Model Studio API key | @@ -956,7 +1013,7 @@ CN Alibaba Cloud Qwen Omni Realtime vendor (`mllm.vendor`: `"qwen_omni"`). Impor ### `GeminiLive` -`GeminiLive` supports existing Gemini Live models and both public Gemini 3.8 voice models. The 3.8 IDs select the preview gateway with `agora-feature: gemini-live`; older model IDs keep the production route. See [Preview Endpoint](../guides/preview-endpoint.md). +`GeminiLive` supports existing Gemini Live models and both public Gemini 3.8 voice models on the production gateway. Preview-era imports remain available as compatibility aliases. | Parameter | Type | Required | Default | Description | |---|---|---|---|---| diff --git a/pyproject.toml b/pyproject.toml index 3e39dc1..d0db875 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ dynamic = ["version"] [tool.poetry] name = "agora-agents" -version = "v2.9.0" +version = "v2.10.0" description = "" readme = "README.md" authors = [] diff --git a/src/agora_agent/__init__.py b/src/agora_agent/__init__.py index 5a1ad53..9c97381 100644 --- a/src/agora_agent/__init__.py +++ b/src/agora_agent/__init__.py @@ -20,6 +20,7 @@ AgentSessionOptions, CNAgent, GlobalAgent, + McpServerConfig, RegionalAgent, generate_rtc_token, GenerateTokenOptions, @@ -70,6 +71,8 @@ RimeTTS, SarvamSTT, SarvamTTS, + SmallestAISTT, + SmallestAITTS, SpeechmaticsSTT, VertexAI, VertexAILLM, diff --git a/src/agora_agent/agentkit/__init__.py b/src/agora_agent/agentkit/__init__.py index 924431b..c45c514 100644 --- a/src/agora_agent/agentkit/__init__.py +++ b/src/agora_agent/agentkit/__init__.py @@ -67,6 +67,7 @@ LlmToolFunctionConfig, LlmToolFunctionParametersConfig, LlmToolServerConfig, + McpServerConfig, McpServersItem, SessionInfo, SessionListResponse, @@ -154,8 +155,8 @@ OpenAITtsPresetModels, normalize_preset_input, ) -from .preview import OpenAIGPTLive, GeminiLiveModels, GEMINI_MLLM_DEFAULT_MODEL from .vendors import ( + GEMINI_MLLM_DEFAULT_MODEL, AkoolAvatar, AmazonBedrock, AmazonSTT, @@ -184,6 +185,7 @@ GeminiSTT, GeminiSTTModels, GeminiLive, + GeminiLiveModels, GenericAvatar, GoogleSTT, GoogleTTS, @@ -202,6 +204,7 @@ MurfTTS, OpenAI, OpenAIRealtime, + OpenAIGPTLive, OpenAISampleRate, OpenAISTT, OpenAITTS, @@ -211,6 +214,8 @@ SampleRate, SarvamSTT, SarvamTTS, + SmallestAISTT, + SmallestAITTS, TypecastTTS, SpeechmaticsSTT, XaiSTT, @@ -345,6 +350,7 @@ "LlmToolFunctionConfig", "LlmToolFunctionParametersConfig", "LlmToolServerConfig", + "McpServerConfig", "McpServersItem", "AgentSession", "AsyncAgentSession", @@ -436,6 +442,7 @@ "StepFunTTS", "MurfTTS", "SarvamTTS", + "SmallestAITTS", "XaiTTS", "SpeechmaticsSTT", "DeepgramSTT", @@ -449,6 +456,7 @@ "AssemblyAISTT", "AresSTT", "SarvamSTT", + "SmallestAISTT", "XaiSTT", "TencentSTT", "FengmingSTT", diff --git a/src/agora_agent/agentkit/agent.py b/src/agora_agent/agentkit/agent.py index 2ea93c7..5eb5659 100644 --- a/src/agora_agent/agentkit/agent.py +++ b/src/agora_agent/agentkit/agent.py @@ -73,6 +73,7 @@ from ..types.llm_tool_function import LlmToolFunction from ..types.llm_tool_function_parameters import LlmToolFunctionParameters from ..types.llm_tool_server import LlmToolServer +from ..types.mcp_server import McpServer from ..agent_management.types.agent_think_agent_management_request_on_listening_action import ( AgentThinkAgentManagementRequestOnListeningAction, ) @@ -228,7 +229,9 @@ def _start_properties_from_mapping( # LLM sub-type aliases LlmGreetingConfigs = typing.Dict[str, typing.Any] LlmGreetingConfigsMode = typing.Any -McpServersItem = typing.Dict[str, typing.Any] +McpServerConfig = McpServer +# Backward-compatible name retained for callers that used the old dict alias. +McpServersItem = typing.Union[typing.Dict[str, typing.Any], McpServer] # Additional top-level config aliases GeofenceConfig = StartAgentsRequestPropertiesGeofence diff --git a/src/agora_agent/agentkit/agent_session.py b/src/agora_agent/agentkit/agent_session.py index e2b2604..18a2bac 100644 --- a/src/agora_agent/agentkit/agent_session.py +++ b/src/agora_agent/agentkit/agent_session.py @@ -43,7 +43,7 @@ normalize_preset_input, resolve_session_presets, ) -from .preview.client import apply_preview_shape, create_preview_session_clients, required_preview_features +from .preview.client import create_preview_session_clients, required_preview_features from .token import _parse_numeric_uid, generate_convo_ai_token @@ -632,7 +632,6 @@ def start(self) -> str: properties, ) - apply_preview_shape(resolved_properties) self._bind_session_clients(required_preview_features(resolved_properties)) if self._debug: @@ -1000,7 +999,6 @@ async def start(self) -> str: properties, ) - apply_preview_shape(resolved_properties) self._bind_session_clients(required_preview_features(resolved_properties)) if self._debug: diff --git a/src/agora_agent/agentkit/preview/__init__.py b/src/agora_agent/agentkit/preview/__init__.py index 70240ec..db67a05 100644 --- a/src/agora_agent/agentkit/preview/__init__.py +++ b/src/agora_agent/agentkit/preview/__init__.py @@ -17,6 +17,7 @@ GEMINI_MLLM_DEFAULT_MODEL, GEMINI_PREVIEW_MLLM_URL, GEMINI_THINKING_LEVELS, + GeminiLive, GeminiLiveModels, GeminiSTT, GeminiSTTModels, @@ -34,6 +35,7 @@ "GEMINI_PREVIEW_MLLM_URL", "GEMINI_THINKING_LEVELS", "GeminiLiveModels", + "GeminiLive", "GeminiThinkingLevel", "apply_preview_shape", "PreviewFeature", diff --git a/src/agora_agent/agentkit/preview/client.py b/src/agora_agent/agentkit/preview/client.py index 6f496eb..ecaa19f 100644 --- a/src/agora_agent/agentkit/preview/client.py +++ b/src/agora_agent/agentkit/preview/client.py @@ -18,7 +18,6 @@ from ...agent_management.client import AgentManagementClient, AsyncAgentManagementClient from ...agents.client import AgentsClient, AsyncAgentsClient from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper -from .vendors import GEMINI_PREVIEW_MLLM_URL #: Base URL that serves the preview providers. PREVIEW_API_BASE_URL = "https://partner.ai.agora.io/preview/api/conversational-ai-agent" @@ -38,7 +37,7 @@ class PreviewFeatures: vendors on the preview endpoint. """ - #: Gemini preview MLLM gate. Gemini ASR uses the production endpoint. + #: Deprecated compatibility value. Gemini Live now uses the production endpoint. GEMINI_LIVE = "gemini-live" LIVE_MODELS = "live-models" @@ -135,86 +134,12 @@ def create_preview_session_clients( #: ASR vendors served only by the preview endpoint. _PREVIEW_FEATURES_BY_CATEGORY: typing.Dict[str, typing.Dict[str, PreviewFeature]] = { "asr": {}, - "mllm": {"openai_gpt_live": PreviewFeatures.LIVE_MODELS}, + "mllm": {}, } -_PREVIEW_MLLM_MODELS = frozenset( - { - "models/gemini-3.8-live", - "models/gemini-3.8-live-extended-thinking", - } -) - - -def _has_preview_mllm_envelope(mllm: typing.Mapping[str, typing.Any]) -> bool: - """Whether a config carries the envelope the preview MLLM classes emit. - - That envelope is a top-level ``mllm.api_key`` plus a ``url`` on the Gemini - Developer API host. ``GeminiLive`` configs for older model IDs use a - different URL (an empty string or WebSocket endpoint). - - This is the second recognition path, and it exists because keying only off - :data:`_PREVIEW_MLLM_MODELS` makes an unrecognised model name fail silently: - :func:`apply_preview_shape` would stop retargeting ``greeting_message``, the - greeting would land in a field these models ignore, and the agent would - simply never greet. A model name we have not listed yet is reachable by - following this SDK's own advice to override ``model`` when Google renames - one ahead of a release, so the failure has to not be silent. - """ - api_key = mllm.get("api_key") - url = mllm.get("url") - return isinstance(api_key, str) and isinstance(url, str) and url.startswith(GEMINI_PREVIEW_MLLM_URL) - - -def _is_preview_mllm(mllm: typing.Any) -> bool: - """Whether an MLLM config targets a preview model. - - Recognised by model name, or by the wire envelope only the preview vendor - classes produce. - """ - if not isinstance(mllm, dict) or mllm.get("vendor") != "gemini": - return False - params = mllm.get("params") - model = params.get("model") if isinstance(params, dict) else None - if isinstance(model, str) and model in _PREVIEW_MLLM_MODELS: - return True - return _has_preview_mllm_envelope(mllm) - - -#: MLLM wire keys the preview route spells differently from the Agora schema, -#: as production spelling -> preview spelling. -#: -#: ``failure_message`` is deliberately absent: it is an Agora engine feature -#: rather than a Gemini one, so it keeps its schema spelling. -_PREVIEW_MLLM_FIELD_RENAMES = {"greeting_message": "greeting"} - - def apply_preview_shape(properties: typing.MutableMapping[str, typing.Any]) -> None: - """Retarget MLLM fields the shared builder wrote with production spellings. - - ``Agent`` fills ``mllm.greeting_message`` from an agent-level ``greeting`` - whenever the vendor has not set that key — correct for every GA vendor, but - the preview Gemini models read ``greeting``, so the value would land in a - field they ignore and the agent would silently never greet. - - Rather than teach the shared builder about preview providers, the - translation lives here and disappears with this package at GA. The vendor's - own value wins; the production-spelled one is the fallback, which also - migrates a hand-written ``greeting_message`` onto the preview key so an - existing config keeps working after only swapping the model. - - Mutates ``properties["mllm"]`` in place. Safe because the builder hands this - a fresh copy of the MLLM config rather than the Agent's stored one. - """ - mllm = properties.get("mllm") - if not isinstance(mllm, dict) or not _is_preview_mllm(mllm): - return - for production, preview in _PREVIEW_MLLM_FIELD_RENAMES.items(): - if production not in mllm: - continue - value = mllm.pop(production) - mllm.setdefault(preview, value) + """Deprecated no-op retained for preview helper import compatibility.""" def required_preview_features(properties: typing.Mapping[str, typing.Any]) -> typing.List[str]: @@ -234,8 +159,6 @@ def required_preview_features(properties: typing.Mapping[str, typing.Any]) -> ty feature = vendors.get(vendor) if feature is not None and feature not in features: features.append(feature) - if _is_preview_mllm(properties.get("mllm")) and PreviewFeatures.GEMINI_LIVE not in features: - features.append(PreviewFeatures.GEMINI_LIVE) return features diff --git a/src/agora_agent/agentkit/preview/vendors.py b/src/agora_agent/agentkit/preview/vendors.py index 1305007..523ec51 100644 --- a/src/agora_agent/agentkit/preview/vendors.py +++ b/src/agora_agent/agentkit/preview/vendors.py @@ -1,143 +1,23 @@ -"""Preview provider vendor classes. - -These follow the same shape as the GA vendor classes in ``vendors/`` — snake_case -constructor options in, snake_case wire config out — so they work with the -corresponding ``Agent`` builder method. Sessions that use a preview-only vendor -route to the preview endpoint automatically. -""" - -import json -import warnings -from typing import Any, Dict, List, Optional -from urllib.parse import urlsplit, urlunsplit - -from ..vendors.base import BaseMLLM, ensure_mcp_transport -from ..vendors.mllm import MllmTurnDetectionConfig +"""Compatibility aliases for preview-era provider imports.""" + +from ..vendors.mllm import ( + GEMINI_MLLM_DEFAULT_MODEL, + GEMINI_MLLM_URL, + GEMINI_THINKING_LEVELS, + GeminiLive, + GeminiLiveModels, + GeminiThinkingLevel, + OpenAIGPTLive, +) from ..vendors.stt import GeminiSTT, GeminiSTTModels -from pydantic import ConfigDict, Field -from typing_extensions import Literal - -_OpenAIApiKey = Field(..., min_length=1, description="OpenAI API key") - - -class OpenAIGPTLive(BaseMLLM): - """GPT Live v3 alpha configuration. Not for production traffic. - Explicit options override params. Unset tuning options retain provider defaults. - instructions is a compatibility alias for prompt; prompt takes precedence. - """ +# Deprecated name retained for callers that imported the preview endpoint URL. +GEMINI_PREVIEW_MLLM_URL = GEMINI_MLLM_URL - model_config = ConfigDict(extra="forbid") - api_key: str = _OpenAIApiKey - url: Optional[str] = None - instructions: Optional[str] = None - greeting: Optional[str] = None - failure_message: Optional[str] = None - input_modalities: Optional[List[str]] = None - output_modalities: Optional[List[str]] = None - messages: Optional[List[Dict[str, Any]]] = None - mcp_servers: Optional[List[Dict[str, Any]]] = Field( - default=None, - description="MCP servers exposed to GPT Live. Requires Agent.with_tools().", - ) - params: Optional[Dict[str, Any]] = None - # Legacy options retained to diagnose unsupported v2 configuration. - input_audio_transcription: Optional[Dict[str, Any]] = None - turn_detection: Optional[Any] = None - model: Optional[str] = Field(default=None, description="Defaults to gpt-live-1.") - voice: Optional[str] = Field(default=None, description="Output voice; provider default marin. Custom voice objects require PR #1522; use params after rollout.") - prompt: Optional[str] = Field(default=None, description="Session instructions.") - base_url: Optional[str] = Field(default=None, description="Host when url is omitted; default wss://api.openai.com.") - path: Optional[str] = Field(default=None, description="WebSocket path; default /v1/live/sessions.") - alpha_selector: Optional[str] = Field(default=None, description="Optional OpenAI-Alpha selector for preview contracts; omitted by default.") - headers: Optional[str] = Field(default=None, description="Extra provider request headers as a JSON string; protocol headers win.") - output_idle_end_ms: Optional[int] = Field(default=None, description="Assistant silence boundary in ms; provider default 600. Zero disables inference.") - input_idle_end_ms: Optional[int] = Field(default=None, description="Caller silence boundary in ms; provider default 1500.") - output_silence_peak: Optional[int] = Field(default=None, description="Speech amplitude threshold on the 16-bit scale; provider default 50.") - output_sample_rate: Optional[int] = Field(default=None, description="Graph PCM sample rate; provider default 24000.") - output_buffer_ms: Optional[int] = Field(default=None, description="Initial audio cushion; provider default 0. Negative disables pacing.") - input_batch_ms: Optional[int] = Field(default=None, description="Mic append batching in ms. Join default 0; extension class default 100.") - tool_enabled: Optional[bool] = Field(default=None, description="Advertise graph tools; provider default false. Does not control delegate built-ins.") - delegation: Optional[Literal["client", "responses"]] = Field(default=None, description="Tool delegation mode; provider default responses. Fixed for the session.") - responses_model: Optional[str] = Field(default=None, description="Tool delegate model; provider default gpt-5.6-sol.") - interrupt_on_user_turn: Optional[bool] = Field(default=None, description="Interrupt playback on caller speech; provider default false.") - session_params: Optional[Dict[str, Any]] = Field(default=None, description="Unmodelled v3 session fields. Cannot override model, delegation, audio, instructions or input.") - def to_config(self) -> Dict[str, Any]: - params: Dict[str, Any] = { - "model": "gpt-live-1", - **(self.params or {}), - } - if self.instructions is not None: - params["prompt"] = self.instructions - for name in ( - "model", - "voice", - "prompt", - "base_url", - "path", - "alpha_selector", - "headers", - "output_idle_end_ms", - "input_idle_end_ms", - "output_silence_peak", - "output_sample_rate", - "output_buffer_ms", - "input_batch_ms", - "tool_enabled", - "delegation", - "responses_model", - "interrupt_on_user_turn", - "session_params", - ): - value = getattr(self, name) - if value is not None: - params[name] = value - if self.input_audio_transcription is not None or "input_audio_transcription" in params: - raise ValueError("GPT Live v3 does not support input_audio_transcription") - if self.turn_detection is not None or "turn_detection" in params: - warnings.warn("GPT Live v3 ignores turn_detection; endpointing is internal", UserWarning, stacklevel=2) - params.pop("turn_detection", None) - if "delegation" in params and params["delegation"] not in ("client", "responses"): - raise ValueError("GPT Live delegation must be client or responses") - if "headers" in params: - try: - headers = json.loads(params["headers"]) - except (TypeError, ValueError) as exc: - raise ValueError("GPT Live headers must be a JSON object string") from exc - if not isinstance(headers, dict): - raise ValueError("GPT Live headers must be a JSON object string") - session = params.get("session_params", {}) - if not isinstance(session, dict): - raise ValueError("GPT Live session_params must be an object") - for name in ("model", "delegation", "audio", "instructions", "input"): - if name in session: - raise ValueError(f"GPT Live session_params cannot override {name}") - url = self.url or ( - str(params.get("base_url", "wss://api.openai.com")).rstrip("/") - + "/" + str(params.get("path", "/v1/live/sessions")).lstrip("/") - ) - try: - parsed = urlsplit(url) - hostname = parsed.hostname - except ValueError as exc: - raise ValueError("GPT Live url must be a full ws:// or wss:// endpoint") from exc - if parsed.scheme not in ("ws", "wss") or not hostname: - raise ValueError("GPT Live url must be a full ws:// or wss:// endpoint") - if parsed.hostname == "api.openai.com" and parsed.path == "/v1/live": - url = urlunsplit(parsed._replace(path="/v1/live/sessions")) - config: Dict[str, Any] = { - "vendor": "openai_gpt_live", "api_key": self.api_key, "url": url, "params": params, - } - for name in ("failure_message", "input_modalities", "output_modalities", "messages"): - value = getattr(self, name) - if value is not None: - config[name] = value - if self.greeting is not None: - config["greeting_message"] = self.greeting - if self.mcp_servers is not None: - config["mcp_servers"] = ensure_mcp_transport(self.mcp_servers) - return config +def build_gemini_preview_config(vendor: GeminiLive): + """Deprecated compatibility wrapper for the production serializer.""" + return vendor.to_config() __all__ = [ @@ -148,88 +28,7 @@ def to_config(self) -> Dict[str, Any]: "GeminiLiveModels", "build_gemini_preview_config", "GeminiThinkingLevel", + "GeminiLive", "GeminiSTTModels", "GeminiSTT", ] - - -class GeminiLiveModels: - """Preview MLLM model names. - - The ``models/`` prefix is part of each model ID. - """ - - LIVE_38 = "models/gemini-3.8-live" - LIVE_38_EXTENDED_THINKING = "models/gemini-3.8-live-extended-thinking" - - -#: The model name the Gemini MLLM sends by default. -#: -#: Low-latency Gemini voice is the default. -GEMINI_MLLM_DEFAULT_MODEL = GeminiLiveModels.LIVE_38 - - -GEMINI_THINKING_LEVELS = ("low", "medium", "high") - -GeminiThinkingLevel = Literal["low", "medium", "high"] - -#: The preview MLLM talks to the Gemini Developer API rather than a WebSocket host. -GEMINI_PREVIEW_MLLM_URL = "https://generativelanguage.googleapis.com" - - -def build_gemini_preview_config(self: Any) -> Dict[str, Any]: - """Serialize GeminiLive options for the Gemini 3.8 preview gateway.""" - model = (self.model or "").strip() or GEMINI_MLLM_DEFAULT_MODEL - voice = self.voice if self.voice is not None else "Puck" - url = self.url if self.url is not None else GEMINI_PREVIEW_MLLM_URL - - params: Dict[str, Any] = dict(self.additional_params or {}) - params.pop("api_key", None) - params["model"] = model - params["voice"] = voice - if model == GeminiLiveModels.LIVE_38_EXTENDED_THINKING: - if self.thinking_level is not None: - params["thinking_level"] = self.thinking_level - else: - params.pop("thinking_level", None) - # Plural array, and omitted when unset. The singular ``params.language`` - # belongs to xAI Grok in the Agora schema, and the production Gemini Live - # provider sends no language field at all. - if self.language_codes is not None: - params["language_codes"] = list(self.language_codes) - - if self.instructions is not None: - params["instructions"] = self.instructions - if self.transcribe_agent is not None: - params["transcribe_agent"] = self.transcribe_agent - if self.transcribe_user is not None: - params["transcribe_user"] = self.transcribe_user - if self.affective_dialog is not None: - params["affective_dialog"] = self.affective_dialog - if self.proactive_audio is not None: - params["proactive_audio"] = self.proactive_audio - if self.http_options is not None: - params["http_options"] = self.http_options - - config: Dict[str, Any] = { - "vendor": "gemini", - "api_key": self.api_key, - "url": url, - "params": params, - } - if self.messages is not None: - config["messages"] = self.messages - # ``greeting``, not ``greeting_message``: the preview Gemini models read - # this spelling. See the preview-endpoint guide. - if self.greeting_message is not None: - config["greeting"] = self.greeting_message - if self.failure_message is not None: - config["failure_message"] = self.failure_message - if self.input_modalities is not None: - config["input_modalities"] = self.input_modalities - if self.output_modalities is not None: - config["output_modalities"] = self.output_modalities - if self.turn_detection is not None: - config["turn_detection"] = self.turn_detection - - return config diff --git a/src/agora_agent/agentkit/regional_agent.py b/src/agora_agent/agentkit/regional_agent.py index 8a9b3f1..8658154 100644 --- a/src/agora_agent/agentkit/regional_agent.py +++ b/src/agora_agent/agentkit/regional_agent.py @@ -35,6 +35,7 @@ MicrosoftSTT, OpenAISTT, SarvamSTT, + SmallestAISTT, SpeechmaticsSTT, XaiSTT, ) @@ -49,7 +50,7 @@ OpenAI, VertexAILLM, ) -from .vendors.mllm import AzureOpenAIRealtime, GeminiLive, OpenAIRealtime, VertexAI, XaiGrok +from .vendors.mllm import AzureOpenAIRealtime, GeminiLive, OpenAIGPTLive, OpenAIRealtime, VertexAI, XaiGrok from .vendors.tts import ( AmazonTTS, CartesiaTTS, @@ -67,6 +68,7 @@ GenericTTS, RimeTTS, SarvamTTS, + SmallestAITTS, TypecastTTS, XaiTTS, ) @@ -89,6 +91,7 @@ AssemblyAISTT, SpeechmaticsSTT, SarvamSTT, + SmallestAISTT, XaiSTT, ] GlobalTTS = typing.Union[ @@ -104,6 +107,7 @@ GoogleTTS, AmazonTTS, SarvamTTS, + SmallestAITTS, GenericTTS, XaiTTS, DeepgramTTS, @@ -122,7 +126,7 @@ Dify, CustomLLM, ] -GlobalMLLM = typing.Union[OpenAIRealtime, AzureOpenAIRealtime, GeminiLive, VertexAI, XaiGrok] +GlobalMLLM = typing.Union[OpenAIRealtime, OpenAIGPTLive, AzureOpenAIRealtime, GeminiLive, VertexAI, XaiGrok] GlobalAvatar = typing.Union[AkoolAvatar, LiveAvatarAvatar, AnamAvatar, GenericAvatar, HeyGenAvatar] diff --git a/src/agora_agent/agentkit/vendors/__init__.py b/src/agora_agent/agentkit/vendors/__init__.py index 643ae17..4d6e7c4 100644 --- a/src/agora_agent/agentkit/vendors/__init__.py +++ b/src/agora_agent/agentkit/vendors/__init__.py @@ -13,7 +13,19 @@ ) from .avatar import AkoolAvatar, AnamAvatar, GenericAvatar, HeyGenAvatar, LiveAvatarAvatar from .llm import AmazonBedrock, Anthropic, AzureOpenAI, CustomLLM, Dify, Gemini, Groq, OpenAI, VertexAILLM -from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIRealtime, VertexAI, XaiGrok +from .mllm import ( + GEMINI_MLLM_DEFAULT_MODEL, + GEMINI_MLLM_URL, + GEMINI_THINKING_LEVELS, + AzureOpenAIRealtime, + GeminiLive, + GeminiLiveModels, + GeminiThinkingLevel, + OpenAIGPTLive, + OpenAIRealtime, + VertexAI, + XaiGrok, +) from .cn import ( AliyunLLM, BytedanceDuplexTTS, @@ -47,6 +59,7 @@ MicrosoftSTT, OpenAISTT, SarvamSTT, + SmallestAISTT, SpeechmaticsSTT, XaiSTT, ) @@ -67,6 +80,7 @@ GenericTTS, RimeTTS, SarvamTTS, + SmallestAITTS, TypecastTTS, XaiTTS, ) @@ -109,6 +123,7 @@ "TypecastTTS", "MurfTTS", "SarvamTTS", + "SmallestAITTS", "GenericTTS", "XaiTTS", "MicrosoftCNTTS", @@ -129,6 +144,7 @@ "AssemblyAISTT", "AresSTT", "SarvamSTT", + "SmallestAISTT", "XaiSTT", "TencentSTT", "FengmingSTT", @@ -140,8 +156,14 @@ "DeepSeekLLM", "TencentLLM", "OpenAIRealtime", + "OpenAIGPTLive", "AzureOpenAIRealtime", "GeminiLive", + "GeminiLiveModels", + "GeminiThinkingLevel", + "GEMINI_MLLM_DEFAULT_MODEL", + "GEMINI_MLLM_URL", + "GEMINI_THINKING_LEVELS", "QwenOmni", "VertexAI", "XaiGrok", diff --git a/src/agora_agent/agentkit/vendors/base.py b/src/agora_agent/agentkit/vendors/base.py index 41cb332..225e96f 100644 --- a/src/agora_agent/agentkit/vendors/base.py +++ b/src/agora_agent/agentkit/vendors/base.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union +from ...types.mcp_server import McpServer from pydantic import BaseModel from typing_extensions import Literal @@ -14,13 +15,27 @@ OpenAISampleRate = Literal[24000] CartesiaSampleRate = Literal[8000, 16000, 22050, 24000, 44100, 48000] GoogleTTSSampleRate = Literal[8000, 16000, 22050, 24000, 44100, 48000] +McpServerInput = Union[Dict[str, Any], McpServer] -def ensure_mcp_transport(servers: List[Dict[str, Any]]) -> List[Dict[str, Any]]: +def dump_config_models(value: Any) -> Any: + """Recursively serialize generated Pydantic config models.""" + if hasattr(value, "model_dump"): + return value.model_dump(exclude_none=True) + if hasattr(value, "dict"): + return value.dict(exclude_none=True) + if isinstance(value, list): + return [dump_config_models(item) for item in value] + if isinstance(value, dict): + return {key: dump_config_models(item) for key, item in value.items()} + return value + + +def ensure_mcp_transport(servers: List[McpServerInput]) -> List[Dict[str, Any]]: """Copy MCP server configs and supply the transport required by the API.""" result = [] for server in servers: - item = dict(server) + item = dump_config_models(server) if item.get("transport") is None: item["transport"] = "streamable_http" result.append(item) diff --git a/src/agora_agent/agentkit/vendors/catalog.py b/src/agora_agent/agentkit/vendors/catalog.py index 6c63d08..ebfb918 100644 --- a/src/agora_agent/agentkit/vendors/catalog.py +++ b/src/agora_agent/agentkit/vendors/catalog.py @@ -16,7 +16,7 @@ OpenAI, VertexAILLM, ) -from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIRealtime, VertexAI, XaiGrok +from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIGPTLive, OpenAIRealtime, VertexAI, XaiGrok from .stt import ( AmazonSTT, AresSTT, @@ -27,6 +27,7 @@ MicrosoftSTT, OpenAISTT, SarvamSTT, + SmallestAISTT, SpeechmaticsSTT, XaiSTT, ) @@ -47,6 +48,7 @@ GenericTTS, RimeTTS, SarvamTTS, + SmallestAITTS, TypecastTTS, XaiTTS, ) @@ -81,6 +83,7 @@ def __init__( "assemblyai": AssemblyAISTT, "speechmatics": SpeechmaticsSTT, "sarvam": SarvamSTT, + "smallestai": SmallestAISTT, "xai": XaiSTT, }, llm={ @@ -96,6 +99,7 @@ def __init__( }, mllm={ "openai": OpenAIRealtime, + "openai_gpt_live": OpenAIGPTLive, "azure": AzureOpenAIRealtime, "gemini": GeminiLive, "vertexai": VertexAI, @@ -114,6 +118,7 @@ def __init__( "google": GoogleTTS, "amazon": AmazonTTS, "sarvam": SarvamTTS, + "smallestai": SmallestAITTS, "generic": GenericTTS, "xai": XaiTTS, "deepgram": DeepgramTTS, diff --git a/src/agora_agent/agentkit/vendors/cn.py b/src/agora_agent/agentkit/vendors/cn.py index 5142667..3675051 100644 --- a/src/agora_agent/agentkit/vendors/cn.py +++ b/src/agora_agent/agentkit/vendors/cn.py @@ -4,7 +4,7 @@ from ...types.mllm_turn_detection import MllmTurnDetection from .avatar import BaseAvatar -from .base import BaseLLM, BaseMLLM, ensure_mcp_transport +from .base import BaseLLM, BaseMLLM, McpServerInput, ensure_mcp_transport from .llm import ( _OPENAI_MANAGED_MODELS, LlmGreetingConfigs, @@ -522,7 +522,7 @@ class AliyunLLMOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="aliyun") - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -609,7 +609,7 @@ class BytedanceLLMOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="bytedance") - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -696,7 +696,7 @@ class DeepSeekLLMOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="deepseek") - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -783,7 +783,7 @@ class TencentLLMOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="tencent") - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -870,6 +870,8 @@ class QwenOmniOptions(BaseModel): params: Optional[Dict[str, Any]] = Field(default=None, description="Additional Qwen Omni parameters") turn_detection: Optional[MllmTurnDetection] = Field(default=None, description="MLLM turn detection configuration") failure_message: Optional[str] = Field(default=None, description="Message played on failure") + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) + tools: Optional[List[LlmToolInput]] = Field(default=None) class QwenOmni(QwenOmniOptions, BaseMLLM): @@ -905,6 +907,10 @@ def to_config(self) -> Dict[str, Any]: config["failure_message"] = self.failure_message if self.turn_detection is not None: config["turn_detection"] = self.turn_detection + if self.mcp_servers is not None: + config["mcp_servers"] = ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) return config diff --git a/src/agora_agent/agentkit/vendors/llm.py b/src/agora_agent/agentkit/vendors/llm.py index bad10b4..9398931 100644 --- a/src/agora_agent/agentkit/vendors/llm.py +++ b/src/agora_agent/agentkit/vendors/llm.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Optional, Union from ...types.llm_tool import LlmTool -from .base import BaseLLM, ensure_mcp_transport +from .base import BaseLLM, McpServerInput, dump_config_models, ensure_mcp_transport from pydantic import BaseModel, ConfigDict, Field, model_validator LlmGreetingConfigs = Dict[str, Any] @@ -9,16 +9,7 @@ _OPENAI_MANAGED_MODELS = {"gpt-4o-mini", "gpt-4.1-mini", "gpt-5-nano", "gpt-5-mini"} -def _dump_optional_model(value: Any) -> Any: - if hasattr(value, "model_dump"): - return value.model_dump(exclude_none=True) - if hasattr(value, "dict"): - return value.dict(exclude_none=True) - if isinstance(value, list): - return [_dump_optional_model(item) for item in value] - if isinstance(value, dict): - return {key: _dump_optional_model(item) for key, item in value.items()} - return value +_dump_optional_model = dump_config_models class OpenAIOptions(BaseModel): @@ -41,7 +32,7 @@ class OpenAIOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -133,7 +124,7 @@ class AzureOpenAIOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -210,7 +201,7 @@ class AnthropicOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -282,7 +273,7 @@ class GeminiOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -358,7 +349,7 @@ class GroqOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -437,7 +428,7 @@ class CustomLLMOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -520,7 +511,7 @@ class VertexAILLMOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -602,7 +593,7 @@ class AmazonBedrockOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -673,7 +664,7 @@ class DifyOptions(BaseModel): greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None) template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) - mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) tools: Optional[List[LlmToolInput]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0) diff --git a/src/agora_agent/agentkit/vendors/mllm.py b/src/agora_agent/agentkit/vendors/mllm.py index e0d00a7..8f0eff0 100644 --- a/src/agora_agent/agentkit/vendors/mllm.py +++ b/src/agora_agent/agentkit/vendors/mllm.py @@ -1,11 +1,130 @@ -from typing import Any, Dict, List, Optional -from typing_extensions import Literal +import json +import warnings +from typing import Any, Dict, List, Optional, Union +from urllib.parse import urlsplit, urlunsplit +from ...types.llm_tool import LlmTool from ...types.mllm_turn_detection import MllmTurnDetection -from .base import BaseMLLM +from .base import BaseMLLM, McpServerInput, dump_config_models, ensure_mcp_transport from pydantic import BaseModel, ConfigDict, Field, field_validator +from typing_extensions import Literal MllmTurnDetectionConfig = MllmTurnDetection +MllmToolInput = Union[Dict[str, Any], LlmTool] + + +class GeminiLiveModels: + """Supported Gemini Live model names.""" + + LIVE_38 = "models/gemini-3.8-live" + LIVE_38_EXTENDED_THINKING = "models/gemini-3.8-live-extended-thinking" + + +GEMINI_MLLM_DEFAULT_MODEL = GeminiLiveModels.LIVE_38 +GEMINI_THINKING_LEVELS = ("low", "medium", "high") +GeminiThinkingLevel = Literal["low", "medium", "high"] +GEMINI_MLLM_URL = "https://generativelanguage.googleapis.com" + + +class OpenAIGPTLive(BaseMLLM): + """OpenAI GPT Live v3 MLLM configuration.""" + + model_config = ConfigDict(extra="forbid") + api_key: str = Field(..., min_length=1, description="OpenAI API key") + url: Optional[str] = None + instructions: Optional[str] = None + greeting: Optional[str] = None + failure_message: Optional[str] = None + input_modalities: Optional[List[str]] = None + output_modalities: Optional[List[str]] = None + messages: Optional[List[Dict[str, Any]]] = None + mcp_servers: Optional[List[McpServerInput]] = None + tools: Optional[List[MllmToolInput]] = None + params: Optional[Dict[str, Any]] = None + input_audio_transcription: Optional[Dict[str, Any]] = None + turn_detection: Optional[Any] = None + model: Optional[str] = Field(default=None, description="Defaults to gpt-live-1.") + voice: Optional[str] = None + prompt: Optional[str] = None + base_url: Optional[str] = None + path: Optional[str] = None + alpha_selector: Optional[str] = None + headers: Optional[str] = None + output_idle_end_ms: Optional[int] = None + input_idle_end_ms: Optional[int] = None + output_silence_peak: Optional[int] = None + output_sample_rate: Optional[int] = None + output_buffer_ms: Optional[int] = None + input_batch_ms: Optional[int] = None + tool_enabled: Optional[bool] = None + delegation: Optional[Literal["client", "responses"]] = None + responses_model: Optional[str] = None + interrupt_on_user_turn: Optional[bool] = None + session_params: Optional[Dict[str, Any]] = None + + def to_config(self) -> Dict[str, Any]: + params: Dict[str, Any] = {"model": "gpt-live-1", **(self.params or {})} + if self.instructions is not None: + params["prompt"] = self.instructions + for name in ( + "model", "voice", "prompt", "base_url", "path", "alpha_selector", "headers", + "output_idle_end_ms", "input_idle_end_ms", "output_silence_peak", "output_sample_rate", + "output_buffer_ms", "input_batch_ms", "tool_enabled", "delegation", "responses_model", + "interrupt_on_user_turn", "session_params", + ): + value = getattr(self, name) + if value is not None: + params[name] = value + if self.input_audio_transcription is not None or "input_audio_transcription" in params: + raise ValueError("GPT Live v3 does not support input_audio_transcription") + if self.turn_detection is not None or "turn_detection" in params: + warnings.warn("GPT Live v3 ignores turn_detection; endpointing is internal", UserWarning, stacklevel=2) + params.pop("turn_detection", None) + if "delegation" in params and params["delegation"] not in ("client", "responses"): + raise ValueError("GPT Live delegation must be client or responses") + if "headers" in params: + try: + headers = json.loads(params["headers"]) + except (TypeError, ValueError) as exc: + raise ValueError("GPT Live headers must be a JSON object string") from exc + if not isinstance(headers, dict): + raise ValueError("GPT Live headers must be a JSON object string") + session = params.get("session_params", {}) + if not isinstance(session, dict): + raise ValueError("GPT Live session_params must be an object") + for name in ("model", "delegation", "audio", "instructions", "input"): + if name in session: + raise ValueError(f"GPT Live session_params cannot override {name}") + url = self.url or (str(params.get("base_url", "wss://api.openai.com")).rstrip("/") + "/" + str(params.get("path", "/v1/live/sessions")).lstrip("/")) + try: + parsed = urlsplit(url) + hostname = parsed.hostname + except ValueError as exc: + raise ValueError("GPT Live url must be a full ws:// or wss:// endpoint") from exc + if parsed.scheme not in ("ws", "wss") or not hostname: + raise ValueError("GPT Live url must be a full ws:// or wss:// endpoint") + if parsed.hostname == "api.openai.com" and parsed.path == "/v1/live": + url = urlunsplit(parsed._replace(path="/v1/live/sessions")) + config: Dict[str, Any] = {"vendor": "openai_gpt_live", "api_key": self.api_key, "url": url, "params": params} + for name in ("failure_message", "input_modalities", "output_modalities", "messages"): + value = getattr(self, name) + if value is not None: + config[name] = value + if self.greeting is not None: + config["greeting_message"] = self.greeting + _add_tool_configs(config, self.mcp_servers, self.tools) + return config + + +def _add_tool_configs( + config: Dict[str, Any], + mcp_servers: Optional[List[McpServerInput]], + tools: Optional[List[MllmToolInput]], +) -> None: + if mcp_servers is not None: + config["mcp_servers"] = ensure_mcp_transport(mcp_servers) + if tools is not None: + config["tools"] = dump_config_models(tools) class OpenAIRealtimeOptions(BaseModel): @@ -27,6 +146,8 @@ class OpenAIRealtimeOptions(BaseModel): params: Optional[Dict[str, Any]] = Field(default=None, description="Additional parameters") turn_detection: Optional[MllmTurnDetectionConfig] = Field(default=None, description="MLLM turn detection configuration") failure_message: Optional[str] = Field(default=None, description="Message played on failure") + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) + tools: Optional[List[MllmToolInput]] = Field(default=None) class OpenAIRealtime(OpenAIRealtimeOptions, BaseMLLM): @@ -68,6 +189,7 @@ def to_config(self) -> Dict[str, Any]: config["failure_message"] = self.failure_message if self.turn_detection is not None: config["turn_detection"] = self.turn_detection + _add_tool_configs(config, self.mcp_servers, self.tools) return config @@ -94,6 +216,8 @@ class AzureOpenAIRealtimeOptions(BaseModel): params: Optional[Dict[str, Any]] = Field(default=None, description="Additional Azure OpenAI parameters") turn_detection: MllmTurnDetectionConfig = Field(..., description="MLLM turn detection configuration") failure_message: Optional[str] = Field(default=None, description="Message played on failure") + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) + tools: Optional[List[MllmToolInput]] = Field(default=None) class AzureOpenAIRealtime(AzureOpenAIRealtimeOptions, BaseMLLM): @@ -128,6 +252,7 @@ def to_config(self) -> Dict[str, Any]: if self.failure_message is not None: config["failure_message"] = self.failure_message config["turn_detection"] = self.turn_detection + _add_tool_configs(config, self.mcp_servers, self.tools) return config @@ -152,6 +277,8 @@ class XaiGrokOptions(BaseModel): params: Optional[Dict[str, Any]] = Field(default=None, description="Additional xAI parameters") turn_detection: Optional[MllmTurnDetectionConfig] = Field(default=None, description="MLLM turn detection configuration") failure_message: Optional[str] = Field(default=None, description="Message played on failure") + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) + tools: Optional[List[MllmToolInput]] = Field(default=None) class XaiGrok(XaiGrokOptions, BaseMLLM): @@ -185,6 +312,7 @@ def to_config(self) -> Dict[str, Any]: config["failure_message"] = self.failure_message if self.turn_detection is not None: config["turn_detection"] = self.turn_detection + _add_tool_configs(config, self.mcp_servers, self.tools) return config @@ -211,6 +339,8 @@ class VertexAIOptions(BaseModel): additional_params: Optional[Dict[str, Any]] = Field(default=None, description="Additional parameters") turn_detection: Optional[MllmTurnDetectionConfig] = Field(default=None, description="MLLM turn detection configuration") failure_message: Optional[str] = Field(default=None, description="Message played on failure") + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) + tools: Optional[List[MllmToolInput]] = Field(default=None) class VertexAI(VertexAIOptions, BaseMLLM): @@ -254,6 +384,7 @@ def to_config(self) -> Dict[str, Any]: config["failure_message"] = self.failure_message if self.turn_detection is not None: config["turn_detection"] = self.turn_detection + _add_tool_configs(config, self.mcp_servers, self.tools) return config @@ -263,7 +394,7 @@ class GeminiLiveOptions(BaseModel): api_key: str = Field(..., description="Google API key") model: str = Field(default="models/gemini-3.8-live", description="Gemini Live model name") - thinking_level: Optional[Literal["low", "medium", "high"]] = Field( + thinking_level: Optional[GeminiThinkingLevel] = Field( default=None, description="Reasoning budget for the 3.8 extended-thinking model" ) language_codes: Optional[List[str]] = Field(default=None, description="Languages for Gemini 3.8") @@ -282,6 +413,8 @@ class GeminiLiveOptions(BaseModel): additional_params: Optional[Dict[str, Any]] = Field(default=None, description="Additional parameters") turn_detection: Optional[MllmTurnDetectionConfig] = Field(default=None, description="MLLM turn detection configuration") failure_message: Optional[str] = Field(default=None, description="Message played on failure") + mcp_servers: Optional[List[McpServerInput]] = Field(default=None) + tools: Optional[List[MllmToolInput]] = Field(default=None) @field_validator("api_key") @classmethod @@ -294,19 +427,25 @@ def _validate_api_key(cls, value: str) -> str: class GeminiLive(GeminiLiveOptions, BaseMLLM): def to_config(self) -> Dict[str, Any]: - from ..preview.vendors import GeminiLiveModels, build_gemini_preview_config - - selected_model = self.model.strip() or GeminiLiveModels.LIVE_38 - if selected_model in (GeminiLiveModels.LIVE_38, GeminiLiveModels.LIVE_38_EXTENDED_THINKING): - return build_gemini_preview_config(self) - - inner_params: Dict[str, Any] = {} - if self.additional_params is not None: - inner_params.update(self.additional_params) + selected_model = self.model.strip() or GEMINI_MLLM_DEFAULT_MODEL + inner_params: Dict[str, Any] = dict(self.additional_params or {}) inner_params["model"] = selected_model + if selected_model in (GeminiLiveModels.LIVE_38, GeminiLiveModels.LIVE_38_EXTENDED_THINKING): + inner_params.pop("api_key", None) + inner_params["voice"] = self.voice if self.voice is not None else "Puck" + if self.language_codes is not None: + inner_params["language_codes"] = list(self.language_codes) + if selected_model == GeminiLiveModels.LIVE_38_EXTENDED_THINKING: + if self.thinking_level is not None: + inner_params["thinking_level"] = self.thinking_level + else: + inner_params.pop("thinking_level", None) if self.instructions is not None: inner_params["instructions"] = self.instructions - if self.voice is not None: + if self.voice is not None and selected_model not in ( + GeminiLiveModels.LIVE_38, + GeminiLiveModels.LIVE_38_EXTENDED_THINKING, + ): inner_params["voice"] = self.voice if self.affective_dialog is not None: inner_params["affective_dialog"] = self.affective_dialog @@ -322,7 +461,11 @@ def to_config(self) -> Dict[str, Any]: config: Dict[str, Any] = { "vendor": "gemini", "api_key": self.api_key, - "url": self.url if self.url is not None else "", + "url": self.url if self.url is not None else ( + GEMINI_MLLM_URL + if selected_model in (GeminiLiveModels.LIVE_38, GeminiLiveModels.LIVE_38_EXTENDED_THINKING) + else "" + ), "params": inner_params, } if self.greeting_message is not None: @@ -337,5 +480,6 @@ def to_config(self) -> Dict[str, Any]: config["failure_message"] = self.failure_message if self.turn_detection is not None: config["turn_detection"] = self.turn_detection + _add_tool_configs(config, self.mcp_servers, self.tools) return config diff --git a/src/agora_agent/agentkit/vendors/namespaces.py b/src/agora_agent/agentkit/vendors/namespaces.py index c5d7c89..782c5aa 100644 --- a/src/agora_agent/agentkit/vendors/namespaces.py +++ b/src/agora_agent/agentkit/vendors/namespaces.py @@ -13,7 +13,7 @@ OpenAI, VertexAILLM, ) -from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIRealtime, VertexAI, XaiGrok +from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIGPTLive, OpenAIRealtime, VertexAI, XaiGrok from .stt import ( AmazonSTT, AresSTT, @@ -24,6 +24,7 @@ MicrosoftSTT, OpenAISTT, SarvamSTT, + SmallestAISTT, SpeechmaticsSTT, XaiSTT, ) @@ -44,6 +45,7 @@ GenericTTS, RimeTTS, SarvamTTS, + SmallestAITTS, TypecastTTS, XaiTTS, ) @@ -60,6 +62,7 @@ class GlobalSTTVendors: assemblyai = AssemblyAISTT speechmatics = SpeechmaticsSTT sarvam = SarvamSTT + smallestai = SmallestAISTT xai = XaiSTT @@ -93,6 +96,7 @@ class CNLLMVendors: class GlobalMLLMVendors: openai = OpenAIRealtime + openai_gpt_live = OpenAIGPTLive azure = AzureOpenAIRealtime gemini = GeminiLive vertexai = VertexAI @@ -116,6 +120,7 @@ class GlobalTTSVendors: google = GoogleTTS amazon = AmazonTTS sarvam = SarvamTTS + smallestai = SmallestAITTS generic = GenericTTS xai = XaiTTS deepgram = DeepgramTTS diff --git a/src/agora_agent/agentkit/vendors/region.py b/src/agora_agent/agentkit/vendors/region.py index 0e5b473..495c8bd 100644 --- a/src/agora_agent/agentkit/vendors/region.py +++ b/src/agora_agent/agentkit/vendors/region.py @@ -28,6 +28,7 @@ "assemblyai", "speechmatics", "sarvam", + "smallestai", "xai", ) @@ -54,6 +55,7 @@ "google", "amazon", "sarvam", + "smallestai", "generic", "xai", "deepgram", diff --git a/src/agora_agent/agentkit/vendors/stt.py b/src/agora_agent/agentkit/vendors/stt.py index dc95564..f05fe79 100644 --- a/src/agora_agent/agentkit/vendors/stt.py +++ b/src/agora_agent/agentkit/vendors/stt.py @@ -6,6 +6,21 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator _DEEPGRAM_MANAGED_MODELS = {"nova-2", "nova-3"} +_SMALLEST_AI_BOOLEAN_PARAMS = ( + "word_timestamps", + "sentence_timestamps", + "diarize", + "vad_events", + "endpointing", + "format", + "finalize_on_words", + "punctuate", + "capitalize", + "itn_normalize", + "full_transcript", + "redact_pii", + "redact_pci", +) class SpeechmaticsSTTOptions(BaseModel): @@ -432,3 +447,55 @@ def to_config(self) -> Dict[str, Any]: "params": params, } return config + + +class SmallestAISTTOptions(BaseModel): + model_config = ConfigDict(extra="forbid") + + api_key: str = Field(..., min_length=1, description="Smallest AI API key") + language: Optional[str] = Field(default=None, description="Language code for speech recognition") + url: Optional[str] = Field(default=None, description="Streaming WebSocket endpoint") + sample_rate: Optional[int] = Field(default=None, gt=0, description="Input audio sample rate in Hz") + encoding: Optional[str] = Field(default=None, description="Input audio encoding") + word_timestamps: Optional[bool] = Field(default=None) + sentence_timestamps: Optional[bool] = Field(default=None) + diarize: Optional[bool] = Field(default=None) + vad_events: Optional[bool] = Field(default=None) + endpointing: Optional[bool] = Field(default=None) + eou_timeout_ms: Optional[int] = Field(default=None, ge=0) + format: Optional[bool] = Field(default=None) + finalize_on_words: Optional[bool] = Field(default=None) + max_words: Optional[str] = Field(default=None) + punctuate: Optional[bool] = Field(default=None) + capitalize: Optional[bool] = Field(default=None) + itn_normalize: Optional[bool] = Field(default=None) + full_transcript: Optional[bool] = Field(default=None) + keywords: Optional[str] = Field(default=None) + redact_pii: Optional[bool] = Field(default=None) + redact_pci: Optional[bool] = Field(default=None) + additional_params: Optional[Dict[str, Any]] = Field(default=None) + + +class SmallestAISTT(SmallestAISTTOptions, BaseSTT): + """Smallest AI streaming speech-to-text provider.""" + + def to_config(self) -> Dict[str, Any]: + params: Dict[str, Any] = dict(self.additional_params or {}) + params["api_key"] = self.api_key + for name in ( + "language", + "url", + "sample_rate", + "encoding", + "eou_timeout_ms", + "max_words", + "keywords", + ): + value = getattr(self, name) + if value is not None: + params[name] = value + for name in _SMALLEST_AI_BOOLEAN_PARAMS: + value = getattr(self, name) + if value is not None: + params[name] = "true" if value else "false" + return {"vendor": "smallestai", "params": params} diff --git a/src/agora_agent/agentkit/vendors/tts.py b/src/agora_agent/agentkit/vendors/tts.py index 3289901..0d8d3ff 100644 --- a/src/agora_agent/agentkit/vendors/tts.py +++ b/src/agora_agent/agentkit/vendors/tts.py @@ -720,3 +720,50 @@ def to_config(self) -> Dict[str, Any]: if self.skip_patterns is not None: result["skip_patterns"] = self.skip_patterns return result + + +class SmallestAITTSOptions(BaseModel): + model_config = ConfigDict(extra="forbid") + + api_key: str = Field(..., min_length=1, description="Smallest AI API key") + url: Optional[str] = Field(default=None, description="Streaming HTTP endpoint") + model: Optional[str] = Field(default=None, description="TTS model name") + voice_id: Optional[str] = Field(default=None, description="Voice identifier") + sample_rate: Optional[int] = Field(default=None, gt=0, description="Output audio sample rate in Hz") + speed: Optional[float] = Field(default=None, gt=0, description="Speech rate multiplier") + language: Optional[str] = Field(default=None, description="Language code for speech synthesis") + number_pronunciation_language: Optional[str] = Field(default=None) + math_notation: Optional[bool] = Field(default=None) + pronunciation_dicts: Optional[List[str]] = Field(default=None) + session_id: Optional[str] = Field(default=None) + request_id: Optional[str] = Field(default=None) + additional_params: Optional[Dict[str, Any]] = Field(default=None) + skip_patterns: Optional[List[int]] = Field(default=None) + + +class SmallestAITTS(SmallestAITTSOptions, BaseTTS): + """Smallest AI streaming text-to-speech provider.""" + + def to_config(self) -> Dict[str, Any]: + params: Dict[str, Any] = dict(self.additional_params or {}) + params["api_key"] = self.api_key + for name in ( + "url", + "model", + "voice_id", + "sample_rate", + "speed", + "language", + "number_pronunciation_language", + "math_notation", + "pronunciation_dicts", + "session_id", + "request_id", + ): + value = getattr(self, name) + if value is not None: + params[name] = value + result: Dict[str, Any] = {"vendor": "smallestai", "params": params} + if self.skip_patterns is not None: + result["skip_patterns"] = self.skip_patterns + return result diff --git a/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content.py b/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content.py index 5e13764..10f0454 100644 --- a/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content.py +++ b/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content.py @@ -25,7 +25,7 @@ class StartAgentsRequestPropertiesFillerWordsContent(UncheckedBaseModel): """ Filler word content mode: - `static`: Static filler words. Uses a predefined list of filler words. - - `generated`: LLM-generated filler words based on the last user message. + - `generated`: LLM-generated filler words based on recent conversation context. """ static_config: typing.Optional[StartAgentsRequestPropertiesFillerWordsContentStaticConfig] = pydantic.Field( diff --git a/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content_generated_config.py b/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content_generated_config.py index b51ac8e..4bb3735 100644 --- a/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content_generated_config.py +++ b/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content_generated_config.py @@ -19,12 +19,12 @@ class StartAgentsRequestPropertiesFillerWordsContentGeneratedConfig(UncheckedBas pydantic.Field(default=None) ) """ - OpenAI-compatible LLM provider used to generate filler words. Runs in parallel with the main business LLM and only uses the last user message as input. + OpenAI-compatible LLM provider used to generate filler words. Runs in parallel with the main business LLM. """ prompt: typing.Optional[str] = pydantic.Field(default=None) """ - System prompt used to generate a short filler phrase based on the last user message. The generated text should be conversational and must not answer the user's question. + System prompt used to generate a short filler phrase based on recent conversation context. The generated text should be conversational and must not answer the user's question. """ fallback_strategy: typing.Optional[typing.Literal["static"]] = pydantic.Field(default=None) @@ -32,6 +32,16 @@ class StartAgentsRequestPropertiesFillerWordsContentGeneratedConfig(UncheckedBas Fallback strategy when generated filler text is not ready, fails, or returns empty text. Phase 1 only supports `static`. """ + context_message_limit: typing.Optional[int] = pydantic.Field(default=None) + """ + Maximum number of recent conversation messages used to generate a filler word. + """ + + history_character_limit: typing.Optional[int] = pydantic.Field(default=None) + """ + Maximum number of characters from conversation history used to generate a filler word. + """ + if IS_PYDANTIC_V2: model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 else: diff --git a/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content_generated_config_llm_provider.py b/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content_generated_config_llm_provider.py index 34d3391..55c434c 100644 --- a/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content_generated_config_llm_provider.py +++ b/src/agora_agent/agents/types/start_agents_request_properties_filler_words_content_generated_config_llm_provider.py @@ -9,7 +9,7 @@ class StartAgentsRequestPropertiesFillerWordsContentGeneratedConfigLlmProvider(UncheckedBaseModel): """ - OpenAI-compatible LLM provider used to generate filler words. Runs in parallel with the main business LLM and only uses the last user message as input. + OpenAI-compatible LLM provider used to generate filler words. Runs in parallel with the main business LLM. """ url: str = pydantic.Field() diff --git a/src/agora_agent/core/client_wrapper.py b/src/agora_agent/core/client_wrapper.py index 728eb53..ac8bacd 100644 --- a/src/agora_agent/core/client_wrapper.py +++ b/src/agora_agent/core/client_wrapper.py @@ -26,10 +26,10 @@ def __init__( def get_headers(self) -> typing.Dict[str, str]: headers: typing.Dict[str, str] = { - "User-Agent": "agora-agents/v2.9.0", + "User-Agent": "agora-agents/v2.10.0", "X-Fern-Language": "Python", "X-Fern-SDK-Name": "agora-agents", - "X-Fern-SDK-Version": "v2.9.0", + "X-Fern-SDK-Version": "v2.10.0", **(self.get_custom_headers() or {}), } headers["Authorization"] = httpx.BasicAuth(self._get_username(), self._get_password())._auth_header diff --git a/src/agora_agent/types/anam_avatar_params.py b/src/agora_agent/types/anam_avatar_params.py new file mode 100644 index 0000000..8da3d0b --- /dev/null +++ b/src/agora_agent/types/anam_avatar_params.py @@ -0,0 +1,74 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel +from .anam_avatar_params_quality import AnamAvatarParamsQuality +from .anam_avatar_params_video_encoding import AnamAvatarParamsVideoEncoding + + +class AnamAvatarParams(UncheckedBaseModel): + """ + Anam Avatar configuration parameters. + """ + + api_key: str = pydantic.Field() + """ + Anam API key. + """ + + avatar_id: str = pydantic.Field() + """ + The unique identifier for the Anam avatar you want to use. + """ + + agora_uid: str = pydantic.Field() + """ + The unique identifier for the avatar's RTC connection. This must be different from other participants in the channel. + """ + + agora_token: str = pydantic.Field() + """ + The RTC token that authorizes the avatar to join the video channel. + """ + + sample_rate: typing.Optional[int] = pydantic.Field(default=None) + """ + The audio sample rate in Hz. + """ + + quality: typing.Optional[AnamAvatarParamsQuality] = pydantic.Field(default=None) + """ + The video quality level. + """ + + video_encoding: typing.Optional[AnamAvatarParamsVideoEncoding] = pydantic.Field(default=None) + """ + The video encoding format. + """ + + avatar_model: typing.Optional[str] = pydantic.Field(default=None) + """ + Anam avatar model, such as `cara_mk4` for Cara 4 portrait mode. + """ + + video_width: typing.Optional[int] = pydantic.Field(default=None) + """ + Anam output video width in pixels. Set together with `video_height`; omit both to use the model default resolution. + """ + + video_height: typing.Optional[int] = pydantic.Field(default=None) + """ + Anam output video height in pixels. Set together with `video_width`; omit both to use the model default resolution. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/anam_avatar_params_quality.py b/src/agora_agent/types/anam_avatar_params_quality.py new file mode 100644 index 0000000..167f503 --- /dev/null +++ b/src/agora_agent/types/anam_avatar_params_quality.py @@ -0,0 +1,5 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +AnamAvatarParamsQuality = typing.Union[typing.Literal["high", "medium", "low"], typing.Any] diff --git a/src/agora_agent/types/anam_avatar_params_video_encoding.py b/src/agora_agent/types/anam_avatar_params_video_encoding.py new file mode 100644 index 0000000..11c0b12 --- /dev/null +++ b/src/agora_agent/types/anam_avatar_params_video_encoding.py @@ -0,0 +1,5 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +AnamAvatarParamsVideoEncoding = typing.Union[typing.Literal["H264", "AV1"], typing.Any] diff --git a/src/agora_agent/types/asr.py b/src/agora_agent/types/asr.py index 2f62cc6..838ea54 100644 --- a/src/agora_agent/types/asr.py +++ b/src/agora_agent/types/asr.py @@ -19,6 +19,7 @@ from .microsoft_asr_params import MicrosoftAsrParams from .open_ai_asr_params import OpenAiAsrParams from .sarvam_asr_params import SarvamAsrParams +from .smallest_ai_asr_params import SmallestAiAsrParams from .speechmatics_asr_params import SpeechmaticsAsrParams from .tencent_asr_params import TencentAsrParams from .x_ai_asr_params import XAiAsrParams @@ -269,6 +270,21 @@ class Config: extra = pydantic.Extra.allow +class Asr_Smallestai(UncheckedBaseModel): + vendor: typing.Literal["smallestai"] = "smallestai" + language: typing.Optional[str] = None + params: SmallestAiAsrParams + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow + + Asr = typing_extensions.Annotated[ typing.Union[ Asr_Ares, @@ -287,6 +303,7 @@ class Config: Asr_Xfyun, Asr_XfyunBigmodel, Asr_XfyunDialect, + Asr_Smallestai, ], UnionMetadata(discriminant="vendor"), ] diff --git a/src/agora_agent/types/llm_tool.py b/src/agora_agent/types/llm_tool.py index 2ab8e40..b5b190e 100644 --- a/src/agora_agent/types/llm_tool.py +++ b/src/agora_agent/types/llm_tool.py @@ -21,9 +21,22 @@ class LlmTool(UncheckedBaseModel): Tool type. Must be `function`. """ - function: LlmToolFunction - execution: typing.Optional[LlmToolExecution] = None - server: LlmToolServer + function: LlmToolFunction = pydantic.Field() + """ + Tool interface exposed to the model. `parameters` is the JSON Schema for LLM arguments, not the HTTP request shape. + """ + + execution: typing.Optional[LlmToolExecution] = pydantic.Field(default=None) + """ + Tool execution configuration. Defaults to `{"mode": "sync"}`. Phase 1a only allows `sync`. + """ + + server: LlmToolServer = pydantic.Field() + """ + Actual HTTP request configuration for this REST tool. + Does not use top-level `parameters`, `path_params`, standalone `query`, + `response`, `json_path`, or `max_chars`. + """ if IS_PYDANTIC_V2: model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 diff --git a/src/agora_agent/types/mcp_server.py b/src/agora_agent/types/mcp_server.py new file mode 100644 index 0000000..c5920e5 --- /dev/null +++ b/src/agora_agent/types/mcp_server.py @@ -0,0 +1,54 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel + + +class McpServer(UncheckedBaseModel): + name: str = pydantic.Field() + """ + A unique identifier for the MCP server. Maximum 48 characters. Accepts only English letters and numbers. + """ + + endpoint: str = pydantic.Field() + """ + The endpoint address of the MCP server. The agent uses this to communicate with the MCP server. + """ + + transport: typing.Optional[typing.Literal["streamable_http"]] = pydantic.Field(default=None) + """ + Transport protocol type. + - `streamable_http`: Streaming HTTP protocol + """ + + headers: typing.Optional[typing.Dict[str, str]] = pydantic.Field(default=None) + """ + HTTP header information to include when requesting the MCP server, such as authentication information. + """ + + allowed_tools: typing.Optional[typing.List[str]] = pydantic.Field(default=None) + """ + A list of tools that the agent is allowed to invoke. The agent can only use tools on this list. + - Empty or omitted: All tools are enabled. + - Empty array `[]`: No tools are enabled. + - `["*"]`: All tools are enabled. + - Specific tools `["aa", "bb"]`: Only listed tools are enabled. + - Mix with wildcard `["aa", "*"]`: All tools are enabled (wildcard takes precedence). + """ + + timeout_ms: typing.Optional[int] = pydantic.Field(default=None) + """ + The MCP server request timeout in milliseconds. After timeout, the agent stops waiting for the MCP server's response and continues executing subsequent logic. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/mllm.py b/src/agora_agent/types/mllm.py index acc7171..655f2e8 100644 --- a/src/agora_agent/types/mllm.py +++ b/src/agora_agent/types/mllm.py @@ -5,6 +5,8 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2 from ..core.unchecked_base_model import UncheckedBaseModel +from .llm_tool import LlmTool +from .mcp_server import McpServer from .mllm_params import MllmParams from .mllm_turn_detection import MllmTurnDetection from .mllm_vendor import MllmVendor @@ -76,6 +78,24 @@ class Mllm(UncheckedBaseModel): Agent failure message. """ + tools: typing.Optional[typing.List[LlmTool]] = pydantic.Field(default=None) + """ + Inline REST (pass-through sync) tool definitions for standard text LLM function calling. + Required fields per tool: `type`, `function.name`, `function.parameters` + (`type: object` with `properties`), `server.method` (`GET` or `POST`), and `server.url`. + The combination of `type: function` and `server` identifies a REST tool. + Phase 1a supports GET and POST only; `execution.mode` defaults to and only accepts `sync`. + Template rules: + - Values must be a constant, or exactly one single-level placeholder. + - `{{args.}}`: `server.url` and `server.body` only; not allowed in headers. + - `{{template_variables.}}` and `{{tool_call_id}}`: `server.url`, `server.headers`, and `server.body`. + """ + + mcp_servers: typing.Optional[typing.List[McpServer]] = pydantic.Field(default=None) + """ + MCP server configuration. + """ + vendor: typing.Optional[MllmVendor] = pydantic.Field(default=None) """ MLLM provider. diff --git a/src/agora_agent/types/smallest_ai_asr.py b/src/agora_agent/types/smallest_ai_asr.py new file mode 100644 index 0000000..e38f417 --- /dev/null +++ b/src/agora_agent/types/smallest_ai_asr.py @@ -0,0 +1,30 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel +from .smallest_ai_asr_params import SmallestAiAsrParams + + +class SmallestAiAsr(UncheckedBaseModel): + """ + Smallest AI ASR configuration. + """ + + language: typing.Optional[str] = pydantic.Field(default=None) + """ + Language code for speech recognition. + """ + + params: SmallestAiAsrParams + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/smallest_ai_asr_params.py b/src/agora_agent/types/smallest_ai_asr_params.py new file mode 100644 index 0000000..a888b11 --- /dev/null +++ b/src/agora_agent/types/smallest_ai_asr_params.py @@ -0,0 +1,127 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel + + +class SmallestAiAsrParams(UncheckedBaseModel): + """ + Smallest AI ASR configuration parameters. + """ + + language: typing.Optional[str] = pydantic.Field(default=None) + """ + Language code for speech recognition. This takes precedence over the top-level `asr.language` value. + """ + + url: typing.Optional[str] = pydantic.Field(default=None) + """ + WebSocket endpoint for the Smallest AI streaming STT API. + """ + + api_key: str = pydantic.Field() + """ + Smallest AI API key. + """ + + sample_rate: typing.Optional[int] = pydantic.Field(default=None) + """ + Input audio sample rate in Hz. + """ + + encoding: typing.Optional[str] = pydantic.Field(default=None) + """ + Input audio encoding. + """ + + word_timestamps: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to include word-level timestamps. + """ + + sentence_timestamps: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to include sentence-level timestamps. + """ + + diarize: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to enable speaker diarization. + """ + + vad_events: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to return voice activity detection events. + """ + + endpointing: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to enable endpoint detection. + """ + + eou_timeout_ms: typing.Optional[int] = pydantic.Field(default=None) + """ + End-of-utterance timeout in milliseconds. + """ + + format: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to format the transcript. + """ + + finalize_on_words: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to finalize results based on recognized words. + """ + + max_words: typing.Optional[str] = pydantic.Field(default=None) + """ + Maximum number of words per result. + """ + + punctuate: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to add punctuation. + """ + + capitalize: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to capitalize transcript text. + """ + + itn_normalize: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to enable inverse text normalization. + """ + + full_transcript: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to return the full transcript. + """ + + keywords: typing.Optional[str] = pydantic.Field(default=None) + """ + Comma-separated keyword boosts in `keyword:weight` format. + """ + + redact_pii: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to redact personally identifiable information. + """ + + redact_pci: typing.Optional[str] = pydantic.Field(default=None) + """ + Whether to redact payment card information. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/smallest_ai_tts.py b/src/agora_agent/types/smallest_ai_tts.py new file mode 100644 index 0000000..2046a84 --- /dev/null +++ b/src/agora_agent/types/smallest_ai_tts.py @@ -0,0 +1,29 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel +from .smallest_ai_tts_params import SmallestAiTtsParams + + +class SmallestAiTts(UncheckedBaseModel): + """ + Smallest AI Text-to-Speech configuration. + """ + + params: SmallestAiTtsParams + skip_patterns: typing.Optional[typing.List[int]] = pydantic.Field(default=None) + """ + Controls whether the TTS module skips bracketed content when reading LLM response text. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/smallest_ai_tts_params.py b/src/agora_agent/types/smallest_ai_tts_params.py new file mode 100644 index 0000000..9800e7d --- /dev/null +++ b/src/agora_agent/types/smallest_ai_tts_params.py @@ -0,0 +1,82 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel + + +class SmallestAiTtsParams(UncheckedBaseModel): + """ + Smallest AI TTS configuration parameters. + """ + + url: typing.Optional[str] = pydantic.Field(default=None) + """ + HTTP endpoint for the Smallest AI streaming TTS API. + """ + + api_key: str = pydantic.Field() + """ + Smallest AI API key. + """ + + model: typing.Optional[str] = pydantic.Field(default=None) + """ + Smallest AI TTS model name. + """ + + voice_id: typing.Optional[str] = pydantic.Field(default=None) + """ + Smallest AI voice identifier. + """ + + sample_rate: typing.Optional[int] = pydantic.Field(default=None) + """ + Output audio sample rate in Hz. + """ + + speed: typing.Optional[float] = pydantic.Field(default=None) + """ + Speech rate multiplier. + """ + + language: typing.Optional[str] = pydantic.Field(default=None) + """ + Language code for speech synthesis. + """ + + number_pronunciation_language: typing.Optional[str] = pydantic.Field(default=None) + """ + Language used to pronounce numbers. + """ + + math_notation: typing.Optional[bool] = pydantic.Field(default=None) + """ + Whether to verbalize mathematical notation. + """ + + pronunciation_dicts: typing.Optional[typing.List[str]] = pydantic.Field(default=None) + """ + Pronunciation dictionaries applied during synthesis. + """ + + session_id: typing.Optional[str] = pydantic.Field(default=None) + """ + Client-provided session identifier. + """ + + request_id: typing.Optional[str] = pydantic.Field(default=None) + """ + Client-provided request identifier. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/tts.py b/src/agora_agent/types/tts.py index 662a81c..ce737ad 100644 --- a/src/agora_agent/types/tts.py +++ b/src/agora_agent/types/tts.py @@ -28,6 +28,7 @@ from .rime_tts_credential_mode import RimeTtsCredentialMode from .rime_tts_params import RimeTtsParams from .sarvam_tts_params import SarvamTtsParams +from .smallest_ai_tts_params import SmallestAiTtsParams from .stepfun_tts_params import StepfunTtsParams from .tencent_tts_params import TencentTtsParams from .typecast_tts_params import TypecastTtsParams @@ -382,6 +383,21 @@ class Config: extra = pydantic.Extra.allow +class Tts_Smallestai(UncheckedBaseModel): + vendor: typing.Literal["smallestai"] = "smallestai" + params: SmallestAiTtsParams + skip_patterns: typing.Optional[typing.List[int]] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow + + Tts = typing_extensions.Annotated[ typing.Union[ Tts_Tencent, @@ -407,6 +423,7 @@ class Config: Tts_Gradium, Tts_Mistral, Tts_Typecast, + Tts_Smallestai, ], UnionMetadata(discriminant="vendor"), ] diff --git a/tests/custom/test_agentkit_vendors.py b/tests/custom/test_agentkit_vendors.py index d47fa7a..0fd81a0 100644 --- a/tests/custom/test_agentkit_vendors.py +++ b/tests/custom/test_agentkit_vendors.py @@ -1,19 +1,69 @@ import pytest from pydantic import ValidationError -from agora_agent.agentkit import LlmGreetingConfigs +from agora_agent import OpenAIGPTLive from agora_agent.agentkit.vendors import ( AnamAvatar, GenericAvatar, GenericTTS, OpenAI, OpenAIRealtime, + OpenAIGPTLive as ProductionOpenAIGPTLive, + SmallestAISTT, SpatiusAvatar, XaiGrok, XaiSTT, XaiTTS, ) -from agora_agent import OpenAIGPTLive + + +def test_smallest_ai_stt_serializes_generated_contract() -> None: + config = SmallestAISTT( + api_key="smallest-key", + language="en-US", + sample_rate=16000, + word_timestamps=True, + sentence_timestamps=False, + diarize=True, + vad_events=False, + endpointing=True, + eou_timeout_ms=480, + format=True, + finalize_on_words=False, + max_words="100", + punctuate=True, + capitalize=False, + itn_normalize=True, + full_transcript=False, + redact_pii=False, + redact_pci=True, + additional_params={"language": "overridden", "punctuate": "false", "custom": "value"}, + ).to_config() + + assert config == { + "vendor": "smallestai", + "params": { + "api_key": "smallest-key", + "language": "en-US", + "sample_rate": 16000, + "word_timestamps": "true", + "sentence_timestamps": "false", + "diarize": "true", + "vad_events": "false", + "endpointing": "true", + "eou_timeout_ms": 480, + "format": "true", + "finalize_on_words": "false", + "max_words": "100", + "punctuate": "true", + "capitalize": "false", + "itn_normalize": "true", + "full_transcript": "false", + "redact_pii": "false", + "redact_pci": "true", + "custom": "value", + }, + } def test_xai_grok_serializes_v27_shape_without_style(): @@ -59,7 +109,7 @@ def test_openai_realtime_rejects_none_url(): OpenAIRealtime(api_key="openai-key", url=None) -def test_openai_gpt_live_uses_its_preview_vendor_and_url(): +def test_openai_gpt_live_uses_production_vendor_and_url(): config = OpenAIGPTLive(api_key="openai-key", greeting="Hello from GPT Live").to_config() assert config == { @@ -73,6 +123,12 @@ def test_openai_gpt_live_uses_its_preview_vendor_and_url(): } +def test_openai_gpt_live_preview_import_is_production_alias(): + from agora_agent.agentkit.preview import OpenAIGPTLive as PreviewOpenAIGPTLive + + assert PreviewOpenAIGPTLive is ProductionOpenAIGPTLive + + def test_openai_realtime_rejects_the_removed_live_mode_switch(): with pytest.raises(ValidationError): OpenAIRealtime(api_key="openai-key", mode="live") diff --git a/tests/custom/test_llm_tools.py b/tests/custom/test_llm_tools.py index 7fc54db..626568d 100644 --- a/tests/custom/test_llm_tools.py +++ b/tests/custom/test_llm_tools.py @@ -7,9 +7,12 @@ LlmToolFunctionConfig, LlmToolFunctionParametersConfig, LlmToolServerConfig, + McpServerConfig, OpenAI, + OpenAIGPTLive, + OpenAIRealtime, ) -from agora_agent.agentkit.vendors.cn import AliyunLLM +from agora_agent.agentkit.vendors.cn import AliyunLLM, BytedanceLLM, DeepSeekLLM, QwenOmni, TencentLLM def _tool() -> dict: @@ -93,3 +96,93 @@ def test_with_tools_preserves_other_advanced_features() -> None: assert agent.advanced_features is not None assert agent.advanced_features.enable_sal is True assert agent.advanced_features.enable_tools is True + + +def test_typed_mcp_server_is_supported_by_llm() -> None: + server = McpServerConfig( + name="orders1", + endpoint="https://example.com/mcp", + headers={"Authorization": "Bearer token"}, + allowed_tools=["lookup_order"], + timeout_ms=2500, + ) + + config = OpenAI(model="gpt-4o-mini", mcp_servers=[server]).to_config() + + assert config["mcp_servers"] == [ + { + "name": "orders1", + "endpoint": "https://example.com/mcp", + "transport": "streamable_http", + "headers": {"Authorization": "Bearer token"}, + "allowed_tools": ["lookup_order"], + "timeout_ms": 2500, + } + ] + + +def test_mllm_supports_typed_tools_and_mcp_servers() -> None: + config = OpenAIRealtime( + api_key="openai-key", + tools=[_typed_tool()], + mcp_servers=[McpServerConfig(name="orders1", endpoint="https://example.com/mcp")], + ).to_config() + + assert config["tools"][0]["function"]["name"] == "lookup_order" + assert config["mcp_servers"] == [ + { + "name": "orders1", + "endpoint": "https://example.com/mcp", + "transport": "streamable_http", + } + ] + assert "tools" not in config.get("params", {}) + assert "mcp_servers" not in config.get("params", {}) + + +def test_preview_mllm_supports_typed_tools() -> None: + config = OpenAIGPTLive(api_key="openai-key", tools=[_typed_tool()]).to_config() + + assert config["tools"][0]["function"]["name"] == "lookup_order" + assert "tools" not in config["params"] + + +def test_all_cn_llms_and_qwen_support_typed_tools_and_mcp_servers() -> None: + server = McpServerConfig(name="orders1", endpoint="https://example.com/mcp") + llm_configs = [ + vendor( + api_key="cn-key", + base_url="https://cn-llm.example.com/v1/chat/completions", + model="cn-model", + tools=[_typed_tool()], + mcp_servers=[server], + ).to_config() + for vendor in (AliyunLLM, BytedanceLLM, DeepSeekLLM, TencentLLM) + ] + qwen_config = QwenOmni( + api_key="aliyun-key", + url="wss://dashscope.example.com/realtime", + model="qwen-omni-turbo-realtime", + tools=[_typed_tool()], + mcp_servers=[server], + ).to_config() + + for config in (*llm_configs, qwen_config): + assert config["tools"][0]["function"]["name"] == "lookup_order" + assert config["mcp_servers"] == [ + { + "name": "orders1", + "endpoint": "https://example.com/mcp", + "transport": "streamable_http", + } + ] + assert "tools" not in config["params"] + assert "mcp_servers" not in config["params"] + + assert [config["vendor"] for config in llm_configs] == [ + "aliyun", + "bytedance", + "deepseek", + "tencent", + ] + assert qwen_config["vendor"] == "qwen_omni" diff --git a/tests/custom/test_preview.py b/tests/custom/test_preview.py index ca08b8b..7e568d5 100644 --- a/tests/custom/test_preview.py +++ b/tests/custom/test_preview.py @@ -67,6 +67,10 @@ def test_gemini_is_not_registered_for_preview_routing() -> None: assert required_preview_features({"asr": {"vendor": "gemini"}}) == [] +def test_gpt_live_is_not_registered_for_preview_routing() -> None: + assert required_preview_features({"mllm": {"vendor": "openai_gpt_live"}}) == [] + + def test_registered_provider_routes_session_to_preview(registered_preview_asr) -> None: recorder = _Recorder() client = Agora( diff --git a/tests/custom/test_preview_gemini_merge.py b/tests/custom/test_preview_gemini_merge.py index 6772d15..93f9230 100644 --- a/tests/custom/test_preview_gemini_merge.py +++ b/tests/custom/test_preview_gemini_merge.py @@ -4,31 +4,28 @@ from agora_agent.agentkit.preview import ( GeminiLiveModels, OpenAIGPTLive, - PreviewFeatures, apply_preview_shape, required_preview_features, ) -from agora_agent.agentkit.vendors.mllm import GeminiLive +from agora_agent.agentkit.vendors.mllm import GEMINI_MLLM_URL, GeminiLive -def test_gemini_and_gpt_live_use_their_own_feature_gates(): +def test_gemini_live_models_use_production_routing(): for vendor in (GeminiLive(api_key="test-key", model=GeminiLiveModels.LIVE_38), GeminiLive(api_key="test-key", model=GeminiLiveModels.LIVE_38_EXTENDED_THINKING)): properties = {"mllm": vendor.to_config()} - assert required_preview_features(properties) == [PreviewFeatures.GEMINI_LIVE] - assert required_preview_features({"mllm": OpenAIGPTLive(api_key="test-key").to_config()}) == [ - PreviewFeatures.LIVE_MODELS - ] + assert required_preview_features(properties) == [] + assert required_preview_features({"mllm": OpenAIGPTLive(api_key="test-key").to_config()}) == [] -def test_gemini_greeting_uses_preview_wire_field(): +def test_gemini_greeting_uses_production_wire_field(): properties = {"mllm": GeminiLive(api_key="test-key", model=GeminiLiveModels.LIVE_38).to_config()} properties["mllm"]["greeting_message"] = "Hello" apply_preview_shape(properties) - assert properties["mllm"]["greeting"] == "Hello" - assert "greeting_message" not in properties["mllm"] + assert properties["mllm"]["greeting_message"] == "Hello" + assert "greeting" not in properties["mllm"] -def test_one_gemini_preview_class_supports_public_38_ids(): +def test_gemini_production_class_supports_public_38_ids(): assert GeminiLive(api_key="test-key", model=" ").to_config()["params"]["model"] == GeminiLiveModels.LIVE_38 models = ( (GeminiLiveModels.LIVE_38, "medium"), @@ -40,14 +37,11 @@ def test_one_gemini_preview_class_supports_public_38_ids(): assert config["api_key"] == "test-key" assert "api_key" not in config["params"] assert config["params"]["model"] == model + assert config["url"] == GEMINI_MLLM_URL expected_thinking = thinking if model == GeminiLiveModels.LIVE_38_EXTENDED_THINKING else None assert config["params"].get("thinking_level") == expected_thinking - assert required_preview_features({"mllm": config}) == [PreviewFeatures.GEMINI_LIVE] - # Explicit IDs must route even in a hand-written config without the - # SDK vendor's preview envelope. - assert required_preview_features({"mllm": {"vendor": "gemini", "params": {"model": model}}}) == [ - PreviewFeatures.GEMINI_LIVE - ] + assert required_preview_features({"mllm": config}) == [] + assert required_preview_features({"mllm": {"vendor": "gemini", "params": {"model": model}}}) == [] def test_gemini_mllm_rejects_blank_api_key(): @@ -55,11 +49,35 @@ def test_gemini_mllm_rejects_blank_api_key(): GeminiLive(api_key=" ") -def test_unknown_gemini_model_keeps_preview_greeting_without_nested_api_key(): +def test_unknown_gemini_model_keeps_production_greeting_without_nested_api_key(): properties = {"mllm": GeminiLive( api_key="test-key", model="future-live-model", url="https://generativelanguage.googleapis.com" ).to_config()} properties["mllm"]["greeting_message"] = "Hello" apply_preview_shape(properties) - assert properties["mllm"]["greeting"] == "Hello" - assert "greeting_message" not in properties["mllm"] + assert properties["mllm"]["greeting_message"] == "Hello" + assert "greeting" not in properties["mllm"] + + +def test_existing_gemini_models_preserve_additional_params(): + config = GeminiLive( + api_key="test-key", + model="gemini-live-2.5-flash", + additional_params={"api_key": "legacy-nested-key", "thinking_level": "legacy-value"}, + ).to_config() + + assert config["url"] == "" + assert config["params"]["api_key"] == "legacy-nested-key" + assert config["params"]["thinking_level"] == "legacy-value" + + +def test_preview_gemini_imports_alias_production_objects(): + from agora_agent.agentkit.preview import GeminiLive as PreviewGeminiLive + from agora_agent.agentkit.preview import GeminiLiveModels as PreviewGeminiLiveModels + from agora_agent.agentkit.preview.vendors import build_gemini_preview_config + + assert PreviewGeminiLive is GeminiLive + assert PreviewGeminiLiveModels is GeminiLiveModels + vendor = PreviewGeminiLive(api_key="test-key", greeting_message="Hello") + assert build_gemini_preview_config(vendor) == vendor.to_config() + assert vendor.to_config()["greeting_message"] == "Hello" diff --git a/tests/custom/test_regional_vendors.py b/tests/custom/test_regional_vendors.py index 0a3ea66..92ff328 100644 --- a/tests/custom/test_regional_vendors.py +++ b/tests/custom/test_regional_vendors.py @@ -1,8 +1,10 @@ +from typing import get_args + import pytest from agora_agent import ( - AgentClient, Agent, + AgentClient, Area, DeepgramSTT, GenericTTS, @@ -11,14 +13,17 @@ MiniMaxTTS, MistralTTS, OpenAI, + SmallestAISTT, + SmallestAITTS, SpatiusAvatar, TencentSTT, + XaiGrok, XaiSTT, XaiTTS, - XaiGrok, ) -from agora_agent.agentkit.vendors.catalog import GLOBAL_VENDOR_NAMESPACE -from agora_agent.agentkit.vendors.namespaces import GlobalTTSVendors +from agora_agent.agentkit.regional_agent import GlobalSTT, GlobalTTS +from agora_agent.agentkit.vendors.catalog import CN_VENDOR_NAMESPACE, GLOBAL_VENDOR_NAMESPACE +from agora_agent.agentkit.vendors.namespaces import CNSTTVendors, CNTTSVendors, GlobalSTTVendors, GlobalTTSVendors from agora_agent.agentkit.vendors.region import ( CN_ASR_VENDORS, CN_AVATAR_VENDORS, @@ -222,3 +227,50 @@ def test_xai_grok_remains_mllm_vendor() -> None: assert agent.__class__.__name__ == "GlobalAgent" assert agent.mllm is not None and agent.mllm["vendor"] == "xai" + + +def test_smallest_ai_is_registered_as_global_only() -> None: + assert "smallestai" in GLOBAL_ASR_VENDORS + assert "smallestai" in GLOBAL_TTS_VENDORS + assert GLOBAL_VENDOR_NAMESPACE.asr["smallestai"] is SmallestAISTT + assert GLOBAL_VENDOR_NAMESPACE.tts["smallestai"] is SmallestAITTS + assert GlobalSTTVendors.smallestai is SmallestAISTT + assert GlobalTTSVendors.smallestai is SmallestAITTS + assert SmallestAISTT in get_args(GlobalSTT) + assert SmallestAITTS in get_args(GlobalTTS) + + assert "smallestai" not in CN_ASR_VENDORS + assert "smallestai" not in CN_TTS_VENDORS + assert "smallestai" not in CN_VENDOR_NAMESPACE.asr + assert "smallestai" not in CN_VENDOR_NAMESPACE.tts + assert not hasattr(CNSTTVendors, "smallestai") + assert not hasattr(CNTTSVendors, "smallestai") + + +def test_smallest_ai_configs_reach_global_request_properties() -> None: + properties = ( + Agent(_client(Area.US)) + .with_stt( + SmallestAISTT( + api_key="stt-key", + language="en-US", + word_timestamps=True, + sentence_timestamps=False, + eou_timeout_ms=480, + ) + ) + .with_tts(SmallestAITTS(api_key="tts-key", voice_id="emily")) + .to_properties( + channel="room", + agent_uid="1", + remote_uids=["2"], + token="rtc-token", + allow_missing_vendor_categories={"llm"}, + ) + ) + + assert properties.asr is not None and properties.asr.vendor == "smallestai" + assert properties.asr.params.word_timestamps == "true" + assert properties.asr.params.sentence_timestamps == "false" + assert properties.asr.params.eou_timeout_ms == 480 + assert properties.tts is not None and properties.tts.vendor == "smallestai" diff --git a/tests/custom/test_request_body.py b/tests/custom/test_request_body.py index a859d78..f5bab70 100644 --- a/tests/custom/test_request_body.py +++ b/tests/custom/test_request_body.py @@ -22,15 +22,16 @@ import httpx import pytest from pydantic import ValidationError +from test_helpers import test_client from agora_agent import ( Agent, Agora, - Area, AmazonBedrock, AmazonSTT, AmazonTTS, Anthropic, + Area, AresSTT, AssemblyAISTT, AzureOpenAI, @@ -42,6 +43,11 @@ DeepgramTTS, Dify, ElevenLabsTTS, + FillerWordsConfig, + FillerWordsContent, + FillerWordsContentGeneratedConfig, + FillerWordsGeneratedLlmProvider, + FillerWordsTrigger, FishAudioTTS, Gemini, GeminiLive, @@ -70,8 +76,6 @@ from agora_agent.agentkit.presets import resolve_session_presets from agora_agent.cn import QwenOmni from agora_agent.types.speechmatics_asr_params import SpeechmaticsAsrParams -from test_helpers import test_client - # --------------------------------------------------------------------------- # Shared helpers @@ -1533,7 +1537,8 @@ def record(request): with pytest.warns(UserWarning, match="ignores agent-level turn_detection"): agent.create_session(channel="test", agent_uid="1", remote_uids=["2"], token="token").start() request = requests[0] - assert request.headers["agora-feature"] == "live-models" + assert "agora-feature" not in request.headers + assert str(request.url).startswith("https://api-us-west-1.agora.io/") props = json.loads(request.content)["properties"] assert props["mllm"]["mcp_servers"] == servers assert props["llm"] is None @@ -1545,3 +1550,40 @@ def record(request): assert "turn_detection" not in props assert "mcp_servers" not in props["mllm"]["params"] assert "silence_config" not in props["mllm"]["params"] + + +def test_generated_filler_words_context_limits_reach_request_properties() -> None: + filler_words = FillerWordsConfig( + enable=True, + trigger=FillerWordsTrigger(mode="fixed_time"), + content=FillerWordsContent( + mode="generated", + generated_config=FillerWordsContentGeneratedConfig( + llm_provider=FillerWordsGeneratedLlmProvider( + url="https://example.com/chat/completions", + api_key="key", + model="gpt-4o-mini", + ), + prompt="Generate a brief acknowledgement.", + fallback_strategy="static", + context_message_limit=6, + history_character_limit=1200, + ), + ), + ) + + properties = Agent(test_client()).with_filler_words(filler_words).to_properties( + channel="room", + agent_uid="1", + remote_uids=["2"], + token="rtc-token", + allow_missing_vendor_categories={"asr", "llm", "tts"}, + ) + + assert properties.filler_words is not None + content = properties.filler_words.content + assert content is not None + generated = content.generated_config + assert generated is not None + assert generated.context_message_limit == 6 + assert generated.history_character_limit == 1200 diff --git a/tests/custom/test_tts_vendors.py b/tests/custom/test_tts_vendors.py index 81306ca..43388bb 100644 --- a/tests/custom/test_tts_vendors.py +++ b/tests/custom/test_tts_vendors.py @@ -25,10 +25,39 @@ OpenAITTS, RimeTTS, SarvamTTS, + SmallestAITTS, TypecastTTS, ) +def test_smallest_ai_tts_serializes_generated_contract() -> None: + tts = SmallestAITTS( + api_key="smallest-key", + model="lightning-v3.1", + voice_id="emily", + sample_rate=24000, + speed=1.1, + math_notation=True, + pronunciation_dicts=["medical"], + skip_patterns=[1, 2], + ) + + assert tts.resolved_sample_rate == 24000 + assert tts.to_config() == { + "vendor": "smallestai", + "params": { + "api_key": "smallest-key", + "model": "lightning-v3.1", + "voice_id": "emily", + "sample_rate": 24000, + "speed": 1.1, + "math_notation": True, + "pronunciation_dicts": ["medical"], + }, + "skip_patterns": [1, 2], + } + + def test_tts_vendor_params_match_generated_core_shapes() -> None: assert MicrosoftTTS(key="ms-key", region="eastus", voice_name="en-US-JennyNeural").to_config()["params"] == { "key": "ms-key",