diff --git a/apps/sim/app/(interfaces)/chat/[identifier]/chat.tsx b/apps/sim/app/(interfaces)/chat/[identifier]/chat.tsx index 41d5da0125d..2d896898f9d 100644 --- a/apps/sim/app/(interfaces)/chat/[identifier]/chat.tsx +++ b/apps/sim/app/(interfaces)/chat/[identifier]/chat.tsx @@ -3,8 +3,6 @@ import { type RefObject, useCallback, useEffect, useMemo, useRef, useState } from 'react' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' -import { DEFAULT_TTS_VOICE_ID } from '@/lib/api/contracts/media/tts-stream' -import { noop } from '@/lib/core/utils/request' import { AGENT_STREAM_PROTOCOL_HEADER, AGENT_STREAM_PROTOCOL_V1, @@ -19,23 +17,15 @@ import { ChatMessageContainer, EmailAuth, PasswordAuth, - VoiceInterface, } from '@/app/(interfaces)/chat/components' import { CHAT_ERROR_MESSAGES, CHAT_REQUEST_TIMEOUT_MS } from '@/app/(interfaces)/chat/constants' -import { useAudioStreaming, useChatStreaming } from '@/app/(interfaces)/chat/hooks' +import { useChatStreaming } from '@/app/(interfaces)/chat/hooks' import SSOAuth from '@/ee/sso/components/sso-auth' import { useDeployedChatConfig } from '@/hooks/queries/chats' import { useGitHubStars } from '@/hooks/queries/github-stars' -import { useVoiceSettings } from '@/hooks/queries/voice-settings' const logger = createLogger('ChatClient') -interface AudioStreamingOptions { - voiceId: string - chatId: string - onError: (error: Error) => void -} - interface ChatRequestFile { name: string size: number @@ -49,10 +39,6 @@ interface ChatRequestPayload { files?: ChatRequestFile[] } -const DEFAULT_VOICE_SETTINGS = { - voiceId: DEFAULT_TTS_VOICE_ID, -} - /** * Converts a File object to a base64 data URL */ @@ -65,33 +51,6 @@ function fileToBase64(file: File): Promise { }) } -/** - * Creates an audio stream handler for text-to-speech conversion - * @param streamTextToAudio - Function to stream text to audio - * @param voiceId - The voice ID to use for TTS - * @param chatId - Optional chat ID for deployed chat authentication - * @returns Audio stream handler function or undefined - */ -function createAudioStreamHandler( - streamTextToAudio: (text: string, options: AudioStreamingOptions) => Promise, - voiceId: string, - chatId: string -) { - return async (text: string) => { - try { - await streamTextToAudio(text, { - voiceId, - chatId, - onError: (error: Error) => { - logger.error('Audio streaming error:', error) - }, - }) - } catch (error) { - logger.error('TTS error:', error) - } - } -} - export default function ChatClient({ identifier }: { identifier: string }) { const [messages, setMessages] = useState([]) const [inputValue, setInputValue] = useState('') @@ -105,13 +64,9 @@ export default function ChatClient({ identifier }: { identifier: string }) { const stickToBottomRef = useRef(true) const ignoreScrollRef = useRef(false) - const [isVoiceFirstMode, setIsVoiceFirstMode] = useState(false) - const { data: chatConfigResult, error: chatConfigError } = useDeployedChatConfig(identifier) - const { data: voiceSettings } = useVoiceSettings() const { data: starCount } = useGitHubStars() - const sttAvailable = voiceSettings?.sttAvailable === true const authRequired = chatConfigResult?.kind === 'auth' ? chatConfigResult.authType : null const chatConfig = chatConfigResult?.kind === 'config' ? chatConfigResult.config : null @@ -135,8 +90,6 @@ export default function ChatClient({ identifier }: { identifier: string }) { const { isStreamingResponse, abortControllerRef, stopStreaming, handleStreamedResponse } = useChatStreaming() - const audioContextRef = useRef(null) - const { isPlayingAudio, streamTextToAudio, stopAudio } = useAudioStreaming(audioContextRef) const NEAR_BOTTOM_THRESHOLD_PX = 100 @@ -208,11 +161,10 @@ export default function ChatClient({ identifier }: { identifier: string }) { container.addEventListener('scroll', handleScroll, { passive: true }) return () => container.removeEventListener('scroll', handleScroll) - }, [chatConfig, isVoiceFirstMode, authRequired]) + }, [chatConfig, authRequired]) const handleSendMessage = async ( messageParam?: string, - isVoiceInput = false, files?: Array<{ id: string name: string @@ -227,7 +179,6 @@ export default function ChatClient({ identifier }: { identifier: string }) { logger.info('Sending message:', { messageToSend, - isVoiceInput, conversationId, filesCount: files?.length, }) @@ -316,30 +267,12 @@ export default function ChatClient({ identifier }: { identifier: string }) { throw new Error('Response body is missing') } - const shouldPlayAudio = isVoiceInput || isVoiceFirstMode - const audioHandler = - shouldPlayAudio && chatConfig?.id - ? createAudioStreamHandler( - streamTextToAudio, - DEFAULT_VOICE_SETTINGS.voiceId, - chatConfig.id - ) - : undefined - - logger.info('Starting to handle streamed response:', { shouldPlayAudio }) - await handleStreamedResponse( response, setMessages, setIsLoading, () => scrollToBottom({ behavior: 'auto' }), { - voiceSettings: { - isVoiceEnabled: shouldPlayAudio, - voiceId: DEFAULT_VOICE_SETTINGS.voiceId, - autoPlayResponses: shouldPlayAudio, - }, - audioStreamHandler: audioHandler, outputConfigs: chatConfig?.outputConfigs, abortController, } @@ -365,41 +298,6 @@ export default function ChatClient({ identifier }: { identifier: string }) { } } - useEffect(() => { - return () => { - stopAudio() - if (audioContextRef.current && audioContextRef.current.state !== 'closed') { - audioContextRef.current.close() - } - } - }, [stopAudio]) - - const handleVoiceInterruption = useCallback(() => { - stopAudio() - - if (isStreamingResponse) { - stopStreaming(setMessages) - } - }, [isStreamingResponse, stopStreaming, setMessages, stopAudio]) - - const handleVoiceStart = useCallback(() => { - if (!sttAvailable) return - setIsVoiceFirstMode(true) - }, [sttAvailable]) - - const handleExitVoiceMode = useCallback(() => { - setIsVoiceFirstMode(false) - stopAudio() - }, [stopAudio]) - - const handleVoiceTranscript = useCallback( - (transcript: string) => { - logger.info('Received voice transcript:', transcript) - handleSendMessage(transcript, true) - }, - [handleSendMessage] - ) - if (chatConfigError) { logger.error('Error fetching chat config:', chatConfigError) return @@ -421,26 +319,6 @@ export default function ChatClient({ identifier }: { identifier: string }) { return } - if (isVoiceFirstMode) { - return ( - ({ - content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content), - type: msg.type, - }))} - /> - ) - } - return (
@@ -463,13 +341,11 @@ export default function ChatClient({ identifier }: { identifier: string }) {
{ - void handleSendMessage(value, isVoiceInput, files) + onSubmit={(value, files) => { + void handleSendMessage(value, files) }} isStreaming={isStreamingResponse} onStopStreaming={() => stopStreaming(setMessages)} - onVoiceStart={handleVoiceStart} - sttAvailable={sttAvailable} />
diff --git a/apps/sim/app/(interfaces)/chat/components/index.ts b/apps/sim/app/(interfaces)/chat/components/index.ts index eef5a82c465..cec1cdc451c 100644 --- a/apps/sim/app/(interfaces)/chat/components/index.ts +++ b/apps/sim/app/(interfaces)/chat/components/index.ts @@ -6,4 +6,3 @@ export { ChatInput } from './input/input' export { ChatLoadingState } from './loading-state/loading-state' export type { ChatMessage } from './message/message' export { ChatMessageContainer } from './message-container/message-container' -export { VoiceInterface } from './voice-interface/voice-interface' diff --git a/apps/sim/app/(interfaces)/chat/components/input/input.tsx b/apps/sim/app/(interfaces)/chat/components/input/input.tsx index 63efde5b042..49d18f47fd2 100644 --- a/apps/sim/app/(interfaces)/chat/components/input/input.tsx +++ b/apps/sim/app/(interfaces)/chat/components/input/input.tsx @@ -5,9 +5,8 @@ import { useCallback, useLayoutEffect, useRef, useState } from 'react' import { Badge, Button, cn, handleKeyboardActivation, Tooltip } from '@sim/emcn' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' -import { ArrowUp, Mic, Paperclip, X } from 'lucide-react' +import { ArrowUp, Paperclip, X } from 'lucide-react' import { CHAT_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation' -import { VoiceInput } from '@/app/(interfaces)/chat/components/input/voice-input' const logger = createLogger('ChatInput') @@ -23,20 +22,10 @@ interface AttachedFile { } export const ChatInput: React.FC<{ - onSubmit?: (value: string, isVoiceInput?: boolean, files?: AttachedFile[]) => void + onSubmit?: (value: string, files?: AttachedFile[]) => void isStreaming?: boolean onStopStreaming?: () => void - onVoiceStart?: () => void - voiceOnly?: boolean - sttAvailable?: boolean -}> = ({ - onSubmit, - isStreaming = false, - onStopStreaming, - onVoiceStart, - voiceOnly = false, - sttAvailable = false, -}) => { +}> = ({ onSubmit, isStreaming = false, onStopStreaming }) => { const fileInputRef = useRef(null) const textareaRef = useRef(null) const [inputValue, setInputValue] = useState('') @@ -114,7 +103,7 @@ export const ChatInput: React.FC<{ const handleSubmit = useCallback(() => { if (isStreaming) return if (!inputValue.trim() && attachedFiles.length === 0) return - onSubmit?.(inputValue.trim(), false, attachedFiles) + onSubmit?.(inputValue.trim(), attachedFiles) setInputValue('') setAttachedFiles([]) setUploadErrors([]) @@ -141,31 +130,6 @@ export const ChatInput: React.FC<{ const canSubmit = (inputValue.trim().length > 0 || attachedFiles.length > 0) && !isStreaming - if (voiceOnly) { - return ( - -
- {sttAvailable && ( - - -
- {})} - disabled={isStreaming} - large={true} - /> -
- - -

Start voice conversation

- - - )} -
- - ) - } - return (
@@ -302,26 +266,8 @@ export const ChatInput: React.FC<{ />
- {/* Right: mic + send */} + {/* Right: send */}
- {sttAvailable && ( - - - - - -

Start voice conversation

- - - )} - {isStreaming ? ( - ) - } - - if (large) { - return ( -
- - - - - -
- ) - } - - return ( -
- - - - - - -
- ) -} diff --git a/apps/sim/app/(interfaces)/chat/components/voice-interface/components/particles.tsx b/apps/sim/app/(interfaces)/chat/components/voice-interface/components/particles.tsx deleted file mode 100644 index 3b206e3369c..00000000000 --- a/apps/sim/app/(interfaces)/chat/components/voice-interface/components/particles.tsx +++ /dev/null @@ -1,503 +0,0 @@ -'use client' - -import { useCallback, useEffect, useRef } from 'react' -import { createLogger } from '@sim/logger' -import * as THREE from 'three' - -const logger = createLogger('Particles') - -interface ShaderUniforms { - u_time: { type: string; value: number } - u_frequency: { type: string; value: number } - u_red: { type: string; value: number } - u_green: { type: string; value: number } - u_blue: { type: string; value: number } -} - -interface ParticlesProps { - audioLevels: number[] - isListening: boolean - isPlayingAudio: boolean - isStreaming: boolean - isMuted: boolean - isProcessingInterruption?: boolean - className?: string -} - -class SimpleBloomComposer { - private renderer: THREE.WebGLRenderer - private scene: THREE.Scene - private camera: THREE.Camera - private bloomScene: THREE.Scene - private bloomMaterial: THREE.ShaderMaterial - private renderTarget: THREE.WebGLRenderTarget - private quad: THREE.Mesh - - constructor(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera) { - this.renderer = renderer - this.scene = scene - this.camera = camera - - this.bloomScene = new THREE.Scene() - - this.renderTarget = new THREE.WebGLRenderTarget( - renderer.domElement.width, - renderer.domElement.height, - { - minFilter: THREE.LinearFilter, - magFilter: THREE.LinearFilter, - format: THREE.RGBAFormat, - } - ) - - this.bloomMaterial = new THREE.ShaderMaterial({ - uniforms: { - tDiffuse: { value: null }, - strength: { value: 1.5 }, - threshold: { value: 0.3 }, - radius: { value: 0.8 }, - }, - vertexShader: ` - varying vec2 vUv; - void main() { - vUv = uv; - gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); - } - `, - fragmentShader: ` - uniform sampler2D tDiffuse; - uniform float strength; - uniform float threshold; - uniform float radius; - varying vec2 vUv; - - void main() { - vec4 color = texture2D(tDiffuse, vUv); - - // Simple bloom effect - float brightness = dot(color.rgb, vec3(0.299, 0.587, 0.114)); - if (brightness > threshold) { - color.rgb *= strength; - } - - gl_FragColor = color; - } - `, - }) - - const geometry = new THREE.PlaneGeometry(2, 2) - this.quad = new THREE.Mesh(geometry, this.bloomMaterial) - this.bloomScene.add(this.quad) - } - - render() { - this.renderer.setRenderTarget(this.renderTarget) - this.renderer.render(this.scene, this.camera) - - this.bloomMaterial.uniforms.tDiffuse.value = this.renderTarget.texture - this.renderer.setRenderTarget(null) - this.renderer.render(this.bloomScene, new THREE.OrthographicCamera(-1, 1, 1, -1, 0, 1)) - } - - setSize(width: number, height: number) { - this.renderTarget.setSize(width, height) - } - - dispose() { - this.renderTarget.dispose() - this.bloomMaterial.dispose() - } -} - -const vertexShader = ` -vec3 mod289(vec3 x) -{ - return x - floor(x * (1.0 / 289.0)) * 289.0; -} - -vec4 mod289(vec4 x) -{ - return x - floor(x * (1.0 / 289.0)) * 289.0; -} - -vec4 permute(vec4 x) -{ - return mod289(((x*34.0)+10.0)*x); -} - -vec4 taylorInvSqrt(vec4 r) -{ - return 1.79284291400159 - 0.85373472095314 * r; -} - -vec3 fade(vec3 t) { - return t*t*t*(t*(t*6.0-15.0)+10.0); -} - -float pnoise(vec3 P, vec3 rep) -{ - vec3 Pi0 = mod(floor(P), rep); // Integer part, modulo period - vec3 Pi1 = mod(Pi0 + vec3(1.0), rep); // Integer part + 1, mod period - Pi0 = mod289(Pi0); - Pi1 = mod289(Pi1); - vec3 Pf0 = fract(P); // Fractional part for interpolation - vec3 Pf1 = Pf0 - vec3(1.0); // Fractional part - 1.0 - vec4 ix = vec4(Pi0.x, Pi1.x, Pi0.x, Pi1.x); - vec4 iy = vec4(Pi0.yy, Pi1.yy); - vec4 iz0 = Pi0.zzzz; - vec4 iz1 = Pi1.zzzz; - - vec4 ixy = permute(permute(ix) + iy); - vec4 ixy0 = permute(ixy + iz0); - vec4 ixy1 = permute(ixy + iz1); - - vec4 gx0 = ixy0 * (1.0 / 7.0); - vec4 gy0 = fract(floor(gx0) * (1.0 / 7.0)) - 0.5; - gx0 = fract(gx0); - vec4 gz0 = vec4(0.5) - abs(gx0) - abs(gy0); - vec4 sz0 = step(gz0, vec4(0.0)); - gx0 -= sz0 * (step(0.0, gx0) - 0.5); - gy0 -= sz0 * (step(0.0, gy0) - 0.5); - - vec4 gx1 = ixy1 * (1.0 / 7.0); - vec4 gy1 = fract(floor(gx1) * (1.0 / 7.0)) - 0.5; - gx1 = fract(gx1); - vec4 gz1 = vec4(0.5) - abs(gx1) - abs(gy1); - vec4 sz1 = step(gz1, vec4(0.0)); - gx1 -= sz1 * (step(0.0, gx1) - 0.5); - gy1 -= sz1 * (step(0.0, gy1) - 0.5); - - vec3 g000 = vec3(gx0.x,gy0.x,gz0.x); - vec3 g100 = vec3(gx0.y,gy0.y,gz0.y); - vec3 g010 = vec3(gx0.z,gy0.z,gz0.z); - vec3 g110 = vec3(gx0.w,gy0.w,gz0.w); - vec3 g001 = vec3(gx1.x,gy1.x,gz1.x); - vec3 g101 = vec3(gx1.y,gy1.y,gz1.y); - vec3 g011 = vec3(gx1.z,gy1.z,gz1.z); - vec3 g111 = vec3(gx1.w,gy1.w,gz1.w); - - vec4 norm0 = taylorInvSqrt(vec4(dot(g000, g000), dot(g010, g010), dot(g100, g100), dot(g110, g110))); - g000 *= norm0.x; - g010 *= norm0.y; - g100 *= norm0.z; - g110 *= norm0.w; - vec4 norm1 = taylorInvSqrt(vec4(dot(g001, g001), dot(g011, g011), dot(g101, g101), dot(g111, g111))); - g001 *= norm1.x; - g011 *= norm1.y; - g101 *= norm1.z; - g111 *= norm1.w; - - float n000 = dot(g000, Pf0); - float n100 = dot(g100, vec3(Pf1.x, Pf0.yz)); - float n010 = dot(g010, vec3(Pf0.x, Pf1.y, Pf0.z)); - float n110 = dot(g110, vec3(Pf1.xy, Pf0.z)); - float n001 = dot(g001, vec3(Pf0.xy, Pf1.z)); - float n101 = dot(g101, vec3(Pf1.x, Pf0.y, Pf1.z)); - float n011 = dot(g011, vec3(Pf0.x, Pf1.yz)); - float n111 = dot(g111, Pf1); - - vec3 fade_xyz = fade(Pf0); - vec4 n_z = mix(vec4(n000, n100, n010, n110), vec4(n001, n101, n011, n111), fade_xyz.z); - vec2 n_yz = mix(n_z.xy, n_z.zw, fade_xyz.y); - float n_xyz = mix(n_yz.x, n_yz.y, fade_xyz.x); - return 2.2 * n_xyz; -} - -uniform float u_time; -uniform float u_frequency; - -void main() { - float noise = 5. * pnoise(position + u_time, vec3(10.)); - - float displacement = (u_frequency / 30.) * (noise / 10.); - - vec3 newPosition = position + normal * displacement; - gl_Position = projectionMatrix * modelViewMatrix * vec4(newPosition, 1.0); -} -` - -const fragmentShader = ` -uniform float u_red; -uniform float u_blue; -uniform float u_green; - -void main() { - gl_FragColor = vec4(vec3(u_red, u_green, u_blue), 1.0); -} -` - -export function ParticlesVisualization({ - audioLevels, - isListening, - isPlayingAudio, - isStreaming, - isMuted, - isProcessingInterruption, - className, -}: ParticlesProps) { - const containerRef = useRef(null) - const rendererRef = useRef(null) - const sceneRef = useRef(null) - const cameraRef = useRef(null) - const meshRef = useRef(null) - const uniformsRef = useRef(null) - const clockRef = useRef(null) - const bloomComposerRef = useRef(null) - const animationFrameRef = useRef(0) - const mouseRef = useRef({ x: 0, y: 0 }) - const isInitializedRef = useRef(false) - - const cleanup = useCallback(() => { - if (animationFrameRef.current) { - cancelAnimationFrame(animationFrameRef.current) - animationFrameRef.current = 0 - } - - if (bloomComposerRef.current) { - bloomComposerRef.current.dispose() - bloomComposerRef.current = null - } - - if (rendererRef.current) { - if (rendererRef.current.domElement?.parentNode) { - rendererRef.current.domElement.parentNode.removeChild(rendererRef.current.domElement) - } - rendererRef.current.dispose() - rendererRef.current = null - } - - sceneRef.current = null - cameraRef.current = null - meshRef.current = null - uniformsRef.current = null - clockRef.current = null - isInitializedRef.current = false - }, []) - - useEffect(() => { - if (!containerRef.current || isInitializedRef.current) return - - const container = containerRef.current - const containerWidth = 400 - const containerHeight = 400 - - isInitializedRef.current = true - - while (container.firstChild) { - container.removeChild(container.firstChild) - } - - const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true }) - renderer.setSize(containerWidth, containerHeight) - renderer.setClearColor(0x000000, 0) - renderer.outputColorSpace = THREE.SRGBColorSpace - container.appendChild(renderer.domElement) - rendererRef.current = renderer - - const scene = new THREE.Scene() - sceneRef.current = scene - - const camera = new THREE.PerspectiveCamera(45, containerWidth / containerHeight, 0.1, 1000) - camera.position.set(0, -2, 14) - camera.lookAt(0, 0, 0) - cameraRef.current = camera - - const uniforms = { - u_time: { type: 'f', value: 0.0 }, - u_frequency: { type: 'f', value: 0.0 }, - u_red: { type: 'f', value: 0.8 }, - u_green: { type: 'f', value: 0.6 }, - u_blue: { type: 'f', value: 1.0 }, - } - uniformsRef.current = uniforms - - let mat: THREE.Material - try { - mat = new THREE.ShaderMaterial({ - uniforms, - vertexShader, - fragmentShader, - }) - } catch (error) { - logger.error('❌ Shader compilation error, using fallback material:', error) - mat = new THREE.MeshBasicMaterial({ - color: 0xb794f6, // Light purple color - wireframe: true, - }) - } - - const geo = new THREE.IcosahedronGeometry(4, 30) // Match tutorial: radius 4, subdivisions 30 - const mesh = new THREE.Mesh(geo, mat) - - if (mat instanceof THREE.ShaderMaterial || mat instanceof THREE.MeshBasicMaterial) { - mat.wireframe = true - } - - scene.add(mesh) - meshRef.current = mesh - - const bloomComposer = new SimpleBloomComposer(renderer, scene, camera) - bloomComposerRef.current = bloomComposer - - const clock = new THREE.Clock() - clockRef.current = clock - - const handleMouseMove = (e: MouseEvent) => { - const rect = container.getBoundingClientRect() - const windowHalfX = containerWidth / 2 - const windowHalfY = containerHeight / 2 - mouseRef.current.x = (e.clientX - rect.left - windowHalfX) / 100 - mouseRef.current.y = (e.clientY - rect.top - windowHalfY) / 100 - } - - container.addEventListener('mousemove', handleMouseMove) - - const updateCameraPosition = () => { - if (!camera || !scene) return - camera.position.x += (mouseRef.current.x - camera.position.x) * 0.05 - camera.position.y += (-mouseRef.current.y - camera.position.y) * 0.5 - camera.lookAt(scene.position) - } - - const calculateAudioIntensity = (elapsedTime: number, avgLevel: number) => { - const baselineIntensity = 8 + Math.sin(elapsedTime * 0.5) * 3 - let audioIntensity = baselineIntensity - - if (isMuted) { - // When muted, only show minimal baseline animation - audioIntensity = baselineIntensity * 0.2 - } else if (isProcessingInterruption) { - // Special pulsing effect during interruption processing - audioIntensity = 35 + Math.sin(elapsedTime * 4) * 10 - } else if (isPlayingAudio) { - // Strong animation when AI is speaking - use simulated levels + enhancement - const aiIntensity = 60 + Math.sin(elapsedTime * 3) * 20 - audioIntensity = Math.max(avgLevel * 0.8, aiIntensity) - } else if (isStreaming) { - // Pulsing animation when AI is thinking/streaming - audioIntensity = 40 + Math.sin(elapsedTime * 2) * 15 - } else if (isListening && avgLevel > 0) { - // Scale user input more dramatically for better visual feedback - const userVoiceIntensity = avgLevel * 2.5 // Amplify user voice significantly - audioIntensity = Math.max(userVoiceIntensity, baselineIntensity * 1.5) - - // Add some dynamic variation based on audio levels - const variationFactor = Math.min(avgLevel / 20, 1) // Cap at reasonable level - audioIntensity += Math.sin(elapsedTime * 8) * (10 * variationFactor) - } else { - // Idle state - subtle breathing animation - audioIntensity = baselineIntensity - } - - // Clamp to reasonable range - audioIntensity = Math.max(audioIntensity, 3) // Never completely still - audioIntensity = Math.min(audioIntensity, 120) // Prevent excessive animation - - return audioIntensity - } - - const updateShaderColors = ( - uniforms: ShaderUniforms, - elapsedTime: number, - avgLevel: number - ) => { - if (isMuted) { - // Muted: dim purple-gray - uniforms.u_red.value = 0.25 - uniforms.u_green.value = 0.1 - uniforms.u_blue.value = 0.5 - } else if (isProcessingInterruption) { - // Interruption: bright purple - uniforms.u_red.value = 0.6 - uniforms.u_green.value = 0.2 - uniforms.u_blue.value = 0.9 - } else if (isPlayingAudio) { - // AI speaking: brand purple (#701FFC) - uniforms.u_red.value = 0.44 - uniforms.u_green.value = 0.12 - uniforms.u_blue.value = 0.99 - } else if (isListening && avgLevel > 10) { - // User speaking: lighter purple with intensity-based variation - const intensity = Math.min(avgLevel / 50, 1) - uniforms.u_red.value = 0.35 + intensity * 0.15 - uniforms.u_green.value = 0.1 + intensity * 0.1 - uniforms.u_blue.value = 0.8 + intensity * 0.2 - } else if (isStreaming) { - // AI thinking: pulsing brand purple - const pulse = (Math.sin(elapsedTime * 2) + 1) / 2 - uniforms.u_red.value = 0.35 + pulse * 0.15 - uniforms.u_green.value = 0.08 + pulse * 0.08 - uniforms.u_blue.value = 0.95 + pulse * 0.05 - } else { - // Default idle: soft brand purple - uniforms.u_red.value = 0.4 - uniforms.u_green.value = 0.15 - uniforms.u_blue.value = 0.9 - } - } - - const animate = () => { - if (!camera || !clock || !scene || !bloomComposer || !isInitializedRef.current) return - - updateCameraPosition() - - if (uniforms) { - const elapsedTime = clock.getElapsedTime() - const avgLevel = audioLevels.reduce((sum, level) => sum + level, 0) / audioLevels.length - - uniforms.u_time.value = elapsedTime - - const audioIntensity = calculateAudioIntensity(elapsedTime, avgLevel) - updateShaderColors(uniforms, elapsedTime, avgLevel) - - uniforms.u_frequency.value = audioIntensity - } - - bloomComposer.render() - animationFrameRef.current = requestAnimationFrame(animate) - } - - animate() - - return () => { - container.removeEventListener('mousemove', handleMouseMove) - cleanup() - } - }, []) - - useEffect(() => { - const handleResize = () => { - if ( - rendererRef.current && - cameraRef.current && - bloomComposerRef.current && - containerRef.current - ) { - const containerWidth = 400 - const containerHeight = 400 - - cameraRef.current.aspect = containerWidth / containerHeight - cameraRef.current.updateProjectionMatrix() - rendererRef.current.setSize(containerWidth, containerHeight) - bloomComposerRef.current.setSize(containerWidth, containerHeight) - } - } - - window.addEventListener('resize', handleResize) - return () => window.removeEventListener('resize', handleResize) - }, []) - - return ( -
- ) -} diff --git a/apps/sim/app/(interfaces)/chat/components/voice-interface/voice-interface.tsx b/apps/sim/app/(interfaces)/chat/components/voice-interface/voice-interface.tsx deleted file mode 100644 index 242ea8bdff1..00000000000 --- a/apps/sim/app/(interfaces)/chat/components/voice-interface/voice-interface.tsx +++ /dev/null @@ -1,582 +0,0 @@ -'use client' - -import { type RefObject, useCallback, useEffect, useRef, useState } from 'react' -import { Button, cn } from '@sim/emcn' -import { createLogger } from '@sim/logger' -import { Mic, MicOff, Phone } from 'lucide-react' -import dynamic from 'next/dynamic' -import { requestJson } from '@/lib/api/client/request' -import { speechTokenContract } from '@/lib/api/contracts/media/speech' -import { arrayBufferToBase64, floatTo16BitPCM } from '@/lib/speech/audio' -import { - CHUNK_SEND_INTERVAL_MS, - ELEVENLABS_WS_URL, - MAX_CHAT_SESSION_MS, - SAMPLE_RATE, -} from '@/lib/speech/config' -import { DesktopTitleBarLane } from '@/app/_shell/desktop-title-bar' - -const ParticlesVisualization = dynamic( - () => - import('@/app/(interfaces)/chat/components/voice-interface/components/particles').then( - (mod) => mod.ParticlesVisualization - ), - { ssr: false } -) - -const logger = createLogger('VoiceInterface') - -interface VoiceInterfaceProps { - onCallEnd?: () => void - onVoiceTranscript?: (transcript: string) => void - onVoiceStart?: () => void - onVoiceEnd?: () => void - onInterrupt?: () => void - isStreaming?: boolean - isPlayingAudio?: boolean - audioContextRef?: RefObject - messages?: Array<{ content: string; type: 'user' | 'assistant' }> - className?: string - chatId?: string -} - -const EMPTY_MESSAGES: Array<{ content: string; type: 'user' | 'assistant' }> = [] - -export function VoiceInterface({ - onCallEnd, - onVoiceTranscript, - onVoiceStart, - onVoiceEnd, - onInterrupt, - isStreaming = false, - isPlayingAudio = false, - audioContextRef: sharedAudioContextRef, - messages = EMPTY_MESSAGES, - className, - chatId, -}: VoiceInterfaceProps) { - const [state, setState] = useState<'idle' | 'listening' | 'agent_speaking'>('idle') - const [isInitialized, setIsInitialized] = useState(false) - const [isMuted, setIsMuted] = useState(false) - const [audioLevels, setAudioLevels] = useState(() => new Array(200).fill(0)) - const permissionStatusRef = useRef<'prompt' | 'granted' | 'denied'>('prompt') - const [currentTranscript, setCurrentTranscript] = useState('') - - const currentStateRef = useRef<'idle' | 'listening' | 'agent_speaking'>('idle') - const isCallEndedRef = useRef(false) - - const updateState = useCallback((next: 'idle' | 'listening' | 'agent_speaking') => { - setState(next) - currentStateRef.current = next - }, []) - - const mediaStreamRef = useRef(null) - const audioContextRef = useRef(null) - const analyserRef = useRef(null) - const animationFrameRef = useRef(null) - const isMutedRef = useRef(false) - - const wsRef = useRef(null) - const processorRef = useRef(null) - const pcmBufferRef = useRef([]) - const sendIntervalRef = useRef | null>(null) - const sessionTimerRef = useRef | null>(null) - const committedTextRef = useRef('') - const lastPartialRef = useRef('') - const onVoiceTranscriptRef = useRef(onVoiceTranscript) - - onVoiceTranscriptRef.current = onVoiceTranscript - - const updateIsMuted = useCallback((next: boolean) => { - setIsMuted(next) - isMutedRef.current = next - }, []) - - const stopSendingAudio = useCallback(() => { - if (sessionTimerRef.current) { - clearTimeout(sessionTimerRef.current) - sessionTimerRef.current = null - } - if (sendIntervalRef.current) { - clearInterval(sendIntervalRef.current) - sendIntervalRef.current = null - } - pcmBufferRef.current = [] - }, []) - - const flushAudioBuffer = useCallback(() => { - const ws = wsRef.current - if (!ws || ws.readyState !== WebSocket.OPEN) return - - const chunks = pcmBufferRef.current - if (chunks.length === 0) return - pcmBufferRef.current = [] - - let totalLength = 0 - for (const chunk of chunks) totalLength += chunk.length - const merged = new Float32Array(totalLength) - let offset = 0 - for (const chunk of chunks) { - merged.set(chunk, offset) - offset += chunk.length - } - - const pcm16 = floatTo16BitPCM(merged) - - ws.send( - JSON.stringify({ - message_type: 'input_audio_chunk', - audio_base_64: arrayBufferToBase64(pcm16), - sample_rate: SAMPLE_RATE, - commit: false, - }) - ) - }, []) - - const startSendingAudio = useCallback(() => { - if (sendIntervalRef.current) return - pcmBufferRef.current = [] - sendIntervalRef.current = setInterval(flushAudioBuffer, CHUNK_SEND_INTERVAL_MS) - }, [flushAudioBuffer]) - - const closeWebSocket = useCallback(() => { - stopSendingAudio() - if (wsRef.current) { - if ( - wsRef.current.readyState === WebSocket.OPEN || - wsRef.current.readyState === WebSocket.CONNECTING - ) { - wsRef.current.close() - } - wsRef.current = null - } - }, [stopSendingAudio]) - - const connectWebSocket = useCallback(async (): Promise => { - try { - let tokenData: Awaited>> - try { - tokenData = await requestJson(speechTokenContract, { - body: chatId ? { chatId } : {}, - }) - } catch (err) { - logger.error('Failed to get STT token', err) - return false - } - - const token = typeof tokenData.token === 'string' ? tokenData.token : undefined - if (!token) { - logger.error('STT token missing from response') - return false - } - - const params = new URLSearchParams({ - token, - model_id: 'scribe_v2_realtime', - audio_format: 'pcm_16000', - commit_strategy: 'vad', - vad_silence_threshold_secs: '1.0', - }) - - const ws = new WebSocket(`${ELEVENLABS_WS_URL}?${params.toString()}`) - wsRef.current = ws - committedTextRef.current = '' - - return new Promise((resolve) => { - ws.onopen = () => resolve(true) - ws.onerror = () => { - logger.error('STT WebSocket connection error') - resolve(false) - } - - ws.onmessage = (event) => { - if (isCallEndedRef.current) return - - try { - const msg = JSON.parse(event.data) - - if (msg.message_type === 'partial_transcript') { - if (msg.text) { - lastPartialRef.current = msg.text - setCurrentTranscript(msg.text) - } - } else if ( - msg.message_type === 'committed_transcript' || - msg.message_type === 'committed_transcript_with_timestamps' - ) { - const finalText = msg.text || lastPartialRef.current - lastPartialRef.current = '' - if (finalText) { - committedTextRef.current = committedTextRef.current - ? `${committedTextRef.current} ${finalText}` - : finalText - setCurrentTranscript('') - onVoiceTranscriptRef.current?.(finalText) - } - } else if ( - msg.message_type === 'error' || - msg.message_type === 'auth_error' || - msg.message_type === 'quota_exceeded' - ) { - logger.error('ElevenLabs STT error', { type: msg.message_type, error: msg.error }) - } - } catch { - // Ignore non-JSON messages - } - } - - ws.onclose = () => { - wsRef.current = null - if (currentStateRef.current === 'listening' && !isCallEndedRef.current) { - stopSendingAudio() - updateState('idle') - } - } - }) - } catch (error) { - logger.error('Failed to connect STT WebSocket', error) - return false - } - }, [chatId]) - - const setupAudioPipeline = useCallback(async () => { - try { - const stream = await navigator.mediaDevices.getUserMedia({ - audio: { - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - channelCount: 1, - sampleRate: SAMPLE_RATE, - }, - }) - - permissionStatusRef.current = 'granted' - mediaStreamRef.current = stream - - const ac = new AudioContext({ sampleRate: SAMPLE_RATE }) - audioContextRef.current = ac - - if (ac.state === 'suspended') { - await ac.resume() - } - - const source = ac.createMediaStreamSource(stream) - - const analyser = ac.createAnalyser() - analyser.fftSize = 256 - analyser.smoothingTimeConstant = 0.8 - source.connect(analyser) - analyserRef.current = analyser - - const processor = ac.createScriptProcessor(4096, 1, 1) - processor.onaudioprocess = (e) => { - if (!isMutedRef.current && currentStateRef.current === 'listening') { - pcmBufferRef.current.push(new Float32Array(e.inputBuffer.getChannelData(0))) - } - } - source.connect(processor) - processor.connect(ac.destination) - processorRef.current = processor - - const updateVisualization = () => { - if (!analyserRef.current) return - const bufferLength = analyserRef.current.frequencyBinCount - const dataArray = new Uint8Array(bufferLength) - analyserRef.current.getByteFrequencyData(dataArray) - - const levels = [] - for (let i = 0; i < 200; i++) { - const dataIndex = Math.floor((i / 200) * bufferLength) - const value = dataArray[dataIndex] || 0 - levels.push((value / 255) * 100) - } - - setAudioLevels(levels) - animationFrameRef.current = requestAnimationFrame(updateVisualization) - } - updateVisualization() - - return true - } catch (error) { - logger.error('Error setting up audio pipeline:', error) - permissionStatusRef.current = 'denied' - return false - } - }, []) - - const startListening = useCallback(async () => { - if (currentStateRef.current !== 'idle' || isMutedRef.current || isCallEndedRef.current) return - - if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) { - const connected = await connectWebSocket() - if (!connected || isCallEndedRef.current) return - } - - updateState('listening') - setCurrentTranscript('') - startSendingAudio() - - sessionTimerRef.current = setTimeout(() => { - logger.info('Voice session reached max duration, stopping') - stopSendingAudio() - closeWebSocket() - updateState('idle') - }, MAX_CHAT_SESSION_MS) - }, [connectWebSocket, updateState, startSendingAudio, stopSendingAudio, closeWebSocket]) - - const stopListening = useCallback(() => { - stopSendingAudio() - updateState('idle') - setCurrentTranscript('') - }, [updateState, stopSendingAudio]) - - useEffect(() => { - if (isPlayingAudio && state === 'listening') { - stopSendingAudio() - closeWebSocket() - updateState('agent_speaking') - setCurrentTranscript('') - - updateIsMuted(true) - if (mediaStreamRef.current) { - mediaStreamRef.current.getAudioTracks().forEach((track) => { - track.enabled = false - }) - } - } else if (!isPlayingAudio && state === 'agent_speaking') { - updateState('idle') - setCurrentTranscript('') - - updateIsMuted(false) - if (mediaStreamRef.current) { - mediaStreamRef.current.getAudioTracks().forEach((track) => { - track.enabled = true - }) - } - } - }, [isPlayingAudio, state, updateState, updateIsMuted, stopSendingAudio, closeWebSocket]) - - const handleInterrupt = useCallback(() => { - if (state === 'agent_speaking') { - onInterrupt?.() - - updateIsMuted(false) - if (mediaStreamRef.current) { - mediaStreamRef.current.getAudioTracks().forEach((track) => { - track.enabled = true - }) - } - - updateState('idle') - setCurrentTranscript('') - } - }, [state, onInterrupt, updateState, updateIsMuted]) - - const handleCallEnd = useCallback(() => { - isCallEndedRef.current = true - - stopSendingAudio() - closeWebSocket() - updateState('idle') - setCurrentTranscript('') - updateIsMuted(false) - - if (processorRef.current) { - processorRef.current.disconnect() - processorRef.current = null - } - - if (mediaStreamRef.current) { - mediaStreamRef.current.getTracks().forEach((track) => track.stop()) - mediaStreamRef.current = null - } - - if (audioContextRef.current && audioContextRef.current.state !== 'closed') { - audioContextRef.current.close().catch(() => {}) - audioContextRef.current = null - } - - if (animationFrameRef.current) { - cancelAnimationFrame(animationFrameRef.current) - animationFrameRef.current = null - } - - onInterrupt?.() - onCallEnd?.() - }, [onCallEnd, onInterrupt, updateState, updateIsMuted, stopSendingAudio, closeWebSocket]) - - useEffect(() => { - const handleKeyDown = (event: KeyboardEvent) => { - if (event.code === 'Space') { - event.preventDefault() - handleInterrupt() - } - } - - document.addEventListener('keydown', handleKeyDown) - return () => document.removeEventListener('keydown', handleKeyDown) - }, [handleInterrupt]) - - const toggleMute = useCallback(() => { - if (state === 'agent_speaking') { - handleInterrupt() - return - } - - const newMutedState = !isMuted - updateIsMuted(newMutedState) - - if (mediaStreamRef.current) { - mediaStreamRef.current.getAudioTracks().forEach((track) => { - track.enabled = !newMutedState - }) - } - - if (newMutedState) { - stopListening() - } else if (state === 'idle') { - startListening() - } - }, [isMuted, state, handleInterrupt, stopListening, startListening, updateIsMuted]) - - useEffect(() => { - isCallEndedRef.current = false - let cancelled = false - - async function init() { - const audioOk = await setupAudioPipeline() - if (!audioOk || cancelled) return - - const wsOk = await connectWebSocket() - if (!wsOk || cancelled) return - - setIsInitialized(true) - } - - init() - - return () => { - cancelled = true - } - }, [setupAudioPipeline, connectWebSocket]) - - useEffect(() => { - if (isInitialized && !isMuted && state === 'idle') { - startListening() - } - }, [isInitialized, isMuted, state, startListening]) - - useEffect(() => { - return () => { - isCallEndedRef.current = true - - stopSendingAudio() - - if (wsRef.current) { - wsRef.current.close() - wsRef.current = null - } - - if (processorRef.current) { - processorRef.current.disconnect() - processorRef.current = null - } - - if (mediaStreamRef.current) { - mediaStreamRef.current.getTracks().forEach((track) => track.stop()) - mediaStreamRef.current = null - } - - if (audioContextRef.current) { - audioContextRef.current.close() - audioContextRef.current = null - } - - if (animationFrameRef.current) { - cancelAnimationFrame(animationFrameRef.current) - animationFrameRef.current = null - } - } - }, [stopSendingAudio]) - - const getStatusText = () => { - switch (state) { - case 'listening': - return 'Listening...' - case 'agent_speaking': - return 'Press Space or tap to interrupt' - default: - return isInitialized ? 'Ready' : 'Initializing...' - } - } - - const getButtonContent = () => { - if (state === 'agent_speaking') { - return ( - - - - ) - } - return isMuted ? : - } - - return ( -
- -
-
- -
- -
- {currentTranscript && ( -
-

- {currentTranscript} -

-
- )} -
- -

- {getStatusText()} - {isMuted && (Muted)} -

-
- -
-
- - - -
-
-
- ) -} diff --git a/apps/sim/app/(interfaces)/chat/hooks/index.ts b/apps/sim/app/(interfaces)/chat/hooks/index.ts index d1a9f6d2934..ec79160ad76 100644 --- a/apps/sim/app/(interfaces)/chat/hooks/index.ts +++ b/apps/sim/app/(interfaces)/chat/hooks/index.ts @@ -1,2 +1 @@ -export { useAudioStreaming } from './use-audio-streaming' export { useChatStreaming } from './use-chat-streaming' diff --git a/apps/sim/app/(interfaces)/chat/hooks/use-audio-streaming.test.ts b/apps/sim/app/(interfaces)/chat/hooks/use-audio-streaming.test.ts deleted file mode 100644 index 6d1a53177a9..00000000000 --- a/apps/sim/app/(interfaces)/chat/hooks/use-audio-streaming.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -/** - * @vitest-environment node - */ -import { describe, expect, it } from 'vitest' -import { MAX_TTS_TEXT_LENGTH } from '@/lib/api/contracts/media/tts-stream' -import { splitForSynthesis } from '@/app/(interfaces)/chat/hooks/use-audio-streaming' - -describe('splitForSynthesis', () => { - it('leaves text within the relay cap untouched', () => { - expect(splitForSynthesis('Short answer.')).toEqual(['Short answer.']) - }) - - /** - * The caller only sentence-splits on Western `.!?`, so CJK punctuation never - * matches and the whole answer arrives as one block. Before splitting, the - * relay rejected it and the message played no audio at all. - */ - it('splits CJK text that never matches the Western sentence split', () => { - const text = '这是一个很长的回答。'.repeat(400) - expect(text.length).toBeGreaterThan(MAX_TTS_TEXT_LENGTH) - - const chunks = splitForSynthesis(text) - - expect(chunks.length).toBeGreaterThan(1) - for (const chunk of chunks) { - expect(chunk.length).toBeLessThanOrEqual(MAX_TTS_TEXT_LENGTH) - } - }) - - it('splits a long list that has no terminal punctuation', () => { - const text = Array.from({ length: 300 }, (_, i) => `- item number ${i}`).join('\n') - expect(text.length).toBeGreaterThan(MAX_TTS_TEXT_LENGTH) - - const chunks = splitForSynthesis(text) - - for (const chunk of chunks) { - expect(chunk.length).toBeLessThanOrEqual(MAX_TTS_TEXT_LENGTH) - } - }) - - it('preserves the spoken content across chunks', () => { - const text = Array.from({ length: 500 }, (_, i) => `word${i}`).join(' ') - - const chunks = splitForSynthesis(text) - - expect(chunks.join(' ').replace(/\s+/g, ' ')).toBe(text) - }) - - it('still caps text with no break opportunity at all', () => { - const chunks = splitForSynthesis('a'.repeat(MAX_TTS_TEXT_LENGTH * 2 + 5)) - - expect(chunks.length).toBe(3) - for (const chunk of chunks) { - expect(chunk.length).toBeLessThanOrEqual(MAX_TTS_TEXT_LENGTH) - } - }) -}) diff --git a/apps/sim/app/(interfaces)/chat/hooks/use-audio-streaming.ts b/apps/sim/app/(interfaces)/chat/hooks/use-audio-streaming.ts deleted file mode 100644 index ec54d4f2c6d..00000000000 --- a/apps/sim/app/(interfaces)/chat/hooks/use-audio-streaming.ts +++ /dev/null @@ -1,210 +0,0 @@ -'use client' - -import { type RefObject, useCallback, useRef, useState } from 'react' -import { createLogger } from '@sim/logger' -import { DEFAULT_TTS_MODEL_ID, MAX_TTS_TEXT_LENGTH } from '@/lib/api/contracts/media/tts-stream' - -const logger = createLogger('UseAudioStreaming') - -/** Prefer breaking on a boundary this far into the chunk before splitting mid-word. */ -const MIN_SPLIT_RATIO = 0.6 - -/** - * Splits text into pieces the TTS relay will accept. - * - * The caller sentence-splits on Western `.!?` only, so text that never matches - * — CJK punctuation, or a list with no terminal punctuation — reaches this hook - * as one accumulated block that can exceed the relay's per-request cap. Without - * splitting, the relay rejects it and the whole message plays no audio. - */ -export function splitForSynthesis(text: string, max: number = MAX_TTS_TEXT_LENGTH): string[] { - if (text.length <= max) return [text] - - const chunks: string[] = [] - let rest = text - - while (rest.length > max) { - const window = rest.slice(0, max) - const boundary = Math.max( - window.lastIndexOf(' '), - window.lastIndexOf('\n'), - window.lastIndexOf('。'), - window.lastIndexOf('，'), - window.lastIndexOf('、') - ) - const cut = boundary >= max * MIN_SPLIT_RATIO ? boundary + 1 : max - const piece = rest.slice(0, cut).trim() - if (piece) chunks.push(piece) - rest = rest.slice(cut) - } - - const tail = rest.trim() - if (tail) chunks.push(tail) - return chunks -} - -declare global { - interface Window { - webkitAudioContext?: typeof AudioContext - } -} - -interface AudioStreamingOptions { - voiceId: string - modelId?: string - chatId: string - onAudioStart?: () => void - onAudioEnd?: () => void - onError?: (error: Error) => void -} - -interface AudioQueueItem { - text: string - options: AudioStreamingOptions -} - -export function useAudioStreaming(sharedAudioContextRef?: RefObject) { - const [isPlayingAudio, setIsPlayingAudio] = useState(false) - const localAudioContextRef = useRef(null) - const audioContextRef = sharedAudioContextRef || localAudioContextRef - const currentSourceRef = useRef(null) - const abortControllerRef = useRef(null) - const audioQueueRef = useRef([]) - const isProcessingQueueRef = useRef(false) - - const getAudioContext = useCallback(() => { - if (!audioContextRef.current) { - const AudioContextConstructor = window.AudioContext || window.webkitAudioContext - if (!AudioContextConstructor) { - throw new Error('AudioContext is not supported in this browser') - } - audioContextRef.current = new AudioContextConstructor() - } - return audioContextRef.current - }, []) - - const stopAudio = useCallback(() => { - abortControllerRef.current?.abort() - - if (currentSourceRef.current) { - try { - currentSourceRef.current.stop() - } catch (e) { - // Already stopped - } - currentSourceRef.current = null - } - - audioQueueRef.current = [] - isProcessingQueueRef.current = false - - setIsPlayingAudio(false) - }, []) - - const processAudioQueue = useCallback(async () => { - if (isProcessingQueueRef.current || audioQueueRef.current.length === 0) { - return - } - - isProcessingQueueRef.current = true - const item = audioQueueRef.current.shift() - - if (!item) { - isProcessingQueueRef.current = false - return - } - - const { text, options } = item - const { - voiceId, - modelId = DEFAULT_TTS_MODEL_ID, - chatId, - onAudioStart, - onAudioEnd, - onError, - } = options - - try { - const audioContext = getAudioContext() - - if (audioContext.state === 'suspended') { - await audioContext.resume() - } - // boundary-raw-fetch: TTS proxy returns raw audio bytes consumed via response.arrayBuffer() and decoded by AudioContext.decodeAudioData - const response = await fetch('/api/proxy/tts/stream', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - text, - voiceId, - modelId, - chatId, - }), - signal: abortControllerRef.current?.signal, - }) - - if (!response.ok) { - const errorText = await response.text().catch(() => '') - throw new Error(errorText || `TTS request failed: ${response.statusText}`) - } - - const arrayBuffer = await response.arrayBuffer() - const audioBuffer = await audioContext.decodeAudioData(arrayBuffer) - - const source = audioContext.createBufferSource() - source.buffer = audioBuffer - source.connect(audioContext.destination) - source.onended = () => { - currentSourceRef.current = null - onAudioEnd?.() - - isProcessingQueueRef.current = false - - if (audioQueueRef.current.length === 0) { - setIsPlayingAudio(false) - } - - setTimeout(() => processAudioQueue(), 0) - } - - currentSourceRef.current = source - source.start(0) - setIsPlayingAudio(true) - onAudioStart?.() - } catch (error) { - if (error instanceof Error && error.name !== 'AbortError') { - logger.error('Audio streaming error:', error) - onError?.(error) - } - - isProcessingQueueRef.current = false - setTimeout(() => processAudioQueue(), 0) - } - }, [getAudioContext]) - - const streamTextToAudio = useCallback( - async (text: string, options: AudioStreamingOptions) => { - if (!text.trim()) { - return - } - - if (!abortControllerRef.current || abortControllerRef.current.signal.aborted) { - abortControllerRef.current = new AbortController() - } - - for (const piece of splitForSynthesis(text)) { - audioQueueRef.current.push({ text: piece, options }) - } - processAudioQueue() - }, - [processAudioQueue] - ) - - return { - isPlayingAudio, - streamTextToAudio, - stopAudio, - } -} diff --git a/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.test.tsx b/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.test.tsx index 34c01db49e5..4362096144c 100644 --- a/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.test.tsx +++ b/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.test.tsx @@ -344,47 +344,6 @@ describe('useChatStreaming thinking + abort', () => { expect(assistant?.thinking).toBe('real thought') }) - it('TTS audioStreamHandler receives answer text only', async () => { - const audioStreamHandler = vi.fn().mockResolvedValue(undefined) - - mockReadSSEEvents.mockImplementation(async (_source, options) => { - await options.onEvent({ - blockId: 'agent-1', - event: 'thinking', - data: 'secret internal monologue that must not be spoken.', - }) - await options.onEvent({ - blockId: 'agent-1', - chunk: 'Hello world.', - }) - await options.onEvent({ - event: 'final', - data: { success: true, output: {} }, - }) - }) - - await act(async () => { - await handle - .latest() - .handleStreamedResponse(makeSseResponse(), setMessages, vi.fn(), vi.fn(), { - voiceSettings: { - isVoiceEnabled: true, - voiceId: 'voice-1', - autoPlayResponses: true, - }, - audioStreamHandler, - }) - }) - await flushUiBatch() - - expect(audioStreamHandler).toHaveBeenCalled() - for (const call of audioStreamHandler.mock.calls) { - expect(String(call[0])).not.toContain('secret') - expect(String(call[0])).not.toContain('monologue') - } - expect(audioStreamHandler.mock.calls.some((c) => String(c[0]).includes('Hello'))).toBe(true) - }) - it('stopStreaming preserves thinking and aborts the shared controller', async () => { const abortController = new AbortController() let resolveStream!: () => void diff --git a/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.ts b/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.ts index b7fdb139d67..5a366b46a5f 100644 --- a/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.ts +++ b/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.ts @@ -69,20 +69,7 @@ function extractFilesFromData( return files } -interface VoiceSettings { - isVoiceEnabled: boolean - voiceId: string - autoPlayResponses: boolean - voiceFirstMode?: boolean - textStreamingInVoiceMode?: 'hidden' | 'synced' | 'normal' - conversationMode?: boolean -} - export interface StreamingOptions { - voiceSettings?: VoiceSettings - onAudioStart?: () => void - onAudioEnd?: () => void - audioStreamHandler?: (text: string) => Promise outputConfigs?: Array<{ blockId: string; path?: string }> /** * Shared AbortController for fetch + SSE body reads. When provided (preferred), @@ -104,9 +91,6 @@ export function useChatStreaming() { const accumulatedTextRef = useRef('') const accumulatedThinkingRef = useRef('') const accumulatedToolCallsRef = useRef([]) - const lastStreamedPositionRef = useRef(0) - const audioStreamingActiveRef = useRef(false) - const lastDisplayedPositionRef = useRef(0) const stopStreaming = (setMessages: React.Dispatch>) => { if (abortControllerRef.current) { @@ -149,9 +133,6 @@ export function useChatStreaming() { accumulatedTextRef.current = '' accumulatedThinkingRef.current = '' accumulatedToolCallsRef.current = [] - lastStreamedPositionRef.current = 0 - lastDisplayedPositionRef.current = 0 - audioStreamingActiveRef.current = false } } @@ -172,11 +153,6 @@ export function useChatStreaming() { abortControllerRef.current = new AbortController() } - const shouldPlayAudio = - streamingOptions?.voiceSettings?.isVoiceEnabled && - streamingOptions?.voiceSettings?.autoPlayResponses && - streamingOptions?.audioStreamHandler - if (!response.body) { setIsLoading(false) setIsStreamingResponse(false) @@ -198,7 +174,6 @@ export function useChatStreaming() { } let accumulatedThinking = '' let isThinkingStreaming = false - let lastAudioPosition = 0 const toolCallsMap = new Map() const toolCallOrder: string[] = [] @@ -543,9 +518,6 @@ export function useChatStreaming() { accumulatedTextRef.current = '' accumulatedThinkingRef.current = '' accumulatedToolCallsRef.current = [] - lastStreamedPositionRef.current = 0 - lastDisplayedPositionRef.current = 0 - audioStreamingActiveRef.current = false terminated = true return true @@ -565,8 +537,6 @@ export function useChatStreaming() { blockTextOrder.splice(orderIndex, 1) } recomputeAccumulatedText() - // Spoken audio cannot be unplayed; clamp so slicing stays valid. - lastAudioPosition = Math.min(lastAudioPosition, accumulatedText.length) uiDirty = true scheduleUIFlush() } @@ -600,32 +570,6 @@ export function useChatStreaming() { }) uiDirty = true scheduleUIFlush() - - if (shouldPlayAudio && streamingOptions?.audioStreamHandler) { - const newText = accumulatedText.substring(lastAudioPosition) - const sentenceEndings = ['. ', '! ', '? ', '.\n', '!\n', '?\n', '.', '!', '?'] - let sentenceEnd = -1 - - for (const ending of sentenceEndings) { - const index = newText.indexOf(ending) - if (index > 0) { - sentenceEnd = index + ending.length - break - } - } - - if (sentenceEnd > 0) { - const sentence = newText.substring(0, sentenceEnd).trim() - if (sentence && sentence.length >= 3) { - try { - await streamingOptions.audioStreamHandler(sentence) - lastAudioPosition += sentenceEnd - } catch (error) { - logger.error('TTS error:', error) - } - } - } - } } }, }) @@ -665,25 +609,10 @@ export function useChatStreaming() { } }) ) - if ( - !wasAborted && - shouldPlayAudio && - streamingOptions?.audioStreamHandler && - accumulatedText.length > lastAudioPosition - ) { - const remainingText = accumulatedText.substring(lastAudioPosition).trim() - if (remainingText) { - try { - await streamingOptions.audioStreamHandler(remainingText) - } catch (error) { - logger.error('TTS error for remaining text:', error) - } - } - } } } catch (error) { // Stop / timeout abort the shared fetch controller; body read then throws AbortError. - // Match chat.tsx + use-audio-streaming: expected cancel, not a hard failure. + // Expected cancel, not a hard failure. if (error instanceof Error && error.name === 'AbortError') { logger.info('Stream aborted by user or timeout') settleRunningToolCalls(toolCallsMap, 'cancelled') @@ -718,10 +647,6 @@ export function useChatStreaming() { setTimeout(() => { scrollToBottom() }, 300) - - if (shouldPlayAudio) { - streamingOptions?.onAudioEnd?.() - } } } diff --git a/apps/sim/app/api/proxy/tts/stream/route.test.ts b/apps/sim/app/api/proxy/tts/stream/route.test.ts deleted file mode 100644 index 0ce1496b49b..00000000000 --- a/apps/sim/app/api/proxy/tts/stream/route.test.ts +++ /dev/null @@ -1,296 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - queueTableRows, - resetDbChainMock, - resetEnvMock, - schemaMock, - setEnv, -} from '@sim/testing' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockRecordUsage, - mockCheckActorUsageLimits, - mockResolveSystemBillingAttribution, - mockCheckAttributedUsageLimits, - mockToBillingContext, - mockEnforceIpRateLimit, - mockEnforceChatRateLimit, -} = vi.hoisted(() => ({ - mockRecordUsage: vi.fn(), - mockCheckActorUsageLimits: vi.fn(), - mockResolveSystemBillingAttribution: vi.fn(), - mockCheckAttributedUsageLimits: vi.fn(), - mockToBillingContext: vi.fn(), - mockEnforceIpRateLimit: vi.fn(), - mockEnforceChatRateLimit: vi.fn(), -})) - -const SYSTEM_BILLING_ATTRIBUTION = { - actorUserId: 'payer-1', - workspaceId: 'ws-1', - organizationId: 'org-1', - billedAccountUserId: 'payer-1', - billingEntity: { type: 'organization' as const, id: 'org-1' }, - billingPeriod: { - start: '2026-07-01T00:00:00.000Z', - end: '2026-08-01T00:00:00.000Z', - }, - payerSubscription: null, -} - -vi.mock('@/lib/billing/core/usage-log', () => ({ recordUsage: mockRecordUsage })) - -vi.mock('@/lib/billing/core/billing-attribution', () => ({ - resolveSystemBillingAttribution: mockResolveSystemBillingAttribution, - checkAttributedUsageLimits: mockCheckAttributedUsageLimits, - toBillingContext: mockToBillingContext, -})) - -vi.mock('@/lib/billing/calculations/usage-monitor', () => ({ - checkActorUsageLimits: mockCheckActorUsageLimits, -})) - -vi.mock('@/lib/core/rate-limiter/route-helpers', () => ({ - enforceIpRateLimit: mockEnforceIpRateLimit, - enforceChatRateLimit: mockEnforceChatRateLimit, -})) - -vi.mock('@/lib/core/security/deployment', () => ({ validateAuthToken: vi.fn(() => false) })) - -import { DEFAULT_TTS_VOICE_ID, MAX_TTS_TEXT_LENGTH } from '@/lib/api/contracts/media/tts-stream' -import { POST } from '@/app/api/proxy/tts/stream/route' - -const publicChatRow = { - id: 'chat-1', - userId: 'owner-1', - isActive: true, - authType: 'public', - password: null, - workspaceId: 'ws-1', -} - -function validBody(overrides: Record = {}) { - return { - text: 'Hello from the deployed chat.', - voiceId: DEFAULT_TTS_VOICE_ID, - chatId: 'chat-1', - ...overrides, - } -} - -/** - * Minimal ElevenLabs stub returning a readable audio body. Returns the `cancel` - * spy so tests can assert the vendor stream is released when we reject. - */ -function mockElevenLabsAudio() { - const stream = new ReadableStream({ - start(controller) { - controller.enqueue(new Uint8Array([0x49, 0x44, 0x33])) - controller.close() - }, - }) - const cancel = vi.fn(() => stream.cancel()) - global.fetch = vi.fn().mockResolvedValue({ - ok: true, - status: 200, - statusText: 'OK', - body: { getReader: () => stream.getReader(), cancel }, - // double-cast-allowed: minimal fetch stub for the ElevenLabs stream call - }) as unknown as typeof fetch - return cancel -} - -beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - setEnv({ ELEVENLABS_API_KEY: 'test-key' }) - mockEnforceIpRateLimit.mockResolvedValue(null) - mockEnforceChatRateLimit.mockResolvedValue(null) - mockRecordUsage.mockResolvedValue(undefined) - mockCheckActorUsageLimits.mockResolvedValue({ isExceeded: false }) - mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: false }) - mockResolveSystemBillingAttribution.mockResolvedValue(SYSTEM_BILLING_ATTRIBUTION) - mockToBillingContext.mockImplementation( - (attribution: { billingEntity: { type: 'organization' | 'user'; id: string } }) => ({ - billingEntity: attribution.billingEntity, - billingPeriod: { - start: new Date('2026-07-01T00:00:00.000Z'), - end: new Date('2026-08-01T00:00:00.000Z'), - }, - }) - ) - mockElevenLabsAudio() -}) - -afterAll(() => { - resetDbChainMock() - resetEnvMock() -}) - -describe('POST /api/proxy/tts/stream — spend controls', () => { - it('caps text length so one request cannot bill unbounded characters', async () => { - const res = await POST( - createMockRequest('POST', validBody({ text: 'a'.repeat(MAX_TTS_TEXT_LENGTH + 1) })) - ) - - expect(res.status).toBe(400) - expect(global.fetch).not.toHaveBeenCalled() - expect(mockRecordUsage).not.toHaveBeenCalled() - }) - - it('rejects an oversized body before buffering it', async () => { - const res = await POST(createMockRequest('POST', validBody({ padding: 'x'.repeat(32 * 1024) }))) - - expect(res.status).toBe(413) - expect(global.fetch).not.toHaveBeenCalled() - expect(mockRecordUsage).not.toHaveBeenCalled() - }) - - it('rejects a voice outside the allowlist so the caller cannot pick a premium voice', async () => { - const res = await POST( - createMockRequest('POST', validBody({ voiceId: '21m00Tcm4TlvDq8ikWAM' })) - ) - - expect(res.status).toBe(400) - expect(global.fetch).not.toHaveBeenCalled() - }) - - it('rejects a model outside the allowlist so the caller cannot pick the billing model', async () => { - const res = await POST( - createMockRequest('POST', validBody({ modelId: 'eleven_multilingual_v2' })) - ) - - expect(res.status).toBe(400) - expect(global.fetch).not.toHaveBeenCalled() - }) - - it('throttles per IP before any chat lookup so a flood cannot be amplified into queries', async () => { - mockEnforceIpRateLimit.mockResolvedValue(new Response('Rate limit exceeded', { status: 429 })) - - const res = await POST(createMockRequest('POST', validBody())) - - expect(res.status).toBe(429) - expect(mockEnforceIpRateLimit).toHaveBeenCalledWith('tts-stream', expect.anything(), { - maxTokens: 60, - refillRate: 30, - refillIntervalMs: 60_000, - }) - expect(mockEnforceChatRateLimit).not.toHaveBeenCalled() - expect(global.fetch).not.toHaveBeenCalled() - }) - - it('throttles per chat so many callers cannot drain one public chat', async () => { - queueTableRows(schemaMock.chat, [publicChatRow]) - mockEnforceChatRateLimit.mockResolvedValue(new Response('Rate limit exceeded', { status: 429 })) - - const res = await POST(createMockRequest('POST', validBody())) - - expect(res.status).toBe(429) - expect(mockEnforceChatRateLimit).toHaveBeenCalledWith('tts-stream', 'chat-1', { - maxTokens: 120, - refillRate: 60, - refillIntervalMs: 60_000, - }) - expect(global.fetch).not.toHaveBeenCalled() - expect(mockRecordUsage).not.toHaveBeenCalled() - }) -}) - -describe('POST /api/proxy/tts/stream — attribution', () => { - it('meters synthesized characters against the chat workspace payer', async () => { - queueTableRows(schemaMock.chat, [publicChatRow]) - const text = 'a'.repeat(1000) - - const res = await POST(createMockRequest('POST', validBody({ text }))) - - expect(res.status).toBe(200) - expect(mockResolveSystemBillingAttribution).toHaveBeenCalledWith('ws-1') - expect(mockCheckAttributedUsageLimits).toHaveBeenCalledWith(SYSTEM_BILLING_ATTRIBUTION) - expect(mockRecordUsage).toHaveBeenCalledTimes(1) - expect(mockRecordUsage.mock.calls[0][0]).toMatchObject({ - userId: 'payer-1', - workspaceId: 'ws-1', - billingEntity: { type: 'organization', id: 'org-1' }, - }) - expect(mockRecordUsage.mock.calls[0][0].entries[0]).toMatchObject({ - category: 'fixed', - source: 'voice-output', - cost: 0.05, - }) - }) - - it('gives each call a unique source reference so equal-length calls are not deduplicated', async () => { - const text = 'Same length text.' - - queueTableRows(schemaMock.chat, [publicChatRow]) - await POST(createMockRequest('POST', validBody({ text }))) - queueTableRows(schemaMock.chat, [publicChatRow]) - await POST(createMockRequest('POST', validBody({ text }))) - - expect(mockRecordUsage).toHaveBeenCalledTimes(2) - const first = mockRecordUsage.mock.calls[0][0].entries[0].sourceReference - const second = mockRecordUsage.mock.calls[1][0].entries[0].sourceReference - expect(first).toBeDefined() - expect(first).not.toBe(second) - }) - - it('falls back to the chat owner when the workflow has no workspace', async () => { - queueTableRows(schemaMock.chat, [{ ...publicChatRow, workspaceId: null }]) - - const res = await POST(createMockRequest('POST', validBody())) - - expect(res.status).toBe(200) - expect(mockResolveSystemBillingAttribution).not.toHaveBeenCalled() - expect(mockCheckActorUsageLimits).toHaveBeenCalledWith('owner-1') - expect(mockRecordUsage.mock.calls[0][0]).toMatchObject({ userId: 'owner-1' }) - }) - - it('refuses to spend when the payer is over its usage limit', async () => { - queueTableRows(schemaMock.chat, [publicChatRow]) - mockCheckAttributedUsageLimits.mockResolvedValue({ - isExceeded: true, - message: 'Usage limit exceeded.', - }) - - const res = await POST(createMockRequest('POST', validBody())) - - expect(res.status).toBe(402) - expect(global.fetch).not.toHaveBeenCalled() - expect(mockRecordUsage).not.toHaveBeenCalled() - }) - - it('refuses to stream audio it could not record a charge for, releasing the vendor stream', async () => { - const cancel = mockElevenLabsAudio() - queueTableRows(schemaMock.chat, [publicChatRow]) - mockRecordUsage.mockRejectedValue(new Error('ledger unavailable')) - - const res = await POST(createMockRequest('POST', validBody())) - - expect(res.status).toBe(500) - expect(res.headers.get('Content-Type')).not.toBe('audio/mpeg') - expect(cancel).toHaveBeenCalledTimes(1) - }) - - it('rejects an unknown chat without touching the platform key', async () => { - queueTableRows(schemaMock.chat, []) - - const res = await POST(createMockRequest('POST', validBody())) - - expect(res.status).toBe(401) - expect(global.fetch).not.toHaveBeenCalled() - expect(mockRecordUsage).not.toHaveBeenCalled() - }) - - it('does not expose the audio stream to arbitrary origins', async () => { - queueTableRows(schemaMock.chat, [publicChatRow]) - - const res = await POST(createMockRequest('POST', validBody())) - - expect(res.status).toBe(200) - expect(res.headers.get('Access-Control-Allow-Origin')).toBeNull() - }) -}) diff --git a/apps/sim/app/api/proxy/tts/stream/route.ts b/apps/sim/app/api/proxy/tts/stream/route.ts deleted file mode 100644 index c3cb988a59c..00000000000 --- a/apps/sim/app/api/proxy/tts/stream/route.ts +++ /dev/null @@ -1,243 +0,0 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { ttsStreamContract } from '@/lib/api/contracts/media/tts-stream' -import { parseRequest } from '@/lib/api/server' -import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor' -import { - checkAttributedUsageLimits, - resolveSystemBillingAttribution, - toBillingContext, -} from '@/lib/billing/core/billing-attribution' -import { recordUsage } from '@/lib/billing/core/usage-log' -import { resolveDeployedChatCaller } from '@/lib/chat/deployed-chat-caller' -import { env } from '@/lib/core/config/env' -import { getCostMultiplier } from '@/lib/core/config/env-flags' -import { enforceChatRateLimit, enforceIpRateLimit } from '@/lib/core/rate-limiter/route-helpers' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('ProxyTTSStreamAPI') - -/** - * Filters naive floods only: `getClientIp` trusts the leftmost - * `X-Forwarded-For` value, which the caller controls, so a deliberate attacker - * rotates past this bucket. See {@link TTS_CHAT_RATE_LIMIT}. - * - * Deployed chat synthesizes sentence by sentence, so a real conversation issues - * several requests per answer — hence the generous burst. - */ -const TTS_IP_RATE_LIMIT = { - maxTokens: 60, - refillRate: 30, - refillIntervalMs: 60 * 1000, -} as const - -/** - * The load-bearing spend control. Public chats hand their id to every visitor, - * so the id alone cannot gate use of the platform ElevenLabs key. This bucket - * is keyed on server-held state, bounding total spend per chat regardless of - * how many source addresses the traffic claims to come from. - */ -const TTS_CHAT_RATE_LIMIT = { - maxTokens: 120, - refillRate: 60, - refillIntervalMs: 60 * 1000, -} as const - -/** - * Published ElevenLabs API rate for Flash/Turbo text-to-speech, in USD per - * 1,000 characters. This is the vendor cost; `getCostMultiplier()` applies the - * platform markup, matching how other metered sources are priced. - */ -const TTS_COST_PER_1K_CHARS = 0.05 - -/** - * The body carries at most `MAX_TTS_TEXT_LENGTH` characters of text plus two - * short ids, so a tight cap keeps an anonymous caller from making the route - * buffer a large payload before validation runs. Without it the shared default - * is 50 MB. - */ -const MAX_TTS_BODY_BYTES = 16 * 1024 - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - // Throttle per IP before any database work so a flood cannot be amplified into chat lookups. - const ipLimited = await enforceIpRateLimit('tts-stream', request, TTS_IP_RATE_LIMIT) - if (ipLimited) return ipLimited - - const parsed = await parseRequest( - ttsStreamContract, - request, - {}, - { - maxBodyBytes: MAX_TTS_BODY_BYTES, - invalidJsonResponse: () => new NextResponse('Invalid request body', { status: 400 }), - validationErrorResponse: (error) => { - if (error.issues.some((issue) => issue.path[0] === 'chatId')) { - return new NextResponse('chatId is required', { status: 400 }) - } - return new NextResponse('Missing required parameters', { status: 400 }) - }, - } - ) - if (!parsed.success) return parsed.response - - const { text, voiceId, modelId, chatId } = parsed.data.body - - const caller = await resolveDeployedChatCaller(request, chatId) - if (!caller.authorized) { - logger.warn('Chat authentication failed for TTS, chatId:', chatId) - return new Response('Unauthorized', { status: 401 }) - } - - const chatLimited = await enforceChatRateLimit('tts-stream', chatId, TTS_CHAT_RATE_LIMIT) - if (chatLimited) return chatLimited - - // Anonymous deployed chats have no human request actor, so the workspace payer is charged. - const workspaceId = caller.workspaceId ?? undefined - const billingAttribution = workspaceId - ? await resolveSystemBillingAttribution(workspaceId) - : undefined - const actorUserId = billingAttribution?.actorUserId ?? caller.ownerId - - const usageCheck = billingAttribution - ? await checkAttributedUsageLimits(billingAttribution) - : await checkActorUsageLimits(actorUserId) - if (usageCheck.isExceeded) { - return new Response(usageCheck.message || 'Usage limit exceeded.', { status: 402 }) - } - - const apiKey = env.ELEVENLABS_API_KEY - if (!apiKey) { - logger.error('ELEVENLABS_API_KEY not configured on server') - return new Response('ElevenLabs service not configured', { status: 503 }) - } - - const query = new URLSearchParams({ output_format: 'mp3_44100_128' }) - const endpoint = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream?${query.toString()}` - - const response = await fetch(endpoint, { - method: 'POST', - headers: { - Accept: 'audio/mpeg', - 'Content-Type': 'application/json', - 'xi-api-key': apiKey, - }, - body: JSON.stringify({ - text, - model_id: modelId, - voice_settings: { - stability: 0.5, - similarity_boost: 0.8, - style: 0.0, - use_speaker_boost: false, - }, - apply_text_normalization: 'auto', - }), - }) - - if (!response.ok) { - logger.error(`Failed to generate Stream TTS: ${response.status} ${response.statusText}`) - return new Response(`Failed to generate TTS: ${response.status} ${response.statusText}`, { - status: response.status, - }) - } - - if (!response.body) { - logger.error('No response body received from ElevenLabs') - return new Response('No audio stream received', { status: 422 }) - } - - /** - * Meter once ElevenLabs has accepted the request — the characters are billed - * to us at that point regardless of whether the client drains the stream. - * - * `sourceReference` must be unique per call. `usage_log.event_key` is - * unique and inserts conflict-do-nothing, and the key is derived from the - * entry's stable fields — without this, two synthesis calls of equal length - * in the same workspace would collide and the second would silently go - * unbilled. Each call is a separate charge from ElevenLabs, so each needs - * its own row rather than being deduplicated. `generateId` rather than - * `generateRequestId`, whose fallback truncates to 8 characters. - * - * A ledger failure fails the request rather than streaming anyway: the - * caller is anonymous, so serving audio we could not charge for is exactly - * the unmetered spend this route exists to prevent. The vendor call is - * already paid for at this point, but the caller gains nothing from it, so - * there is no incentive to farm ledger outages. - * - * No threshold settlement here: it runs per metered event elsewhere and is - * far too heavy for a per-sentence realtime path. The workflow execution - * that produced this text already settles the payer. - */ - try { - await recordUsage({ - userId: actorUserId, - workspaceId, - ...(billingAttribution ? toBillingContext(billingAttribution) : {}), - entries: [ - { - category: 'fixed', - source: 'voice-output', - description: `Voice output (${text.length} characters)`, - cost: (text.length / 1000) * TTS_COST_PER_1K_CHARS * getCostMultiplier(), - sourceReference: `voice-output:${chatId}:${generateId()}`, - }, - ], - }) - } catch (err) { - logger.error('Failed to record voice output usage, refusing to stream:', err) - // Release the open vendor stream; nothing will drain it once we reject. - await response.body?.cancel().catch(() => {}) - return new Response('Unable to record usage for this request', { status: 500 }) - } - - const { readable, writable } = new TransformStream({ - transform(chunk, controller) { - controller.enqueue(chunk) - }, - flush(controller) { - controller.terminate() - }, - }) - - const writer = writable.getWriter() - const reader = response.body.getReader() - - ;(async () => { - try { - while (true) { - const { done, value } = await reader.read() - if (done) { - await writer.close() - break - } - writer.write(value).catch(logger.error) - } - } catch (error) { - logger.error('Error during Stream streaming:', error) - await writer.abort(error) - } - })() - - return new Response(readable, { - headers: { - 'Content-Type': 'audio/mpeg', - 'Transfer-Encoding': 'chunked', - 'Cache-Control': 'no-cache, no-store, must-revalidate', - Pragma: 'no-cache', - Expires: '0', - 'X-Content-Type-Options': 'nosniff', - Connection: 'keep-alive', - 'X-Accel-Buffering': 'no', - 'X-Stream-Type': 'real-time', - }, - }) - } catch (error) { - logger.error('Error in Stream TTS:', error) - - return new Response('Internal Server Error', { - status: 500, - }) - } -}) diff --git a/apps/sim/app/api/speech/token/route.test.ts b/apps/sim/app/api/speech/token/route.test.ts index cf599da66c0..218af7fb1e4 100644 --- a/apps/sim/app/api/speech/token/route.test.ts +++ b/apps/sim/app/api/speech/token/route.test.ts @@ -4,10 +4,8 @@ import { authMockFns, createMockRequest, - queueTableRows, resetDbChainMock, resetEnvMock, - schemaMock, setEnv, } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' @@ -17,7 +15,6 @@ const { mockCheckActorUsageLimits, mockVerifyWorkspaceMembership, mockResolveBillingAttribution, - mockResolveSystemBillingAttribution, mockCheckAttributedUsageLimits, mockToBillingContext, mockCheckAndBillPayerOverageThreshold, @@ -26,30 +23,15 @@ const { mockCheckActorUsageLimits: vi.fn(), mockVerifyWorkspaceMembership: vi.fn(), mockResolveBillingAttribution: vi.fn(), - mockResolveSystemBillingAttribution: vi.fn(), mockCheckAttributedUsageLimits: vi.fn(), mockToBillingContext: vi.fn(), mockCheckAndBillPayerOverageThreshold: vi.fn(), })) -const SYSTEM_BILLING_ATTRIBUTION = { - actorUserId: 'owner-after-transfer', - workspaceId: 'ws-1', - organizationId: 'org-after-transfer', - billedAccountUserId: 'owner-after-transfer', - billingEntity: { type: 'organization' as const, id: 'org-after-transfer' }, - billingPeriod: { - start: '2026-07-01T00:00:00.000Z', - end: '2026-08-01T00:00:00.000Z', - }, - payerSubscription: null, -} - vi.mock('@/lib/billing/core/usage-log', () => ({ recordUsage: mockRecordUsage })) vi.mock('@/lib/billing/core/billing-attribution', () => ({ resolveBillingAttribution: mockResolveBillingAttribution, - resolveSystemBillingAttribution: mockResolveSystemBillingAttribution, checkAttributedUsageLimits: mockCheckAttributedUsageLimits, toBillingContext: mockToBillingContext, })) @@ -72,21 +54,10 @@ vi.mock('@/lib/core/rate-limiter', () => ({ }, })) -vi.mock('@/lib/core/security/deployment', () => ({ validateAuthToken: vi.fn(() => false) })) - import { POST } from '@/app/api/speech/token/route' const mockGetSession = authMockFns.mockGetSession -const publicChatRow = { - id: 'chat-1', - userId: 'owner-1', - isActive: true, - authType: 'public', - password: null, - workspaceId: 'ws-1', -} - beforeEach(() => { vi.clearAllMocks() resetDbChainMock() @@ -102,7 +73,6 @@ beforeEach(() => { billingEntity: { type: 'organization', id: 'org-1' }, }) ) - mockResolveSystemBillingAttribution.mockResolvedValue(SYSTEM_BILLING_ATTRIBUTION) mockToBillingContext.mockImplementation( (attribution: { billingEntity: { type: 'organization' | 'user'; id: string } }) => ({ billingEntity: attribution.billingEntity, @@ -140,7 +110,6 @@ describe('POST /api/speech/token — usage attribution', () => { actorUserId: 'member-1', workspaceId: 'ws-1', }) - expect(mockResolveSystemBillingAttribution).not.toHaveBeenCalled() expect(mockCheckAndBillPayerOverageThreshold).toHaveBeenCalledWith({ type: 'organization', id: 'org-1', @@ -156,44 +125,8 @@ describe('POST /api/speech/token — usage attribution', () => { expect(mockRecordUsage).not.toHaveBeenCalled() }) - it('deployed chat: uses one atomic system actor and payer snapshot', async () => { - queueTableRows(schemaMock.chat, [publicChatRow]) - - const res = await POST(createMockRequest('POST', { chatId: 'chat-1' })) - - expect(res.status).toBe(200) - expect(mockGetSession).not.toHaveBeenCalled() - expect(mockResolveSystemBillingAttribution).toHaveBeenCalledWith('ws-1') - expect(mockResolveBillingAttribution).not.toHaveBeenCalled() - expect(mockCheckAttributedUsageLimits).toHaveBeenCalledWith(SYSTEM_BILLING_ATTRIBUTION) - expect(mockToBillingContext).toHaveBeenCalledWith(SYSTEM_BILLING_ATTRIBUTION) - expect(mockRecordUsage.mock.calls[0][0]).toMatchObject({ - userId: 'owner-after-transfer', - workspaceId: 'ws-1', - billingEntity: { type: 'organization', id: 'org-after-transfer' }, - }) - expect(mockCheckAndBillPayerOverageThreshold).toHaveBeenCalledWith({ - type: 'organization', - id: 'org-after-transfer', - }) - }) - - it('deployed chat: uses the chat owner only when no workspace exists', async () => { - queueTableRows(schemaMock.chat, [{ ...publicChatRow, workspaceId: null }]) - - const res = await POST(createMockRequest('POST', { chatId: 'chat-1' })) - - expect(res.status).toBe(200) - expect(mockResolveSystemBillingAttribution).not.toHaveBeenCalled() - expect(mockResolveBillingAttribution).not.toHaveBeenCalled() - expect(mockRecordUsage.mock.calls[0][0]).toMatchObject({ - userId: 'owner-1', - }) - expect(mockRecordUsage.mock.calls[0][0].workspaceId).toBeUndefined() - }) - it('rejects an oversized body before any auth/billing work runs', async () => { - const oversizedBody = { chatId: 'x'.repeat(64 * 1024) } + const oversizedBody = { workspaceId: 'x'.repeat(64 * 1024) } const res = await POST(createMockRequest('POST', oversizedBody)) expect(res.status).toBe(413) diff --git a/apps/sim/app/api/speech/token/route.ts b/apps/sim/app/api/speech/token/route.ts index c5465f75318..f659bf8fc49 100644 --- a/apps/sim/app/api/speech/token/route.ts +++ b/apps/sim/app/api/speech/token/route.ts @@ -10,16 +10,13 @@ import { type BillingAttributionSnapshot, checkAttributedUsageLimits, resolveBillingAttribution, - resolveSystemBillingAttribution, toBillingContext, } from '@/lib/billing/core/billing-attribution' import { recordUsage } from '@/lib/billing/core/usage-log' import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' -import { resolveDeployedChatCaller } from '@/lib/chat/deployed-chat-caller' import { env } from '@/lib/core/config/env' import { getCostMultiplier, isBillingEnabled } from '@/lib/core/config/env-flags' import { RateLimiter } from '@/lib/core/rate-limiter' -import { getClientIp } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' @@ -31,7 +28,6 @@ const ELEVENLABS_TOKEN_URL = 'https://api.elevenlabs.io/v1/single-use-token/real const VOICE_SESSION_COST_PER_MIN = 0.008 const WORKSPACE_SESSION_MAX_MINUTES = 3 -const CHAT_SESSION_MAX_MINUTES = 1 const STT_TOKEN_RATE_LIMIT = { maxTokens: 30, @@ -40,7 +36,7 @@ const STT_TOKEN_RATE_LIMIT = { } as const /** - * This body only ever carries an optional chatId/workspaceId string, so a + * This body only ever carries an optional workspaceId string, so a * tight cap keeps an unauthenticated caller from forcing a large in-memory * allocation before the auth checks below run. */ @@ -57,57 +53,34 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const parsedBody = await parseOptionalJsonBody(request, MAX_SPEECH_TOKEN_BODY_BYTES) if (!parsedBody.success) return parsedBody.response const body = speechTokenBodySchema.safeParse(parsedBody.data ?? {}) - const chatId = - body.success && typeof body.data.chatId === 'string' ? body.data.chatId : undefined - let actorUserId: string | undefined let workspaceId: string | undefined let billingAttribution: BillingAttributionSnapshot | undefined - if (chatId) { - const chatAuth = await resolveDeployedChatCaller(request, chatId) - if (!chatAuth.authorized) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - /** - * Anonymous deployed chats have no human request actor, so resolve the - * system actor and immutable workspace payer together. - */ - workspaceId = chatAuth.workspaceId ?? undefined - if (workspaceId) { - billingAttribution = await resolveSystemBillingAttribution(workspaceId) - actorUserId = billingAttribution.actorUserId - } else { - actorUserId = chatAuth.ownerId - } - } else { - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - actorUserId = session.user.id - /** - * Editor voice accepts only a workspace the caller belongs to, preventing - * client-supplied IDs from misattributing or bypassing member usage. - */ - const requestedWorkspaceId = - body.success && typeof body.data.workspaceId === 'string' - ? body.data.workspaceId - : undefined - if (requestedWorkspaceId) { - const permission = await verifyWorkspaceMembership(session.user.id, requestedWorkspaceId) - if (permission) workspaceId = requestedWorkspaceId - } - /** - * Editor voice is workspace-scoped so every charge has a payer and member - * cap attribution. - */ - if (!workspaceId) { - return NextResponse.json({ error: 'Workspace context is required.' }, { status: 400 }) - } + const session = await getSession() + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + const actorUserId = session.user.id + /** + * Editor voice accepts only a workspace the caller belongs to, preventing + * client-supplied IDs from misattributing or bypassing member usage. + */ + const requestedWorkspaceId = + body.success && typeof body.data.workspaceId === 'string' ? body.data.workspaceId : undefined + if (requestedWorkspaceId) { + const permission = await verifyWorkspaceMembership(session.user.id, requestedWorkspaceId) + if (permission) workspaceId = requestedWorkspaceId + } + /** + * Editor voice is workspace-scoped so every charge has a payer and member + * cap attribution. + */ + if (!workspaceId) { + return NextResponse.json({ error: 'Workspace context is required.' }, { status: 400 }) } - if (!billingAttribution && actorUserId && workspaceId) { + if (!billingAttribution) { billingAttribution = await resolveBillingAttribution({ actorUserId, workspaceId, @@ -115,11 +88,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } if (isBillingEnabled) { - const rateLimitKey = chatId - ? `stt-token:chat:${chatId}:${getClientIp(request)}` - : `stt-token:user:${actorUserId}` - - const rateCheck = await rateLimiter.checkRateLimitDirect(rateLimitKey, STT_TOKEN_RATE_LIMIT) + const rateCheck = await rateLimiter.checkRateLimitDirect( + `stt-token:user:${actorUserId}`, + STT_TOKEN_RATE_LIMIT + ) if (!rateCheck.allowed) { return NextResponse.json( { error: 'Voice input rate limit exceeded. Please try again later.' }, @@ -133,20 +105,28 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } } - if (actorUserId) { - const usageCheck = billingAttribution - ? await checkAttributedUsageLimits(billingAttribution) - : await checkActorUsageLimits(actorUserId) - if (usageCheck.isExceeded) { - return NextResponse.json( - { - error: - usageCheck.message || 'Usage limit exceeded. Please upgrade your plan to continue.', - scope: usageCheck.scope, - }, - { status: 402 } - ) - } + /** + * Read-then-act, as every metered route in the repo does: concurrent calls + * can pass against the same balance and overshoot the payer's limit. Making + * this atomic needs a reservation primitive in `lib/billing` that fixes the + * class everywhere, not a one-off here. + * + * Accepted as bounded rather than eliminated: this route is session-gated + * and rate limited per user, so the overshoot is a knowable ceiling against + * an identified payer. + */ + const usageCheck = billingAttribution + ? await checkAttributedUsageLimits(billingAttribution) + : await checkActorUsageLimits(actorUserId) + if (usageCheck.isExceeded) { + return NextResponse.json( + { + error: + usageCheck.message || 'Usage limit exceeded. Please upgrade your plan to continue.', + scope: usageCheck.scope, + }, + { status: 402 } + ) } const apiKey = env.ELEVENLABS_API_KEY @@ -172,31 +152,28 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const data = await response.json() - if (actorUserId) { - const maxMinutes = chatId ? CHAT_SESSION_MAX_MINUTES : WORKSPACE_SESSION_MAX_MINUTES - const sessionCost = VOICE_SESSION_COST_PER_MIN * maxMinutes - - try { - await recordUsage({ - userId: actorUserId, - workspaceId, - ...(billingAttribution ? toBillingContext(billingAttribution) : {}), - entries: [ - { - category: 'fixed', - source: 'voice-input', - description: `Voice input session (${maxMinutes} min)`, - cost: sessionCost * getCostMultiplier(), - sourceReference: `voice-input:${hashVoiceToken(data.token)}`, - }, - ], - }) - if (billingAttribution) { - await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) - } - } catch (err) { - logger.warn('Failed to record voice input usage, continuing:', err) + const sessionCost = VOICE_SESSION_COST_PER_MIN * WORKSPACE_SESSION_MAX_MINUTES + + try { + await recordUsage({ + userId: actorUserId, + workspaceId, + ...(billingAttribution ? toBillingContext(billingAttribution) : {}), + entries: [ + { + category: 'fixed', + source: 'voice-input', + description: `Voice input session (${WORKSPACE_SESSION_MAX_MINUTES} min)`, + cost: sessionCost * getCostMultiplier(), + sourceReference: `voice-input:${hashVoiceToken(data.token)}`, + }, + ], + }) + if (billingAttribution) { + await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) } + } catch (err) { + logger.warn('Failed to record voice input usage, continuing:', err) } return NextResponse.json({ token: data.token }) diff --git a/apps/sim/hooks/queries/voice-settings.ts b/apps/sim/hooks/queries/voice-settings.ts deleted file mode 100644 index 31bd9f874e0..00000000000 --- a/apps/sim/hooks/queries/voice-settings.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { useQuery } from '@tanstack/react-query' -import { requestJson } from '@/lib/api/client/request' -import type { ContractJsonResponse } from '@/lib/api/contracts' -import { getVoiceSettingsContract } from '@/lib/api/contracts' - -/** - * Query key factory for voice settings queries - */ -export const VOICE_SETTINGS_STALE_TIME = 5 * 60 * 1000 - -export const voiceSettingsKeys = { - all: ['voiceSettings'] as const, - availability: () => [...voiceSettingsKeys.all, 'availability'] as const, -} - -type VoiceSettingsResponse = ContractJsonResponse - -async function fetchVoiceSettings(signal?: AbortSignal): Promise { - try { - return await requestJson(getVoiceSettingsContract, { signal }) - } catch { - return { sttAvailable: false } - } -} - -/** - * Loads the server-side voice configuration so clients can conditionally - * enable voice input. Returns `{ sttAvailable: false }` on failure rather - * than throwing, since STT is an optional capability. - */ -export function useVoiceSettings() { - return useQuery({ - queryKey: voiceSettingsKeys.availability(), - queryFn: ({ signal }) => fetchVoiceSettings(signal), - staleTime: VOICE_SETTINGS_STALE_TIME, - }) -} diff --git a/apps/sim/lib/api/contracts/media/index.ts b/apps/sim/lib/api/contracts/media/index.ts index 8e8688d0ea2..abbdf2e2725 100644 --- a/apps/sim/lib/api/contracts/media/index.ts +++ b/apps/sim/lib/api/contracts/media/index.ts @@ -1,2 +1 @@ export * from '@/lib/api/contracts/media/speech' -export * from '@/lib/api/contracts/media/tts-stream' diff --git a/apps/sim/lib/api/contracts/media/speech.ts b/apps/sim/lib/api/contracts/media/speech.ts index f01e47b213a..40461bc5470 100644 --- a/apps/sim/lib/api/contracts/media/speech.ts +++ b/apps/sim/lib/api/contracts/media/speech.ts @@ -3,7 +3,6 @@ import { defineRouteContract } from '@/lib/api/contracts/types' export const speechTokenBodySchema = z .object({ - chatId: z.string().optional(), /** Editor/workspace voice: the workspace the session user is recording in. */ workspaceId: z.string().optional(), }) diff --git a/apps/sim/lib/api/contracts/media/tts-stream.ts b/apps/sim/lib/api/contracts/media/tts-stream.ts deleted file mode 100644 index 49a0515e80d..00000000000 --- a/apps/sim/lib/api/contracts/media/tts-stream.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { z } from 'zod' -import { defineRouteContract } from '@/lib/api/contracts/types' - -/** Default ElevenLabs voice (Jessica) — Flash v2.5-optimized. */ -export const DEFAULT_TTS_VOICE_ID = 'cgSgspJ2msm6clMCkdW9' - -export const DEFAULT_TTS_MODEL_ID = 'eleven_flash_v2_5' - -/** - * Voices and models the deployed-chat relay may spend the platform ElevenLabs - * key on. Anonymous visitors of a public chat can reach the relay, so these are - * allowlists rather than free-form strings — otherwise the caller chooses the - * (possibly premium or cloned) voice and the billing model. - */ -const ALLOWED_TTS_VOICE_IDS = [DEFAULT_TTS_VOICE_ID] as const -const ALLOWED_TTS_MODEL_IDS = [DEFAULT_TTS_MODEL_ID] as const - -/** - * ElevenLabs bills per character, so an uncapped `text` is an uncapped charge. - * Chat TTS synthesizes the streamed answer sentence by sentence, keeping real - * requests far below this ceiling. - */ -export const MAX_TTS_TEXT_LENGTH = 2000 - -export const ttsStreamBodySchema = z.object({ - text: z.string().min(1).max(MAX_TTS_TEXT_LENGTH), - voiceId: z.enum(ALLOWED_TTS_VOICE_IDS), - modelId: z.enum(ALLOWED_TTS_MODEL_IDS).optional().default(DEFAULT_TTS_MODEL_ID), - chatId: z.string().min(1), -}) - -export const ttsStreamContract = defineRouteContract({ - method: 'POST', - path: '/api/proxy/tts/stream', - body: ttsStreamBodySchema, - response: { mode: 'stream' }, -}) diff --git a/apps/sim/lib/chat/deployed-chat-caller.ts b/apps/sim/lib/chat/deployed-chat-caller.ts deleted file mode 100644 index 3a019adc840..00000000000 --- a/apps/sim/lib/chat/deployed-chat-caller.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { db } from '@sim/db' -import { chat, workflow } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { and, eq, isNull } from 'drizzle-orm' -import type { NextRequest } from 'next/server' -import { validateAuthToken } from '@/lib/core/security/deployment' - -const logger = createLogger('DeployedChatCaller') - -export type DeployedChatCaller = - | { authorized: false } - | { authorized: true; ownerId: string; workspaceId: string | null } - -/** - * Resolves whether a caller may act on a deployed chat, and who pays for it. - * - * Anonymous visitors of a public chat are authorized, so callers that spend a - * platform credential must meter against the returned payer rather than treat - * authorization as permission to spend freely. - * - * Shared by every deployed-chat surface that bills a vendor call, so the gate - * and the payer are resolved together and cannot drift apart per route. - */ -export async function resolveDeployedChatCaller( - request: NextRequest, - chatId: string -): Promise { - try { - const rows = await db - .select({ - userId: chat.userId, - isActive: chat.isActive, - authType: chat.authType, - password: chat.password, - workspaceId: workflow.workspaceId, - }) - .from(chat) - .leftJoin(workflow, eq(workflow.id, chat.workflowId)) - .where(and(eq(chat.id, chatId), isNull(chat.archivedAt))) - .limit(1) - - if (rows.length === 0 || !rows[0].isActive) { - logger.warn('Chat not found, archived or inactive', { chatId }) - return { authorized: false } - } - - const chatData = rows[0] - const authorized = { - authorized: true, - ownerId: chatData.userId, - workspaceId: chatData.workspaceId, - } as const - - if (chatData.authType === 'public') { - return authorized - } - - const authCookie = request.cookies.get(`chat_auth_${chatId}`) - if ( - authCookie && - validateAuthToken(authCookie.value, chatId, chatData.authType, chatData.password) - ) { - return authorized - } - - return { authorized: false } - } catch (error) { - logger.error('Error resolving deployed chat caller', { chatId, error }) - return { authorized: false } - } -} diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 939942670e4..eb79f5fc0d4 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -183,7 +183,7 @@ export const env = createEnv({ COHERE_API_KEY_1: z.string().min(1).optional(), // Primary Cohere API key for rotation COHERE_API_KEY_2: z.string().min(1).optional(), // Additional Cohere API key for load balancing COHERE_API_KEY_3: z.string().min(1).optional(), // Additional Cohere API key for load balancing - ELEVENLABS_API_KEY: z.string().min(1).optional(), // ElevenLabs API key for text-to-speech in deployed chat + ELEVENLABS_API_KEY: z.string().min(1).optional(), // ElevenLabs API key for workspace speech-to-text SERPER_API_KEY: z.string().min(1).optional(), // Serper API key for online search EXA_API_KEY: z.string().min(1).optional(), // Exa AI API key for enhanced online search BLACKLISTED_PROVIDERS: z.string().optional(), // Comma-separated provider IDs to hide (e.g., "openai,anthropic") diff --git a/apps/sim/lib/core/utils/request.ts b/apps/sim/lib/core/utils/request.ts index 3634c2f38c9..84150f4a38c 100644 --- a/apps/sim/lib/core/utils/request.ts +++ b/apps/sim/lib/core/utils/request.ts @@ -20,8 +20,3 @@ export function getClientIp(request: { headers: { get(name: string): string | nu 'unknown' ) } - -/** - * No-operation function for use as default callback - */ -export const noop = () => {} diff --git a/apps/sim/lib/speech/config.ts b/apps/sim/lib/speech/config.ts index ec4d520a7f7..4e7a831a63b 100644 --- a/apps/sim/lib/speech/config.ts +++ b/apps/sim/lib/speech/config.ts @@ -4,7 +4,6 @@ export const ELEVENLABS_WS_URL = 'wss://api.elevenlabs.io/v1/speech-to-text/real export const SAMPLE_RATE = 16000 export const CHUNK_SEND_INTERVAL_MS = 250 export const MAX_SESSION_MS = 3 * 60 * 1000 -export const MAX_CHAT_SESSION_MS = 1 * 60 * 1000 /** * Whether a speech-to-text provider is configured.