AEC in Unity Audio

README.md

@@ -311,6 +311,22 @@ IEnumerator PublishLocalMicrophoneUnity(Room room)
 }
 ```
 
+#### Unity Audio Processing
+
+Unity's `Microphone` path does not go through a platform audio device module, so on its own it has no echo cancellation. Pass `AudioProcessingOptions` to run libwebrtc's audio processing (AEC3 echo cancellation, noise suppression, gain control, high-pass filter) over the captured audio before it reaches the track:
+
+```cs
+var processing = new AudioProcessingOptions
+{
+    EchoCancellation = true,
+    NoiseSuppression = true,
+    AutoGainControl = true
+};
+var rtcSource = new MicrophoneSource(Microphone.devices[0], microphoneObject, processing);
+```
+
+Echo cancellation takes its reference from the final mix Unity plays, so it covers every remote `AudioStream` as well as the game's own audio. The SDK attaches a `PlayoutReference` component to the active `AudioListener` for that; adding it to the listener yourself does the same. `rtcSource.AudioProcessingStats` reports whether the reference is attached and how many chunks flowed. Unity's output sample rate must be a multiple of 100 Hz (48000, 44100 and 24000 all are); otherwise processing is bypassed with a warning and the raw microphone is published.
+
 #### Unity Audio Output
 
 ```cs

Runtime/Plugins/iOS/LiveKitAudioSession.mm

@@ -67,4 +67,18 @@ void LiveKit_RestoreDefaultAudioSession() {
     }
 }
 
+/// AVAudioSession latency terms, in seconds. Used to seed the echo canceller's stream delay in
+/// Unity-audio mode (see AudioProcessingDelayHint.cs). All report 0 until the session is active.
+double LiveKit_AudioSessionOutputLatency() {
+    return [[AVAudioSession sharedInstance] outputLatency];
+}
+
+double LiveKit_AudioSessionInputLatency() {
+    return [[AVAudioSession sharedInstance] inputLatency];
+}
+
+double LiveKit_AudioSessionIOBufferDuration() {
+    return [[AVAudioSession sharedInstance] IOBufferDuration];
+}
+
 }

Runtime/Scripts/Audio/MicrophoneSource.cs

@@ -11,6 +11,13 @@ namespace LiveKit
     /// </summary>
     /// <remarks>
     /// Ensure microphone permissions are granted before calling <see cref="Start"/>.
+    ///
+    /// Unity's <c>Microphone</c> path does not go through a platform audio device module, so on
+    /// its own it has no echo cancellation. Construct the source with
+    /// <see cref="AudioProcessingOptions"/> to run libwebrtc's audio processing over the capture;
+    /// echo cancellation then uses the mix Unity plays as its reference (see
+    /// <see cref="PlayoutReference"/>), which covers every remote <see cref="AudioStream"/> and the
+    /// application's own audio.
     /// </remarks>
     sealed public class MicrophoneSource : RtcAudioSource
     {
@@ -35,6 +42,26 @@ public MicrophoneSource(string deviceName, GameObject sourceObject) : base(RtcAu
             _sourceObject = sourceObject;
         }
 
+        /// <summary>
+        /// Creates a microphone source whose capture is run through libwebrtc's audio processing
+        /// (AEC3 echo cancellation, noise suppression, gain control, high-pass filter) before it
+        /// reaches the track.
+        /// </summary>
+        /// <param name="deviceName">The name of the device to capture from. Use <see cref="Microphone.devices"/> to
+        /// get the list of available devices.</param>
+        /// <param name="sourceObject">The GameObject to attach the AudioSource to. The object must be kept in the scene
+        /// for the duration of the source's lifetime.</param>
+        /// <param name="processing">Which stages to enable. With <see cref="AudioProcessingOptions.EchoCancellation"/>
+        /// the SDK attaches a <see cref="PlayoutReference"/> to the active <see cref="AudioListener"/> to obtain the
+        /// far-end reference. Requires Unity's output sample rate to be a multiple of 100 Hz; otherwise processing is
+        /// bypassed with a warning. See <see cref="RtcAudioSource.AudioProcessingStats"/> for diagnostics.</param>
+        public MicrophoneSource(string deviceName, GameObject sourceObject, AudioProcessingOptions processing)
+            : base(RtcAudioSourceType.AudioSourceMicrophone, processing)
+        {
+            _deviceName = deviceName;
+            _sourceObject = sourceObject;
+        }
+
         /// <summary>
         /// Begins capturing audio from the microphone.
         /// </summary>
@@ -207,6 +234,8 @@ private IEnumerator RestartMicrophone()
             // recover from interruption. Poll for readiness instead of using arbitrary delay.
             yield return WaitForMicrophoneReady();
 
+            // A resume is a new audio path: drop whatever the processing stage buffered before.
+            ResetAudioProcessing();
             yield return StartMicrophone();
         }
 

Runtime/Scripts/Audio/PlatformAudioSource.cs

@@ -7,7 +7,10 @@
 namespace LiveKit
 {
     /// <summary>
-    /// Options for audio processing when creating a PlatformAudioSource.
+    /// Options for libwebrtc's audio processing. Used by <see cref="PlatformAudioSource"/>, where
+    /// the ADM applies them, and by Unity-audio sources such as <see cref="MicrophoneSource"/>
+    /// created with options, where the SDK runs libwebrtc's audio processing module over the
+    /// capture with the mix Unity plays as the echo reference (see <see cref="PlayoutReference"/>).
     /// </summary>
     public struct AudioProcessingOptions
     {
@@ -17,7 +20,10 @@ public struct AudioProcessingOptions
         public bool NoiseSuppression;
         /// <summary>Enable automatic gain control (AGC). Default: true.</summary>
         public bool AutoGainControl;
-        /// <summary>Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. Default: true.</summary>
+        /// <summary>
+        /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency.
+        /// <see cref="PlatformAudioSource"/> only. Default: true.
+        /// </summary>
         public bool PreferHardware;
 
         /// <summary>
@@ -30,6 +36,9 @@ public struct AudioProcessingOptions
             AutoGainControl = true,
             PreferHardware = true
         };
+
+        /// <summary>Whether any stage of the Unity-audio processing pipeline is enabled.</summary>
+        internal bool AnyProcessingEnabled => EchoCancellation || NoiseSuppression || AutoGainControl;
     }
 
     /// <summary>

Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs

@@ -0,0 +1,78 @@
+using System;
+using System.Runtime.InteropServices;
+using UnityEngine;
+
+namespace LiveKit
+{
+    /// <summary>
+    /// Estimates the render-to-capture delay hint handed to
+    /// <see cref="AudioProcessingModule.SetStreamDelayMs"/>.
+    /// </summary>
+    /// <remarks>
+    /// AEC3 runs its own correlation-based delay estimator; the hint only sets the initial
+    /// alignment after a reset, so it has to be in the right ballpark rather than exact. The echo
+    /// path in Unity-audio mode is: <see cref="PlayoutReference"/> tap → Unity's output queue (a
+    /// few DSP blocks) → device output → air → device input → <c>Microphone</c> clip → the
+    /// AudioSource reading that clip → capture probe. Only the DSP block size and, on iOS, the
+    /// audio session latencies are readable; the remaining terms are constants.
+    ///
+    /// Main thread only: reads <see cref="AudioSettings"/>.
+    /// </remarks>
+    internal static class AudioProcessingDelayHint
+    {
+#if UNITY_IOS && !UNITY_EDITOR
+        [DllImport("__Internal")] private static extern double LiveKit_AudioSessionOutputLatency();
+        [DllImport("__Internal")] private static extern double LiveKit_AudioSessionInputLatency();
+        [DllImport("__Internal")] private static extern double LiveKit_AudioSessionIOBufferDuration();
+#endif
+
+        internal const int MinDelayMs = 0;
+        internal const int MaxDelayMs = 500;
+
+        /// <summary>Output queue depth assumed between the listener tap and the device, in DSP blocks.</summary>
+        internal const int OutputQueueBlocks = 2;
+
+        /// <summary>
+        /// How far the AudioSource reading the microphone clip trails the clip's write head.
+        /// <see cref="MicrophoneSource"/> starts reading once <c>Microphone.GetPosition</c> first
+        /// reports data, polled at 50 ms, and that offset persists for the life of the clip.
+        /// </summary>
+        internal const int MicrophoneReadBehindMs = 50;
+
+        /// <summary>Device input plus output latency when the platform does not report it.</summary>
+        internal const int FallbackDeviceLatencyMs = 30;
+
+        public static int EstimateMs()
+        {
+            var config = AudioSettings.GetConfiguration();
+            return EstimateMs(config.dspBufferSize, config.sampleRate, PlatformLatencyMs());
+        }
+
+        internal static int EstimateMs(int dspBufferSize, int sampleRate, double deviceLatencyMs)
+        {
+            var blockMs = sampleRate > 0 ? dspBufferSize * 1000.0 / sampleRate : 0.0;
+            var estimate = blockMs * OutputQueueBlocks + deviceLatencyMs + MicrophoneReadBehindMs;
+            return (int)Math.Round(Math.Min(MaxDelayMs, Math.Max(MinDelayMs, estimate)));
+        }
+
+        internal static double PlatformLatencyMs()
+        {
+#if UNITY_IOS && !UNITY_EDITOR
+            try
+            {
+                // AVAudioSession reports zero until the session is active, hence the fallback.
+                var output = LiveKit_AudioSessionOutputLatency();
+                var input = LiveKit_AudioSessionInputLatency();
+                var ioBuffer = LiveKit_AudioSessionIOBufferDuration();
+                if (output > 0d || input > 0d)
+                    return (output + input + ioBuffer) * 1000d;
+            }
+            catch (Exception)
+            {
+                // Fall through to the constant.
+            }
+#endif
+            return FallbackDeviceLatencyMs;
+        }
+    }
+}

Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs

@@ -0,0 +1,167 @@
+using System;
+using LiveKit.Internal.FFI;
+using LiveKit.Internal.FFI.Requests;
+using LiveKit.Proto;
+
+namespace LiveKit
+{
+    /// <summary>
+    /// libwebrtc's <c>AudioProcessingModule</c> (AEC3 echo cancellation, noise suppression, gain
+    /// control, high-pass filter), driven over the FFI.
+    /// </summary>
+    /// <remarks>
+    /// Use this to run echo cancellation over a capture path that does not go through the
+    /// platform audio device module (e.g. Unity's <c>Microphone</c>): feed the audio that is
+    /// played out of the loudspeaker to <see cref="ProcessReverseStream"/> and the captured
+    /// microphone audio to <see cref="ProcessStream"/>, which processes it in place.
+    ///
+    /// Both accept exactly one 10 ms chunk of interleaved int16 PCM (<see cref="FrameSizeFor"/>
+    /// samples per channel) and nothing else. libwebrtc's own contract is a capture thread calling
+    /// <see cref="ProcessStream"/> and a render thread calling <see cref="ProcessReverseStream"/>;
+    /// the native module is internally synchronised for exactly that split, and the SDK's request
+    /// plumbing is safe to use from both.
+    /// </remarks>
+    internal sealed class AudioProcessingModule : IDisposable
+    {
+        /// <summary>libwebrtc's <c>kChunkSizeMs</c> — the APM accepts nothing else.</summary>
+        public const int ChunkSizeMs = 10;
+
+        /// <summary>
+        /// libwebrtc's internal processing rates. The APM resamples any other API rate onto one of
+        /// these itself, so this list is diagnostic information — NOT an admission requirement. See
+        /// <see cref="IsSupportedApiRate"/>.
+        /// </summary>
+        private static readonly int[] NativeSampleRates = { 8000, 16000, 32000, 48000 };
+
+        private readonly FfiHandle _handle;
+        private bool _disposed;
+
+        /// <summary>The native handle id, for diagnostics.</summary>
+        public ulong Handle => (ulong)_handle.DangerousGetHandle();
+
+        public AudioProcessingModule(
+            bool echoCancellerEnabled,
+            bool gainControllerEnabled,
+            bool highPassFilterEnabled,
+            bool noiseSuppressionEnabled)
+        {
+            using var request = FFIBridge.Instance.NewRequest<NewApmRequest>();
+            var newApm = request.request;
+            newApm.EchoCancellerEnabled = echoCancellerEnabled;
+            newApm.GainControllerEnabled = gainControllerEnabled;
+            newApm.HighPassFilterEnabled = highPassFilterEnabled;
+            newApm.NoiseSuppressionEnabled = noiseSuppressionEnabled;
+
+            using var response = request.Send();
+            FfiResponse res = response;
+            var owned = res.NewApm?.Apm;
+            if (owned?.Handle == null || owned.Handle.Id == 0)
+                throw new InvalidOperationException("FFI returned no APM handle");
+
+            _handle = FfiHandle.FromOwnedHandle(owned.Handle);
+        }
+
+        public static bool IsNativeSampleRate(int sampleRate)
+        {
+            foreach (var rate in NativeSampleRates)
+                if (rate == sampleRate) return true;
+            return false;
+        }
+
+        /// <summary>
+        /// Whether the APM accepts this rate on its API surface.
+        /// </summary>
+        /// <remarks>
+        /// The only hard requirement is that one 10 ms chunk is a whole number of samples: both
+        /// <see cref="FrameSizeFor"/> here and libwebrtc's own <c>StreamConfig::num_frames()</c>
+        /// derive the chunk with integer division, so a rate that is not a multiple of 100 Hz would
+        /// short every chunk and drift the two feeds apart.
+        ///
+        /// A non-native rate is NOT rejected — the rate goes straight into a <c>StreamConfig</c>
+        /// and libwebrtc resamples to a native processing rate internally. A 24 kHz output rate
+        /// (iPad) is cancelled just as well as 48 kHz.
+        /// </remarks>
+        public static bool IsSupportedApiRate(int sampleRate) =>
+            sampleRate > 0 && sampleRate % (1000 / ChunkSizeMs) == 0;
+
+        /// <summary>Samples per channel in one APM chunk at the given rate.</summary>
+        public static int FrameSizeFor(int sampleRate) => sampleRate / (1000 / ChunkSizeMs);
+
+        /// <summary>
+        /// Processes the near-end (capture) stream in place. <paramref name="byteCount"/> is bytes,
+        /// not samples — the buffer is interleaved int16. Returns the FFI error, or null on success.
+        /// </summary>
+        public string ProcessStream(IntPtr dataPtr, int byteCount, int sampleRate, int channels)
+        {
+            if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule));
+
+            using var request = FFIBridge.Instance.NewRequest<ApmProcessStreamRequest>();
+            var process = request.request;
+            process.ApmHandle = Handle;
+            process.DataPtr = (ulong)dataPtr.ToInt64();
+            process.Size = (uint)byteCount;
+            process.SampleRate = (uint)sampleRate;
+            process.NumChannels = (uint)channels;
+
+            using var response = request.Send();
+            FfiResponse res = response;
+            return ErrorOrNull(res.ApmProcessStream?.Error);
+        }
+
+        /// <summary>
+        /// Processes the far-end (render) reference stream in place. Same buffer contract as
+        /// <see cref="ProcessStream"/>.
+        /// </summary>
+        public string ProcessReverseStream(IntPtr dataPtr, int byteCount, int sampleRate, int channels)
+        {
+            if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule));
+
+            using var request = FFIBridge.Instance.NewRequest<ApmProcessReverseStreamRequest>();
+            var reverse = request.request;
+            reverse.ApmHandle = Handle;
+            reverse.DataPtr = (ulong)dataPtr.ToInt64();
+            reverse.Size = (uint)byteCount;
+            reverse.SampleRate = (uint)sampleRate;
+            reverse.NumChannels = (uint)channels;
+
+            using var response = request.Send();
+            FfiResponse res = response;
+            return ErrorOrNull(res.ApmProcessReverseStream?.Error);
+        }
+
+        /// <summary>
+        /// Seeds the render/capture delay. AEC3 runs its own correlation estimator, so this is a
+        /// convergence hint rather than a hard alignment. Returns the FFI error, or null on success.
+        /// </summary>
+        public string SetStreamDelayMs(int delayMs)
+        {
+            if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule));
+
+            using var request = FFIBridge.Instance.NewRequest<ApmSetStreamDelayRequest>();
+            var delay = request.request;
+            delay.ApmHandle = Handle;
+            delay.DelayMs = delayMs;
+
+            using var response = request.Send();
+            FfiResponse res = response;
+            return ErrorOrNull(res.ApmSetStreamDelay?.Error);
+        }
+
+        private static string ErrorOrNull(string error) => string.IsNullOrEmpty(error) ? null : error;
+
+        public void Dispose()
+        {
+            if (_disposed) return;
+            _disposed = true;
+            _handle.Dispose();
+            GC.SuppressFinalize(this);
+        }
+
+        ~AudioProcessingModule()
+        {
+            if (_disposed) return;
+            _disposed = true;
+            _handle.Dispose();
+        }
+    }
+}