From 2e9eee08efa4c029be6bc3da3146adcf08fe7bc9 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:24:43 +0200 Subject: [PATCH 1/4] The first version is already working in Meet on MacOS --- .../Scripts/Audio/AudioProcessingModule.cs | 167 ++++++++++ .../Audio/AudioProcessingModule.cs.meta | 11 + Runtime/Scripts/Core/FfiFrameObserver.cs | 160 +++++++++ Runtime/Scripts/Core/FfiFrameObserver.cs.meta | 11 + Runtime/Scripts/Internal/FFI/FFIClient.cs | 4 + .../Internal/FFI/FfiRequestExtensions.cs | 13 + .../Meet/Assets/Editor/MeetManagerEditor.cs | 23 ++ Samples~/Meet/Assets/Plugins/iOS.meta | 8 + .../Assets/Plugins/iOS/AudioSessionLatency.mm | 18 ++ .../Plugins/iOS/AudioSessionLatency.mm.meta | 33 ++ Samples~/Meet/Assets/Runtime/Audio.meta | 8 + .../Runtime/Audio/AcousticEchoCanceller.cs | 220 +++++++++++++ .../Audio/AcousticEchoCanceller.cs.meta | 11 + .../Assets/Runtime/Audio/AecAudioProbe.cs | 51 +++ .../Runtime/Audio/AecAudioProbe.cs.meta | 11 + .../Assets/Runtime/Audio/AecMicrophoneHost.cs | 22 ++ .../Runtime/Audio/AecMicrophoneHost.cs.meta | 11 + .../Runtime/Audio/AecMicrophoneSource.cs | 304 ++++++++++++++++++ .../Runtime/Audio/AecMicrophoneSource.cs.meta | 11 + .../Meet/Assets/Runtime/Audio/ApmChunkPump.cs | 168 ++++++++++ .../Assets/Runtime/Audio/ApmChunkPump.cs.meta | 11 + .../Runtime/Audio/AudioProcessingDelaySeed.cs | 64 ++++ .../Audio/AudioProcessingDelaySeed.cs.meta | 11 + .../Runtime/Audio/AudioProcessingSmokeTest.cs | 61 ++++ .../Audio/AudioProcessingSmokeTest.cs.meta | 11 + .../Assets/Runtime/Audio/PcmRingBuffer.cs | 100 ++++++ .../Runtime/Audio/PcmRingBuffer.cs.meta | 11 + Samples~/Meet/Assets/Runtime/MeetManager.cs | 39 ++- Samples~/Meet/Assets/Scenes/MeetApp.unity | 6 +- 29 files changed, 1572 insertions(+), 7 deletions(-) create mode 100644 Runtime/Scripts/Audio/AudioProcessingModule.cs create mode 100644 Runtime/Scripts/Audio/AudioProcessingModule.cs.meta create mode 100644 Runtime/Scripts/Core/FfiFrameObserver.cs create mode 100644 Runtime/Scripts/Core/FfiFrameObserver.cs.meta create mode 100644 Samples~/Meet/Assets/Plugins/iOS.meta create mode 100644 Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm create mode 100644 Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta diff --git a/Runtime/Scripts/Audio/AudioProcessingModule.cs b/Runtime/Scripts/Audio/AudioProcessingModule.cs new file mode 100644 index 00000000..d83f83be --- /dev/null +++ b/Runtime/Scripts/Audio/AudioProcessingModule.cs @@ -0,0 +1,167 @@ +using System; +using LiveKit.Internal.FFI; +using LiveKit.Internal.FFI.Requests; +using LiveKit.Proto; + +namespace LiveKit +{ + /// + /// libwebrtc's AudioProcessingModule (AEC3 echo cancellation, noise suppression, gain + /// control, high-pass filter), driven over the FFI. + /// + /// + /// Use this to run echo cancellation over a capture path that does not go through the + /// platform audio device module (e.g. Unity's Microphone): feed the audio that is + /// played out of the loudspeaker to and the captured + /// microphone audio to , which processes it in place. + /// + /// Both accept exactly one 10 ms chunk of interleaved int16 PCM ( + /// samples per channel) and nothing else. libwebrtc's own contract is a capture thread calling + /// and a render thread calling ; + /// the native module is internally synchronised for exactly that split, and the SDK's request + /// plumbing is safe to use from both. + /// + public sealed class AudioProcessingModule : IDisposable + { + /// libwebrtc's kChunkSizeMs — the APM accepts nothing else. + public const int ChunkSizeMs = 10; + + /// + /// libwebrtc's internal processing rates. The APM resamples any other API rate onto one of + /// these itself, so this list is diagnostic information — NOT an admission requirement. See + /// . + /// + private static readonly int[] NativeSampleRates = { 8000, 16000, 32000, 48000 }; + + private readonly FfiHandle _handle; + private bool _disposed; + + /// The native handle id, for diagnostics. + public ulong Handle => (ulong)_handle.DangerousGetHandle(); + + public AudioProcessingModule( + bool echoCancellerEnabled, + bool gainControllerEnabled, + bool highPassFilterEnabled, + bool noiseSuppressionEnabled) + { + using var request = FFIBridge.Instance.NewRequest(); + var newApm = request.request; + newApm.EchoCancellerEnabled = echoCancellerEnabled; + newApm.GainControllerEnabled = gainControllerEnabled; + newApm.HighPassFilterEnabled = highPassFilterEnabled; + newApm.NoiseSuppressionEnabled = noiseSuppressionEnabled; + + using var response = request.Send(); + FfiResponse res = response; + var owned = res.NewApm?.Apm; + if (owned?.Handle == null || owned.Handle.Id == 0) + throw new InvalidOperationException("FFI returned no APM handle"); + + _handle = FfiHandle.FromOwnedHandle(owned.Handle); + } + + public static bool IsNativeSampleRate(int sampleRate) + { + foreach (var rate in NativeSampleRates) + if (rate == sampleRate) return true; + return false; + } + + /// + /// Whether the APM accepts this rate on its API surface. + /// + /// + /// The only hard requirement is that one 10 ms chunk is a whole number of samples: both + /// here and libwebrtc's own StreamConfig::num_frames() + /// derive the chunk with integer division, so a rate that is not a multiple of 100 Hz would + /// short every chunk and drift the two feeds apart. + /// + /// A non-native rate is NOT rejected — the rate goes straight into a StreamConfig + /// and libwebrtc resamples to a native processing rate internally. A 24 kHz output rate + /// (iPad) is cancelled just as well as 48 kHz. + /// + public static bool IsSupportedApiRate(int sampleRate) => + sampleRate > 0 && sampleRate % (1000 / ChunkSizeMs) == 0; + + /// Samples per channel in one APM chunk at the given rate. + public static int FrameSizeFor(int sampleRate) => sampleRate / (1000 / ChunkSizeMs); + + /// + /// Processes the near-end (capture) stream in place. is bytes, + /// not samples — the buffer is interleaved int16. Returns the FFI error, or null on success. + /// + public string ProcessStream(IntPtr dataPtr, int byteCount, int sampleRate, int channels) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var process = request.request; + process.ApmHandle = Handle; + process.DataPtr = (ulong)dataPtr.ToInt64(); + process.Size = (uint)byteCount; + process.SampleRate = (uint)sampleRate; + process.NumChannels = (uint)channels; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmProcessStream?.Error); + } + + /// + /// Processes the far-end (render) reference stream in place. Same buffer contract as + /// . + /// + public string ProcessReverseStream(IntPtr dataPtr, int byteCount, int sampleRate, int channels) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var reverse = request.request; + reverse.ApmHandle = Handle; + reverse.DataPtr = (ulong)dataPtr.ToInt64(); + reverse.Size = (uint)byteCount; + reverse.SampleRate = (uint)sampleRate; + reverse.NumChannels = (uint)channels; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmProcessReverseStream?.Error); + } + + /// + /// Seeds the render/capture delay. AEC3 runs its own correlation estimator, so this is a + /// convergence hint rather than a hard alignment. Returns the FFI error, or null on success. + /// + public string SetStreamDelayMs(int delayMs) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var delay = request.request; + delay.ApmHandle = Handle; + delay.DelayMs = delayMs; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmSetStreamDelay?.Error); + } + + private static string ErrorOrNull(string error) => string.IsNullOrEmpty(error) ? null : error; + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _handle.Dispose(); + GC.SuppressFinalize(this); + } + + ~AudioProcessingModule() + { + if (_disposed) return; + _disposed = true; + _handle.Dispose(); + } + } +} diff --git a/Runtime/Scripts/Audio/AudioProcessingModule.cs.meta b/Runtime/Scripts/Audio/AudioProcessingModule.cs.meta new file mode 100644 index 00000000..4263c783 --- /dev/null +++ b/Runtime/Scripts/Audio/AudioProcessingModule.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 033fc8514082a43a18e018ca3267a9b5 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Core/FfiFrameObserver.cs b/Runtime/Scripts/Core/FfiFrameObserver.cs new file mode 100644 index 00000000..cdb46fef --- /dev/null +++ b/Runtime/Scripts/Core/FfiFrameObserver.cs @@ -0,0 +1,160 @@ +using System; +using LiveKit.Internal; +using LiveKit.Proto; + +namespace LiveKit +{ + /// + /// A decoded video frame, extracted from the raw FFI event into a protobuf-free value. + /// Carries the native plane pointers and geometry only — no managed wrappers and no + /// Google.Protobuf surface, so consumers can use frames without inheriting a + /// compile-time protobuf dependency. + /// + /// + /// The DataPtr* values point at native buffers that are valid for the duration of + /// the callback ONLY — copy out + /// synchronously if you need them past return. + /// + public readonly struct RawVideoFrame + { + /// Stream handle the frame arrived on, for correlating to a . + public readonly ulong StreamHandle; + public readonly IntPtr DataPtrY, DataPtrU, DataPtrV; + public readonly int StrideY, StrideU, StrideV; + public readonly int Width, Height; + + public RawVideoFrame( + ulong streamHandle, + IntPtr dataPtrY, int strideY, + IntPtr dataPtrU, int strideU, + IntPtr dataPtrV, int strideV, + int width, int height) + { + StreamHandle = streamHandle; + DataPtrY = dataPtrY; StrideY = strideY; + DataPtrU = dataPtrU; StrideU = strideU; + DataPtrV = dataPtrV; StrideV = strideV; + Width = width; Height = height; + } + } + + /// + /// A decoded audio frame, extracted from the raw FFI event into a protobuf-free value. + /// points at interleaved S16 PCM valid for the callback duration ONLY. + /// + public readonly struct RawAudioFrame + { + /// Stream handle the frame arrived on, for correlating to an . + public readonly ulong StreamHandle; + public readonly IntPtr DataPtr; + public readonly int SamplesPerChannel, NumChannels, SampleRate; + + public RawAudioFrame(ulong streamHandle, IntPtr dataPtr, int samplesPerChannel, int numChannels, int sampleRate) + { + StreamHandle = streamHandle; + DataPtr = dataPtr; + SamplesPerChannel = samplesPerChannel; + NumChannels = numChannels; + SampleRate = sampleRate; + } + } + + /// + /// Opt-in extension point for raw decoded frames. + /// + /// + /// / are invoked + /// synchronously on the FFI callback thread from the event router, BEFORE the event's + /// FfiHandles wrap or free the underlying native buffers — so the DataPtr + /// values each frame carries are valid for the duration of the callback ONLY. + /// + /// The SDK extracts the protobuf event into the plain / + /// structs here, on the FFI thread, so subscribers consume + /// decoded frames WITHOUT a compile-time dependency on Google.Protobuf. + /// + /// Subscriber contract: + /// + /// Runs on the FFI thread, not Unity's main loop — do not touch Unity APIs. + /// Must be non-blocking; it sits in the frame-delivery hot path. + /// Must NOT retain any DataPtr past return — copy out synchronously if needed. + /// + /// + /// No subscriber == zero cost: extraction is skipped entirely when the matching delegate + /// is null. This lets consumers build native Picture-in-Picture, echo-cancellation + /// references, frame capture, custom GPU upload, or analytics on top of the decoded + /// stream without patching the SDK. + /// + public static class FfiFrameObserver + { + public static event Action VideoFrameReceived; + public static event Action AudioFrameReceived; + + internal static void Dispatch(FfiEvent ev) + { + switch (ev.MessageCase) + { + case FfiEvent.MessageOneofCase.VideoStreamEvent: + ExtractVideo(ev.VideoStreamEvent); + break; + case FfiEvent.MessageOneofCase.AudioStreamEvent: + ExtractAudio(ev.AudioStreamEvent); + break; + } + } + + private static void ExtractVideo(VideoStreamEvent vse) + { + var handler = VideoFrameReceived; + if (handler == null) return; + if (vse.MessageCase != VideoStreamEvent.MessageOneofCase.FrameReceived) return; + + var buf = vse.FrameReceived?.Buffer; + if (buf?.Info == null || buf.Info.Components.Count < 3) return; + + var info = buf.Info; + var yc = info.Components[0]; + var uc = info.Components[1]; + var vc = info.Components[2]; + if (yc.DataPtr == 0 || uc.DataPtr == 0 || vc.DataPtr == 0) return; + + Invoke(handler, new RawVideoFrame( + vse.StreamHandle, + (IntPtr)(long)yc.DataPtr, (int)yc.Stride, + (IntPtr)(long)uc.DataPtr, (int)uc.Stride, + (IntPtr)(long)vc.DataPtr, (int)vc.Stride, + (int)info.Width, (int)info.Height)); + } + + private static void ExtractAudio(AudioStreamEvent ase) + { + var handler = AudioFrameReceived; + if (handler == null) return; + if (ase.MessageCase != AudioStreamEvent.MessageOneofCase.FrameReceived) return; + + var frame = ase.FrameReceived?.Frame; + if (frame?.Info == null || frame.Info.DataPtr == 0) return; + + var info = frame.Info; + Invoke(handler, new RawAudioFrame( + ase.StreamHandle, + (IntPtr)(long)info.DataPtr, + (int)info.SamplesPerChannel, + (int)info.NumChannels, + (int)info.SampleRate)); + } + + // A subscriber exception must not escape the native callback: this runs on the FFI + // thread inside a reverse P/Invoke, where an unhandled managed exception is fatal. + private static void Invoke(Action handler, T frame) + { + try + { + handler(frame); + } + catch (Exception e) + { + Utils.Error($"FfiFrameObserver subscriber threw: {e}"); + } + } + } +} diff --git a/Runtime/Scripts/Core/FfiFrameObserver.cs.meta b/Runtime/Scripts/Core/FfiFrameObserver.cs.meta new file mode 100644 index 00000000..e35c2080 --- /dev/null +++ b/Runtime/Scripts/Core/FfiFrameObserver.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 50292b03c8d4441118cfe20a865e0e27 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Internal/FFI/FFIClient.cs b/Runtime/Scripts/Internal/FFI/FFIClient.cs index 213ee1f4..ec03b01a 100644 --- a/Runtime/Scripts/Internal/FFI/FFIClient.cs +++ b/Runtime/Scripts/Internal/FFI/FFIClient.cs @@ -375,6 +375,10 @@ internal static void RouteFfiEvent(FfiEvent response) { if (_isDisposed) return; + // Raw decoded-frame hook. Runs first, on this thread, so subscribers see the native + // buffers before any FfiHandle below wraps or frees them. No-op when unsubscribed. + FfiFrameObserver.Dispatch(response); + // Audio stream events are handled directly on the FFI callback thread // to bypass the main thread, since the audio thread consumes the data if (response.MessageCase == FfiEvent.MessageOneofCase.AudioStreamEvent) diff --git a/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs b/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs index b31f4dc0..fb9ed957 100644 --- a/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs +++ b/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs @@ -152,6 +152,19 @@ public static void Inject(this FfiRequest ffiRequest, T request) case RemixAndResampleRequest remixAndResampleRequest: ffiRequest.RemixAndResample = remixAndResampleRequest; break; + // Audio processing module + case NewApmRequest newApmRequest: + ffiRequest.NewApm = newApmRequest; + break; + case ApmProcessStreamRequest apmProcessStreamRequest: + ffiRequest.ApmProcessStream = apmProcessStreamRequest; + break; + case ApmProcessReverseStreamRequest apmProcessReverseStreamRequest: + ffiRequest.ApmProcessReverseStream = apmProcessReverseStreamRequest; + break; + case ApmSetStreamDelayRequest apmSetStreamDelayRequest: + ffiRequest.ApmSetStreamDelay = apmSetStreamDelayRequest; + break; // PlatformAudio case NewPlatformAudioRequest newPlatformAudioRequest: ffiRequest.NewPlatformAudio = newPlatformAudioRequest; diff --git a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs index 1ca6c412..8211dbc2 100644 --- a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs +++ b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs @@ -13,6 +13,8 @@ public class MeetManagerEditor : Editor private SerializedProperty noiseSuppression; private SerializedProperty autoGainControl; private SerializedProperty preferHardwareProcessing; + private SerializedProperty unityEchoCancellation; + private SerializedProperty remoteAudioGain; private void OnEnable() { @@ -25,6 +27,8 @@ private void OnEnable() noiseSuppression = serializedObject.FindProperty("noiseSuppression"); autoGainControl = serializedObject.FindProperty("autoGainControl"); preferHardwareProcessing = serializedObject.FindProperty("preferHardwareProcessing"); + unityEchoCancellation = serializedObject.FindProperty("unityEchoCancellation"); + remoteAudioGain = serializedObject.FindProperty("remoteAudioGain"); } public override void OnInspectorGUI() @@ -69,6 +73,25 @@ public override void OnInspectorGUI() "Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")); } + EditorGUILayout.Space(); + EditorGUILayout.LabelField("Unity Audio (PlatformAudio off)", EditorStyles.boldLabel); + + // Gray out Unity audio options when PlatformAudio is enabled + using (new EditorGUI.DisabledGroupScope(platformAudioEnabled)) + { + if (platformAudioEnabled) + { + EditorGUILayout.HelpBox("Unity audio options are only used when 'Use Platform Audio' is disabled.", MessageType.Info); + } + + EditorGUILayout.PropertyField(unityEchoCancellation, new GUIContent("Echo Cancellation (AEC3)", + "Run libwebrtc's AEC3 over Unity microphone capture, using the decoded remote audio frames as the " + + "echo reference. Assumes a single remote audio stream.")); + EditorGUILayout.PropertyField(remoteAudioGain, new GUIContent("Remote Audio Gain", + "Playback gain for every remote AudioSource. Below 1 keeps headroom so full-volume playout does not " + + "distort or overload the echo canceller. 0.7 is -3.1 dB.")); + } + serializedObject.ApplyModifiedProperties(); } } diff --git a/Samples~/Meet/Assets/Plugins/iOS.meta b/Samples~/Meet/Assets/Plugins/iOS.meta new file mode 100644 index 00000000..1205da93 --- /dev/null +++ b/Samples~/Meet/Assets/Plugins/iOS.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 2f128f6a519a14fc0aa42bbc2d20f447 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm new file mode 100644 index 00000000..e8749f1e --- /dev/null +++ b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm @@ -0,0 +1,18 @@ +#import + +// Latency terms for seeding libwebrtc's AEC3 stream delay; consumed by +// Assets/Runtime/Audio/AudioProcessingDelaySeed.cs. AVAudioSession only reports meaningful +// values once the session is active; it returns 0 before that. +extern "C" { + double MeetSample_AudioSessionOutputLatency() { + return [[AVAudioSession sharedInstance] outputLatency]; + } + + double MeetSample_AudioSessionInputLatency() { + return [[AVAudioSession sharedInstance] inputLatency]; + } + + double MeetSample_AudioSessionIOBufferDuration() { + return [[AVAudioSession sharedInstance] IOBufferDuration]; + } +} diff --git a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta new file mode 100644 index 00000000..3483a3d7 --- /dev/null +++ b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta @@ -0,0 +1,33 @@ +fileFormatVersion: 2 +guid: 25d054283a4734b5a817eb1b23cbea61 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + iPhone: iOS + second: + enabled: 1 + settings: + AddToEmbeddedBinaries: false + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio.meta b/Samples~/Meet/Assets/Runtime/Audio.meta new file mode 100644 index 00000000..deab7576 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 7ed59392f993648e8ae9d202fb567a84 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs b/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs new file mode 100644 index 00000000..54da05d1 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs @@ -0,0 +1,220 @@ +using System; +using System.Diagnostics; +using System.Threading; +using LiveKit; +using UnityEngine; +using Debug = UnityEngine.Debug; + +/// +/// Runs libwebrtc's AEC3 over the local microphone capture, using the decoded remote audio as +/// the echo reference. +/// +/// The loudspeaker plays the remote audio, the microphone re-captures it, and the published +/// track would carry it back to everyone. Unity's Microphone path has no echo canceller +/// anywhere (capture never goes through a platform audio device module), so cancellation is +/// done here against the two streams already available in managed code: the FFI render frames +/// (far end, via ) and OnAudioFilterRead (near end). +/// +/// Threading: runs on the Unity audio thread and +/// on the FFI callback thread. Each owns its own pump, and +/// libwebrtc's APM is built for exactly that capture/render thread split. Nothing here touches +/// a Unity API from those threads; diagnostics use , not +/// UnityEngine.Time. +/// +/// Limitation: the far-end tap is not filtered by stream, so it assumes a SINGLE remote audio +/// stream. With several remote speakers the reference becomes the interleaving of all their +/// frames and AEC3 will not converge. The remote audio must also play through Unity +/// (AudioStream) — in PlatformAudio mode no FFI audio streams exist and nothing arrives. +/// +internal sealed class AcousticEchoCanceller : IDisposable +{ + private const int DiagnosticIntervalMs = 5000; + private const int SeedIntervalMs = 2000; + private const int SeedChangeThresholdMs = 5; + + private readonly AudioProcessingModule _apm; + private readonly ApmChunkPump _capturePump; + private readonly ApmChunkPump _renderPump; + private readonly Stopwatch _clock = new Stopwatch(); + + private long _nextDiagnosticMs; + private long _nextSeedMs; + private int _seededDelayMs = -1; + private int _farEndFrames; + private int _unsupportedRateWarned; + private bool _subscribed; + private bool _disposed; + + /// Processed 10 ms capture chunks, raised on the Unity audio thread. + public event ProcessedChunkHandler CaptureProcessed; + + private AcousticEchoCanceller(AudioProcessingModule apm) + { + _apm = apm; + _capturePump = new ApmChunkPump(_apm.ProcessStream, RaiseCaptureProcessed); + _renderPump = new ApmChunkPump(_apm.ProcessReverseStream); + } + + /// + /// Creates the canceller, or returns null when the FFI cannot hand out an APM handle — the + /// caller then publishes the unprocessed microphone rather than failing to publish at all. + /// + public static AcousticEchoCanceller TryCreate() + { + try + { + var apm = new AudioProcessingModule( + echoCancellerEnabled: true, + gainControllerEnabled: true, + highPassFilterEnabled: true, + noiseSuppressionEnabled: true); + + Debug.Log($"[AEC] APM created handle={apm.Handle}"); + return new AcousticEchoCanceller(apm); + } + catch (Exception e) + { + Debug.LogWarning($"[AEC] APM unavailable, publishing microphone unprocessed: {e.Message}"); + return null; + } + } + + public void Start() + { + if (_disposed || _subscribed) return; + + _capturePump.Reset(); + _renderPump.Reset(); + _clock.Restart(); + _nextDiagnosticMs = DiagnosticIntervalMs; + _nextSeedMs = SeedIntervalMs; + + // Forget the delay seeded before the last Stop(). A resume restarts the whole audio path, + // so the previous value describes an acoustic path that no longer exists — and carrying it + // over lets the SeedChangeThresholdMs guard in SeedStreamDelay silently skip the reseed + // below whenever the new estimate lands within 5 ms of the stale one. + _seededDelayMs = -1; + + FfiFrameObserver.AudioFrameReceived += OnFarEndFrame; + _subscribed = true; + + SeedStreamDelay(0d); + } + + public void Stop() + { + if (!_subscribed) return; + + FfiFrameObserver.AudioFrameReceived -= OnFarEndFrame; + _subscribed = false; + _clock.Reset(); + } + + /// + /// Feeds near-end capture. Unity audio thread. Returns false when the block cannot be + /// processed, and the caller must publish it unchanged — the APM needs a rate whose 10 ms + /// chunk is a whole number of samples, and device audio backends do run at odd rates. + /// + public bool TryPushCapture(float[] data, int channels, int sampleRate) + { + if (_disposed || data == null || channels <= 0 || sampleRate <= 0) return false; + + if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) + { + WarnUnsupportedRateOnce(sampleRate); + return false; + } + + _capturePump.Push(data, channels, sampleRate); + + var captureBlockMs = data.Length / (double)channels * 1000d / sampleRate; + MaybeReseed(captureBlockMs); + MaybeLogDiagnostics(channels, sampleRate); + return true; + } + + private void WarnUnsupportedRateOnce(int sampleRate) + { + if (Interlocked.Exchange(ref _unsupportedRateWarned, 1) == 1) return; + + Debug.LogWarning( + $"[AEC] capture rate {sampleRate} has no whole-sample 10 ms chunk — " + + "echo cancellation disabled, publishing microphone unprocessed"); + } + + // The far-end DataPtr is valid for the duration of this callback ONLY; the pump copies out + // before doing anything else. + private void OnFarEndFrame(RawAudioFrame frame) + { + if (_disposed) return; + + Interlocked.Increment(ref _farEndFrames); + _renderPump.Push(frame.DataPtr, frame.SamplesPerChannel, frame.NumChannels, frame.SampleRate); + } + + private void RaiseCaptureProcessed(float[] data, int channels, int sampleRate) + { + CaptureProcessed?.Invoke(data, channels, sampleRate); + } + + // AVAudioSession reports zero latency until the session goes active, so the seed is + // re-evaluated on a slow cadence rather than only once at Start(). + private void MaybeReseed(double captureBlockMs) + { + if (!_clock.IsRunning) return; + + var elapsedMs = _clock.ElapsedMilliseconds; + if (elapsedMs < _nextSeedMs) return; + _nextSeedMs = elapsedMs + SeedIntervalMs; + + SeedStreamDelay(captureBlockMs); + } + + private void SeedStreamDelay(double captureBlockMs) + { + var delayMs = AudioProcessingDelaySeed.Estimate(captureBlockMs); + if (_seededDelayMs >= 0 && Math.Abs(delayMs - _seededDelayMs) < SeedChangeThresholdMs) return; + + var error = _apm.SetStreamDelayMs(delayMs); + if (error != null) + { + Debug.LogWarning($"[AEC] set_stream_delay_ms({delayMs}) failed: {error}"); + return; + } + + _seededDelayMs = delayMs; + Debug.Log($"[AEC] stream delay seeded to {delayMs}ms (captureBlock={captureBlockMs:F1}ms)"); + } + + // Reports the measured frame geometry both feeds are actually running at. Stopwatch, not + // UnityEngine.Time: this is the audio thread. + private void MaybeLogDiagnostics(int channels, int sampleRate) + { + if (!_clock.IsRunning) return; + + var elapsedMs = _clock.ElapsedMilliseconds; + if (elapsedMs < _nextDiagnosticMs) return; + _nextDiagnosticMs = elapsedMs + DiagnosticIntervalMs; + + Debug.Log( + $"[AEC] capture {channels}ch@{sampleRate} native={AudioProcessingModule.IsNativeSampleRate(sampleRate)} " + + $"chunks={_capturePump.ProcessedChunkCount} dropped={_capturePump.DroppedSamples} " + + $"failed={_capturePump.FailedChunkCount} err={_capturePump.LastError ?? "-"} | " + + $"render {_renderPump.Channels}ch@{_renderPump.SampleRate} frames={_farEndFrames} " + + $"chunks={_renderPump.ProcessedChunkCount} dropped={_renderPump.DroppedSamples} " + + $"failed={_renderPump.FailedChunkCount} err={_renderPump.LastError ?? "-"} | " + + $"delay={_seededDelayMs}ms"); + } + + public void Dispose() + { + if (_disposed) return; + + Stop(); + _disposed = true; + CaptureProcessed = null; + _capturePump.Dispose(); + _renderPump.Dispose(); + _apm.Dispose(); + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta new file mode 100644 index 00000000..39aae659 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6d2e02f70cc454f188b929269a8acbc7 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs b/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs new file mode 100644 index 00000000..3ae50e5b --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs @@ -0,0 +1,51 @@ +using System; +using UnityEngine; + +/// +/// Intercepts the microphone clip's audio on the Unity audio thread. +/// +/// Sample-side re-implementation of the SDK's AudioProbe, which is internal. Behaviour is +/// deliberately identical, including — without it the +/// microphone is played back through the local loudspeaker. +/// +internal sealed class AecAudioProbe : MonoBehaviour +{ + public delegate void OnAudioDelegate(float[] data, int channels, int sampleRate); + + public event OnAudioDelegate AudioRead; + + private int _sampleRate; + private volatile bool _clearAfterInvocation; + + public void ClearAfterInvocation() + { + _clearAfterInvocation = true; + } + + private void OnEnable() + { + OnAudioConfigurationChanged(false); + AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; + } + + private void OnDisable() + { + AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; + } + + private void OnAudioConfigurationChanged(bool deviceWasChanged) + { + _sampleRate = AudioSettings.outputSampleRate; + } + + private void OnAudioFilterRead(float[] data, int channels) + { + AudioRead?.Invoke(data, channels, _sampleRate); + if (_clearAfterInvocation) data.AsSpan().Clear(); + } + + private void OnDestroy() + { + AudioRead = null; + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta new file mode 100644 index 00000000..6c3e6728 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 21b265f6c78d54d0ebae5f213cee36a1 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs new file mode 100644 index 00000000..3f4a21f4 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs @@ -0,0 +1,22 @@ +using System; +using UnityEngine; + +/// +/// Coroutine runner and application-pause relay for , attached +/// to the microphone GameObject. Stands in for the SDK's internal MonoBehaviourContext, +/// which sample code cannot reach. +/// +internal sealed class AecMicrophoneHost : MonoBehaviour +{ + public event Action Paused; + + private void OnApplicationPause(bool pause) + { + Paused?.Invoke(pause); + } + + private void OnDestroy() + { + Paused = null; + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta new file mode 100644 index 00000000..0a43fb48 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 4e837510d3f214ed8bc1ee291df8f7a4 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs new file mode 100644 index 00000000..c9f99ef3 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs @@ -0,0 +1,304 @@ +using System; +using System.Collections; +using LiveKit; +using UnityEngine; + +/// +/// Microphone capture source that runs AEC3 over the captured PCM before publishing it. +/// +/// The SDK's MicrophoneSource is sealed and its AudioProbe and +/// MonoBehaviourContext are internal, so the capture path is re-implemented here. Its +/// behaviours are load-bearing and preserved: clear-after-invocation (so the microphone is not +/// played back locally), the duplicate-component guard, the Microphone.GetPosition +/// readiness poll, and the pause/resume stop-restart cycle. +/// +/// The only behavioural difference is that carries APM-processed 10 ms +/// chunks instead of raw DSP blocks. When the APM is unavailable the raw blocks pass straight +/// through, so publishing never fails because of the canceller. +/// +internal sealed class AecMicrophoneSource : RtcAudioSource +{ + private readonly GameObject _sourceObject; + private readonly string _deviceName; + private readonly AecMicrophoneHost _host; + private readonly AcousticEchoCanceller _canceller; + + public override event Action AudioRead; + + private bool _disposed; + private bool _started; + + public bool EchoCancellationActive => _canceller != null; + + private AecMicrophoneSource( + string deviceName, + GameObject sourceObject, + AecMicrophoneHost host, + AcousticEchoCanceller canceller) + : base(RtcAudioSourceType.AudioSourceMicrophone) + { + _deviceName = deviceName; + _sourceObject = sourceObject; + _host = host; + _canceller = canceller; + + if (_canceller != null) _canceller.CaptureProcessed += OnProcessedAudio; + } + + /// + /// Builds the source. The base constructor configures the native source from Unity's current + /// output configuration, which is also the format OnAudioFilterRead delivers, so the + /// published track's metadata matches the capture geometry without further alignment. + /// + /// One of . + /// GameObject that hosts the AudioSource, probe and coroutine + /// runner. Must stay alive for the source's lifetime. + public static AecMicrophoneSource Create(string deviceName, GameObject sourceObject) + { + if (sourceObject == null) throw new ArgumentNullException(nameof(sourceObject)); + + var host = sourceObject.GetComponent(); + if (host == null) host = sourceObject.AddComponent(); + + return new AecMicrophoneSource(deviceName, sourceObject, host, AcousticEchoCanceller.TryCreate()); + } + + public override void Start() + { + base.Start(); + if (_started) return; + + if (!Application.HasUserAuthorization(UserAuthorization.Microphone)) + throw new InvalidOperationException("Microphone access not authorized"); + + _host.Paused += OnApplicationPause; + _canceller?.Start(); + RunCoroutine(StartMicrophone()); + + _started = true; + } + + public override void Stop() + { + base.Stop(); + RunCoroutine(StopMicrophone()); + if (_host != null) _host.Paused -= OnApplicationPause; + _canceller?.Stop(); + _started = false; + } + + private IEnumerator StartMicrophone() + { + if (_sourceObject == null) + { + Debug.LogError("[AEC] microphone GameObject is null, cannot start"); + yield break; + } + + if (!Application.HasUserAuthorization(UserAuthorization.Microphone)) + { + Debug.LogError("[AEC] microphone authorization lost"); + yield break; + } + + AudioClip clip = null; + try + { + clip = Microphone.Start( + _deviceName, + loop: true, + lengthSec: 1, + frequency: SupportedCaptureFrequency()); + } + catch (Exception e) + { + Debug.LogError($"[AEC] exception starting microphone: {e.Message}"); + yield break; + } + + if (clip == null) + { + Debug.LogError("[AEC] Microphone.Start returned null, audio session may not be ready"); + yield break; + } + + // Unity's Destroy is deferred, so a resume can land here while the previous pair is still + // alive. Duplicates would double every captured block into the APM. + var existingSource = _sourceObject.GetComponent(); + if (existingSource != null) UnityEngine.Object.DestroyImmediate(existingSource); + + var existingProbe = _sourceObject.GetComponent(); + if (existingProbe != null) + { + existingProbe.AudioRead -= OnCapturedAudio; + UnityEngine.Object.DestroyImmediate(existingProbe); + } + + var source = _sourceObject.AddComponent(); + source.clip = clip; + source.loop = true; + + var probe = _sourceObject.AddComponent(); + probe.ClearAfterInvocation(); + probe.AudioRead += OnCapturedAudio; + + const float timeout = 2f; + var elapsed = 0f; + while (Microphone.GetPosition(_deviceName) <= 0 && elapsed < timeout) + { + yield return new WaitForSeconds(0.05f); + elapsed += 0.05f; + } + + if (Microphone.GetPosition(_deviceName) <= 0) + { + Debug.LogError($"[AEC] microphone did not start producing data after {timeout}s"); + yield break; + } + + source.Play(); + Debug.Log($"[AEC] microphone '{_deviceName}' started at {clip.frequency}Hz, echo cancellation={EchoCancellationActive}"); + } + + // The requested rate is Unity's output rate, which the device may not accept as a capture + // rate. Clamping keeps Microphone.Start working; the DSP graph resamples the clip anyway, so + // OnAudioFilterRead still delivers the output rate either way. + private int SupportedCaptureFrequency() + { + var requested = AudioSettings.outputSampleRate; + Microphone.GetDeviceCaps(_deviceName, out var min, out var max); + + // Unity reports 0/0 when the device accepts any frequency. + if (min == 0 && max == 0) return requested; + + var clamped = Mathf.Clamp(requested, min, max); + if (clamped != requested) + Debug.Log($"[AEC] capture rate clamped {requested} -> {clamped} (device caps {min}-{max})"); + + return clamped; + } + + private IEnumerator StopMicrophone() + { + if (Microphone.IsRecording(_deviceName)) + Microphone.End(_deviceName); + + if (_sourceObject != null) + { + var probe = _sourceObject.GetComponent(); + if (probe != null) + { + probe.AudioRead -= OnCapturedAudio; + UnityEngine.Object.Destroy(probe); + } + + var source = _sourceObject.GetComponent(); + if (source != null) + UnityEngine.Object.Destroy(source); + } + + Debug.Log($"[AEC] microphone '{_deviceName}' stopped"); + yield return null; + } + + // Unity audio thread. A block the canceller cannot take is published unchanged rather than + // dropped — an un-cancelled participant beats a silent one. + private void OnCapturedAudio(float[] data, int channels, int sampleRate) + { + if (_canceller != null && _canceller.TryPushCapture(data, channels, sampleRate)) return; + + AudioRead?.Invoke(data, channels, sampleRate); + } + + // Unity audio thread, via the capture pump. + private void OnProcessedAudio(float[] data, int channels, int sampleRate) + { + AudioRead?.Invoke(data, channels, sampleRate); + } + + private void OnApplicationPause(bool pause) + { + if (!_started) return; + + if (pause) + { + // Backgrounded, release the audio resources — leaving them open trips + // AVAudioSession interruption errors (FigCaptureSourceRemote -17281). + _canceller?.Stop(); + RunCoroutine(StopMicrophone()); + } + else + { + RunCoroutine(RestartMicrophone()); + } + } + + private IEnumerator RestartMicrophone() + { + yield return StopMicrophone(); + + // After a resume the iOS audio session needs time to recover from interruption. Poll for + // actual readiness instead of guessing a delay. + yield return WaitForMicrophoneReady(); + + _canceller?.Start(); + yield return StartMicrophone(); + } + + private IEnumerator WaitForMicrophoneReady() + { + const float timeout = 2f; + var elapsed = 0f; + + while (Microphone.devices.Length == 0 && elapsed < timeout) + { + yield return new WaitForSeconds(0.05f); + elapsed += 0.05f; + } + + if (Microphone.devices.Length == 0) + { + Debug.LogError($"[AEC] microphone devices not available after {timeout}s"); + yield break; + } + + yield return null; + } + + // The host is a component on the (caller-owned) microphone GameObject. If that object is + // already gone — scene unload, app quit — drain the coroutine synchronously so Microphone.End + // and the component cleanup still run, as the SDK's MonoBehaviourContext does. + private void RunCoroutine(IEnumerator coroutine) + { + if (_host != null) + { + _host.StartCoroutine(coroutine); + return; + } + + while (coroutine.MoveNext()) + { + if (coroutine.Current is IEnumerator nested) + RunCoroutine(nested); + } + } + + protected override void Dispose(bool disposing) + { + if (!_disposed && disposing) Stop(); + _disposed = true; + + if (_canceller != null) + { + _canceller.CaptureProcessed -= OnProcessedAudio; + _canceller.Dispose(); + } + + base.Dispose(disposing); + } + + ~AecMicrophoneSource() + { + Dispose(false); + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta new file mode 100644 index 00000000..83f422c2 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 582d502ba6935413ebc5cc0683c21059 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs new file mode 100644 index 00000000..7159b8d8 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs @@ -0,0 +1,168 @@ +using System; +using System.Runtime.InteropServices; +using LiveKit; + +/// Processes one 10 ms interleaved int16 chunk in place at . +internal delegate string ApmChunkHandler(IntPtr dataPtr, int byteCount, int sampleRate, int channels); + +/// Receives one processed 10 ms chunk as interleaved floats. +internal delegate void ProcessedChunkHandler(float[] data, int channels, int sampleRate); + +/// +/// Re-chunks a variable-size PCM feed into the fixed 10 ms frames the APM requires, processes +/// each one in place, and optionally hands the result on. +/// +/// Neither feed is 10 ms natively: capture arrives in DSP-buffer-sized blocks (1024 frames +/// ≈ 21.3 ms at 48 kHz) and render frames arrive at whatever the decoder emits. Geometry is +/// taken from the incoming buffers, never from declared constants. +/// +/// Allocation-free once the format has settled. The chunk buffer stays pinned for the pump's +/// lifetime so the APM has a stable address to process in place. +/// +internal sealed class ApmChunkPump : IDisposable +{ + private readonly ApmChunkHandler _process; + private readonly ProcessedChunkHandler _onProcessed; + private readonly int _bufferedChunks; + + private PcmRingBuffer _ring; + private short[] _chunk; + private GCHandle _chunkPin; + private IntPtr _chunkPtr; + private int _chunkSamples; + private int _chunkBytes; + + private short[] _staging; + private float[] _processed; + + private int _sampleRate; + private int _channels; + private bool _disposed; + + public int SampleRate => _sampleRate; + public int Channels => _channels; + public int ChunkSamples => _chunkSamples; + public int ProcessedChunkCount { get; private set; } + public int FailedChunkCount { get; private set; } + public int DroppedSamples => _ring?.OverflowSamples ?? 0; + public string LastError { get; private set; } + + /// Processes one chunk in place; returns an error string or null. + /// Optional consumer of the processed chunk (capture side only). + /// + /// Ring capacity in 10 ms chunks. Sets the worst-case added latency, so keep it just large + /// enough to absorb one input block plus jitter. + /// + public ApmChunkPump(ApmChunkHandler process, ProcessedChunkHandler onProcessed = null, int bufferedChunks = 8) + { + if (bufferedChunks <= 0) throw new ArgumentOutOfRangeException(nameof(bufferedChunks)); + _process = process ?? throw new ArgumentNullException(nameof(process)); + _onProcessed = onProcessed; + _bufferedChunks = bufferedChunks; + } + + /// Feeds interleaved floats (Unity capture path). + public void Push(float[] data, int channels, int sampleRate) + { + if (_disposed || data == null || data.Length == 0) return; + if (!EnsureFormat(sampleRate, channels, data.Length)) return; + + if (_staging == null || _staging.Length < data.Length) _staging = new short[data.Length]; + for (var i = 0; i < data.Length; i++) _staging[i] = FloatToS16(data[i]); + + _ring.Write(_staging, 0, data.Length); + Drain(); + } + + /// Feeds interleaved int16 straight from a native buffer (FFI render path). + public void Push(IntPtr dataPtr, int samplesPerChannel, int channels, int sampleRate) + { + if (_disposed || dataPtr == IntPtr.Zero || samplesPerChannel <= 0 || channels <= 0) return; + + var total = samplesPerChannel * channels; + if (!EnsureFormat(sampleRate, channels, total)) return; + + if (_staging == null || _staging.Length < total) _staging = new short[total]; + Marshal.Copy(dataPtr, _staging, 0, total); + + _ring.Write(_staging, 0, total); + Drain(); + } + + public void Reset() => _ring?.Clear(); + + // An FFI failure must not escape: on the capture side this runs inside OnAudioFilterRead, and + // an exception there takes out Unity's audio callback. The unprocessed chunk is forwarded + // instead, so a broken APM degrades to no cancellation rather than to no audio. + private void Drain() + { + while (_ring.TryDrain(_chunk, _chunkSamples)) + { + try + { + var error = _process(_chunkPtr, _chunkBytes, _sampleRate, _channels); + if (error != null) LastError = error; + } + catch (Exception e) + { + LastError = e.Message; + FailedChunkCount++; + } + + ProcessedChunkCount++; + + if (_onProcessed == null) continue; + + for (var i = 0; i < _chunkSamples; i++) _processed[i] = _chunk[i] / 32768f; + _onProcessed(_processed, _channels, _sampleRate); + } + } + + private bool EnsureFormat(int sampleRate, int channels, int incomingSamples) + { + if (sampleRate <= 0 || channels <= 0) return false; + if (sampleRate == _sampleRate && channels == _channels) return true; + + var frameSize = AudioProcessingModule.FrameSizeFor(sampleRate); + if (frameSize <= 0) return false; + + ReleaseChunk(); + + _sampleRate = sampleRate; + _channels = channels; + _chunkSamples = frameSize * channels; + _chunkBytes = _chunkSamples * sizeof(short); + + _chunk = new short[_chunkSamples]; + _chunkPin = GCHandle.Alloc(_chunk, GCHandleType.Pinned); + _chunkPtr = _chunkPin.AddrOfPinnedObject(); + _processed = new float[_chunkSamples]; + + // Never smaller than one input block, or a large block would immediately overflow. + var capacity = Math.Max(_chunkSamples * _bufferedChunks, incomingSamples + _chunkSamples); + _ring = new PcmRingBuffer(capacity); + return true; + } + + private void ReleaseChunk() + { + if (_chunkPin.IsAllocated) _chunkPin.Free(); + _chunkPtr = IntPtr.Zero; + _chunk = null; + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + ReleaseChunk(); + } + + private static short FloatToS16(float v) + { + v *= 32768f; + if (v > 32767f) v = 32767f; + else if (v < -32768f) v = -32768f; + return (short)(v + Math.Sign(v) * 0.5f); + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta new file mode 100644 index 00000000..99d2ca85 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 3213055d7f9af4da4b525a2d8b9f9a1d +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs new file mode 100644 index 00000000..3a4cdeb2 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs @@ -0,0 +1,64 @@ +using System; +using System.Runtime.InteropServices; + +/// +/// Computes the set_stream_delay_ms seed: +/// (t_render - t_analyze) + (t_process - t_capture). +/// +/// AEC3 runs its own correlation-based delay estimator +/// (use_external_delay_estimator = false), so this is a convergence hint, not a hard +/// alignment — but a wildly wrong value slows convergence. iOS sources the platform terms from +/// AVAudioSession (see Plugins/iOS/AudioSessionLatency.mm); Android has no +/// equivalent accessor, so it starts from the buffering this pipeline adds and lets AEC3 find +/// the rest. +/// +/// Known gap: the reference is tapped on the decoder side, so any playout queue between the +/// decoder and the loudspeaker (Unity's AudioStream ring buffer, ~30-200 ms) is not part of +/// this estimate. +/// +internal static class AudioProcessingDelaySeed +{ +#if UNITY_IOS && !UNITY_EDITOR + [DllImport("__Internal")] private static extern double MeetSample_AudioSessionOutputLatency(); + [DllImport("__Internal")] private static extern double MeetSample_AudioSessionInputLatency(); + [DllImport("__Internal")] private static extern double MeetSample_AudioSessionIOBufferDuration(); +#endif + + private const int MinDelayMs = 0; + private const int MaxDelayMs = 500; + + /// Seed used when no platform latency is readable (Android, or an inactive session). + private const int FallbackDelayMs = 60; + + /// + /// Measured duration of one OnAudioFilterRead block, which is how far behind + /// real-time the capture stream already is when it reaches the APM. + /// + public static int Estimate(double captureBlockMs) + { + var platformMs = PlatformLatencyMs(); + var seed = (int)Math.Round(platformMs + captureBlockMs); + return Math.Min(MaxDelayMs, Math.Max(MinDelayMs, seed)); + } + + public static double PlatformLatencyMs() + { +#if UNITY_IOS && !UNITY_EDITOR + try + { + var output = MeetSample_AudioSessionOutputLatency(); + var input = MeetSample_AudioSessionInputLatency(); + var ioBuffer = MeetSample_AudioSessionIOBufferDuration(); + if (output > 0d || input > 0d) + return (output + input + ioBuffer) * 1000d; + } + catch (Exception) + { + // Session not yet active; fall through to the platform-agnostic seed. + } + return FallbackDelayMs; +#else + return FallbackDelayMs; +#endif + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta new file mode 100644 index 00000000..086e91bd --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9636ad91efd914976b3ed12df25e7e8c +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs new file mode 100644 index 00000000..859260bd --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs @@ -0,0 +1,61 @@ +using System; +using System.Runtime.InteropServices; +using LiveKit; +using UnityEngine; + +/// +/// Go/no-go check for the FFI , runnable without joining a +/// room. Exercises the full call shape: create a handle, seed a delay, and process one 10 ms +/// chunk in each direction. The APM is compiled into each platform's FFI binary separately, so +/// a per-platform surprise is far cheaper to find here than after the pipeline is wired. +/// +internal static class AudioProcessingSmokeTest +{ + public static string Run() + { + var rate = AudioSettings.outputSampleRate > 0 ? AudioSettings.outputSampleRate : 48000; + var frameSize = AudioProcessingModule.FrameSizeFor(rate); + + AudioProcessingModule apm; + try + { + apm = new AudioProcessingModule( + echoCancellerEnabled: true, + gainControllerEnabled: false, + highPassFilterEnabled: false, + noiseSuppressionEnabled: false); + } + catch (Exception e) + { + return $"FAIL create_apm: {e.GetType().Name}: {e.Message}"; + } + + using (apm) + { + var delayError = apm.SetStreamDelayMs(100); + if (delayError != null) return $"FAIL set_stream_delay_ms: {delayError}"; + + var chunk = new short[frameSize]; + var pin = GCHandle.Alloc(chunk, GCHandleType.Pinned); + try + { + var ptr = pin.AddrOfPinnedObject(); + var bytes = chunk.Length * sizeof(short); + + var reverseError = apm.ProcessReverseStream(ptr, bytes, rate, 1); + if (reverseError != null) return $"FAIL process_reverse_stream: {reverseError}"; + + var processError = apm.ProcessStream(ptr, bytes, rate, 1); + if (processError != null) return $"FAIL process_stream: {processError}"; + } + finally + { + pin.Free(); + } + + return $"PASS handle={apm.Handle} rate={rate} frameSize={frameSize} " + + $"nativeRate={AudioProcessingModule.IsNativeSampleRate(rate)} " + + $"delaySeed={AudioProcessingDelaySeed.Estimate(0d)}ms"; + } + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta new file mode 100644 index 00000000..cf3748bf --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 78f2bfe939fd54d97b797b6b28398e88 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs b/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs new file mode 100644 index 00000000..f4b3e8fd --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs @@ -0,0 +1,100 @@ +using System; + +/// +/// Fixed-capacity interleaved int16 PCM ring buffer with a fixed-size drain. +/// +/// Allocation-free after construction — both feeds run on audio-priority threads. Sized in +/// samples (interleaved, i.e. frames × channels), not frames. +/// +/// Single producer / single consumer per direction; the near-end and far-end feeds each own +/// their own instance, so no synchronisation is needed inside. +/// +internal sealed class PcmRingBuffer +{ + private readonly short[] _buffer; + private int _readIndex; + private int _writeIndex; + private int _count; + + /// Samples dropped because the buffer was full, since construction. + public int OverflowSamples { get; private set; } + + public int Capacity => _buffer.Length; + public int Available => _count; + + public PcmRingBuffer(int capacitySamples) + { + if (capacitySamples <= 0) throw new ArgumentOutOfRangeException(nameof(capacitySamples)); + _buffer = new short[capacitySamples]; + } + + /// + /// Appends samples. When the buffer is full the OLDEST samples are + /// dropped: a stalled consumer must not push the echo reference arbitrarily far out of + /// alignment with the capture stream. + /// + public void Write(short[] source, int offset, int count) + { + if (source == null) throw new ArgumentNullException(nameof(source)); + if (offset < 0 || count < 0 || offset + count > source.Length) + throw new ArgumentOutOfRangeException(nameof(count)); + + if (count >= _buffer.Length) + { + OverflowSamples += _count + count - _buffer.Length; + offset += count - _buffer.Length; + count = _buffer.Length; + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + else + { + var free = _buffer.Length - _count; + if (count > free) Discard(count - free); + } + + for (var i = 0; i < count; i++) + { + _buffer[_writeIndex] = source[offset + i]; + _writeIndex = _writeIndex + 1 == _buffer.Length ? 0 : _writeIndex + 1; + } + + _count += count; + } + + /// + /// Copies exactly samples into and + /// consumes them. Returns false and consumes nothing when fewer are available. + /// + public bool TryDrain(short[] destination, int count) + { + if (destination == null) throw new ArgumentNullException(nameof(destination)); + if (count < 0 || count > destination.Length) throw new ArgumentOutOfRangeException(nameof(count)); + if (_count < count) return false; + + for (var i = 0; i < count; i++) + { + destination[i] = _buffer[_readIndex]; + _readIndex = _readIndex + 1 == _buffer.Length ? 0 : _readIndex + 1; + } + + _count -= count; + return true; + } + + public void Clear() + { + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + + private void Discard(int count) + { + if (count > _count) count = _count; + _readIndex = (_readIndex + count) % _buffer.Length; + _count -= count; + OverflowSamples += count; + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta new file mode 100644 index 00000000..8c579352 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6a4b6b6ecf98e4447ab88686c0058b9a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index 1dc27c71..c0bf6c0a 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -13,7 +13,9 @@ /// - PlatformAudio (default): Uses WebRTC's ADM for microphone capture and automatic /// speaker playout. Provides echo cancellation (AEC), AGC, and noise suppression. /// - Unity Audio: Uses Unity's Microphone API and AudioStream for manual audio handling. -/// No AEC support but gives more control over audio processing. +/// Gives more control over audio processing. Optionally runs libwebrtc's AEC3 over the +/// captured audio (), using the decoded remote audio frames +/// as the echo reference; without it there is no echo cancellation in this mode. /// [RequireComponent(typeof(TokenSourceComponent))] public class MeetManager : MonoBehaviour @@ -44,6 +46,18 @@ public class MeetManager : MonoBehaviour [Tooltip("Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] [SerializeField] private bool preferHardwareProcessing = true; + [Header("Unity Audio (PlatformAudio off)")] + [Tooltip("Run libwebrtc's AEC3 (the FFI AudioProcessingModule) over Unity microphone capture, " + + "using the decoded remote audio frames as the echo reference. Assumes a single remote " + + "audio stream: with several remote speakers the reference is the interleaving of all of " + + "them and the canceller will not converge.")] + [SerializeField] private bool unityEchoCancellation = true; + [Tooltip("Playback gain for every remote AudioSource in Unity audio mode. Kept below 1 so a device " + + "at full speaker volume keeps headroom: full-scale playout distorts on Android and feeds " + + "the echo canceller more echo than it can remove. Linear amplitude, 0.7 is -3.1 dB.")] + [Range(0f, 1f)] + [SerializeField] private float remoteAudioGain = 0.7f; + private const string PlaceholderTextureResourceName = "PlaceholderTileSquare"; private Texture _placeholderTexture; @@ -90,6 +104,8 @@ private void Start() if (usePlatformAudio) InitializePlatformAudio(); + else if (unityEchoCancellation) + Debug.Log($"AEC smoke test: {AudioProcessingSmokeTest.Run()}"); } private void InitializePlatformAudio() @@ -372,6 +388,7 @@ private void AddRemoteAudioTrack(RemoteAudioTrack audioTrack) audioObject.transform.SetParent(_audioTrackParent); var source = audioObject.AddComponent(); + source.volume = remoteAudioGain; var audiostream = new AudioStream(audioTrack, source); _audioStreams.Add(sid, audiostream); @@ -605,15 +622,22 @@ private IEnumerator PublishLocalMicrophonePlatform() private IEnumerator PublishLocalMicrophoneUnity() { - Debug.Log("Publishing microphone using Unity Microphone API"); + Debug.Log($"Publishing microphone using Unity Microphone API (AEC3: {unityEchoCancellation})"); // Start the microphone here for early iOS permission request and android getting access to Microphone.devices Microphone.Start(null, true, 10, 44100); - + var audioObject = new GameObject($"My Microphone: {Microphone.devices[0]}"); audioObject.transform.SetParent(_audioTrackParent); - var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject); + // AecMicrophoneSource re-implements MicrophoneSource with an APM stage in between: + // captured blocks go through AEC3 (reference = decoded remote frames) before they reach + // the track. If the APM cannot be created it publishes the raw microphone instead. + RtcAudioSource rtcSource; + if (unityEchoCancellation) + rtcSource = AecMicrophoneSource.Create(Microphone.devices[0], audioObject); + else + rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject); _localAudioTrack = LocalAudioTrack.CreateAudioTrack(LocalAudioTrackName, rtcSource, _room); @@ -628,6 +652,9 @@ private IEnumerator PublishLocalMicrophoneUnity() if (publish.IsError) { + // Dispose before destroying the host object so the source (and, for AEC, its APM + // handle) is released now rather than by the finalizer. + rtcSource.Dispose(); Destroy(audioObject); _localAudioTrack = null; yield break; @@ -638,7 +665,9 @@ private IEnumerator PublishLocalMicrophoneUnity() _localRtcAudioSource = rtcSource; rtcSource.Start(); - Debug.Log("Microphone published via Unity Microphone API (no AEC)"); + Debug.Log(rtcSource is AecMicrophoneSource { EchoCancellationActive: true } + ? "Microphone published via Unity Microphone API (AEC3 active)" + : "Microphone published via Unity Microphone API (no AEC)"); } private void UnpublishLocalMicrophone() diff --git a/Samples~/Meet/Assets/Scenes/MeetApp.unity b/Samples~/Meet/Assets/Scenes/MeetApp.unity index 91fc1e07..3122a7c1 100644 --- a/Samples~/Meet/Assets/Scenes/MeetApp.unity +++ b/Samples~/Meet/Assets/Scenes/MeetApp.unity @@ -902,11 +902,13 @@ MonoBehaviour: videoTrackParent: {fileID: 2128321498} participantTilePrefab: {fileID: 4315784896331113596, guid: bec493bbc3d574c07b5bbf8dd2be26b3, type: 3} frameRate: 30 - usePlatformAudio: 1 + usePlatformAudio: 0 echoCancellation: 1 noiseSuppression: 1 autoGainControl: 1 preferHardwareProcessing: 1 + unityEchoCancellation: 1 + remoteAudioGain: 0.7 --- !u!114 &1478206705 MonoBehaviour: m_ObjectHideFlags: 0 @@ -919,7 +921,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: a498c208deeab40c39b4ba609d7d222c, type: 3} m_Name: m_EditorClassIdentifier: - _config: {fileID: 11400000, guid: 1a1b8efb5101449b280f574f853b7459, type: 2} + _config: {fileID: 11400000, guid: 6d52b5cb4c971436098df568ef2e2c67, type: 2} --- !u!1 &1651282853 GameObject: m_ObjectHideFlags: 0 From 935c8bb84d83fa384c10dbf4183bb342e2492dd1 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:13:47 +0200 Subject: [PATCH 2/4] Code moved to SDK --- README.md | 17 + Runtime/Plugins/iOS/LiveKitAudioSession.mm | 14 + .../Scripts/Audio/AudioProcessingDelayHint.cs | 78 ++++ .../Audio/AudioProcessingDelayHint.cs.meta | 2 +- Runtime/Scripts/Audio/AudioProcessingStats.cs | 87 +++++ .../AudioProcessingStats.cs.meta} | 2 +- Runtime/Scripts/Audio/AudioProcessor.cs | 350 ++++++++++++++++++ .../Scripts/Audio/AudioProcessor.cs.meta | 2 +- Runtime/Scripts/Audio/MicrophoneSource.cs | 29 ++ Runtime/Scripts/Audio/PcmRingBuffer.cs | 118 ++++++ .../Scripts}/Audio/PcmRingBuffer.cs.meta | 2 +- Runtime/Scripts/Audio/PlatformAudioSource.cs | 20 +- Runtime/Scripts/Audio/PlayoutReference.cs | 149 ++++++++ .../Scripts/Audio/PlayoutReference.cs.meta | 2 +- Runtime/Scripts/Audio/RtcAudioSource.cs | 103 ++++-- Runtime/Scripts/Core/FfiFrameObserver.cs | 160 -------- Runtime/Scripts/Internal/FFI/FFIClient.cs | 4 - .../Meet/Assets/Editor/MeetManagerEditor.cs | 39 +- Samples~/Meet/Assets/Plugins/iOS.meta | 8 - .../Assets/Plugins/iOS/AudioSessionLatency.mm | 18 - .../Plugins/iOS/AudioSessionLatency.mm.meta | 33 -- Samples~/Meet/Assets/Runtime/Audio.meta | 8 - .../Runtime/Audio/AcousticEchoCanceller.cs | 220 ----------- .../Assets/Runtime/Audio/AecAudioProbe.cs | 51 --- .../Assets/Runtime/Audio/AecMicrophoneHost.cs | 22 -- .../Runtime/Audio/AecMicrophoneSource.cs | 304 --------------- .../Runtime/Audio/AecMicrophoneSource.cs.meta | 11 - .../Meet/Assets/Runtime/Audio/ApmChunkPump.cs | 168 --------- .../Assets/Runtime/Audio/ApmChunkPump.cs.meta | 11 - .../Runtime/Audio/AudioProcessingDelaySeed.cs | 64 ---- .../Audio/AudioProcessingDelaySeed.cs.meta | 11 - .../Runtime/Audio/AudioProcessingSmokeTest.cs | 61 --- .../Audio/AudioProcessingSmokeTest.cs.meta | 11 - .../Assets/Runtime/Audio/PcmRingBuffer.cs | 100 ----- Samples~/Meet/Assets/Runtime/MeetManager.cs | 67 ++-- Samples~/Meet/Assets/Scenes/MeetApp.unity | 3 +- Tests/EditMode/AudioProcessingTests.cs | 155 ++++++++ Tests/EditMode/AudioProcessingTests.cs.meta | 11 + Tests/EditMode/PlatformAudioTests.cs | 1 + Tests/PlayMode/AudioProcessingTests.cs | 241 ++++++++++++ Tests/PlayMode/AudioProcessingTests.cs.meta | 11 + 41 files changed, 1423 insertions(+), 1345 deletions(-) create mode 100644 Runtime/Scripts/Audio/AudioProcessingDelayHint.cs rename Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta => Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta (83%) create mode 100644 Runtime/Scripts/Audio/AudioProcessingStats.cs rename Runtime/Scripts/{Core/FfiFrameObserver.cs.meta => Audio/AudioProcessingStats.cs.meta} (83%) create mode 100644 Runtime/Scripts/Audio/AudioProcessor.cs rename Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta => Runtime/Scripts/Audio/AudioProcessor.cs.meta (83%) create mode 100644 Runtime/Scripts/Audio/PcmRingBuffer.cs rename {Samples~/Meet/Assets/Runtime => Runtime/Scripts}/Audio/PcmRingBuffer.cs.meta (83%) create mode 100644 Runtime/Scripts/Audio/PlayoutReference.cs rename Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta => Runtime/Scripts/Audio/PlayoutReference.cs.meta (83%) delete mode 100644 Runtime/Scripts/Core/FfiFrameObserver.cs delete mode 100644 Samples~/Meet/Assets/Plugins/iOS.meta delete mode 100644 Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm delete mode 100644 Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs create mode 100644 Tests/EditMode/AudioProcessingTests.cs create mode 100644 Tests/EditMode/AudioProcessingTests.cs.meta create mode 100644 Tests/PlayMode/AudioProcessingTests.cs create mode 100644 Tests/PlayMode/AudioProcessingTests.cs.meta diff --git a/README.md b/README.md index 284dee4c..ce91509a 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,23 @@ IEnumerator PublishLocalMicrophoneUnity(Room room) } ``` +#### Unity Audio Processing + +Unity's `Microphone` path does not go through a platform audio device module, so on its own it has no echo cancellation. Pass `AudioProcessingOptions` to run libwebrtc's audio processing (AEC3 echo cancellation, noise suppression, gain control, high-pass filter) over the captured audio before it reaches the track: + +```cs +var processing = new AudioProcessingOptions +{ + EchoCancellation = true, + NoiseSuppression = true, + AutoGainControl = true, + HighPassFilter = true +}; +var rtcSource = new MicrophoneSource(Microphone.devices[0], microphoneObject, processing); +``` + +Echo cancellation takes its reference from the final mix Unity plays, so it covers every remote `AudioStream` as well as the game's own audio. The SDK attaches a `PlayoutReference` component to the active `AudioListener` for that; adding it to the listener yourself does the same. `rtcSource.AudioProcessingStats` reports whether the reference is attached and how many chunks flowed. Unity's output sample rate must be a multiple of 100 Hz (48000, 44100 and 24000 all are); otherwise processing is bypassed with a warning and the raw microphone is published. + #### Unity Audio Output ```cs diff --git a/Runtime/Plugins/iOS/LiveKitAudioSession.mm b/Runtime/Plugins/iOS/LiveKitAudioSession.mm index f341fc56..c18515ff 100644 --- a/Runtime/Plugins/iOS/LiveKitAudioSession.mm +++ b/Runtime/Plugins/iOS/LiveKitAudioSession.mm @@ -67,4 +67,18 @@ void LiveKit_RestoreDefaultAudioSession() { } } +/// AVAudioSession latency terms, in seconds. Used to seed the echo canceller's stream delay in +/// Unity-audio mode (see AudioProcessingDelayHint.cs). All report 0 until the session is active. +double LiveKit_AudioSessionOutputLatency() { + return [[AVAudioSession sharedInstance] outputLatency]; +} + +double LiveKit_AudioSessionInputLatency() { + return [[AVAudioSession sharedInstance] inputLatency]; +} + +double LiveKit_AudioSessionIOBufferDuration() { + return [[AVAudioSession sharedInstance] IOBufferDuration]; +} + } diff --git a/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs b/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs new file mode 100644 index 00000000..feaf914b --- /dev/null +++ b/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs @@ -0,0 +1,78 @@ +using System; +using System.Runtime.InteropServices; +using UnityEngine; + +namespace LiveKit +{ + /// + /// Estimates the render-to-capture delay hint handed to + /// . + /// + /// + /// AEC3 runs its own correlation-based delay estimator; the hint only sets the initial + /// alignment after a reset, so it has to be in the right ballpark rather than exact. The echo + /// path in Unity-audio mode is: tap → Unity's output queue (a + /// few DSP blocks) → device output → air → device input → Microphone clip → the + /// AudioSource reading that clip → capture probe. Only the DSP block size and, on iOS, the + /// audio session latencies are readable; the remaining terms are constants. + /// + /// Main thread only: reads . + /// + internal static class AudioProcessingDelayHint + { +#if UNITY_IOS && !UNITY_EDITOR + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionOutputLatency(); + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionInputLatency(); + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionIOBufferDuration(); +#endif + + internal const int MinDelayMs = 0; + internal const int MaxDelayMs = 500; + + /// Output queue depth assumed between the listener tap and the device, in DSP blocks. + internal const int OutputQueueBlocks = 2; + + /// + /// How far the AudioSource reading the microphone clip trails the clip's write head. + /// starts reading once Microphone.GetPosition first + /// reports data, polled at 50 ms, and that offset persists for the life of the clip. + /// + internal const int MicrophoneReadBehindMs = 50; + + /// Device input plus output latency when the platform does not report it. + internal const int FallbackDeviceLatencyMs = 30; + + public static int EstimateMs() + { + var config = AudioSettings.GetConfiguration(); + return EstimateMs(config.dspBufferSize, config.sampleRate, PlatformLatencyMs()); + } + + internal static int EstimateMs(int dspBufferSize, int sampleRate, double deviceLatencyMs) + { + var blockMs = sampleRate > 0 ? dspBufferSize * 1000.0 / sampleRate : 0.0; + var estimate = blockMs * OutputQueueBlocks + deviceLatencyMs + MicrophoneReadBehindMs; + return (int)Math.Round(Math.Min(MaxDelayMs, Math.Max(MinDelayMs, estimate))); + } + + internal static double PlatformLatencyMs() + { +#if UNITY_IOS && !UNITY_EDITOR + try + { + // AVAudioSession reports zero until the session is active, hence the fallback. + var output = LiveKit_AudioSessionOutputLatency(); + var input = LiveKit_AudioSessionInputLatency(); + var ioBuffer = LiveKit_AudioSessionIOBufferDuration(); + if (output > 0d || input > 0d) + return (output + input + ioBuffer) * 1000d; + } + catch (Exception) + { + // Fall through to the constant. + } +#endif + return FallbackDeviceLatencyMs; + } + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta b/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta similarity index 83% rename from Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta rename to Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta index 6c3e6728..c97cd183 100644 --- a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta +++ b/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 21b265f6c78d54d0ebae5f213cee36a1 +guid: b1bb08510396f4e28b12569471ea0cf6 MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Runtime/Scripts/Audio/AudioProcessingStats.cs b/Runtime/Scripts/Audio/AudioProcessingStats.cs new file mode 100644 index 00000000..0362a81c --- /dev/null +++ b/Runtime/Scripts/Audio/AudioProcessingStats.cs @@ -0,0 +1,87 @@ +namespace LiveKit +{ + /// + /// Counters from the audio processing stage of an that was + /// created with . A snapshot; read it from any thread via + /// . + /// + public readonly struct AudioProcessingStats + { + /// + /// True while capture is actually being processed. False when the source has no processing + /// stage, is stopped, or bypasses processing because Unity's output sample rate has no + /// whole-sample 10 ms chunk (it is not a multiple of 100 Hz). + /// + public readonly bool Active; + + /// + /// True while a on an enabled AudioListener is feeding + /// the far-end reference. Without it echo cancellation has nothing to cancel; noise + /// suppression, gain control and the high-pass filter still run. + /// + public readonly bool ReferenceAttached; + + /// Format of the capture feed, as delivered by Unity's audio graph. + public readonly int CaptureSampleRate; + public readonly int CaptureChannels; + + /// Format of the playout reference feed. + public readonly int ReferenceSampleRate; + public readonly int ReferenceChannels; + + /// 10 ms capture chunks run through the module. + public readonly long CaptureChunks; + + /// 10 ms reference chunks run through the module. + public readonly long ReferenceChunks; + + /// Samples discarded because a feed outran its buffer. Non-zero means the audio thread stalled. + public readonly long DroppedCaptureSamples; + public readonly long DroppedReferenceSamples; + + /// Chunks the module rejected; those capture chunks were published unprocessed. + public readonly long FailedChunks; + + /// Most recent module error, or null. + public readonly string LastError; + + /// Render-to-capture delay hint last handed to the module, or -1 if none yet. + public readonly int StreamDelayHintMs; + + public AudioProcessingStats( + bool active, + bool referenceAttached, + int captureSampleRate, + int captureChannels, + int referenceSampleRate, + int referenceChannels, + long captureChunks, + long referenceChunks, + long droppedCaptureSamples, + long droppedReferenceSamples, + long failedChunks, + string lastError, + int streamDelayHintMs) + { + Active = active; + ReferenceAttached = referenceAttached; + CaptureSampleRate = captureSampleRate; + CaptureChannels = captureChannels; + ReferenceSampleRate = referenceSampleRate; + ReferenceChannels = referenceChannels; + CaptureChunks = captureChunks; + ReferenceChunks = referenceChunks; + DroppedCaptureSamples = droppedCaptureSamples; + DroppedReferenceSamples = droppedReferenceSamples; + FailedChunks = failedChunks; + LastError = lastError; + StreamDelayHintMs = streamDelayHintMs; + } + + public override string ToString() => + $"active={Active} reference={ReferenceAttached} " + + $"capture={CaptureChannels}ch@{CaptureSampleRate} chunks={CaptureChunks} dropped={DroppedCaptureSamples} | " + + $"reference={ReferenceChannels}ch@{ReferenceSampleRate} chunks={ReferenceChunks} dropped={DroppedReferenceSamples} | " + + $"failed={FailedChunks} lastError={LastError ?? "-"} delayHint={StreamDelayHintMs}ms"; + } +} diff --git a/Runtime/Scripts/Core/FfiFrameObserver.cs.meta b/Runtime/Scripts/Audio/AudioProcessingStats.cs.meta similarity index 83% rename from Runtime/Scripts/Core/FfiFrameObserver.cs.meta rename to Runtime/Scripts/Audio/AudioProcessingStats.cs.meta index e35c2080..3174751e 100644 --- a/Runtime/Scripts/Core/FfiFrameObserver.cs.meta +++ b/Runtime/Scripts/Audio/AudioProcessingStats.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 50292b03c8d4441118cfe20a865e0e27 +guid: 8515244a296964046ae7f54887a9aa1b MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Runtime/Scripts/Audio/AudioProcessor.cs b/Runtime/Scripts/Audio/AudioProcessor.cs new file mode 100644 index 00000000..f9d8d34f --- /dev/null +++ b/Runtime/Scripts/Audio/AudioProcessor.cs @@ -0,0 +1,350 @@ +using System; +using System.Collections; +using System.Runtime.InteropServices; +using System.Threading; +using LiveKit.Internal; +using LiveKit.Internal.FFI; +using LiveKit.Internal.Threading; +using Unity.Collections; +using Unity.Collections.LowLevel.Unsafe; +using UnityEngine; + +namespace LiveKit +{ + /// + /// The processing stage behind on an + /// : owns one , re-chunks the + /// capture and the playout reference into 10 ms frames, and hands every processed capture + /// chunk to the source for the FFI. + /// + /// + /// Threading. (from the source's audio callback) and + /// (from ) both run on the Unity + /// audio thread. Within one DSP tick the capture probes run before the listener tap, so the + /// reference for tick N arrives after the capture of tick N; that is fine because the acoustic + /// echo of tick N's playout only reaches the microphone several ticks later. , + /// and the maintenance coroutine run on the main thread and never touch the + /// ring buffers; they raise flags the audio thread acts on. Nothing here logs from the audio + /// thread — diagnostics are counters, read via . + /// + /// The Rust side asserts (and takes the process down) on a frame that is not a whole multiple + /// of 10 ms, so the chunking here is not optional, and rates whose 10 ms chunk is not a whole + /// number of samples are bypassed entirely. + /// + internal sealed class AudioProcessor : IDisposable + { + /// Receives one processed 10 ms chunk and takes ownership of the array. + internal delegate void ProcessedFrameSink(NativeArray frame, int channels, int sampleRate); + + // Ring capacity in chunks. Bounds the latency added when a DSP block is not a multiple of 10 ms. + private const int BufferedChunks = 8; + private const float MaintenanceIntervalSeconds = 2f; + + private readonly AudioProcessingModule _apm; + private readonly ProcessedFrameSink _sink; + private readonly bool _echoCancellation; + + // Guards the pinned reference chunk against Dispose racing an in-flight callback. The + // capture side needs no lock: each chunk lives in its own NativeArray owned by the sink. + private readonly object _referenceLock = new object(); + + // Capture side. Audio thread only. + private PcmRingBuffer _captureRing; + private short[] _captureStaging; + private short[] _captureChunk; + private int _captureRate; + private int _captureChannels; + private int _captureChunkSamples; + + // Reference side. Audio thread only, under _referenceLock. The chunk stays pinned so the + // module has a stable address to process in place. + private PcmRingBuffer _referenceRing; + private short[] _referenceStaging; + private short[] _referenceChunk; + private GCHandle _referenceChunkPin; + private int _referenceRate; + private int _referenceChannels; + private int _referenceChunkSamples; + + // Cross-thread flags. + private volatile bool _running; + private volatile bool _bypass; + private volatile bool _disposed; + private int _captureResetRequested; + private int _referenceResetRequested; + private int _unsupportedRateWarned; + private int _maintenanceGeneration; + + // Counters. + private long _captureChunks; + private long _referenceChunks; + private long _failedChunks; + private volatile string _lastError; + private volatile int _delayHintMs = -1; + + /// The FFI could not create the module. + public AudioProcessor(AudioProcessingOptions options, ProcessedFrameSink sink) + { + _sink = sink ?? throw new ArgumentNullException(nameof(sink)); + _echoCancellation = options.EchoCancellation; + _apm = new AudioProcessingModule( + echoCancellerEnabled: options.EchoCancellation, + gainControllerEnabled: options.AutoGainControl, + highPassFilterEnabled: options.HighPassFilter, + noiseSuppressionEnabled: options.NoiseSuppression); + } + + /// Main thread. + public void Start() + { + if (_disposed || _running) return; + _running = true; + RequestReset(); + + if (_echoCancellation) + { + PlayoutReference.AudioRead += OnPlayoutAudio; + PlayoutReference.Acquire(); + } + + SeedDelayHint(); + MonoBehaviourContext.RunCoroutine(Maintenance(++_maintenanceGeneration)); + } + + /// Main thread. + public void Stop() + { + if (!_running) return; + _running = false; + + if (_echoCancellation) + { + PlayoutReference.AudioRead -= OnPlayoutAudio; + PlayoutReference.Release(); + } + } + + /// + /// Any thread. Clears both feeds before their next audio callback. Call when the capture + /// path restarts (e.g. a microphone resume) so stale samples do not misalign the canceller. + /// + public void RequestReset() + { + Interlocked.Exchange(ref _captureResetRequested, 1); + Interlocked.Exchange(ref _referenceResetRequested, 1); + } + + // Periodic main-thread upkeep: re-attach the reference after scene or device changes and + // refresh the delay hint (iOS reports zero session latency until the session is active). + private IEnumerator Maintenance(int generation) + { + while (_running && !_disposed && generation == _maintenanceGeneration) + { + if (_echoCancellation) PlayoutReference.EnsureAttached(); + SeedDelayHint(); + yield return new WaitForSeconds(MaintenanceIntervalSeconds); + } + } + + private void SeedDelayHint() + { + if (!_echoCancellation || _disposed) return; + + var hint = AudioProcessingDelayHint.EstimateMs(); + if (hint == _delayHintMs) return; + + string error; + try + { + error = _apm.SetStreamDelayMs(hint); + } + catch (ObjectDisposedException) + { + return; + } + + if (error != null) + { + Utils.Warning($"AudioProcessor: set_stream_delay_ms({hint}) failed: {error}"); + return; + } + + _delayHintMs = hint; + } + + /// + /// Unity audio thread. Runs the block through the module in 10 ms chunks and forwards each + /// chunk to the sink. Returns false when the block must go out unprocessed instead: + /// processing is stopped, or bypassed for this sample rate. + /// + public bool TryProcessCapture(float[] data, int channels, int sampleRate) + { + if (_disposed || !_running || _bypass) return false; + if (data == null || data.Length == 0 || channels <= 0 || sampleRate <= 0) return false; + + if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) + { + _bypass = true; + WarnUnsupportedRate(sampleRate); + return false; + } + + if (Interlocked.Exchange(ref _captureResetRequested, 0) == 1) + _captureRing?.Clear(); + + if (_captureStaging == null || sampleRate != _captureRate || channels != _captureChannels || _captureStaging.Length < data.Length) + ConfigureCapture(sampleRate, channels, data.Length); + + for (var i = 0; i < data.Length; i++) + _captureStaging[i] = PcmConvert.FloatToS16(data[i]); + _captureRing.Write(_captureStaging, 0, data.Length); + + while (_captureRing.TryDrain(_captureChunk, _captureChunkSamples)) + { + var frame = new NativeArray(_captureChunkSamples, Allocator.Persistent); + frame.CopyFrom(_captureChunk); + ProcessCaptureChunk(frame, sampleRate, channels); + Interlocked.Increment(ref _captureChunks); + _sink(frame, channels, sampleRate); + } + + return true; + } + + private void ProcessCaptureChunk(NativeArray frame, int sampleRate, int channels) + { + try + { + IntPtr ptr; + unsafe + { + ptr = (IntPtr)NativeArrayUnsafeUtility.GetUnsafePtr(frame); + } + var error = _apm.ProcessStream(ptr, frame.Length * sizeof(short), sampleRate, channels); + if (error != null) RecordFailure(error); + } + catch (Exception e) + { + // The chunk goes out unprocessed rather than not at all. + RecordFailure(e.Message); + } + } + + // Unity audio thread, from PlayoutReference. Must not modify data. + private void OnPlayoutAudio(float[] data, int channels, int sampleRate) + { + if (_disposed || !_running || _bypass) return; + if (data == null || data.Length == 0 || channels <= 0) return; + if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) return; + + lock (_referenceLock) + { + if (_disposed) return; + + if (Interlocked.Exchange(ref _referenceResetRequested, 0) == 1) + _referenceRing?.Clear(); + + if (_referenceStaging == null || sampleRate != _referenceRate || channels != _referenceChannels || _referenceStaging.Length < data.Length) + ConfigureReference(sampleRate, channels, data.Length); + + for (var i = 0; i < data.Length; i++) + _referenceStaging[i] = PcmConvert.FloatToS16(data[i]); + _referenceRing.Write(_referenceStaging, 0, data.Length); + + var byteCount = _referenceChunkSamples * sizeof(short); + while (_referenceRing.TryDrain(_referenceChunk, _referenceChunkSamples)) + { + try + { + var error = _apm.ProcessReverseStream(_referenceChunkPin.AddrOfPinnedObject(), byteCount, sampleRate, channels); + if (error != null) RecordFailure(error); + } + catch (Exception e) + { + RecordFailure(e.Message); + } + Interlocked.Increment(ref _referenceChunks); + } + } + } + + // Format changes are rare (first block, device switch); the allocations here are accepted + // on the audio thread for the same reason AudioStream sizes its buffers lazily. + private void ConfigureCapture(int sampleRate, int channels, int incomingSamples) + { + var chunkSamples = AudioProcessingModule.FrameSizeFor(sampleRate) * channels; + _captureRate = sampleRate; + _captureChannels = channels; + _captureChunkSamples = chunkSamples; + _captureChunk = new short[chunkSamples]; + _captureStaging = new short[incomingSamples]; + // Never smaller than one input block, or a large block would overflow immediately. + _captureRing = new PcmRingBuffer(Math.Max(chunkSamples * BufferedChunks, incomingSamples + chunkSamples)); + } + + private void ConfigureReference(int sampleRate, int channels, int incomingSamples) + { + var chunkSamples = AudioProcessingModule.FrameSizeFor(sampleRate) * channels; + _referenceRate = sampleRate; + _referenceChannels = channels; + _referenceChunkSamples = chunkSamples; + + if (_referenceChunkPin.IsAllocated) _referenceChunkPin.Free(); + _referenceChunk = new short[chunkSamples]; + _referenceChunkPin = GCHandle.Alloc(_referenceChunk, GCHandleType.Pinned); + + _referenceStaging = new short[incomingSamples]; + _referenceRing = new PcmRingBuffer(Math.Max(chunkSamples * BufferedChunks, incomingSamples + chunkSamples)); + } + + private void RecordFailure(string error) + { + _lastError = error; + Interlocked.Increment(ref _failedChunks); + } + + // Logging is not allowed on the audio thread; hand the message to the main thread. + private void WarnUnsupportedRate(int sampleRate) + { + if (Interlocked.Exchange(ref _unsupportedRateWarned, 1) == 1) return; + + var message = $"AudioProcessor: Unity's output sample rate {sampleRate} Hz has no whole-sample 10 ms chunk; " + + "audio processing is bypassed and the capture is published unprocessed."; + var context = FfiClient.Instance._context; + if (context != null) + context.Post(static m => Utils.Warning(m), message); + else + Utils.Warning(message); + } + + public AudioProcessingStats GetStats() => new AudioProcessingStats( + active: _running && !_bypass && !_disposed, + referenceAttached: _echoCancellation && PlayoutReference.IsAttached, + captureSampleRate: _captureRate, + captureChannels: _captureChannels, + referenceSampleRate: _referenceRate, + referenceChannels: _referenceChannels, + captureChunks: Interlocked.Read(ref _captureChunks), + referenceChunks: Interlocked.Read(ref _referenceChunks), + droppedCaptureSamples: _captureRing?.OverflowSamples ?? 0, + droppedReferenceSamples: _referenceRing?.OverflowSamples ?? 0, + failedChunks: Interlocked.Read(ref _failedChunks), + lastError: _lastError, + streamDelayHintMs: _delayHintMs); + + public void Dispose() + { + if (_disposed) return; + + Stop(); + lock (_referenceLock) + { + _disposed = true; + if (_referenceChunkPin.IsAllocated) _referenceChunkPin.Free(); + _referenceChunk = null; + _referenceRing = null; + } + _apm.Dispose(); + } + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta b/Runtime/Scripts/Audio/AudioProcessor.cs.meta similarity index 83% rename from Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta rename to Runtime/Scripts/Audio/AudioProcessor.cs.meta index 0a43fb48..593e25f5 100644 --- a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta +++ b/Runtime/Scripts/Audio/AudioProcessor.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 4e837510d3f214ed8bc1ee291df8f7a4 +guid: 0154e5d4b40264653a6b7832d5640b7a MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Runtime/Scripts/Audio/MicrophoneSource.cs b/Runtime/Scripts/Audio/MicrophoneSource.cs index 75bec1f0..9f88ad38 100644 --- a/Runtime/Scripts/Audio/MicrophoneSource.cs +++ b/Runtime/Scripts/Audio/MicrophoneSource.cs @@ -11,6 +11,13 @@ namespace LiveKit /// /// /// Ensure microphone permissions are granted before calling . + /// + /// Unity's Microphone path does not go through a platform audio device module, so on + /// its own it has no echo cancellation. Construct the source with + /// to run libwebrtc's audio processing over the capture; + /// echo cancellation then uses the mix Unity plays as its reference (see + /// ), which covers every remote and the + /// application's own audio. /// sealed public class MicrophoneSource : RtcAudioSource { @@ -35,6 +42,26 @@ public MicrophoneSource(string deviceName, GameObject sourceObject) : base(RtcAu _sourceObject = sourceObject; } + /// + /// Creates a microphone source whose capture is run through libwebrtc's audio processing + /// (AEC3 echo cancellation, noise suppression, gain control, high-pass filter) before it + /// reaches the track. + /// + /// The name of the device to capture from. Use to + /// get the list of available devices. + /// The GameObject to attach the AudioSource to. The object must be kept in the scene + /// for the duration of the source's lifetime. + /// Which stages to enable. With + /// the SDK attaches a to the active to obtain the + /// far-end reference. Requires Unity's output sample rate to be a multiple of 100 Hz; otherwise processing is + /// bypassed with a warning. See for diagnostics. + public MicrophoneSource(string deviceName, GameObject sourceObject, AudioProcessingOptions processing) + : base(RtcAudioSourceType.AudioSourceMicrophone, processing) + { + _deviceName = deviceName; + _sourceObject = sourceObject; + } + /// /// Begins capturing audio from the microphone. /// @@ -207,6 +234,8 @@ private IEnumerator RestartMicrophone() // recover from interruption. Poll for readiness instead of using arbitrary delay. yield return WaitForMicrophoneReady(); + // A resume is a new audio path: drop whatever the processing stage buffered before. + ResetAudioProcessing(); yield return StartMicrophone(); } diff --git a/Runtime/Scripts/Audio/PcmRingBuffer.cs b/Runtime/Scripts/Audio/PcmRingBuffer.cs new file mode 100644 index 00000000..a9acddbb --- /dev/null +++ b/Runtime/Scripts/Audio/PcmRingBuffer.cs @@ -0,0 +1,118 @@ +using System; + +namespace LiveKit +{ + /// + /// Fixed-capacity interleaved int16 PCM ring buffer with a fixed-size drain. Re-chunks Unity's + /// DSP-block-sized audio into the 10 ms frames the requires. + /// + /// + /// Allocation-free after construction: both users run on the Unity audio thread. Sized in + /// samples (frames × channels), not frames. Single producer and single consumer per instance; + /// the capture and reference feeds each own one, so there is no synchronisation inside. + /// + internal sealed class PcmRingBuffer + { + private readonly short[] _buffer; + private int _readIndex; + private int _writeIndex; + private int _count; + + /// Samples dropped because the buffer was full, since construction. + public int OverflowSamples { get; private set; } + + public int Capacity => _buffer.Length; + public int Available => _count; + + public PcmRingBuffer(int capacitySamples) + { + if (capacitySamples <= 0) throw new ArgumentOutOfRangeException(nameof(capacitySamples)); + _buffer = new short[capacitySamples]; + } + + /// + /// Appends samples. When the buffer is full the OLDEST samples are + /// dropped: a stalled consumer must not push the echo reference arbitrarily far out of + /// alignment with the capture stream. + /// + public void Write(short[] source, int offset, int count) + { + if (source == null) throw new ArgumentNullException(nameof(source)); + if (offset < 0 || count < 0 || offset + count > source.Length) + throw new ArgumentOutOfRangeException(nameof(count)); + + if (count >= _buffer.Length) + { + OverflowSamples += _count + count - _buffer.Length; + offset += count - _buffer.Length; + count = _buffer.Length; + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + else + { + var free = _buffer.Length - _count; + if (count > free) Discard(count - free); + } + + for (var i = 0; i < count; i++) + { + _buffer[_writeIndex] = source[offset + i]; + _writeIndex = _writeIndex + 1 == _buffer.Length ? 0 : _writeIndex + 1; + } + + _count += count; + } + + /// + /// Copies exactly samples into and + /// consumes them. Returns false and consumes nothing when fewer are available. + /// + public bool TryDrain(short[] destination, int count) + { + if (destination == null) throw new ArgumentNullException(nameof(destination)); + if (count < 0 || count > destination.Length) throw new ArgumentOutOfRangeException(nameof(count)); + if (_count < count) return false; + + for (var i = 0; i < count; i++) + { + destination[i] = _buffer[_readIndex]; + _readIndex = _readIndex + 1 == _buffer.Length ? 0 : _readIndex + 1; + } + + _count -= count; + return true; + } + + public void Clear() + { + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + + private void Discard(int count) + { + if (count > _count) count = _count; + _readIndex = (_readIndex + count) % _buffer.Length; + _count -= count; + OverflowSamples += count; + } + } + + /// Sample format conversions shared by the capture paths. + internal static class PcmConvert + { + /// Float [-1, 1] to int16 with clamping and round-half-away-from-zero. + public static short FloatToS16(float v) + { + v *= 32768f; + if (v > 32767f) v = 32767f; + else if (v < -32768f) v = -32768f; + return (short)(v + Math.Sign(v) * 0.5f); + } + + public static float S16ToFloat(short v) => v / 32768f; + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta b/Runtime/Scripts/Audio/PcmRingBuffer.cs.meta similarity index 83% rename from Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta rename to Runtime/Scripts/Audio/PcmRingBuffer.cs.meta index 8c579352..f096bd2b 100644 --- a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta +++ b/Runtime/Scripts/Audio/PcmRingBuffer.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 6a4b6b6ecf98e4447ab88686c0058b9a +guid: cce610429285242e193622fd7e37bb80 MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Runtime/Scripts/Audio/PlatformAudioSource.cs b/Runtime/Scripts/Audio/PlatformAudioSource.cs index 0b507c4c..5a7066fc 100644 --- a/Runtime/Scripts/Audio/PlatformAudioSource.cs +++ b/Runtime/Scripts/Audio/PlatformAudioSource.cs @@ -7,7 +7,10 @@ namespace LiveKit { /// - /// Options for audio processing when creating a PlatformAudioSource. + /// Options for libwebrtc's audio processing. Used by , where + /// the ADM applies them, and by Unity-audio sources such as + /// created with options, where the SDK runs the over the + /// capture with the mix Unity plays as the echo reference (see ). /// public struct AudioProcessingOptions { @@ -17,7 +20,16 @@ public struct AudioProcessingOptions public bool NoiseSuppression; /// Enable automatic gain control (AGC). Default: true. public bool AutoGainControl; - /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. Default: true. + /// + /// Enable the high-pass filter, which removes DC offset and low-frequency rumble ahead of + /// the other stages. Unity-audio sources only; ignores it. + /// Default: true. + /// + public bool HighPassFilter; + /// + /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. + /// only. Default: true. + /// public bool PreferHardware; /// @@ -28,8 +40,12 @@ public struct AudioProcessingOptions EchoCancellation = true, NoiseSuppression = true, AutoGainControl = true, + HighPassFilter = true, PreferHardware = true }; + + /// Whether any stage of the Unity-audio processing pipeline is enabled. + internal bool AnyProcessingEnabled => EchoCancellation || NoiseSuppression || AutoGainControl || HighPassFilter; } /// diff --git a/Runtime/Scripts/Audio/PlayoutReference.cs b/Runtime/Scripts/Audio/PlayoutReference.cs new file mode 100644 index 00000000..2693b4a7 --- /dev/null +++ b/Runtime/Scripts/Audio/PlayoutReference.cs @@ -0,0 +1,149 @@ +using System.Collections; +using UnityEngine; +using LiveKit.Internal; +using LiveKit.Internal.Threading; + +namespace LiveKit +{ + /// + /// Taps the final mix Unity sends to the audio device and feeds it to the echo canceller as + /// the far-end reference. Lives on the GameObject of the active . + /// + /// + /// An created with + /// attaches this component to the active listener when it starts and re-attaches it after + /// scene loads and audio device changes. Adding it to the listener yourself is supported and + /// does the same thing. + /// + /// Because the tap sits after every AudioSource, mixer group and spatializer, the reference is + /// exactly what the loudspeaker plays: every remote participant plus the game's own audio. + /// The capture probe clears its buffer after reading, so the + /// local microphone never appears in the mix. + /// + /// OnAudioFilterRead runs on the Unity audio thread and must not touch Unity APIs, so + /// the sample rate and listener state are cached on the main thread. + /// + [AddComponentMenu("LiveKit/Playout Reference")] + public sealed class PlayoutReference : MonoBehaviour + { + internal delegate void PlayoutAudioDelegate(float[] data, int channels, int sampleRate); + + /// + /// Raised on the Unity audio thread with the final mix. Subscribers must not modify the + /// buffer: it is on its way to the speaker. + /// + internal static event PlayoutAudioDelegate AudioRead; + + private static PlayoutReference _active; + private static int _consumers; + + private AudioListener _listener; + private volatile int _sampleRate; + private volatile bool _deliver; + + /// Whether a reference on an enabled listener is delivering audio. + internal static bool IsAttached => _active != null && _active._deliver; + + /// Main thread. Registers a consumer and attaches to the listener if possible. + internal static void Acquire() + { + _consumers++; + EnsureAttached(); + } + + /// Main thread. The component stays on the listener; it is inert without consumers. + internal static void Release() + { + if (_consumers > 0) _consumers--; + } + + /// + /// Main thread. Attaches to the active AudioListener unless a working reference already + /// exists. No-op without consumers or without a listener; consumers call this periodically, + /// which is what covers scene loads and a destroyed listener. + /// + internal static void EnsureAttached() + { + if (_consumers == 0) return; + if (_active != null && _active.isActiveAndEnabled && + _active._listener != null && _active._listener.isActiveAndEnabled) + return; + + var listener = FindActiveListener(); + if (listener == null) return; + + var existing = listener.GetComponent(); + _active = existing != null ? existing : listener.gameObject.AddComponent(); + } + + private static AudioListener FindActiveListener() + { + var listeners = FindObjectsByType(FindObjectsSortMode.None); + foreach (var listener in listeners) + { + if (listener.isActiveAndEnabled) return listener; + } + return null; + } + + private void OnEnable() + { + _listener = GetComponent(); + if (_listener == null) + Utils.Warning("PlayoutReference must be on the AudioListener's GameObject; it will not deliver a reference from here."); + + RefreshDeliveryState(); + AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; + if (_active == null) _active = this; + } + + private void OnDisable() + { + AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; + _deliver = false; + if (_active == this) _active = null; + } + + private void Update() + { + RefreshDeliveryState(); + } + + // Listener state and the output rate are Unity APIs; sample them here for the audio thread. + private void RefreshDeliveryState() + { + _sampleRate = AudioSettings.outputSampleRate; + _deliver = _listener != null && _listener.isActiveAndEnabled; + } + + // Unity rebuilds the DSP graph on a device change (or AudioSettings.Reset), which can leave + // filter nodes detached; AudioStream recreates its probe for the same reason. Recreate this + // component so the tap is registered on the new graph. Only done while something consumes + // the reference, so a hand-placed component in an idle scene is left alone. + private void OnAudioConfigurationChanged(bool deviceWasChanged) + { + RefreshDeliveryState(); + if (_consumers == 0) return; + + var host = gameObject; + Destroy(this); + MonoBehaviourContext.RunCoroutine(Reattach(host)); + } + + private static IEnumerator Reattach(GameObject host) + { + // Let the deferred Destroy apply before adding the replacement. + yield return null; + if (host == null || _consumers == 0) yield break; + if (host.GetComponent() == null) + _active = host.AddComponent(); + } + + // Unity audio thread. + private void OnAudioFilterRead(float[] data, int channels) + { + if (!_deliver) return; + AudioRead?.Invoke(data, channels, _sampleRate); + } + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta b/Runtime/Scripts/Audio/PlayoutReference.cs.meta similarity index 83% rename from Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta rename to Runtime/Scripts/Audio/PlayoutReference.cs.meta index 39aae659..a74f1070 100644 --- a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta +++ b/Runtime/Scripts/Audio/PlayoutReference.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 6d2e02f70cc454f188b929269a8acbc7 +guid: 44cbde2518fe54f5ebd5f24f65d8a070 MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Runtime/Scripts/Audio/RtcAudioSource.cs b/Runtime/Scripts/Audio/RtcAudioSource.cs index 9147b431..ee764916 100644 --- a/Runtime/Scripts/Audio/RtcAudioSource.cs +++ b/Runtime/Scripts/Audio/RtcAudioSource.cs @@ -49,12 +49,23 @@ private sealed class PendingAudioFrame private readonly RtcAudioSourceType _sourceType; public RtcAudioSourceType SourceType => _sourceType; + + /// + /// Whether this source runs libwebrtc's audio processing over its capture. False when it was + /// created without , or when the module could not be + /// created and the source fell back to unprocessed capture. + /// + public bool AudioProcessingEnabled => _processor != null; + + /// Counters from the audio processing stage; default when processing is off. + public AudioProcessingStats AudioProcessingStats => _processor?.GetStats() ?? default; private readonly int _debugId = Interlocked.Increment(ref nextDebugId); internal readonly uint _expectedSampleRate; internal readonly uint _expectedChannels; internal readonly FfiHandle Handle; protected AudioSourceInfo _info; + private readonly AudioProcessor _processor; // CaptureAudioFrame is asynchronous: the native side can continue reading from the PCM // pointer after request.Send() returns and encode it later on another queue. Because of @@ -72,16 +83,29 @@ private sealed class PendingAudioFrame private bool _started = false; private volatile bool _disposed = false; private int _audioReadCount = 0; + private int _sentFrameCount = 0; // Device-capture sources (microphone, AudioSource taps) don't know their format ahead of // time — it is whatever Unity's audio graph delivers. They use this constructor, which // configures the native source from Unity's current output configuration. protected RtcAudioSource(RtcAudioSourceType audioSourceType) - : this(audioSourceType, 0, 0) { } + : this(audioSourceType, 0, 0, null) { } + + /// + /// Device-capture source whose audio is run through libwebrtc's audio processing (echo + /// cancellation, noise suppression, gain control, high-pass filter) before it reaches the + /// track. See . If the module cannot be created the + /// source logs a warning and captures unprocessed. + /// + protected RtcAudioSource(RtcAudioSourceType audioSourceType, AudioProcessingOptions processing) + : this(audioSourceType, 0, 0, processing) { } // Sources that generate a fixed, known format (e.g. test signal generators) declare it // directly. Passing 0 for either value falls back to the device configuration. protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, uint channels) + : this(audioSourceType, sampleRate, channels, null) { } + + protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, uint channels, AudioProcessingOptions? processing) { _sourceType = audioSourceType; @@ -110,6 +134,19 @@ protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, ui _info = res.NewAudioSource.Source.Info; Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle); Utils.Debug($"{DebugTag} created handle={Handle.DangerousGetHandle()} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); + + if (processing is { } options && options.AnyProcessingEnabled) + { + try + { + _processor = new AudioProcessor(options, SendProcessedFrame); + } + catch (Exception e) + { + // Publish unprocessed rather than not at all. + Utils.Warning($"{DebugTag} audio processing unavailable, capturing unprocessed: {e.Message}"); + } + } } // Reads Unity's actual output audio configuration. The capture path delivers buffers at the @@ -150,6 +187,7 @@ public virtual void Start() { if (_started) return; AudioRead += OnAudioRead; + _processor?.Start(); _started = true; Utils.Debug($"{DebugTag} start"); } @@ -161,6 +199,7 @@ public virtual void Stop() { if (!_started) return; AudioRead -= OnAudioRead; + _processor?.Stop(); _started = false; var pendingCount = PendingFrameCount(); if (pendingCount > 0) @@ -174,45 +213,59 @@ private void OnAudioRead(float[] data, int channels, int sampleRate) if (_muted) return; if (_disposed) return; - var frameIndex = Interlocked.Increment(ref _audioReadCount); + var readIndex = Interlocked.Increment(ref _audioReadCount); if (channels <= 0) { - Utils.Warning($"{DebugTag} dropping audio frame #{frameIndex} because channels={channels}"); + Utils.Warning($"{DebugTag} dropping audio frame #{readIndex} because channels={channels}"); return; } if (data.Length == 0 || data.Length % channels != 0) { - Utils.Warning($"{DebugTag} audio frame #{frameIndex} has invalid shape samples={data.Length} channels={channels}"); + Utils.Warning($"{DebugTag} audio frame #{readIndex} has invalid shape samples={data.Length} channels={channels}"); return; } if ((uint)sampleRate != _expectedSampleRate || (uint)channels != _expectedChannels) { - Utils.Warning($"{DebugTag} audio frame #{frameIndex} metadata mismatch actualRate={sampleRate} actualChannels={channels} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); + Utils.Warning($"{DebugTag} audio frame #{readIndex} metadata mismatch actualRate={sampleRate} actualChannels={channels} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); } - var pendingBeforeSend = PendingFrameCount(); - if (frameIndex <= 3 || frameIndex % 100 == 0 || pendingBeforeSend >= 3) - { - Utils.Debug($"{DebugTag} capture frame #{frameIndex} samples={data.Length} channels={channels} sampleRate={sampleRate} pendingBeforeSend={pendingBeforeSend} thread={Thread.CurrentThread.ManagedThreadId}"); - } + // Optional processing stage: the block is re-chunked into 10 ms frames, run through the + // module and delivered to SendFrame one chunk at a time via SendProcessedFrame. + if (_processor != null && _processor.TryProcessCapture(data, channels, sampleRate)) + return; // Each captured frame gets its own backing buffer so the native encoder can safely // consume it asynchronously after request.Send() returns. var frameData = new NativeArray(data.Length, Allocator.Persistent); + for (int i = 0; i < data.Length; i++) + frameData[i] = PcmConvert.FloatToS16(data[i]); + + SendFrame(frameData, channels, sampleRate); + } - // Copy from the audio read buffer into the frame buffer, converting - // each sample to a 16-bit signed integer. - static short FloatToS16(float v) + // Audio thread, from the processing stage. Owns the frame from here on. + private void SendProcessedFrame(NativeArray frame, int channels, int sampleRate) + { + if (_disposed || _muted) { - v *= 32768f; - v = Math.Min(v, 32767f); - v = Math.Max(v, -32768f); - return (short)(v + Math.Sign(v) * 0.5f); + frame.Dispose(); + return; + } + SendFrame(frame, channels, sampleRate); + } + + // Hands one int16 frame to the native source. Takes ownership of frameData: it is released + // when the CaptureAudioFrame callback completes, is canceled, or the send fails. + private void SendFrame(NativeArray frameData, int channels, int sampleRate) + { + var frameIndex = Interlocked.Increment(ref _sentFrameCount); + var pendingBeforeSend = PendingFrameCount(); + if (frameIndex <= 3 || frameIndex % 100 == 0 || pendingBeforeSend >= 3) + { + Utils.Debug($"{DebugTag} capture frame #{frameIndex} samples={frameData.Length} channels={channels} sampleRate={sampleRate} pendingBeforeSend={pendingBeforeSend} thread={Thread.CurrentThread.ManagedThreadId}"); } - for (int i = 0; i < data.Length; i++) - frameData[i] = FloatToS16(data[i]); // Capture the frame. using var request = FFIBridge.Instance.NewRequest(); @@ -228,7 +281,7 @@ static short FloatToS16(float v) } pushFrame.Buffer.NumChannels = (uint)channels; pushFrame.Buffer.SampleRate = (uint)sampleRate; - pushFrame.Buffer.SamplesPerChannel = (uint)data.Length / (uint)channels; + pushFrame.Buffer.SamplesPerChannel = (uint)frameData.Length / (uint)channels; // Wait for async callback, log an error if the capture fails. The callback's AsyncId // echoes the RequestAsyncId that Unity wrote onto the request. @@ -239,7 +292,7 @@ static short FloatToS16(float v) FrameIndex = frameIndex, SampleRate = sampleRate, Channels = channels, - SampleCount = data.Length, + SampleCount = frameData.Length, StartedTimestamp = Stopwatch.GetTimestamp(), }; lock (_pendingFrameDataLock) @@ -292,6 +345,13 @@ void OnCanceled() } } + /// + /// Clears the audio processing stage's buffers. Call after the capture path restarts (e.g. a + /// microphone resume) so stale samples do not misalign the echo canceller. No-op without + /// processing. + /// + protected void ResetAudioProcessing() => _processor?.RequestReset(); + /// /// Mutes or unmutes the audio source. /// @@ -328,6 +388,7 @@ protected virtual void Dispose(bool disposing) } _pendingFrameData.Clear(); } + _processor?.Dispose(); Handle?.Dispose(); _disposed = true; Utils.Debug($"{DebugTag} disposed"); diff --git a/Runtime/Scripts/Core/FfiFrameObserver.cs b/Runtime/Scripts/Core/FfiFrameObserver.cs deleted file mode 100644 index cdb46fef..00000000 --- a/Runtime/Scripts/Core/FfiFrameObserver.cs +++ /dev/null @@ -1,160 +0,0 @@ -using System; -using LiveKit.Internal; -using LiveKit.Proto; - -namespace LiveKit -{ - /// - /// A decoded video frame, extracted from the raw FFI event into a protobuf-free value. - /// Carries the native plane pointers and geometry only — no managed wrappers and no - /// Google.Protobuf surface, so consumers can use frames without inheriting a - /// compile-time protobuf dependency. - /// - /// - /// The DataPtr* values point at native buffers that are valid for the duration of - /// the callback ONLY — copy out - /// synchronously if you need them past return. - /// - public readonly struct RawVideoFrame - { - /// Stream handle the frame arrived on, for correlating to a . - public readonly ulong StreamHandle; - public readonly IntPtr DataPtrY, DataPtrU, DataPtrV; - public readonly int StrideY, StrideU, StrideV; - public readonly int Width, Height; - - public RawVideoFrame( - ulong streamHandle, - IntPtr dataPtrY, int strideY, - IntPtr dataPtrU, int strideU, - IntPtr dataPtrV, int strideV, - int width, int height) - { - StreamHandle = streamHandle; - DataPtrY = dataPtrY; StrideY = strideY; - DataPtrU = dataPtrU; StrideU = strideU; - DataPtrV = dataPtrV; StrideV = strideV; - Width = width; Height = height; - } - } - - /// - /// A decoded audio frame, extracted from the raw FFI event into a protobuf-free value. - /// points at interleaved S16 PCM valid for the callback duration ONLY. - /// - public readonly struct RawAudioFrame - { - /// Stream handle the frame arrived on, for correlating to an . - public readonly ulong StreamHandle; - public readonly IntPtr DataPtr; - public readonly int SamplesPerChannel, NumChannels, SampleRate; - - public RawAudioFrame(ulong streamHandle, IntPtr dataPtr, int samplesPerChannel, int numChannels, int sampleRate) - { - StreamHandle = streamHandle; - DataPtr = dataPtr; - SamplesPerChannel = samplesPerChannel; - NumChannels = numChannels; - SampleRate = sampleRate; - } - } - - /// - /// Opt-in extension point for raw decoded frames. - /// - /// - /// / are invoked - /// synchronously on the FFI callback thread from the event router, BEFORE the event's - /// FfiHandles wrap or free the underlying native buffers — so the DataPtr - /// values each frame carries are valid for the duration of the callback ONLY. - /// - /// The SDK extracts the protobuf event into the plain / - /// structs here, on the FFI thread, so subscribers consume - /// decoded frames WITHOUT a compile-time dependency on Google.Protobuf. - /// - /// Subscriber contract: - /// - /// Runs on the FFI thread, not Unity's main loop — do not touch Unity APIs. - /// Must be non-blocking; it sits in the frame-delivery hot path. - /// Must NOT retain any DataPtr past return — copy out synchronously if needed. - /// - /// - /// No subscriber == zero cost: extraction is skipped entirely when the matching delegate - /// is null. This lets consumers build native Picture-in-Picture, echo-cancellation - /// references, frame capture, custom GPU upload, or analytics on top of the decoded - /// stream without patching the SDK. - /// - public static class FfiFrameObserver - { - public static event Action VideoFrameReceived; - public static event Action AudioFrameReceived; - - internal static void Dispatch(FfiEvent ev) - { - switch (ev.MessageCase) - { - case FfiEvent.MessageOneofCase.VideoStreamEvent: - ExtractVideo(ev.VideoStreamEvent); - break; - case FfiEvent.MessageOneofCase.AudioStreamEvent: - ExtractAudio(ev.AudioStreamEvent); - break; - } - } - - private static void ExtractVideo(VideoStreamEvent vse) - { - var handler = VideoFrameReceived; - if (handler == null) return; - if (vse.MessageCase != VideoStreamEvent.MessageOneofCase.FrameReceived) return; - - var buf = vse.FrameReceived?.Buffer; - if (buf?.Info == null || buf.Info.Components.Count < 3) return; - - var info = buf.Info; - var yc = info.Components[0]; - var uc = info.Components[1]; - var vc = info.Components[2]; - if (yc.DataPtr == 0 || uc.DataPtr == 0 || vc.DataPtr == 0) return; - - Invoke(handler, new RawVideoFrame( - vse.StreamHandle, - (IntPtr)(long)yc.DataPtr, (int)yc.Stride, - (IntPtr)(long)uc.DataPtr, (int)uc.Stride, - (IntPtr)(long)vc.DataPtr, (int)vc.Stride, - (int)info.Width, (int)info.Height)); - } - - private static void ExtractAudio(AudioStreamEvent ase) - { - var handler = AudioFrameReceived; - if (handler == null) return; - if (ase.MessageCase != AudioStreamEvent.MessageOneofCase.FrameReceived) return; - - var frame = ase.FrameReceived?.Frame; - if (frame?.Info == null || frame.Info.DataPtr == 0) return; - - var info = frame.Info; - Invoke(handler, new RawAudioFrame( - ase.StreamHandle, - (IntPtr)(long)info.DataPtr, - (int)info.SamplesPerChannel, - (int)info.NumChannels, - (int)info.SampleRate)); - } - - // A subscriber exception must not escape the native callback: this runs on the FFI - // thread inside a reverse P/Invoke, where an unhandled managed exception is fatal. - private static void Invoke(Action handler, T frame) - { - try - { - handler(frame); - } - catch (Exception e) - { - Utils.Error($"FfiFrameObserver subscriber threw: {e}"); - } - } - } -} diff --git a/Runtime/Scripts/Internal/FFI/FFIClient.cs b/Runtime/Scripts/Internal/FFI/FFIClient.cs index ec03b01a..213ee1f4 100644 --- a/Runtime/Scripts/Internal/FFI/FFIClient.cs +++ b/Runtime/Scripts/Internal/FFI/FFIClient.cs @@ -375,10 +375,6 @@ internal static void RouteFfiEvent(FfiEvent response) { if (_isDisposed) return; - // Raw decoded-frame hook. Runs first, on this thread, so subscribers see the native - // buffers before any FfiHandle below wraps or frees them. No-op when unsubscribed. - FfiFrameObserver.Dispatch(response); - // Audio stream events are handled directly on the FFI callback thread // to bypass the main thread, since the audio thread consumes the data if (response.MessageCase == FfiEvent.MessageOneofCase.AudioStreamEvent) diff --git a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs index 8211dbc2..0fb70078 100644 --- a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs +++ b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs @@ -13,7 +13,6 @@ public class MeetManagerEditor : Editor private SerializedProperty noiseSuppression; private SerializedProperty autoGainControl; private SerializedProperty preferHardwareProcessing; - private SerializedProperty unityEchoCancellation; private SerializedProperty remoteAudioGain; private void OnEnable() @@ -27,7 +26,6 @@ private void OnEnable() noiseSuppression = serializedObject.FindProperty("noiseSuppression"); autoGainControl = serializedObject.FindProperty("autoGainControl"); preferHardwareProcessing = serializedObject.FindProperty("preferHardwareProcessing"); - unityEchoCancellation = serializedObject.FindProperty("unityEchoCancellation"); remoteAudioGain = serializedObject.FindProperty("remoteAudioGain"); } @@ -51,26 +49,24 @@ public override void OnInspectorGUI() "Provides AEC, AGC, and NS. Disable to use Unity's Microphone API instead.")); EditorGUILayout.Space(); - EditorGUILayout.LabelField("Audio Processing (PlatformAudio only)", EditorStyles.boldLabel); + EditorGUILayout.LabelField("Audio Processing", EditorStyles.boldLabel); - // Gray out audio processing options when PlatformAudio is disabled bool platformAudioEnabled = usePlatformAudio.boolValue; + EditorGUILayout.PropertyField(echoCancellation, new GUIContent("Echo Cancellation", + "Enable echo cancellation. PlatformAudio: WebRTC's ADM. Unity audio: libwebrtc's AEC3 over the " + + "Microphone capture, with the mix Unity plays as the reference.")); + EditorGUILayout.PropertyField(noiseSuppression, new GUIContent("Noise Suppression", + "Enable noise suppression to remove background noise.")); + EditorGUILayout.PropertyField(autoGainControl, new GUIContent("Auto Gain Control", + "Enable auto gain control to normalize audio levels.")); + + // Hardware processing is an ADM feature; gray it out when PlatformAudio is disabled. using (new EditorGUI.DisabledGroupScope(!platformAudioEnabled)) { - if (!platformAudioEnabled) - { - EditorGUILayout.HelpBox("Audio processing options are only available when 'Use Platform Audio' is enabled.", MessageType.Info); - } - - EditorGUILayout.PropertyField(echoCancellation, new GUIContent("Echo Cancellation", - "Enable echo cancellation to remove echo from speaker playback.")); - EditorGUILayout.PropertyField(noiseSuppression, new GUIContent("Noise Suppression", - "Enable noise suppression to remove background noise.")); - EditorGUILayout.PropertyField(autoGainControl, new GUIContent("Auto Gain Control", - "Enable auto gain control to normalize audio levels.")); EditorGUILayout.PropertyField(preferHardwareProcessing, new GUIContent("Prefer Hardware Processing", - "Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")); + "PlatformAudio only. Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have " + + "different quality characteristics.")); } EditorGUILayout.Space(); @@ -79,14 +75,11 @@ public override void OnInspectorGUI() // Gray out Unity audio options when PlatformAudio is enabled using (new EditorGUI.DisabledGroupScope(platformAudioEnabled)) { - if (platformAudioEnabled) - { - EditorGUILayout.HelpBox("Unity audio options are only used when 'Use Platform Audio' is disabled.", MessageType.Info); - } + EditorGUILayout.HelpBox(platformAudioEnabled + ? "Unity audio options are only used when 'Use Platform Audio' is disabled." + : "Echo cancellation in this mode runs libwebrtc's AEC3 in the SDK; the reference is the mix on the " + + "AudioListener (a PlayoutReference component is attached automatically).", MessageType.Info); - EditorGUILayout.PropertyField(unityEchoCancellation, new GUIContent("Echo Cancellation (AEC3)", - "Run libwebrtc's AEC3 over Unity microphone capture, using the decoded remote audio frames as the " + - "echo reference. Assumes a single remote audio stream.")); EditorGUILayout.PropertyField(remoteAudioGain, new GUIContent("Remote Audio Gain", "Playback gain for every remote AudioSource. Below 1 keeps headroom so full-volume playout does not " + "distort or overload the echo canceller. 0.7 is -3.1 dB.")); diff --git a/Samples~/Meet/Assets/Plugins/iOS.meta b/Samples~/Meet/Assets/Plugins/iOS.meta deleted file mode 100644 index 1205da93..00000000 --- a/Samples~/Meet/Assets/Plugins/iOS.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 2f128f6a519a14fc0aa42bbc2d20f447 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm deleted file mode 100644 index e8749f1e..00000000 --- a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm +++ /dev/null @@ -1,18 +0,0 @@ -#import - -// Latency terms for seeding libwebrtc's AEC3 stream delay; consumed by -// Assets/Runtime/Audio/AudioProcessingDelaySeed.cs. AVAudioSession only reports meaningful -// values once the session is active; it returns 0 before that. -extern "C" { - double MeetSample_AudioSessionOutputLatency() { - return [[AVAudioSession sharedInstance] outputLatency]; - } - - double MeetSample_AudioSessionInputLatency() { - return [[AVAudioSession sharedInstance] inputLatency]; - } - - double MeetSample_AudioSessionIOBufferDuration() { - return [[AVAudioSession sharedInstance] IOBufferDuration]; - } -} diff --git a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta deleted file mode 100644 index 3483a3d7..00000000 --- a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta +++ /dev/null @@ -1,33 +0,0 @@ -fileFormatVersion: 2 -guid: 25d054283a4734b5a817eb1b23cbea61 -PluginImporter: - externalObjects: {} - serializedVersion: 2 - iconMap: {} - executionOrder: {} - defineConstraints: [] - isPreloaded: 0 - isOverridable: 0 - isExplicitlyReferenced: 0 - validateReferences: 1 - platformData: - - first: - Any: - second: - enabled: 0 - settings: {} - - first: - Editor: Editor - second: - enabled: 0 - settings: - DefaultValueInitialized: true - - first: - iPhone: iOS - second: - enabled: 1 - settings: - AddToEmbeddedBinaries: false - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio.meta b/Samples~/Meet/Assets/Runtime/Audio.meta deleted file mode 100644 index deab7576..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 7ed59392f993648e8ae9d202fb567a84 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs b/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs deleted file mode 100644 index 54da05d1..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs +++ /dev/null @@ -1,220 +0,0 @@ -using System; -using System.Diagnostics; -using System.Threading; -using LiveKit; -using UnityEngine; -using Debug = UnityEngine.Debug; - -/// -/// Runs libwebrtc's AEC3 over the local microphone capture, using the decoded remote audio as -/// the echo reference. -/// -/// The loudspeaker plays the remote audio, the microphone re-captures it, and the published -/// track would carry it back to everyone. Unity's Microphone path has no echo canceller -/// anywhere (capture never goes through a platform audio device module), so cancellation is -/// done here against the two streams already available in managed code: the FFI render frames -/// (far end, via ) and OnAudioFilterRead (near end). -/// -/// Threading: runs on the Unity audio thread and -/// on the FFI callback thread. Each owns its own pump, and -/// libwebrtc's APM is built for exactly that capture/render thread split. Nothing here touches -/// a Unity API from those threads; diagnostics use , not -/// UnityEngine.Time. -/// -/// Limitation: the far-end tap is not filtered by stream, so it assumes a SINGLE remote audio -/// stream. With several remote speakers the reference becomes the interleaving of all their -/// frames and AEC3 will not converge. The remote audio must also play through Unity -/// (AudioStream) — in PlatformAudio mode no FFI audio streams exist and nothing arrives. -/// -internal sealed class AcousticEchoCanceller : IDisposable -{ - private const int DiagnosticIntervalMs = 5000; - private const int SeedIntervalMs = 2000; - private const int SeedChangeThresholdMs = 5; - - private readonly AudioProcessingModule _apm; - private readonly ApmChunkPump _capturePump; - private readonly ApmChunkPump _renderPump; - private readonly Stopwatch _clock = new Stopwatch(); - - private long _nextDiagnosticMs; - private long _nextSeedMs; - private int _seededDelayMs = -1; - private int _farEndFrames; - private int _unsupportedRateWarned; - private bool _subscribed; - private bool _disposed; - - /// Processed 10 ms capture chunks, raised on the Unity audio thread. - public event ProcessedChunkHandler CaptureProcessed; - - private AcousticEchoCanceller(AudioProcessingModule apm) - { - _apm = apm; - _capturePump = new ApmChunkPump(_apm.ProcessStream, RaiseCaptureProcessed); - _renderPump = new ApmChunkPump(_apm.ProcessReverseStream); - } - - /// - /// Creates the canceller, or returns null when the FFI cannot hand out an APM handle — the - /// caller then publishes the unprocessed microphone rather than failing to publish at all. - /// - public static AcousticEchoCanceller TryCreate() - { - try - { - var apm = new AudioProcessingModule( - echoCancellerEnabled: true, - gainControllerEnabled: true, - highPassFilterEnabled: true, - noiseSuppressionEnabled: true); - - Debug.Log($"[AEC] APM created handle={apm.Handle}"); - return new AcousticEchoCanceller(apm); - } - catch (Exception e) - { - Debug.LogWarning($"[AEC] APM unavailable, publishing microphone unprocessed: {e.Message}"); - return null; - } - } - - public void Start() - { - if (_disposed || _subscribed) return; - - _capturePump.Reset(); - _renderPump.Reset(); - _clock.Restart(); - _nextDiagnosticMs = DiagnosticIntervalMs; - _nextSeedMs = SeedIntervalMs; - - // Forget the delay seeded before the last Stop(). A resume restarts the whole audio path, - // so the previous value describes an acoustic path that no longer exists — and carrying it - // over lets the SeedChangeThresholdMs guard in SeedStreamDelay silently skip the reseed - // below whenever the new estimate lands within 5 ms of the stale one. - _seededDelayMs = -1; - - FfiFrameObserver.AudioFrameReceived += OnFarEndFrame; - _subscribed = true; - - SeedStreamDelay(0d); - } - - public void Stop() - { - if (!_subscribed) return; - - FfiFrameObserver.AudioFrameReceived -= OnFarEndFrame; - _subscribed = false; - _clock.Reset(); - } - - /// - /// Feeds near-end capture. Unity audio thread. Returns false when the block cannot be - /// processed, and the caller must publish it unchanged — the APM needs a rate whose 10 ms - /// chunk is a whole number of samples, and device audio backends do run at odd rates. - /// - public bool TryPushCapture(float[] data, int channels, int sampleRate) - { - if (_disposed || data == null || channels <= 0 || sampleRate <= 0) return false; - - if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) - { - WarnUnsupportedRateOnce(sampleRate); - return false; - } - - _capturePump.Push(data, channels, sampleRate); - - var captureBlockMs = data.Length / (double)channels * 1000d / sampleRate; - MaybeReseed(captureBlockMs); - MaybeLogDiagnostics(channels, sampleRate); - return true; - } - - private void WarnUnsupportedRateOnce(int sampleRate) - { - if (Interlocked.Exchange(ref _unsupportedRateWarned, 1) == 1) return; - - Debug.LogWarning( - $"[AEC] capture rate {sampleRate} has no whole-sample 10 ms chunk — " + - "echo cancellation disabled, publishing microphone unprocessed"); - } - - // The far-end DataPtr is valid for the duration of this callback ONLY; the pump copies out - // before doing anything else. - private void OnFarEndFrame(RawAudioFrame frame) - { - if (_disposed) return; - - Interlocked.Increment(ref _farEndFrames); - _renderPump.Push(frame.DataPtr, frame.SamplesPerChannel, frame.NumChannels, frame.SampleRate); - } - - private void RaiseCaptureProcessed(float[] data, int channels, int sampleRate) - { - CaptureProcessed?.Invoke(data, channels, sampleRate); - } - - // AVAudioSession reports zero latency until the session goes active, so the seed is - // re-evaluated on a slow cadence rather than only once at Start(). - private void MaybeReseed(double captureBlockMs) - { - if (!_clock.IsRunning) return; - - var elapsedMs = _clock.ElapsedMilliseconds; - if (elapsedMs < _nextSeedMs) return; - _nextSeedMs = elapsedMs + SeedIntervalMs; - - SeedStreamDelay(captureBlockMs); - } - - private void SeedStreamDelay(double captureBlockMs) - { - var delayMs = AudioProcessingDelaySeed.Estimate(captureBlockMs); - if (_seededDelayMs >= 0 && Math.Abs(delayMs - _seededDelayMs) < SeedChangeThresholdMs) return; - - var error = _apm.SetStreamDelayMs(delayMs); - if (error != null) - { - Debug.LogWarning($"[AEC] set_stream_delay_ms({delayMs}) failed: {error}"); - return; - } - - _seededDelayMs = delayMs; - Debug.Log($"[AEC] stream delay seeded to {delayMs}ms (captureBlock={captureBlockMs:F1}ms)"); - } - - // Reports the measured frame geometry both feeds are actually running at. Stopwatch, not - // UnityEngine.Time: this is the audio thread. - private void MaybeLogDiagnostics(int channels, int sampleRate) - { - if (!_clock.IsRunning) return; - - var elapsedMs = _clock.ElapsedMilliseconds; - if (elapsedMs < _nextDiagnosticMs) return; - _nextDiagnosticMs = elapsedMs + DiagnosticIntervalMs; - - Debug.Log( - $"[AEC] capture {channels}ch@{sampleRate} native={AudioProcessingModule.IsNativeSampleRate(sampleRate)} " + - $"chunks={_capturePump.ProcessedChunkCount} dropped={_capturePump.DroppedSamples} " + - $"failed={_capturePump.FailedChunkCount} err={_capturePump.LastError ?? "-"} | " + - $"render {_renderPump.Channels}ch@{_renderPump.SampleRate} frames={_farEndFrames} " + - $"chunks={_renderPump.ProcessedChunkCount} dropped={_renderPump.DroppedSamples} " + - $"failed={_renderPump.FailedChunkCount} err={_renderPump.LastError ?? "-"} | " + - $"delay={_seededDelayMs}ms"); - } - - public void Dispose() - { - if (_disposed) return; - - Stop(); - _disposed = true; - CaptureProcessed = null; - _capturePump.Dispose(); - _renderPump.Dispose(); - _apm.Dispose(); - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs b/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs deleted file mode 100644 index 3ae50e5b..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System; -using UnityEngine; - -/// -/// Intercepts the microphone clip's audio on the Unity audio thread. -/// -/// Sample-side re-implementation of the SDK's AudioProbe, which is internal. Behaviour is -/// deliberately identical, including — without it the -/// microphone is played back through the local loudspeaker. -/// -internal sealed class AecAudioProbe : MonoBehaviour -{ - public delegate void OnAudioDelegate(float[] data, int channels, int sampleRate); - - public event OnAudioDelegate AudioRead; - - private int _sampleRate; - private volatile bool _clearAfterInvocation; - - public void ClearAfterInvocation() - { - _clearAfterInvocation = true; - } - - private void OnEnable() - { - OnAudioConfigurationChanged(false); - AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; - } - - private void OnDisable() - { - AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; - } - - private void OnAudioConfigurationChanged(bool deviceWasChanged) - { - _sampleRate = AudioSettings.outputSampleRate; - } - - private void OnAudioFilterRead(float[] data, int channels) - { - AudioRead?.Invoke(data, channels, _sampleRate); - if (_clearAfterInvocation) data.AsSpan().Clear(); - } - - private void OnDestroy() - { - AudioRead = null; - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs deleted file mode 100644 index 3f4a21f4..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; -using UnityEngine; - -/// -/// Coroutine runner and application-pause relay for , attached -/// to the microphone GameObject. Stands in for the SDK's internal MonoBehaviourContext, -/// which sample code cannot reach. -/// -internal sealed class AecMicrophoneHost : MonoBehaviour -{ - public event Action Paused; - - private void OnApplicationPause(bool pause) - { - Paused?.Invoke(pause); - } - - private void OnDestroy() - { - Paused = null; - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs deleted file mode 100644 index c9f99ef3..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs +++ /dev/null @@ -1,304 +0,0 @@ -using System; -using System.Collections; -using LiveKit; -using UnityEngine; - -/// -/// Microphone capture source that runs AEC3 over the captured PCM before publishing it. -/// -/// The SDK's MicrophoneSource is sealed and its AudioProbe and -/// MonoBehaviourContext are internal, so the capture path is re-implemented here. Its -/// behaviours are load-bearing and preserved: clear-after-invocation (so the microphone is not -/// played back locally), the duplicate-component guard, the Microphone.GetPosition -/// readiness poll, and the pause/resume stop-restart cycle. -/// -/// The only behavioural difference is that carries APM-processed 10 ms -/// chunks instead of raw DSP blocks. When the APM is unavailable the raw blocks pass straight -/// through, so publishing never fails because of the canceller. -/// -internal sealed class AecMicrophoneSource : RtcAudioSource -{ - private readonly GameObject _sourceObject; - private readonly string _deviceName; - private readonly AecMicrophoneHost _host; - private readonly AcousticEchoCanceller _canceller; - - public override event Action AudioRead; - - private bool _disposed; - private bool _started; - - public bool EchoCancellationActive => _canceller != null; - - private AecMicrophoneSource( - string deviceName, - GameObject sourceObject, - AecMicrophoneHost host, - AcousticEchoCanceller canceller) - : base(RtcAudioSourceType.AudioSourceMicrophone) - { - _deviceName = deviceName; - _sourceObject = sourceObject; - _host = host; - _canceller = canceller; - - if (_canceller != null) _canceller.CaptureProcessed += OnProcessedAudio; - } - - /// - /// Builds the source. The base constructor configures the native source from Unity's current - /// output configuration, which is also the format OnAudioFilterRead delivers, so the - /// published track's metadata matches the capture geometry without further alignment. - /// - /// One of . - /// GameObject that hosts the AudioSource, probe and coroutine - /// runner. Must stay alive for the source's lifetime. - public static AecMicrophoneSource Create(string deviceName, GameObject sourceObject) - { - if (sourceObject == null) throw new ArgumentNullException(nameof(sourceObject)); - - var host = sourceObject.GetComponent(); - if (host == null) host = sourceObject.AddComponent(); - - return new AecMicrophoneSource(deviceName, sourceObject, host, AcousticEchoCanceller.TryCreate()); - } - - public override void Start() - { - base.Start(); - if (_started) return; - - if (!Application.HasUserAuthorization(UserAuthorization.Microphone)) - throw new InvalidOperationException("Microphone access not authorized"); - - _host.Paused += OnApplicationPause; - _canceller?.Start(); - RunCoroutine(StartMicrophone()); - - _started = true; - } - - public override void Stop() - { - base.Stop(); - RunCoroutine(StopMicrophone()); - if (_host != null) _host.Paused -= OnApplicationPause; - _canceller?.Stop(); - _started = false; - } - - private IEnumerator StartMicrophone() - { - if (_sourceObject == null) - { - Debug.LogError("[AEC] microphone GameObject is null, cannot start"); - yield break; - } - - if (!Application.HasUserAuthorization(UserAuthorization.Microphone)) - { - Debug.LogError("[AEC] microphone authorization lost"); - yield break; - } - - AudioClip clip = null; - try - { - clip = Microphone.Start( - _deviceName, - loop: true, - lengthSec: 1, - frequency: SupportedCaptureFrequency()); - } - catch (Exception e) - { - Debug.LogError($"[AEC] exception starting microphone: {e.Message}"); - yield break; - } - - if (clip == null) - { - Debug.LogError("[AEC] Microphone.Start returned null, audio session may not be ready"); - yield break; - } - - // Unity's Destroy is deferred, so a resume can land here while the previous pair is still - // alive. Duplicates would double every captured block into the APM. - var existingSource = _sourceObject.GetComponent(); - if (existingSource != null) UnityEngine.Object.DestroyImmediate(existingSource); - - var existingProbe = _sourceObject.GetComponent(); - if (existingProbe != null) - { - existingProbe.AudioRead -= OnCapturedAudio; - UnityEngine.Object.DestroyImmediate(existingProbe); - } - - var source = _sourceObject.AddComponent(); - source.clip = clip; - source.loop = true; - - var probe = _sourceObject.AddComponent(); - probe.ClearAfterInvocation(); - probe.AudioRead += OnCapturedAudio; - - const float timeout = 2f; - var elapsed = 0f; - while (Microphone.GetPosition(_deviceName) <= 0 && elapsed < timeout) - { - yield return new WaitForSeconds(0.05f); - elapsed += 0.05f; - } - - if (Microphone.GetPosition(_deviceName) <= 0) - { - Debug.LogError($"[AEC] microphone did not start producing data after {timeout}s"); - yield break; - } - - source.Play(); - Debug.Log($"[AEC] microphone '{_deviceName}' started at {clip.frequency}Hz, echo cancellation={EchoCancellationActive}"); - } - - // The requested rate is Unity's output rate, which the device may not accept as a capture - // rate. Clamping keeps Microphone.Start working; the DSP graph resamples the clip anyway, so - // OnAudioFilterRead still delivers the output rate either way. - private int SupportedCaptureFrequency() - { - var requested = AudioSettings.outputSampleRate; - Microphone.GetDeviceCaps(_deviceName, out var min, out var max); - - // Unity reports 0/0 when the device accepts any frequency. - if (min == 0 && max == 0) return requested; - - var clamped = Mathf.Clamp(requested, min, max); - if (clamped != requested) - Debug.Log($"[AEC] capture rate clamped {requested} -> {clamped} (device caps {min}-{max})"); - - return clamped; - } - - private IEnumerator StopMicrophone() - { - if (Microphone.IsRecording(_deviceName)) - Microphone.End(_deviceName); - - if (_sourceObject != null) - { - var probe = _sourceObject.GetComponent(); - if (probe != null) - { - probe.AudioRead -= OnCapturedAudio; - UnityEngine.Object.Destroy(probe); - } - - var source = _sourceObject.GetComponent(); - if (source != null) - UnityEngine.Object.Destroy(source); - } - - Debug.Log($"[AEC] microphone '{_deviceName}' stopped"); - yield return null; - } - - // Unity audio thread. A block the canceller cannot take is published unchanged rather than - // dropped — an un-cancelled participant beats a silent one. - private void OnCapturedAudio(float[] data, int channels, int sampleRate) - { - if (_canceller != null && _canceller.TryPushCapture(data, channels, sampleRate)) return; - - AudioRead?.Invoke(data, channels, sampleRate); - } - - // Unity audio thread, via the capture pump. - private void OnProcessedAudio(float[] data, int channels, int sampleRate) - { - AudioRead?.Invoke(data, channels, sampleRate); - } - - private void OnApplicationPause(bool pause) - { - if (!_started) return; - - if (pause) - { - // Backgrounded, release the audio resources — leaving them open trips - // AVAudioSession interruption errors (FigCaptureSourceRemote -17281). - _canceller?.Stop(); - RunCoroutine(StopMicrophone()); - } - else - { - RunCoroutine(RestartMicrophone()); - } - } - - private IEnumerator RestartMicrophone() - { - yield return StopMicrophone(); - - // After a resume the iOS audio session needs time to recover from interruption. Poll for - // actual readiness instead of guessing a delay. - yield return WaitForMicrophoneReady(); - - _canceller?.Start(); - yield return StartMicrophone(); - } - - private IEnumerator WaitForMicrophoneReady() - { - const float timeout = 2f; - var elapsed = 0f; - - while (Microphone.devices.Length == 0 && elapsed < timeout) - { - yield return new WaitForSeconds(0.05f); - elapsed += 0.05f; - } - - if (Microphone.devices.Length == 0) - { - Debug.LogError($"[AEC] microphone devices not available after {timeout}s"); - yield break; - } - - yield return null; - } - - // The host is a component on the (caller-owned) microphone GameObject. If that object is - // already gone — scene unload, app quit — drain the coroutine synchronously so Microphone.End - // and the component cleanup still run, as the SDK's MonoBehaviourContext does. - private void RunCoroutine(IEnumerator coroutine) - { - if (_host != null) - { - _host.StartCoroutine(coroutine); - return; - } - - while (coroutine.MoveNext()) - { - if (coroutine.Current is IEnumerator nested) - RunCoroutine(nested); - } - } - - protected override void Dispose(bool disposing) - { - if (!_disposed && disposing) Stop(); - _disposed = true; - - if (_canceller != null) - { - _canceller.CaptureProcessed -= OnProcessedAudio; - _canceller.Dispose(); - } - - base.Dispose(disposing); - } - - ~AecMicrophoneSource() - { - Dispose(false); - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta deleted file mode 100644 index 83f422c2..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 582d502ba6935413ebc5cc0683c21059 -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs deleted file mode 100644 index 7159b8d8..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs +++ /dev/null @@ -1,168 +0,0 @@ -using System; -using System.Runtime.InteropServices; -using LiveKit; - -/// Processes one 10 ms interleaved int16 chunk in place at . -internal delegate string ApmChunkHandler(IntPtr dataPtr, int byteCount, int sampleRate, int channels); - -/// Receives one processed 10 ms chunk as interleaved floats. -internal delegate void ProcessedChunkHandler(float[] data, int channels, int sampleRate); - -/// -/// Re-chunks a variable-size PCM feed into the fixed 10 ms frames the APM requires, processes -/// each one in place, and optionally hands the result on. -/// -/// Neither feed is 10 ms natively: capture arrives in DSP-buffer-sized blocks (1024 frames -/// ≈ 21.3 ms at 48 kHz) and render frames arrive at whatever the decoder emits. Geometry is -/// taken from the incoming buffers, never from declared constants. -/// -/// Allocation-free once the format has settled. The chunk buffer stays pinned for the pump's -/// lifetime so the APM has a stable address to process in place. -/// -internal sealed class ApmChunkPump : IDisposable -{ - private readonly ApmChunkHandler _process; - private readonly ProcessedChunkHandler _onProcessed; - private readonly int _bufferedChunks; - - private PcmRingBuffer _ring; - private short[] _chunk; - private GCHandle _chunkPin; - private IntPtr _chunkPtr; - private int _chunkSamples; - private int _chunkBytes; - - private short[] _staging; - private float[] _processed; - - private int _sampleRate; - private int _channels; - private bool _disposed; - - public int SampleRate => _sampleRate; - public int Channels => _channels; - public int ChunkSamples => _chunkSamples; - public int ProcessedChunkCount { get; private set; } - public int FailedChunkCount { get; private set; } - public int DroppedSamples => _ring?.OverflowSamples ?? 0; - public string LastError { get; private set; } - - /// Processes one chunk in place; returns an error string or null. - /// Optional consumer of the processed chunk (capture side only). - /// - /// Ring capacity in 10 ms chunks. Sets the worst-case added latency, so keep it just large - /// enough to absorb one input block plus jitter. - /// - public ApmChunkPump(ApmChunkHandler process, ProcessedChunkHandler onProcessed = null, int bufferedChunks = 8) - { - if (bufferedChunks <= 0) throw new ArgumentOutOfRangeException(nameof(bufferedChunks)); - _process = process ?? throw new ArgumentNullException(nameof(process)); - _onProcessed = onProcessed; - _bufferedChunks = bufferedChunks; - } - - /// Feeds interleaved floats (Unity capture path). - public void Push(float[] data, int channels, int sampleRate) - { - if (_disposed || data == null || data.Length == 0) return; - if (!EnsureFormat(sampleRate, channels, data.Length)) return; - - if (_staging == null || _staging.Length < data.Length) _staging = new short[data.Length]; - for (var i = 0; i < data.Length; i++) _staging[i] = FloatToS16(data[i]); - - _ring.Write(_staging, 0, data.Length); - Drain(); - } - - /// Feeds interleaved int16 straight from a native buffer (FFI render path). - public void Push(IntPtr dataPtr, int samplesPerChannel, int channels, int sampleRate) - { - if (_disposed || dataPtr == IntPtr.Zero || samplesPerChannel <= 0 || channels <= 0) return; - - var total = samplesPerChannel * channels; - if (!EnsureFormat(sampleRate, channels, total)) return; - - if (_staging == null || _staging.Length < total) _staging = new short[total]; - Marshal.Copy(dataPtr, _staging, 0, total); - - _ring.Write(_staging, 0, total); - Drain(); - } - - public void Reset() => _ring?.Clear(); - - // An FFI failure must not escape: on the capture side this runs inside OnAudioFilterRead, and - // an exception there takes out Unity's audio callback. The unprocessed chunk is forwarded - // instead, so a broken APM degrades to no cancellation rather than to no audio. - private void Drain() - { - while (_ring.TryDrain(_chunk, _chunkSamples)) - { - try - { - var error = _process(_chunkPtr, _chunkBytes, _sampleRate, _channels); - if (error != null) LastError = error; - } - catch (Exception e) - { - LastError = e.Message; - FailedChunkCount++; - } - - ProcessedChunkCount++; - - if (_onProcessed == null) continue; - - for (var i = 0; i < _chunkSamples; i++) _processed[i] = _chunk[i] / 32768f; - _onProcessed(_processed, _channels, _sampleRate); - } - } - - private bool EnsureFormat(int sampleRate, int channels, int incomingSamples) - { - if (sampleRate <= 0 || channels <= 0) return false; - if (sampleRate == _sampleRate && channels == _channels) return true; - - var frameSize = AudioProcessingModule.FrameSizeFor(sampleRate); - if (frameSize <= 0) return false; - - ReleaseChunk(); - - _sampleRate = sampleRate; - _channels = channels; - _chunkSamples = frameSize * channels; - _chunkBytes = _chunkSamples * sizeof(short); - - _chunk = new short[_chunkSamples]; - _chunkPin = GCHandle.Alloc(_chunk, GCHandleType.Pinned); - _chunkPtr = _chunkPin.AddrOfPinnedObject(); - _processed = new float[_chunkSamples]; - - // Never smaller than one input block, or a large block would immediately overflow. - var capacity = Math.Max(_chunkSamples * _bufferedChunks, incomingSamples + _chunkSamples); - _ring = new PcmRingBuffer(capacity); - return true; - } - - private void ReleaseChunk() - { - if (_chunkPin.IsAllocated) _chunkPin.Free(); - _chunkPtr = IntPtr.Zero; - _chunk = null; - } - - public void Dispose() - { - if (_disposed) return; - _disposed = true; - ReleaseChunk(); - } - - private static short FloatToS16(float v) - { - v *= 32768f; - if (v > 32767f) v = 32767f; - else if (v < -32768f) v = -32768f; - return (short)(v + Math.Sign(v) * 0.5f); - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta deleted file mode 100644 index 99d2ca85..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 3213055d7f9af4da4b525a2d8b9f9a1d -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs deleted file mode 100644 index 3a4cdeb2..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs +++ /dev/null @@ -1,64 +0,0 @@ -using System; -using System.Runtime.InteropServices; - -/// -/// Computes the set_stream_delay_ms seed: -/// (t_render - t_analyze) + (t_process - t_capture). -/// -/// AEC3 runs its own correlation-based delay estimator -/// (use_external_delay_estimator = false), so this is a convergence hint, not a hard -/// alignment — but a wildly wrong value slows convergence. iOS sources the platform terms from -/// AVAudioSession (see Plugins/iOS/AudioSessionLatency.mm); Android has no -/// equivalent accessor, so it starts from the buffering this pipeline adds and lets AEC3 find -/// the rest. -/// -/// Known gap: the reference is tapped on the decoder side, so any playout queue between the -/// decoder and the loudspeaker (Unity's AudioStream ring buffer, ~30-200 ms) is not part of -/// this estimate. -/// -internal static class AudioProcessingDelaySeed -{ -#if UNITY_IOS && !UNITY_EDITOR - [DllImport("__Internal")] private static extern double MeetSample_AudioSessionOutputLatency(); - [DllImport("__Internal")] private static extern double MeetSample_AudioSessionInputLatency(); - [DllImport("__Internal")] private static extern double MeetSample_AudioSessionIOBufferDuration(); -#endif - - private const int MinDelayMs = 0; - private const int MaxDelayMs = 500; - - /// Seed used when no platform latency is readable (Android, or an inactive session). - private const int FallbackDelayMs = 60; - - /// - /// Measured duration of one OnAudioFilterRead block, which is how far behind - /// real-time the capture stream already is when it reaches the APM. - /// - public static int Estimate(double captureBlockMs) - { - var platformMs = PlatformLatencyMs(); - var seed = (int)Math.Round(platformMs + captureBlockMs); - return Math.Min(MaxDelayMs, Math.Max(MinDelayMs, seed)); - } - - public static double PlatformLatencyMs() - { -#if UNITY_IOS && !UNITY_EDITOR - try - { - var output = MeetSample_AudioSessionOutputLatency(); - var input = MeetSample_AudioSessionInputLatency(); - var ioBuffer = MeetSample_AudioSessionIOBufferDuration(); - if (output > 0d || input > 0d) - return (output + input + ioBuffer) * 1000d; - } - catch (Exception) - { - // Session not yet active; fall through to the platform-agnostic seed. - } - return FallbackDelayMs; -#else - return FallbackDelayMs; -#endif - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta deleted file mode 100644 index 086e91bd..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 9636ad91efd914976b3ed12df25e7e8c -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs deleted file mode 100644 index 859260bd..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System; -using System.Runtime.InteropServices; -using LiveKit; -using UnityEngine; - -/// -/// Go/no-go check for the FFI , runnable without joining a -/// room. Exercises the full call shape: create a handle, seed a delay, and process one 10 ms -/// chunk in each direction. The APM is compiled into each platform's FFI binary separately, so -/// a per-platform surprise is far cheaper to find here than after the pipeline is wired. -/// -internal static class AudioProcessingSmokeTest -{ - public static string Run() - { - var rate = AudioSettings.outputSampleRate > 0 ? AudioSettings.outputSampleRate : 48000; - var frameSize = AudioProcessingModule.FrameSizeFor(rate); - - AudioProcessingModule apm; - try - { - apm = new AudioProcessingModule( - echoCancellerEnabled: true, - gainControllerEnabled: false, - highPassFilterEnabled: false, - noiseSuppressionEnabled: false); - } - catch (Exception e) - { - return $"FAIL create_apm: {e.GetType().Name}: {e.Message}"; - } - - using (apm) - { - var delayError = apm.SetStreamDelayMs(100); - if (delayError != null) return $"FAIL set_stream_delay_ms: {delayError}"; - - var chunk = new short[frameSize]; - var pin = GCHandle.Alloc(chunk, GCHandleType.Pinned); - try - { - var ptr = pin.AddrOfPinnedObject(); - var bytes = chunk.Length * sizeof(short); - - var reverseError = apm.ProcessReverseStream(ptr, bytes, rate, 1); - if (reverseError != null) return $"FAIL process_reverse_stream: {reverseError}"; - - var processError = apm.ProcessStream(ptr, bytes, rate, 1); - if (processError != null) return $"FAIL process_stream: {processError}"; - } - finally - { - pin.Free(); - } - - return $"PASS handle={apm.Handle} rate={rate} frameSize={frameSize} " + - $"nativeRate={AudioProcessingModule.IsNativeSampleRate(rate)} " + - $"delaySeed={AudioProcessingDelaySeed.Estimate(0d)}ms"; - } - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta deleted file mode 100644 index cf3748bf..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 78f2bfe939fd54d97b797b6b28398e88 -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs b/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs deleted file mode 100644 index f4b3e8fd..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System; - -/// -/// Fixed-capacity interleaved int16 PCM ring buffer with a fixed-size drain. -/// -/// Allocation-free after construction — both feeds run on audio-priority threads. Sized in -/// samples (interleaved, i.e. frames × channels), not frames. -/// -/// Single producer / single consumer per direction; the near-end and far-end feeds each own -/// their own instance, so no synchronisation is needed inside. -/// -internal sealed class PcmRingBuffer -{ - private readonly short[] _buffer; - private int _readIndex; - private int _writeIndex; - private int _count; - - /// Samples dropped because the buffer was full, since construction. - public int OverflowSamples { get; private set; } - - public int Capacity => _buffer.Length; - public int Available => _count; - - public PcmRingBuffer(int capacitySamples) - { - if (capacitySamples <= 0) throw new ArgumentOutOfRangeException(nameof(capacitySamples)); - _buffer = new short[capacitySamples]; - } - - /// - /// Appends samples. When the buffer is full the OLDEST samples are - /// dropped: a stalled consumer must not push the echo reference arbitrarily far out of - /// alignment with the capture stream. - /// - public void Write(short[] source, int offset, int count) - { - if (source == null) throw new ArgumentNullException(nameof(source)); - if (offset < 0 || count < 0 || offset + count > source.Length) - throw new ArgumentOutOfRangeException(nameof(count)); - - if (count >= _buffer.Length) - { - OverflowSamples += _count + count - _buffer.Length; - offset += count - _buffer.Length; - count = _buffer.Length; - _readIndex = 0; - _writeIndex = 0; - _count = 0; - } - else - { - var free = _buffer.Length - _count; - if (count > free) Discard(count - free); - } - - for (var i = 0; i < count; i++) - { - _buffer[_writeIndex] = source[offset + i]; - _writeIndex = _writeIndex + 1 == _buffer.Length ? 0 : _writeIndex + 1; - } - - _count += count; - } - - /// - /// Copies exactly samples into and - /// consumes them. Returns false and consumes nothing when fewer are available. - /// - public bool TryDrain(short[] destination, int count) - { - if (destination == null) throw new ArgumentNullException(nameof(destination)); - if (count < 0 || count > destination.Length) throw new ArgumentOutOfRangeException(nameof(count)); - if (_count < count) return false; - - for (var i = 0; i < count; i++) - { - destination[i] = _buffer[_readIndex]; - _readIndex = _readIndex + 1 == _buffer.Length ? 0 : _readIndex + 1; - } - - _count -= count; - return true; - } - - public void Clear() - { - _readIndex = 0; - _writeIndex = 0; - _count = 0; - } - - private void Discard(int count) - { - if (count > _count) count = _count; - _readIndex = (_readIndex + count) % _buffer.Length; - _count -= count; - OverflowSamples += count; - } -} diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index c0bf6c0a..b9893fbc 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -13,9 +13,9 @@ /// - PlatformAudio (default): Uses WebRTC's ADM for microphone capture and automatic /// speaker playout. Provides echo cancellation (AEC), AGC, and noise suppression. /// - Unity Audio: Uses Unity's Microphone API and AudioStream for manual audio handling. -/// Gives more control over audio processing. Optionally runs libwebrtc's AEC3 over the -/// captured audio (), using the decoded remote audio frames -/// as the echo reference; without it there is no echo cancellation in this mode. +/// Gives more control over audio processing. The same AEC/NS/AGC toggles apply: the SDK runs +/// libwebrtc's audio processing over the Microphone capture, with the mix Unity plays as the +/// echo reference (see ). /// [RequireComponent(typeof(TokenSourceComponent))] public class MeetManager : MonoBehaviour @@ -36,22 +36,18 @@ public class MeetManager : MonoBehaviour "Provides AEC, AGC, and NS. Disable to use Unity's Microphone API instead.")] [SerializeField] private bool usePlatformAudio = true; - [Header("Audio Processing (PlatformAudio only)")] - [Tooltip("Enable echo cancellation to remove echo from speaker playback.")] + [Header("Audio Processing")] + [Tooltip("Enable echo cancellation. PlatformAudio: WebRTC's ADM. Unity audio: libwebrtc's AEC3 over the " + + "Microphone capture, with the mix Unity plays as the reference.")] [SerializeField] private bool echoCancellation = true; [Tooltip("Enable noise suppression to remove background noise.")] [SerializeField] private bool noiseSuppression = true; [Tooltip("Enable auto gain control to normalize audio levels.")] [SerializeField] private bool autoGainControl = true; - [Tooltip("Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] + [Tooltip("PlatformAudio only. Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] [SerializeField] private bool preferHardwareProcessing = true; [Header("Unity Audio (PlatformAudio off)")] - [Tooltip("Run libwebrtc's AEC3 (the FFI AudioProcessingModule) over Unity microphone capture, " + - "using the decoded remote audio frames as the echo reference. Assumes a single remote " + - "audio stream: with several remote speakers the reference is the interleaving of all of " + - "them and the canceller will not converge.")] - [SerializeField] private bool unityEchoCancellation = true; [Tooltip("Playback gain for every remote AudioSource in Unity audio mode. Kept below 1 so a device " + "at full speaker volume keeps headroom: full-scale playout distorts on Android and feeds " + "the echo canceller more echo than it can remove. Linear amplitude, 0.7 is -3.1 dB.")] @@ -104,8 +100,6 @@ private void Start() if (usePlatformAudio) InitializePlatformAudio(); - else if (unityEchoCancellation) - Debug.Log($"AEC smoke test: {AudioProcessingSmokeTest.Run()}"); } private void InitializePlatformAudio() @@ -622,7 +616,7 @@ private IEnumerator PublishLocalMicrophonePlatform() private IEnumerator PublishLocalMicrophoneUnity() { - Debug.Log($"Publishing microphone using Unity Microphone API (AEC3: {unityEchoCancellation})"); + Debug.Log($"Publishing microphone using Unity Microphone API (AEC={echoCancellation}, NS={noiseSuppression}, AGC={autoGainControl})"); // Start the microphone here for early iOS permission request and android getting access to Microphone.devices Microphone.Start(null, true, 10, 44100); @@ -630,14 +624,18 @@ private IEnumerator PublishLocalMicrophoneUnity() var audioObject = new GameObject($"My Microphone: {Microphone.devices[0]}"); audioObject.transform.SetParent(_audioTrackParent); - // AecMicrophoneSource re-implements MicrophoneSource with an APM stage in between: - // captured blocks go through AEC3 (reference = decoded remote frames) before they reach - // the track. If the APM cannot be created it publishes the raw microphone instead. - RtcAudioSource rtcSource; - if (unityEchoCancellation) - rtcSource = AecMicrophoneSource.Create(Microphone.devices[0], audioObject); - else - rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject); + // With options, MicrophoneSource runs libwebrtc's audio processing over the capture. Echo + // cancellation takes its reference from the mix Unity plays (the SDK attaches a + // PlayoutReference to the AudioListener), so it covers every remote AudioStream and the + // app's own audio. If the module cannot be created the source publishes the raw microphone. + var processing = new AudioProcessingOptions + { + EchoCancellation = echoCancellation, + NoiseSuppression = noiseSuppression, + AutoGainControl = autoGainControl, + HighPassFilter = true + }; + var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject, processing); _localAudioTrack = LocalAudioTrack.CreateAudioTrack(LocalAudioTrackName, rtcSource, _room); @@ -652,8 +650,8 @@ private IEnumerator PublishLocalMicrophoneUnity() if (publish.IsError) { - // Dispose before destroying the host object so the source (and, for AEC, its APM - // handle) is released now rather than by the finalizer. + // Dispose before destroying the host object so the source and its processing module + // are released now rather than by the finalizer. rtcSource.Dispose(); Destroy(audioObject); _localAudioTrack = null; @@ -665,9 +663,24 @@ private IEnumerator PublishLocalMicrophoneUnity() _localRtcAudioSource = rtcSource; rtcSource.Start(); - Debug.Log(rtcSource is AecMicrophoneSource { EchoCancellationActive: true } - ? "Microphone published via Unity Microphone API (AEC3 active)" - : "Microphone published via Unity Microphone API (no AEC)"); + Debug.Log(rtcSource.AudioProcessingEnabled + ? "Microphone published via Unity Microphone API (audio processing active)" + : "Microphone published via Unity Microphone API (no audio processing)"); + + if (rtcSource.AudioProcessingEnabled) + StartCoroutine(LogAudioProcessingStats(rtcSource)); + } + + // Periodic snapshot of the processing stage while the Unity microphone is published: confirms + // the playout reference is attached and that capture and reference chunks are flowing. + private IEnumerator LogAudioProcessingStats(RtcAudioSource source) + { + while (_microphoneActive && ReferenceEquals(_localRtcAudioSource, source)) + { + yield return new WaitForSeconds(5f); + if (!_microphoneActive || !ReferenceEquals(_localRtcAudioSource, source)) yield break; + Debug.Log($"Audio processing: {source.AudioProcessingStats}"); + } } private void UnpublishLocalMicrophone() diff --git a/Samples~/Meet/Assets/Scenes/MeetApp.unity b/Samples~/Meet/Assets/Scenes/MeetApp.unity index 3122a7c1..3e03e2d1 100644 --- a/Samples~/Meet/Assets/Scenes/MeetApp.unity +++ b/Samples~/Meet/Assets/Scenes/MeetApp.unity @@ -907,7 +907,6 @@ MonoBehaviour: noiseSuppression: 1 autoGainControl: 1 preferHardwareProcessing: 1 - unityEchoCancellation: 1 remoteAudioGain: 0.7 --- !u!114 &1478206705 MonoBehaviour: @@ -921,7 +920,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: a498c208deeab40c39b4ba609d7d222c, type: 3} m_Name: m_EditorClassIdentifier: - _config: {fileID: 11400000, guid: 6d52b5cb4c971436098df568ef2e2c67, type: 2} + _config: {fileID: 11400000, guid: aae6c4b1158ca4c929c5d84962c95f91, type: 2} --- !u!1 &1651282853 GameObject: m_ObjectHideFlags: 0 diff --git a/Tests/EditMode/AudioProcessingTests.cs b/Tests/EditMode/AudioProcessingTests.cs new file mode 100644 index 00000000..0295dfbc --- /dev/null +++ b/Tests/EditMode/AudioProcessingTests.cs @@ -0,0 +1,155 @@ +using NUnit.Framework; + +namespace LiveKit.EditModeTests +{ + /// + /// Pure-managed tests for the Unity-audio processing helpers: the 10 ms re-chunking, sample + /// conversion, rate rules and the delay hint. No FFI, no audio device, so they always run. + /// + public class AudioProcessingTests + { + [Test] + public void PcmRingBuffer_DrainsInWriteOrder() + { + var ring = new PcmRingBuffer(8); + ring.Write(new short[] { 1, 2, 3, 4, 5 }, 0, 5); + + var dest = new short[3]; + Assert.IsTrue(ring.TryDrain(dest, 3)); + Assert.AreEqual(new short[] { 1, 2, 3 }, dest); + Assert.AreEqual(2, ring.Available); + + var rest = new short[2]; + Assert.IsTrue(ring.TryDrain(rest, 2)); + Assert.AreEqual(new short[] { 4, 5 }, rest); + Assert.AreEqual(0, ring.Available); + Assert.AreEqual(0, ring.OverflowSamples); + } + + [Test] + public void PcmRingBuffer_TryDrain_WithoutEnoughSamples_ConsumesNothing() + { + var ring = new PcmRingBuffer(8); + ring.Write(new short[] { 1, 2 }, 0, 2); + + Assert.IsFalse(ring.TryDrain(new short[3], 3)); + Assert.AreEqual(2, ring.Available); + } + + [Test] + public void PcmRingBuffer_WhenFull_DropsOldest() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + ring.Write(new short[] { 4, 5 }, 0, 2); + + Assert.AreEqual(1, ring.OverflowSamples); + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 2, 3, 4, 5 }, dest); + } + + [Test] + public void PcmRingBuffer_WriteLargerThanCapacity_KeepsNewest() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2 }, 0, 2); + ring.Write(new short[] { 3, 4, 5, 6, 7 }, 0, 5); + + Assert.AreEqual(3, ring.OverflowSamples); + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 4, 5, 6, 7 }, dest); + } + + [Test] + public void PcmRingBuffer_WrapsAround() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + Assert.IsTrue(ring.TryDrain(new short[2], 2)); + ring.Write(new short[] { 4, 5, 6 }, 0, 3); + + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 3, 4, 5, 6 }, dest); + Assert.AreEqual(0, ring.OverflowSamples); + } + + [Test] + public void PcmRingBuffer_Clear_Empties() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + ring.Clear(); + + Assert.AreEqual(0, ring.Available); + Assert.IsFalse(ring.TryDrain(new short[1], 1)); + } + + [Test] + public void PcmConvert_FloatToS16_ClampsAndRounds() + { + Assert.AreEqual(0, PcmConvert.FloatToS16(0f)); + Assert.AreEqual(16384, PcmConvert.FloatToS16(0.5f)); + Assert.AreEqual(-16384, PcmConvert.FloatToS16(-0.5f)); + Assert.AreEqual(short.MaxValue, PcmConvert.FloatToS16(1f)); + Assert.AreEqual(short.MinValue, PcmConvert.FloatToS16(-1f)); + Assert.AreEqual(short.MaxValue, PcmConvert.FloatToS16(3f)); + Assert.AreEqual(short.MinValue, PcmConvert.FloatToS16(-3f)); + } + + [Test] + public void AudioProcessingModule_SupportedApiRates_NeedWholeSampleChunks() + { + // The Rust side asserts on a frame that is not a whole multiple of 10 ms, so a rate whose + // 10 ms chunk is fractional must be refused up front. + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(48000)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(44100)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(24000)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(16000)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(22050)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(11025)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(0)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(-48000)); + } + + [Test] + public void AudioProcessingModule_FrameSizeFor_IsTenMilliseconds() + { + Assert.AreEqual(480, AudioProcessingModule.FrameSizeFor(48000)); + Assert.AreEqual(441, AudioProcessingModule.FrameSizeFor(44100)); + Assert.AreEqual(240, AudioProcessingModule.FrameSizeFor(24000)); + Assert.IsTrue(AudioProcessingModule.IsNativeSampleRate(48000)); + Assert.IsFalse(AudioProcessingModule.IsNativeSampleRate(44100)); + } + + [Test] + public void AudioProcessingOptions_Default_EnablesHighPassFilter_AndReportsProcessing() + { + Assert.IsTrue(AudioProcessingOptions.Default.HighPassFilter); + Assert.IsTrue(AudioProcessingOptions.Default.AnyProcessingEnabled); + + // An all-false struct means "no processing"; PreferHardware alone is not a stage. + Assert.IsFalse(default(AudioProcessingOptions).AnyProcessingEnabled); + Assert.IsFalse(new AudioProcessingOptions { PreferHardware = true }.AnyProcessingEnabled); + Assert.IsTrue(new AudioProcessingOptions { HighPassFilter = true }.AnyProcessingEnabled); + } + + [Test] + public void DelayHint_SumsQueueDeviceAndMicrophoneTerms() + { + // 1024 frames at 48 kHz = 21.33 ms per block; two queued blocks + 30 ms device + 50 ms + // microphone read-behind = 122.67 ms. + Assert.AreEqual(123, AudioProcessingDelayHint.EstimateMs(1024, 48000, 30)); + Assert.AreEqual(AudioProcessingDelayHint.MicrophoneReadBehindMs, AudioProcessingDelayHint.EstimateMs(0, 0, 0)); + } + + [Test] + public void DelayHint_ClampsToRange() + { + Assert.AreEqual(AudioProcessingDelayHint.MaxDelayMs, AudioProcessingDelayHint.EstimateMs(48000, 48000, 1000)); + Assert.AreEqual(AudioProcessingDelayHint.MinDelayMs, AudioProcessingDelayHint.EstimateMs(0, 0, -1000)); + } + } +} diff --git a/Tests/EditMode/AudioProcessingTests.cs.meta b/Tests/EditMode/AudioProcessingTests.cs.meta new file mode 100644 index 00000000..80c9b2e9 --- /dev/null +++ b/Tests/EditMode/AudioProcessingTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: e9ce682bd85ef402eb8ce57e87d70231 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/EditMode/PlatformAudioTests.cs b/Tests/EditMode/PlatformAudioTests.cs index b7608ac7..8c4bca40 100644 --- a/Tests/EditMode/PlatformAudioTests.cs +++ b/Tests/EditMode/PlatformAudioTests.cs @@ -18,6 +18,7 @@ public void AudioProcessingOptions_Default_EnablesProcessingAndHardware() Assert.IsTrue(options.EchoCancellation, "AEC should be enabled by default"); Assert.IsTrue(options.NoiseSuppression, "NS should be enabled by default"); Assert.IsTrue(options.AutoGainControl, "AGC should be enabled by default"); + Assert.IsTrue(options.HighPassFilter, "HPF should be enabled by default"); // Unlike the C++ defaults (prefer_hardware == false), the Unity default prefers // hardware processing (e.g. iOS VPIO) for lower latency. Assert.IsTrue(options.PreferHardware, "Unity default prefers hardware processing"); diff --git a/Tests/PlayMode/AudioProcessingTests.cs b/Tests/PlayMode/AudioProcessingTests.cs new file mode 100644 index 00000000..f132e77e --- /dev/null +++ b/Tests/PlayMode/AudioProcessingTests.cs @@ -0,0 +1,241 @@ +using System; +using System.Collections; +using System.Runtime.InteropServices; +using NUnit.Framework; +using Unity.Collections; +using UnityEngine; +using UnityEngine.TestTools; + +namespace LiveKit.PlayModeTests +{ + /// + /// Tests for the Unity-audio processing stage. They need the FFI for the module and, for the + /// echo test, Unity's audio thread. No LiveKit server. + /// + class AudioProcessingTests + { + [Test] + public void AudioProcessingModule_AcceptsTenMillisecondChunks() + { + using var apm = new AudioProcessingModule( + echoCancellerEnabled: true, + gainControllerEnabled: true, + highPassFilterEnabled: true, + noiseSuppressionEnabled: true); + + const int rate = 48000; + const int channels = 2; + var chunk = new short[AudioProcessingModule.FrameSizeFor(rate) * channels]; + var pin = GCHandle.Alloc(chunk, GCHandleType.Pinned); + try + { + var ptr = pin.AddrOfPinnedObject(); + var bytes = chunk.Length * sizeof(short); + Assert.IsNull(apm.ProcessReverseStream(ptr, bytes, rate, channels)); + Assert.IsNull(apm.ProcessStream(ptr, bytes, rate, channels)); + Assert.IsNull(apm.SetStreamDelayMs(80)); + } + finally + { + pin.Free(); + } + } + + [UnityTest] + public IEnumerator RtcAudioSource_WithProcessing_CapturesInTenMillisecondChunks() + { + using var source = new PushAudioSource(AudioProcessingOptions.Default); + Assert.IsTrue(source.AudioProcessingEnabled, "module creation failed"); + + var rate = (int)source._expectedSampleRate; + var channels = (int)source._expectedChannels; + if (!AudioProcessingModule.IsSupportedApiRate(rate)) + Assert.Ignore($"output rate {rate} has no whole-sample 10 ms chunk"); + + source.Start(); + + const int blockFrames = 1024; + const int blocks = 10; + var block = new float[blockFrames * channels]; + for (int i = 0; i < block.Length; i++) + block[i] = 0.1f * Mathf.Sin(i * 0.05f); + for (int i = 0; i < blocks; i++) + source.Push(block, channels, rate); + + // Let the capture callbacks return before disposing. + yield return new WaitForSeconds(0.2f); + + var stats = source.AudioProcessingStats; + var expectedChunks = blockFrames * blocks / AudioProcessingModule.FrameSizeFor(rate); + Assert.IsTrue(stats.Active, stats.ToString()); + Assert.AreEqual(0, stats.FailedChunks, stats.LastError); + Assert.That(stats.CaptureChunks, Is.InRange(expectedChunks - 1, expectedChunks), stats.ToString()); + Assert.AreEqual(0, stats.DroppedCaptureSamples, stats.ToString()); + + source.Stop(); + } + + /// + /// End-to-end check of the canceller without hardware: the listener hears a noise source, + /// and a second source plays the same noise 120 ms later, probed as "microphone" and then + /// cleared so the mix contains only the far end. The capture is therefore a pure delayed + /// echo of the playout reference, which AEC3 must learn to remove. + /// + [UnityTest] + public IEnumerator AudioProcessor_CancelsDelayedEchoOfPlayout() + { + var rate = AudioSettings.outputSampleRate; + if (!AudioProcessingModule.IsSupportedApiRate(rate)) + Assert.Ignore($"output rate {rate} has no whole-sample 10 ms chunk"); + + var listenerGo = new GameObject("AecTestListener"); + listenerGo.AddComponent(); + + var clip = NoiseClip(rate, seconds: 2f, seed: 1234, amplitude: 0.3f); + + var farGo = new GameObject("AecTestFarEnd"); + var far = farGo.AddComponent(); + far.clip = clip; + far.loop = true; + + var nearGo = new GameObject("AecTestNearEnd"); + var near = nearGo.AddComponent(); + near.clip = clip; + near.loop = true; + var probe = nearGo.AddComponent(); + probe.ClearAfterInvocation(); + + var meter = new EchoMeter(); + var processor = new AudioProcessor( + new AudioProcessingOptions { EchoCancellation = true, HighPassFilter = true }, + meter.OnProcessed); + probe.AudioRead += (data, channels, sampleRate) => + { + meter.OnRaw(data); + processor.TryProcessCapture(data, channels, sampleRate); + }; + processor.Start(); + + var startTime = AudioSettings.dspTime + 0.2; + far.PlayScheduled(startTime); + near.PlayScheduled(startTime + 0.12); + + AudioProcessingStats stats; + float rawRms, processedRms; + try + { + yield return new WaitForSeconds(1f); + if (meter.RawBlocks == 0) + Assert.Ignore("Unity's audio thread delivered no capture callbacks (no audio device?)"); + Assert.IsTrue(PlayoutReference.IsAttached, "reference not attached to the listener"); + + // Convergence time, then a clean measurement window. + yield return new WaitForSeconds(3f); + meter.ResetWindow(); + yield return new WaitForSeconds(1.5f); + + (rawRms, processedRms) = meter.Window(); + stats = processor.GetStats(); + } + finally + { + processor.Dispose(); + UnityEngine.Object.Destroy(farGo); + UnityEngine.Object.Destroy(nearGo); + UnityEngine.Object.Destroy(listenerGo); + } + + Assert.Greater(rawRms, 0.01f, "near-end source produced no signal"); + Assert.AreEqual(0, stats.FailedChunks, stats.ToString()); + Assert.Greater(stats.ReferenceChunks, 0, stats.ToString()); + + var attenuationDb = 20f * Mathf.Log10(rawRms / Mathf.Max(processedRms, 1e-6f)); + Debug.Log($"AEC3 attenuated the synthetic echo by {attenuationDb:F1} dB ({stats})"); + Assert.GreaterOrEqual(attenuationDb, 6f, $"AEC3 attenuated the echo by only {attenuationDb:F1} dB; {stats}"); + } + + private static AudioClip NoiseClip(int sampleRate, float seconds, int seed, float amplitude) + { + var samples = (int)(sampleRate * seconds); + var data = new float[samples]; + var random = new System.Random(seed); + for (int i = 0; i < samples; i++) + data[i] = amplitude * (float)(random.NextDouble() * 2.0 - 1.0); + + var clip = AudioClip.Create("AecTestNoise", samples, 1, sampleRate, false); + clip.SetData(data, 0); + return clip; + } + + private sealed class PushAudioSource : RtcAudioSource + { + public override event Action AudioRead; + + public PushAudioSource(AudioProcessingOptions options) + : base(RtcAudioSourceType.AudioSourceMicrophone, options) { } + + public void Push(float[] data, int channels, int sampleRate) => AudioRead?.Invoke(data, channels, sampleRate); + } + + // Accumulates energy of the raw near end and of the processed output. Both callbacks run on + // the Unity audio thread; the sink owns and disposes the frames it is handed. + private sealed class EchoMeter + { + private readonly object _lock = new object(); + private double _rawSum; + private long _rawCount; + private double _processedSum; + private long _processedCount; + private int _rawBlocks; + + public int RawBlocks { get { lock (_lock) return _rawBlocks; } } + + public void OnRaw(float[] data) + { + double sum = 0; + for (int i = 0; i < data.Length; i++) sum += data[i] * data[i]; + lock (_lock) + { + _rawSum += sum; + _rawCount += data.Length; + _rawBlocks++; + } + } + + public void OnProcessed(NativeArray frame, int channels, int sampleRate) + { + double sum = 0; + var length = frame.Length; + for (int i = 0; i < length; i++) + { + var v = frame[i] / 32768.0; + sum += v * v; + } + frame.Dispose(); + lock (_lock) + { + _processedSum += sum; + _processedCount += length; + } + } + + public void ResetWindow() + { + lock (_lock) + { + _rawSum = 0; + _rawCount = 0; + _processedSum = 0; + _processedCount = 0; + } + } + + public (float raw, float processed) Window() + { + lock (_lock) return (Rms(_rawSum, _rawCount), Rms(_processedSum, _processedCount)); + } + + private static float Rms(double sum, long count) => count == 0 ? 0f : (float)Math.Sqrt(sum / count); + } + } +} diff --git a/Tests/PlayMode/AudioProcessingTests.cs.meta b/Tests/PlayMode/AudioProcessingTests.cs.meta new file mode 100644 index 00000000..4b6b1e59 --- /dev/null +++ b/Tests/PlayMode/AudioProcessingTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b22944e23e00341eb8cd913bce198b4e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From abeb06dcc3b544a33dd2e78e063a64af31df1265 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:20:29 +0200 Subject: [PATCH 3/4] Created subfolder for audio processing --- Runtime/Scripts/Audio/Processing.meta | 8 ++++++++ .../Audio/{ => Processing}/AudioProcessingDelayHint.cs | 0 .../{ => Processing}/AudioProcessingDelayHint.cs.meta | 0 .../Audio/{ => Processing}/AudioProcessingModule.cs | 0 .../Audio/{ => Processing}/AudioProcessingModule.cs.meta | 0 .../Audio/{ => Processing}/AudioProcessingStats.cs | 0 .../Audio/{ => Processing}/AudioProcessingStats.cs.meta | 0 Runtime/Scripts/Audio/{ => Processing}/AudioProcessor.cs | 0 .../Scripts/Audio/{ => Processing}/AudioProcessor.cs.meta | 0 Runtime/Scripts/Audio/{ => Processing}/PcmRingBuffer.cs | 0 .../Scripts/Audio/{ => Processing}/PcmRingBuffer.cs.meta | 0 .../Scripts/Audio/{ => Processing}/PlayoutReference.cs | 0 .../Audio/{ => Processing}/PlayoutReference.cs.meta | 0 13 files changed, 8 insertions(+) create mode 100644 Runtime/Scripts/Audio/Processing.meta rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingDelayHint.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingDelayHint.cs.meta (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingModule.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingModule.cs.meta (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingStats.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingStats.cs.meta (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessor.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessor.cs.meta (100%) rename Runtime/Scripts/Audio/{ => Processing}/PcmRingBuffer.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/PcmRingBuffer.cs.meta (100%) rename Runtime/Scripts/Audio/{ => Processing}/PlayoutReference.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/PlayoutReference.cs.meta (100%) diff --git a/Runtime/Scripts/Audio/Processing.meta b/Runtime/Scripts/Audio/Processing.meta new file mode 100644 index 00000000..180519b6 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: afb866c8c71a34134bf00d5cd4ee1778 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingDelayHint.cs rename to Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs diff --git a/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta rename to Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs.meta diff --git a/Runtime/Scripts/Audio/AudioProcessingModule.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingModule.cs rename to Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs diff --git a/Runtime/Scripts/Audio/AudioProcessingModule.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingModule.cs.meta rename to Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs.meta diff --git a/Runtime/Scripts/Audio/AudioProcessingStats.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingStats.cs rename to Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs diff --git a/Runtime/Scripts/Audio/AudioProcessingStats.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingStats.cs.meta rename to Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs.meta diff --git a/Runtime/Scripts/Audio/AudioProcessor.cs b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessor.cs rename to Runtime/Scripts/Audio/Processing/AudioProcessor.cs diff --git a/Runtime/Scripts/Audio/AudioProcessor.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessor.cs.meta rename to Runtime/Scripts/Audio/Processing/AudioProcessor.cs.meta diff --git a/Runtime/Scripts/Audio/PcmRingBuffer.cs b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs similarity index 100% rename from Runtime/Scripts/Audio/PcmRingBuffer.cs rename to Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs diff --git a/Runtime/Scripts/Audio/PcmRingBuffer.cs.meta b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/PcmRingBuffer.cs.meta rename to Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs.meta diff --git a/Runtime/Scripts/Audio/PlayoutReference.cs b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs similarity index 100% rename from Runtime/Scripts/Audio/PlayoutReference.cs rename to Runtime/Scripts/Audio/Processing/PlayoutReference.cs diff --git a/Runtime/Scripts/Audio/PlayoutReference.cs.meta b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/PlayoutReference.cs.meta rename to Runtime/Scripts/Audio/Processing/PlayoutReference.cs.meta From be5d9c4651f43590b47807ba699baeeae05b63df Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:53:33 +0200 Subject: [PATCH 4/4] Drop the HighPassFilter option and make AudioProcessingModule internal libwebrtc instantiates the high-pass filter whenever AEC or NS is enabled (echo_canceller.enforce_high_pass_filtering defaults to true) and the platform ADM path always runs it, so the flag only ever mattered with AEC and NS both off. Hardcode it on for the Unity-audio path so both paths match, and keep the FFI wrapper off the public surface: sources configure it through AudioProcessingOptions. Co-Authored-By: Claude Fable 5.1 --- README.md | 3 +-- Runtime/Scripts/Audio/PlatformAudioSource.cs | 11 ++--------- .../Scripts/Audio/Processing/AudioProcessingModule.cs | 2 +- Runtime/Scripts/Audio/Processing/AudioProcessor.cs | 5 ++++- Samples~/Meet/Assets/Runtime/MeetManager.cs | 3 +-- Tests/EditMode/AudioProcessingTests.cs | 5 ++--- Tests/EditMode/PlatformAudioTests.cs | 1 - Tests/PlayMode/AudioProcessingTests.cs | 2 +- 8 files changed, 12 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ce91509a..c4c78532 100644 --- a/README.md +++ b/README.md @@ -320,8 +320,7 @@ var processing = new AudioProcessingOptions { EchoCancellation = true, NoiseSuppression = true, - AutoGainControl = true, - HighPassFilter = true + AutoGainControl = true }; var rtcSource = new MicrophoneSource(Microphone.devices[0], microphoneObject, processing); ``` diff --git a/Runtime/Scripts/Audio/PlatformAudioSource.cs b/Runtime/Scripts/Audio/PlatformAudioSource.cs index 5a7066fc..119adbf1 100644 --- a/Runtime/Scripts/Audio/PlatformAudioSource.cs +++ b/Runtime/Scripts/Audio/PlatformAudioSource.cs @@ -9,7 +9,7 @@ namespace LiveKit /// /// Options for libwebrtc's audio processing. Used by , where /// the ADM applies them, and by Unity-audio sources such as - /// created with options, where the SDK runs the over the + /// created with options, where the SDK runs libwebrtc's audio processing module over the /// capture with the mix Unity plays as the echo reference (see ). /// public struct AudioProcessingOptions @@ -21,12 +21,6 @@ public struct AudioProcessingOptions /// Enable automatic gain control (AGC). Default: true. public bool AutoGainControl; /// - /// Enable the high-pass filter, which removes DC offset and low-frequency rumble ahead of - /// the other stages. Unity-audio sources only; ignores it. - /// Default: true. - /// - public bool HighPassFilter; - /// /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. /// only. Default: true. /// @@ -40,12 +34,11 @@ public struct AudioProcessingOptions EchoCancellation = true, NoiseSuppression = true, AutoGainControl = true, - HighPassFilter = true, PreferHardware = true }; /// Whether any stage of the Unity-audio processing pipeline is enabled. - internal bool AnyProcessingEnabled => EchoCancellation || NoiseSuppression || AutoGainControl || HighPassFilter; + internal bool AnyProcessingEnabled => EchoCancellation || NoiseSuppression || AutoGainControl; } /// diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs index d83f83be..5e264a2b 100644 --- a/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs @@ -21,7 +21,7 @@ namespace LiveKit /// the native module is internally synchronised for exactly that split, and the SDK's request /// plumbing is safe to use from both. /// - public sealed class AudioProcessingModule : IDisposable + internal sealed class AudioProcessingModule : IDisposable { /// libwebrtc's kChunkSizeMs — the APM accepts nothing else. public const int ChunkSizeMs = 10; diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs index f9d8d34f..ca72566c 100644 --- a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs +++ b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs @@ -87,10 +87,13 @@ public AudioProcessor(AudioProcessingOptions options, ProcessedFrameSink sink) { _sink = sink ?? throw new ArgumentNullException(nameof(sink)); _echoCancellation = options.EchoCancellation; + // The high-pass filter is not exposed as an option: libwebrtc instantiates it anyway + // whenever AEC or NS is on, and the platform ADM path always runs it, so both paths + // stay identical. _apm = new AudioProcessingModule( echoCancellerEnabled: options.EchoCancellation, gainControllerEnabled: options.AutoGainControl, - highPassFilterEnabled: options.HighPassFilter, + highPassFilterEnabled: true, noiseSuppressionEnabled: options.NoiseSuppression); } diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index b9893fbc..29314dfa 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -632,8 +632,7 @@ private IEnumerator PublishLocalMicrophoneUnity() { EchoCancellation = echoCancellation, NoiseSuppression = noiseSuppression, - AutoGainControl = autoGainControl, - HighPassFilter = true + AutoGainControl = autoGainControl }; var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject, processing); diff --git a/Tests/EditMode/AudioProcessingTests.cs b/Tests/EditMode/AudioProcessingTests.cs index 0295dfbc..4c29ba3b 100644 --- a/Tests/EditMode/AudioProcessingTests.cs +++ b/Tests/EditMode/AudioProcessingTests.cs @@ -125,15 +125,14 @@ public void AudioProcessingModule_FrameSizeFor_IsTenMilliseconds() } [Test] - public void AudioProcessingOptions_Default_EnablesHighPassFilter_AndReportsProcessing() + public void AudioProcessingOptions_AnyProcessingEnabled_TracksTheProcessingStages() { - Assert.IsTrue(AudioProcessingOptions.Default.HighPassFilter); Assert.IsTrue(AudioProcessingOptions.Default.AnyProcessingEnabled); // An all-false struct means "no processing"; PreferHardware alone is not a stage. Assert.IsFalse(default(AudioProcessingOptions).AnyProcessingEnabled); Assert.IsFalse(new AudioProcessingOptions { PreferHardware = true }.AnyProcessingEnabled); - Assert.IsTrue(new AudioProcessingOptions { HighPassFilter = true }.AnyProcessingEnabled); + Assert.IsTrue(new AudioProcessingOptions { NoiseSuppression = true }.AnyProcessingEnabled); } [Test] diff --git a/Tests/EditMode/PlatformAudioTests.cs b/Tests/EditMode/PlatformAudioTests.cs index 8c4bca40..b7608ac7 100644 --- a/Tests/EditMode/PlatformAudioTests.cs +++ b/Tests/EditMode/PlatformAudioTests.cs @@ -18,7 +18,6 @@ public void AudioProcessingOptions_Default_EnablesProcessingAndHardware() Assert.IsTrue(options.EchoCancellation, "AEC should be enabled by default"); Assert.IsTrue(options.NoiseSuppression, "NS should be enabled by default"); Assert.IsTrue(options.AutoGainControl, "AGC should be enabled by default"); - Assert.IsTrue(options.HighPassFilter, "HPF should be enabled by default"); // Unlike the C++ defaults (prefer_hardware == false), the Unity default prefers // hardware processing (e.g. iOS VPIO) for lower latency. Assert.IsTrue(options.PreferHardware, "Unity default prefers hardware processing"); diff --git a/Tests/PlayMode/AudioProcessingTests.cs b/Tests/PlayMode/AudioProcessingTests.cs index f132e77e..5e5f5c73 100644 --- a/Tests/PlayMode/AudioProcessingTests.cs +++ b/Tests/PlayMode/AudioProcessingTests.cs @@ -107,7 +107,7 @@ public IEnumerator AudioProcessor_CancelsDelayedEchoOfPlayout() var meter = new EchoMeter(); var processor = new AudioProcessor( - new AudioProcessingOptions { EchoCancellation = true, HighPassFilter = true }, + new AudioProcessingOptions { EchoCancellation = true }, meter.OnProcessed); probe.AudioRead += (data, channels, sampleRate) => {