diff --git a/README.md b/README.md index 284dee4c..c4c78532 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,22 @@ IEnumerator PublishLocalMicrophoneUnity(Room room) } ``` +#### Unity Audio Processing + +Unity's `Microphone` path does not go through a platform audio device module, so on its own it has no echo cancellation. Pass `AudioProcessingOptions` to run libwebrtc's audio processing (AEC3 echo cancellation, noise suppression, gain control, high-pass filter) over the captured audio before it reaches the track: + +```cs +var processing = new AudioProcessingOptions +{ + EchoCancellation = true, + NoiseSuppression = true, + AutoGainControl = true +}; +var rtcSource = new MicrophoneSource(Microphone.devices[0], microphoneObject, processing); +``` + +Echo cancellation takes its reference from the final mix Unity plays, so it covers every remote `AudioStream` as well as the game's own audio. The SDK attaches a `PlayoutReference` component to the active `AudioListener` for that; adding it to the listener yourself does the same. `rtcSource.AudioProcessingStats` reports whether the reference is attached and how many chunks flowed. Unity's output sample rate must be a multiple of 100 Hz (48000, 44100 and 24000 all are); otherwise processing is bypassed with a warning and the raw microphone is published. + #### Unity Audio Output ```cs diff --git a/Runtime/Plugins/iOS/LiveKitAudioSession.mm b/Runtime/Plugins/iOS/LiveKitAudioSession.mm index f341fc56..c18515ff 100644 --- a/Runtime/Plugins/iOS/LiveKitAudioSession.mm +++ b/Runtime/Plugins/iOS/LiveKitAudioSession.mm @@ -67,4 +67,18 @@ void LiveKit_RestoreDefaultAudioSession() { } } +/// AVAudioSession latency terms, in seconds. Used to seed the echo canceller's stream delay in +/// Unity-audio mode (see AudioProcessingDelayHint.cs). All report 0 until the session is active. +double LiveKit_AudioSessionOutputLatency() { + return [[AVAudioSession sharedInstance] outputLatency]; +} + +double LiveKit_AudioSessionInputLatency() { + return [[AVAudioSession sharedInstance] inputLatency]; +} + +double LiveKit_AudioSessionIOBufferDuration() { + return [[AVAudioSession sharedInstance] IOBufferDuration]; +} + } diff --git a/Runtime/Scripts/Audio/MicrophoneSource.cs b/Runtime/Scripts/Audio/MicrophoneSource.cs index 75bec1f0..9f88ad38 100644 --- a/Runtime/Scripts/Audio/MicrophoneSource.cs +++ b/Runtime/Scripts/Audio/MicrophoneSource.cs @@ -11,6 +11,13 @@ namespace LiveKit /// /// /// Ensure microphone permissions are granted before calling . + /// + /// Unity's Microphone path does not go through a platform audio device module, so on + /// its own it has no echo cancellation. Construct the source with + /// to run libwebrtc's audio processing over the capture; + /// echo cancellation then uses the mix Unity plays as its reference (see + /// ), which covers every remote and the + /// application's own audio. /// sealed public class MicrophoneSource : RtcAudioSource { @@ -35,6 +42,26 @@ public MicrophoneSource(string deviceName, GameObject sourceObject) : base(RtcAu _sourceObject = sourceObject; } + /// + /// Creates a microphone source whose capture is run through libwebrtc's audio processing + /// (AEC3 echo cancellation, noise suppression, gain control, high-pass filter) before it + /// reaches the track. + /// + /// The name of the device to capture from. Use to + /// get the list of available devices. + /// The GameObject to attach the AudioSource to. The object must be kept in the scene + /// for the duration of the source's lifetime. + /// Which stages to enable. With + /// the SDK attaches a to the active to obtain the + /// far-end reference. Requires Unity's output sample rate to be a multiple of 100 Hz; otherwise processing is + /// bypassed with a warning. See for diagnostics. + public MicrophoneSource(string deviceName, GameObject sourceObject, AudioProcessingOptions processing) + : base(RtcAudioSourceType.AudioSourceMicrophone, processing) + { + _deviceName = deviceName; + _sourceObject = sourceObject; + } + /// /// Begins capturing audio from the microphone. /// @@ -207,6 +234,8 @@ private IEnumerator RestartMicrophone() // recover from interruption. Poll for readiness instead of using arbitrary delay. yield return WaitForMicrophoneReady(); + // A resume is a new audio path: drop whatever the processing stage buffered before. + ResetAudioProcessing(); yield return StartMicrophone(); } diff --git a/Runtime/Scripts/Audio/PlatformAudioSource.cs b/Runtime/Scripts/Audio/PlatformAudioSource.cs index 0b507c4c..119adbf1 100644 --- a/Runtime/Scripts/Audio/PlatformAudioSource.cs +++ b/Runtime/Scripts/Audio/PlatformAudioSource.cs @@ -7,7 +7,10 @@ namespace LiveKit { /// - /// Options for audio processing when creating a PlatformAudioSource. + /// Options for libwebrtc's audio processing. Used by , where + /// the ADM applies them, and by Unity-audio sources such as + /// created with options, where the SDK runs libwebrtc's audio processing module over the + /// capture with the mix Unity plays as the echo reference (see ). /// public struct AudioProcessingOptions { @@ -17,7 +20,10 @@ public struct AudioProcessingOptions public bool NoiseSuppression; /// Enable automatic gain control (AGC). Default: true. public bool AutoGainControl; - /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. Default: true. + /// + /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. + /// only. Default: true. + /// public bool PreferHardware; /// @@ -30,6 +36,9 @@ public struct AudioProcessingOptions AutoGainControl = true, PreferHardware = true }; + + /// Whether any stage of the Unity-audio processing pipeline is enabled. + internal bool AnyProcessingEnabled => EchoCancellation || NoiseSuppression || AutoGainControl; } /// diff --git a/Runtime/Scripts/Audio/Processing.meta b/Runtime/Scripts/Audio/Processing.meta new file mode 100644 index 00000000..180519b6 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: afb866c8c71a34134bf00d5cd4ee1778 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs new file mode 100644 index 00000000..feaf914b --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs @@ -0,0 +1,78 @@ +using System; +using System.Runtime.InteropServices; +using UnityEngine; + +namespace LiveKit +{ + /// + /// Estimates the render-to-capture delay hint handed to + /// . + /// + /// + /// AEC3 runs its own correlation-based delay estimator; the hint only sets the initial + /// alignment after a reset, so it has to be in the right ballpark rather than exact. The echo + /// path in Unity-audio mode is: tap → Unity's output queue (a + /// few DSP blocks) → device output → air → device input → Microphone clip → the + /// AudioSource reading that clip → capture probe. Only the DSP block size and, on iOS, the + /// audio session latencies are readable; the remaining terms are constants. + /// + /// Main thread only: reads . + /// + internal static class AudioProcessingDelayHint + { +#if UNITY_IOS && !UNITY_EDITOR + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionOutputLatency(); + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionInputLatency(); + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionIOBufferDuration(); +#endif + + internal const int MinDelayMs = 0; + internal const int MaxDelayMs = 500; + + /// Output queue depth assumed between the listener tap and the device, in DSP blocks. + internal const int OutputQueueBlocks = 2; + + /// + /// How far the AudioSource reading the microphone clip trails the clip's write head. + /// starts reading once Microphone.GetPosition first + /// reports data, polled at 50 ms, and that offset persists for the life of the clip. + /// + internal const int MicrophoneReadBehindMs = 50; + + /// Device input plus output latency when the platform does not report it. + internal const int FallbackDeviceLatencyMs = 30; + + public static int EstimateMs() + { + var config = AudioSettings.GetConfiguration(); + return EstimateMs(config.dspBufferSize, config.sampleRate, PlatformLatencyMs()); + } + + internal static int EstimateMs(int dspBufferSize, int sampleRate, double deviceLatencyMs) + { + var blockMs = sampleRate > 0 ? dspBufferSize * 1000.0 / sampleRate : 0.0; + var estimate = blockMs * OutputQueueBlocks + deviceLatencyMs + MicrophoneReadBehindMs; + return (int)Math.Round(Math.Min(MaxDelayMs, Math.Max(MinDelayMs, estimate))); + } + + internal static double PlatformLatencyMs() + { +#if UNITY_IOS && !UNITY_EDITOR + try + { + // AVAudioSession reports zero until the session is active, hence the fallback. + var output = LiveKit_AudioSessionOutputLatency(); + var input = LiveKit_AudioSessionInputLatency(); + var ioBuffer = LiveKit_AudioSessionIOBufferDuration(); + if (output > 0d || input > 0d) + return (output + input + ioBuffer) * 1000d; + } + catch (Exception) + { + // Fall through to the constant. + } +#endif + return FallbackDeviceLatencyMs; + } + } +} diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs.meta new file mode 100644 index 00000000..c97cd183 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b1bb08510396f4e28b12569471ea0cf6 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs new file mode 100644 index 00000000..5e264a2b --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs @@ -0,0 +1,167 @@ +using System; +using LiveKit.Internal.FFI; +using LiveKit.Internal.FFI.Requests; +using LiveKit.Proto; + +namespace LiveKit +{ + /// + /// libwebrtc's AudioProcessingModule (AEC3 echo cancellation, noise suppression, gain + /// control, high-pass filter), driven over the FFI. + /// + /// + /// Use this to run echo cancellation over a capture path that does not go through the + /// platform audio device module (e.g. Unity's Microphone): feed the audio that is + /// played out of the loudspeaker to and the captured + /// microphone audio to , which processes it in place. + /// + /// Both accept exactly one 10 ms chunk of interleaved int16 PCM ( + /// samples per channel) and nothing else. libwebrtc's own contract is a capture thread calling + /// and a render thread calling ; + /// the native module is internally synchronised for exactly that split, and the SDK's request + /// plumbing is safe to use from both. + /// + internal sealed class AudioProcessingModule : IDisposable + { + /// libwebrtc's kChunkSizeMs — the APM accepts nothing else. + public const int ChunkSizeMs = 10; + + /// + /// libwebrtc's internal processing rates. The APM resamples any other API rate onto one of + /// these itself, so this list is diagnostic information — NOT an admission requirement. See + /// . + /// + private static readonly int[] NativeSampleRates = { 8000, 16000, 32000, 48000 }; + + private readonly FfiHandle _handle; + private bool _disposed; + + /// The native handle id, for diagnostics. + public ulong Handle => (ulong)_handle.DangerousGetHandle(); + + public AudioProcessingModule( + bool echoCancellerEnabled, + bool gainControllerEnabled, + bool highPassFilterEnabled, + bool noiseSuppressionEnabled) + { + using var request = FFIBridge.Instance.NewRequest(); + var newApm = request.request; + newApm.EchoCancellerEnabled = echoCancellerEnabled; + newApm.GainControllerEnabled = gainControllerEnabled; + newApm.HighPassFilterEnabled = highPassFilterEnabled; + newApm.NoiseSuppressionEnabled = noiseSuppressionEnabled; + + using var response = request.Send(); + FfiResponse res = response; + var owned = res.NewApm?.Apm; + if (owned?.Handle == null || owned.Handle.Id == 0) + throw new InvalidOperationException("FFI returned no APM handle"); + + _handle = FfiHandle.FromOwnedHandle(owned.Handle); + } + + public static bool IsNativeSampleRate(int sampleRate) + { + foreach (var rate in NativeSampleRates) + if (rate == sampleRate) return true; + return false; + } + + /// + /// Whether the APM accepts this rate on its API surface. + /// + /// + /// The only hard requirement is that one 10 ms chunk is a whole number of samples: both + /// here and libwebrtc's own StreamConfig::num_frames() + /// derive the chunk with integer division, so a rate that is not a multiple of 100 Hz would + /// short every chunk and drift the two feeds apart. + /// + /// A non-native rate is NOT rejected — the rate goes straight into a StreamConfig + /// and libwebrtc resamples to a native processing rate internally. A 24 kHz output rate + /// (iPad) is cancelled just as well as 48 kHz. + /// + public static bool IsSupportedApiRate(int sampleRate) => + sampleRate > 0 && sampleRate % (1000 / ChunkSizeMs) == 0; + + /// Samples per channel in one APM chunk at the given rate. + public static int FrameSizeFor(int sampleRate) => sampleRate / (1000 / ChunkSizeMs); + + /// + /// Processes the near-end (capture) stream in place. is bytes, + /// not samples — the buffer is interleaved int16. Returns the FFI error, or null on success. + /// + public string ProcessStream(IntPtr dataPtr, int byteCount, int sampleRate, int channels) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var process = request.request; + process.ApmHandle = Handle; + process.DataPtr = (ulong)dataPtr.ToInt64(); + process.Size = (uint)byteCount; + process.SampleRate = (uint)sampleRate; + process.NumChannels = (uint)channels; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmProcessStream?.Error); + } + + /// + /// Processes the far-end (render) reference stream in place. Same buffer contract as + /// . + /// + public string ProcessReverseStream(IntPtr dataPtr, int byteCount, int sampleRate, int channels) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var reverse = request.request; + reverse.ApmHandle = Handle; + reverse.DataPtr = (ulong)dataPtr.ToInt64(); + reverse.Size = (uint)byteCount; + reverse.SampleRate = (uint)sampleRate; + reverse.NumChannels = (uint)channels; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmProcessReverseStream?.Error); + } + + /// + /// Seeds the render/capture delay. AEC3 runs its own correlation estimator, so this is a + /// convergence hint rather than a hard alignment. Returns the FFI error, or null on success. + /// + public string SetStreamDelayMs(int delayMs) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var delay = request.request; + delay.ApmHandle = Handle; + delay.DelayMs = delayMs; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmSetStreamDelay?.Error); + } + + private static string ErrorOrNull(string error) => string.IsNullOrEmpty(error) ? null : error; + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _handle.Dispose(); + GC.SuppressFinalize(this); + } + + ~AudioProcessingModule() + { + if (_disposed) return; + _disposed = true; + _handle.Dispose(); + } + } +} diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs.meta new file mode 100644 index 00000000..4263c783 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 033fc8514082a43a18e018ca3267a9b5 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs new file mode 100644 index 00000000..0362a81c --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs @@ -0,0 +1,87 @@ +namespace LiveKit +{ + /// + /// Counters from the audio processing stage of an that was + /// created with . A snapshot; read it from any thread via + /// . + /// + public readonly struct AudioProcessingStats + { + /// + /// True while capture is actually being processed. False when the source has no processing + /// stage, is stopped, or bypasses processing because Unity's output sample rate has no + /// whole-sample 10 ms chunk (it is not a multiple of 100 Hz). + /// + public readonly bool Active; + + /// + /// True while a on an enabled AudioListener is feeding + /// the far-end reference. Without it echo cancellation has nothing to cancel; noise + /// suppression, gain control and the high-pass filter still run. + /// + public readonly bool ReferenceAttached; + + /// Format of the capture feed, as delivered by Unity's audio graph. + public readonly int CaptureSampleRate; + public readonly int CaptureChannels; + + /// Format of the playout reference feed. + public readonly int ReferenceSampleRate; + public readonly int ReferenceChannels; + + /// 10 ms capture chunks run through the module. + public readonly long CaptureChunks; + + /// 10 ms reference chunks run through the module. + public readonly long ReferenceChunks; + + /// Samples discarded because a feed outran its buffer. Non-zero means the audio thread stalled. + public readonly long DroppedCaptureSamples; + public readonly long DroppedReferenceSamples; + + /// Chunks the module rejected; those capture chunks were published unprocessed. + public readonly long FailedChunks; + + /// Most recent module error, or null. + public readonly string LastError; + + /// Render-to-capture delay hint last handed to the module, or -1 if none yet. + public readonly int StreamDelayHintMs; + + public AudioProcessingStats( + bool active, + bool referenceAttached, + int captureSampleRate, + int captureChannels, + int referenceSampleRate, + int referenceChannels, + long captureChunks, + long referenceChunks, + long droppedCaptureSamples, + long droppedReferenceSamples, + long failedChunks, + string lastError, + int streamDelayHintMs) + { + Active = active; + ReferenceAttached = referenceAttached; + CaptureSampleRate = captureSampleRate; + CaptureChannels = captureChannels; + ReferenceSampleRate = referenceSampleRate; + ReferenceChannels = referenceChannels; + CaptureChunks = captureChunks; + ReferenceChunks = referenceChunks; + DroppedCaptureSamples = droppedCaptureSamples; + DroppedReferenceSamples = droppedReferenceSamples; + FailedChunks = failedChunks; + LastError = lastError; + StreamDelayHintMs = streamDelayHintMs; + } + + public override string ToString() => + $"active={Active} reference={ReferenceAttached} " + + $"capture={CaptureChannels}ch@{CaptureSampleRate} chunks={CaptureChunks} dropped={DroppedCaptureSamples} | " + + $"reference={ReferenceChannels}ch@{ReferenceSampleRate} chunks={ReferenceChunks} dropped={DroppedReferenceSamples} | " + + $"failed={FailedChunks} lastError={LastError ?? "-"} delayHint={StreamDelayHintMs}ms"; + } +} diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs.meta new file mode 100644 index 00000000..3174751e --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 8515244a296964046ae7f54887a9aa1b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs new file mode 100644 index 00000000..ca72566c --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs @@ -0,0 +1,353 @@ +using System; +using System.Collections; +using System.Runtime.InteropServices; +using System.Threading; +using LiveKit.Internal; +using LiveKit.Internal.FFI; +using LiveKit.Internal.Threading; +using Unity.Collections; +using Unity.Collections.LowLevel.Unsafe; +using UnityEngine; + +namespace LiveKit +{ + /// + /// The processing stage behind on an + /// : owns one , re-chunks the + /// capture and the playout reference into 10 ms frames, and hands every processed capture + /// chunk to the source for the FFI. + /// + /// + /// Threading. (from the source's audio callback) and + /// (from ) both run on the Unity + /// audio thread. Within one DSP tick the capture probes run before the listener tap, so the + /// reference for tick N arrives after the capture of tick N; that is fine because the acoustic + /// echo of tick N's playout only reaches the microphone several ticks later. , + /// and the maintenance coroutine run on the main thread and never touch the + /// ring buffers; they raise flags the audio thread acts on. Nothing here logs from the audio + /// thread — diagnostics are counters, read via . + /// + /// The Rust side asserts (and takes the process down) on a frame that is not a whole multiple + /// of 10 ms, so the chunking here is not optional, and rates whose 10 ms chunk is not a whole + /// number of samples are bypassed entirely. + /// + internal sealed class AudioProcessor : IDisposable + { + /// Receives one processed 10 ms chunk and takes ownership of the array. + internal delegate void ProcessedFrameSink(NativeArray frame, int channels, int sampleRate); + + // Ring capacity in chunks. Bounds the latency added when a DSP block is not a multiple of 10 ms. + private const int BufferedChunks = 8; + private const float MaintenanceIntervalSeconds = 2f; + + private readonly AudioProcessingModule _apm; + private readonly ProcessedFrameSink _sink; + private readonly bool _echoCancellation; + + // Guards the pinned reference chunk against Dispose racing an in-flight callback. The + // capture side needs no lock: each chunk lives in its own NativeArray owned by the sink. + private readonly object _referenceLock = new object(); + + // Capture side. Audio thread only. + private PcmRingBuffer _captureRing; + private short[] _captureStaging; + private short[] _captureChunk; + private int _captureRate; + private int _captureChannels; + private int _captureChunkSamples; + + // Reference side. Audio thread only, under _referenceLock. The chunk stays pinned so the + // module has a stable address to process in place. + private PcmRingBuffer _referenceRing; + private short[] _referenceStaging; + private short[] _referenceChunk; + private GCHandle _referenceChunkPin; + private int _referenceRate; + private int _referenceChannels; + private int _referenceChunkSamples; + + // Cross-thread flags. + private volatile bool _running; + private volatile bool _bypass; + private volatile bool _disposed; + private int _captureResetRequested; + private int _referenceResetRequested; + private int _unsupportedRateWarned; + private int _maintenanceGeneration; + + // Counters. + private long _captureChunks; + private long _referenceChunks; + private long _failedChunks; + private volatile string _lastError; + private volatile int _delayHintMs = -1; + + /// The FFI could not create the module. + public AudioProcessor(AudioProcessingOptions options, ProcessedFrameSink sink) + { + _sink = sink ?? throw new ArgumentNullException(nameof(sink)); + _echoCancellation = options.EchoCancellation; + // The high-pass filter is not exposed as an option: libwebrtc instantiates it anyway + // whenever AEC or NS is on, and the platform ADM path always runs it, so both paths + // stay identical. + _apm = new AudioProcessingModule( + echoCancellerEnabled: options.EchoCancellation, + gainControllerEnabled: options.AutoGainControl, + highPassFilterEnabled: true, + noiseSuppressionEnabled: options.NoiseSuppression); + } + + /// Main thread. + public void Start() + { + if (_disposed || _running) return; + _running = true; + RequestReset(); + + if (_echoCancellation) + { + PlayoutReference.AudioRead += OnPlayoutAudio; + PlayoutReference.Acquire(); + } + + SeedDelayHint(); + MonoBehaviourContext.RunCoroutine(Maintenance(++_maintenanceGeneration)); + } + + /// Main thread. + public void Stop() + { + if (!_running) return; + _running = false; + + if (_echoCancellation) + { + PlayoutReference.AudioRead -= OnPlayoutAudio; + PlayoutReference.Release(); + } + } + + /// + /// Any thread. Clears both feeds before their next audio callback. Call when the capture + /// path restarts (e.g. a microphone resume) so stale samples do not misalign the canceller. + /// + public void RequestReset() + { + Interlocked.Exchange(ref _captureResetRequested, 1); + Interlocked.Exchange(ref _referenceResetRequested, 1); + } + + // Periodic main-thread upkeep: re-attach the reference after scene or device changes and + // refresh the delay hint (iOS reports zero session latency until the session is active). + private IEnumerator Maintenance(int generation) + { + while (_running && !_disposed && generation == _maintenanceGeneration) + { + if (_echoCancellation) PlayoutReference.EnsureAttached(); + SeedDelayHint(); + yield return new WaitForSeconds(MaintenanceIntervalSeconds); + } + } + + private void SeedDelayHint() + { + if (!_echoCancellation || _disposed) return; + + var hint = AudioProcessingDelayHint.EstimateMs(); + if (hint == _delayHintMs) return; + + string error; + try + { + error = _apm.SetStreamDelayMs(hint); + } + catch (ObjectDisposedException) + { + return; + } + + if (error != null) + { + Utils.Warning($"AudioProcessor: set_stream_delay_ms({hint}) failed: {error}"); + return; + } + + _delayHintMs = hint; + } + + /// + /// Unity audio thread. Runs the block through the module in 10 ms chunks and forwards each + /// chunk to the sink. Returns false when the block must go out unprocessed instead: + /// processing is stopped, or bypassed for this sample rate. + /// + public bool TryProcessCapture(float[] data, int channels, int sampleRate) + { + if (_disposed || !_running || _bypass) return false; + if (data == null || data.Length == 0 || channels <= 0 || sampleRate <= 0) return false; + + if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) + { + _bypass = true; + WarnUnsupportedRate(sampleRate); + return false; + } + + if (Interlocked.Exchange(ref _captureResetRequested, 0) == 1) + _captureRing?.Clear(); + + if (_captureStaging == null || sampleRate != _captureRate || channels != _captureChannels || _captureStaging.Length < data.Length) + ConfigureCapture(sampleRate, channels, data.Length); + + for (var i = 0; i < data.Length; i++) + _captureStaging[i] = PcmConvert.FloatToS16(data[i]); + _captureRing.Write(_captureStaging, 0, data.Length); + + while (_captureRing.TryDrain(_captureChunk, _captureChunkSamples)) + { + var frame = new NativeArray(_captureChunkSamples, Allocator.Persistent); + frame.CopyFrom(_captureChunk); + ProcessCaptureChunk(frame, sampleRate, channels); + Interlocked.Increment(ref _captureChunks); + _sink(frame, channels, sampleRate); + } + + return true; + } + + private void ProcessCaptureChunk(NativeArray frame, int sampleRate, int channels) + { + try + { + IntPtr ptr; + unsafe + { + ptr = (IntPtr)NativeArrayUnsafeUtility.GetUnsafePtr(frame); + } + var error = _apm.ProcessStream(ptr, frame.Length * sizeof(short), sampleRate, channels); + if (error != null) RecordFailure(error); + } + catch (Exception e) + { + // The chunk goes out unprocessed rather than not at all. + RecordFailure(e.Message); + } + } + + // Unity audio thread, from PlayoutReference. Must not modify data. + private void OnPlayoutAudio(float[] data, int channels, int sampleRate) + { + if (_disposed || !_running || _bypass) return; + if (data == null || data.Length == 0 || channels <= 0) return; + if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) return; + + lock (_referenceLock) + { + if (_disposed) return; + + if (Interlocked.Exchange(ref _referenceResetRequested, 0) == 1) + _referenceRing?.Clear(); + + if (_referenceStaging == null || sampleRate != _referenceRate || channels != _referenceChannels || _referenceStaging.Length < data.Length) + ConfigureReference(sampleRate, channels, data.Length); + + for (var i = 0; i < data.Length; i++) + _referenceStaging[i] = PcmConvert.FloatToS16(data[i]); + _referenceRing.Write(_referenceStaging, 0, data.Length); + + var byteCount = _referenceChunkSamples * sizeof(short); + while (_referenceRing.TryDrain(_referenceChunk, _referenceChunkSamples)) + { + try + { + var error = _apm.ProcessReverseStream(_referenceChunkPin.AddrOfPinnedObject(), byteCount, sampleRate, channels); + if (error != null) RecordFailure(error); + } + catch (Exception e) + { + RecordFailure(e.Message); + } + Interlocked.Increment(ref _referenceChunks); + } + } + } + + // Format changes are rare (first block, device switch); the allocations here are accepted + // on the audio thread for the same reason AudioStream sizes its buffers lazily. + private void ConfigureCapture(int sampleRate, int channels, int incomingSamples) + { + var chunkSamples = AudioProcessingModule.FrameSizeFor(sampleRate) * channels; + _captureRate = sampleRate; + _captureChannels = channels; + _captureChunkSamples = chunkSamples; + _captureChunk = new short[chunkSamples]; + _captureStaging = new short[incomingSamples]; + // Never smaller than one input block, or a large block would overflow immediately. + _captureRing = new PcmRingBuffer(Math.Max(chunkSamples * BufferedChunks, incomingSamples + chunkSamples)); + } + + private void ConfigureReference(int sampleRate, int channels, int incomingSamples) + { + var chunkSamples = AudioProcessingModule.FrameSizeFor(sampleRate) * channels; + _referenceRate = sampleRate; + _referenceChannels = channels; + _referenceChunkSamples = chunkSamples; + + if (_referenceChunkPin.IsAllocated) _referenceChunkPin.Free(); + _referenceChunk = new short[chunkSamples]; + _referenceChunkPin = GCHandle.Alloc(_referenceChunk, GCHandleType.Pinned); + + _referenceStaging = new short[incomingSamples]; + _referenceRing = new PcmRingBuffer(Math.Max(chunkSamples * BufferedChunks, incomingSamples + chunkSamples)); + } + + private void RecordFailure(string error) + { + _lastError = error; + Interlocked.Increment(ref _failedChunks); + } + + // Logging is not allowed on the audio thread; hand the message to the main thread. + private void WarnUnsupportedRate(int sampleRate) + { + if (Interlocked.Exchange(ref _unsupportedRateWarned, 1) == 1) return; + + var message = $"AudioProcessor: Unity's output sample rate {sampleRate} Hz has no whole-sample 10 ms chunk; " + + "audio processing is bypassed and the capture is published unprocessed."; + var context = FfiClient.Instance._context; + if (context != null) + context.Post(static m => Utils.Warning(m), message); + else + Utils.Warning(message); + } + + public AudioProcessingStats GetStats() => new AudioProcessingStats( + active: _running && !_bypass && !_disposed, + referenceAttached: _echoCancellation && PlayoutReference.IsAttached, + captureSampleRate: _captureRate, + captureChannels: _captureChannels, + referenceSampleRate: _referenceRate, + referenceChannels: _referenceChannels, + captureChunks: Interlocked.Read(ref _captureChunks), + referenceChunks: Interlocked.Read(ref _referenceChunks), + droppedCaptureSamples: _captureRing?.OverflowSamples ?? 0, + droppedReferenceSamples: _referenceRing?.OverflowSamples ?? 0, + failedChunks: Interlocked.Read(ref _failedChunks), + lastError: _lastError, + streamDelayHintMs: _delayHintMs); + + public void Dispose() + { + if (_disposed) return; + + Stop(); + lock (_referenceLock) + { + _disposed = true; + if (_referenceChunkPin.IsAllocated) _referenceChunkPin.Free(); + _referenceChunk = null; + _referenceRing = null; + } + _apm.Dispose(); + } + } +} diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs.meta new file mode 100644 index 00000000..593e25f5 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 0154e5d4b40264653a6b7832d5640b7a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs new file mode 100644 index 00000000..a9acddbb --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs @@ -0,0 +1,118 @@ +using System; + +namespace LiveKit +{ + /// + /// Fixed-capacity interleaved int16 PCM ring buffer with a fixed-size drain. Re-chunks Unity's + /// DSP-block-sized audio into the 10 ms frames the requires. + /// + /// + /// Allocation-free after construction: both users run on the Unity audio thread. Sized in + /// samples (frames × channels), not frames. Single producer and single consumer per instance; + /// the capture and reference feeds each own one, so there is no synchronisation inside. + /// + internal sealed class PcmRingBuffer + { + private readonly short[] _buffer; + private int _readIndex; + private int _writeIndex; + private int _count; + + /// Samples dropped because the buffer was full, since construction. + public int OverflowSamples { get; private set; } + + public int Capacity => _buffer.Length; + public int Available => _count; + + public PcmRingBuffer(int capacitySamples) + { + if (capacitySamples <= 0) throw new ArgumentOutOfRangeException(nameof(capacitySamples)); + _buffer = new short[capacitySamples]; + } + + /// + /// Appends samples. When the buffer is full the OLDEST samples are + /// dropped: a stalled consumer must not push the echo reference arbitrarily far out of + /// alignment with the capture stream. + /// + public void Write(short[] source, int offset, int count) + { + if (source == null) throw new ArgumentNullException(nameof(source)); + if (offset < 0 || count < 0 || offset + count > source.Length) + throw new ArgumentOutOfRangeException(nameof(count)); + + if (count >= _buffer.Length) + { + OverflowSamples += _count + count - _buffer.Length; + offset += count - _buffer.Length; + count = _buffer.Length; + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + else + { + var free = _buffer.Length - _count; + if (count > free) Discard(count - free); + } + + for (var i = 0; i < count; i++) + { + _buffer[_writeIndex] = source[offset + i]; + _writeIndex = _writeIndex + 1 == _buffer.Length ? 0 : _writeIndex + 1; + } + + _count += count; + } + + /// + /// Copies exactly samples into and + /// consumes them. Returns false and consumes nothing when fewer are available. + /// + public bool TryDrain(short[] destination, int count) + { + if (destination == null) throw new ArgumentNullException(nameof(destination)); + if (count < 0 || count > destination.Length) throw new ArgumentOutOfRangeException(nameof(count)); + if (_count < count) return false; + + for (var i = 0; i < count; i++) + { + destination[i] = _buffer[_readIndex]; + _readIndex = _readIndex + 1 == _buffer.Length ? 0 : _readIndex + 1; + } + + _count -= count; + return true; + } + + public void Clear() + { + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + + private void Discard(int count) + { + if (count > _count) count = _count; + _readIndex = (_readIndex + count) % _buffer.Length; + _count -= count; + OverflowSamples += count; + } + } + + /// Sample format conversions shared by the capture paths. + internal static class PcmConvert + { + /// Float [-1, 1] to int16 with clamping and round-half-away-from-zero. + public static short FloatToS16(float v) + { + v *= 32768f; + if (v > 32767f) v = 32767f; + else if (v < -32768f) v = -32768f; + return (short)(v + Math.Sign(v) * 0.5f); + } + + public static float S16ToFloat(short v) => v / 32768f; + } +} diff --git a/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs.meta b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs.meta new file mode 100644 index 00000000..f096bd2b --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: cce610429285242e193622fd7e37bb80 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/PlayoutReference.cs b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs new file mode 100644 index 00000000..2693b4a7 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs @@ -0,0 +1,149 @@ +using System.Collections; +using UnityEngine; +using LiveKit.Internal; +using LiveKit.Internal.Threading; + +namespace LiveKit +{ + /// + /// Taps the final mix Unity sends to the audio device and feeds it to the echo canceller as + /// the far-end reference. Lives on the GameObject of the active . + /// + /// + /// An created with + /// attaches this component to the active listener when it starts and re-attaches it after + /// scene loads and audio device changes. Adding it to the listener yourself is supported and + /// does the same thing. + /// + /// Because the tap sits after every AudioSource, mixer group and spatializer, the reference is + /// exactly what the loudspeaker plays: every remote participant plus the game's own audio. + /// The capture probe clears its buffer after reading, so the + /// local microphone never appears in the mix. + /// + /// OnAudioFilterRead runs on the Unity audio thread and must not touch Unity APIs, so + /// the sample rate and listener state are cached on the main thread. + /// + [AddComponentMenu("LiveKit/Playout Reference")] + public sealed class PlayoutReference : MonoBehaviour + { + internal delegate void PlayoutAudioDelegate(float[] data, int channels, int sampleRate); + + /// + /// Raised on the Unity audio thread with the final mix. Subscribers must not modify the + /// buffer: it is on its way to the speaker. + /// + internal static event PlayoutAudioDelegate AudioRead; + + private static PlayoutReference _active; + private static int _consumers; + + private AudioListener _listener; + private volatile int _sampleRate; + private volatile bool _deliver; + + /// Whether a reference on an enabled listener is delivering audio. + internal static bool IsAttached => _active != null && _active._deliver; + + /// Main thread. Registers a consumer and attaches to the listener if possible. + internal static void Acquire() + { + _consumers++; + EnsureAttached(); + } + + /// Main thread. The component stays on the listener; it is inert without consumers. + internal static void Release() + { + if (_consumers > 0) _consumers--; + } + + /// + /// Main thread. Attaches to the active AudioListener unless a working reference already + /// exists. No-op without consumers or without a listener; consumers call this periodically, + /// which is what covers scene loads and a destroyed listener. + /// + internal static void EnsureAttached() + { + if (_consumers == 0) return; + if (_active != null && _active.isActiveAndEnabled && + _active._listener != null && _active._listener.isActiveAndEnabled) + return; + + var listener = FindActiveListener(); + if (listener == null) return; + + var existing = listener.GetComponent(); + _active = existing != null ? existing : listener.gameObject.AddComponent(); + } + + private static AudioListener FindActiveListener() + { + var listeners = FindObjectsByType(FindObjectsSortMode.None); + foreach (var listener in listeners) + { + if (listener.isActiveAndEnabled) return listener; + } + return null; + } + + private void OnEnable() + { + _listener = GetComponent(); + if (_listener == null) + Utils.Warning("PlayoutReference must be on the AudioListener's GameObject; it will not deliver a reference from here."); + + RefreshDeliveryState(); + AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; + if (_active == null) _active = this; + } + + private void OnDisable() + { + AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; + _deliver = false; + if (_active == this) _active = null; + } + + private void Update() + { + RefreshDeliveryState(); + } + + // Listener state and the output rate are Unity APIs; sample them here for the audio thread. + private void RefreshDeliveryState() + { + _sampleRate = AudioSettings.outputSampleRate; + _deliver = _listener != null && _listener.isActiveAndEnabled; + } + + // Unity rebuilds the DSP graph on a device change (or AudioSettings.Reset), which can leave + // filter nodes detached; AudioStream recreates its probe for the same reason. Recreate this + // component so the tap is registered on the new graph. Only done while something consumes + // the reference, so a hand-placed component in an idle scene is left alone. + private void OnAudioConfigurationChanged(bool deviceWasChanged) + { + RefreshDeliveryState(); + if (_consumers == 0) return; + + var host = gameObject; + Destroy(this); + MonoBehaviourContext.RunCoroutine(Reattach(host)); + } + + private static IEnumerator Reattach(GameObject host) + { + // Let the deferred Destroy apply before adding the replacement. + yield return null; + if (host == null || _consumers == 0) yield break; + if (host.GetComponent() == null) + _active = host.AddComponent(); + } + + // Unity audio thread. + private void OnAudioFilterRead(float[] data, int channels) + { + if (!_deliver) return; + AudioRead?.Invoke(data, channels, _sampleRate); + } + } +} diff --git a/Runtime/Scripts/Audio/Processing/PlayoutReference.cs.meta b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs.meta new file mode 100644 index 00000000..a74f1070 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 44cbde2518fe54f5ebd5f24f65d8a070 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/RtcAudioSource.cs b/Runtime/Scripts/Audio/RtcAudioSource.cs index 9147b431..ee764916 100644 --- a/Runtime/Scripts/Audio/RtcAudioSource.cs +++ b/Runtime/Scripts/Audio/RtcAudioSource.cs @@ -49,12 +49,23 @@ private sealed class PendingAudioFrame private readonly RtcAudioSourceType _sourceType; public RtcAudioSourceType SourceType => _sourceType; + + /// + /// Whether this source runs libwebrtc's audio processing over its capture. False when it was + /// created without , or when the module could not be + /// created and the source fell back to unprocessed capture. + /// + public bool AudioProcessingEnabled => _processor != null; + + /// Counters from the audio processing stage; default when processing is off. + public AudioProcessingStats AudioProcessingStats => _processor?.GetStats() ?? default; private readonly int _debugId = Interlocked.Increment(ref nextDebugId); internal readonly uint _expectedSampleRate; internal readonly uint _expectedChannels; internal readonly FfiHandle Handle; protected AudioSourceInfo _info; + private readonly AudioProcessor _processor; // CaptureAudioFrame is asynchronous: the native side can continue reading from the PCM // pointer after request.Send() returns and encode it later on another queue. Because of @@ -72,16 +83,29 @@ private sealed class PendingAudioFrame private bool _started = false; private volatile bool _disposed = false; private int _audioReadCount = 0; + private int _sentFrameCount = 0; // Device-capture sources (microphone, AudioSource taps) don't know their format ahead of // time — it is whatever Unity's audio graph delivers. They use this constructor, which // configures the native source from Unity's current output configuration. protected RtcAudioSource(RtcAudioSourceType audioSourceType) - : this(audioSourceType, 0, 0) { } + : this(audioSourceType, 0, 0, null) { } + + /// + /// Device-capture source whose audio is run through libwebrtc's audio processing (echo + /// cancellation, noise suppression, gain control, high-pass filter) before it reaches the + /// track. See . If the module cannot be created the + /// source logs a warning and captures unprocessed. + /// + protected RtcAudioSource(RtcAudioSourceType audioSourceType, AudioProcessingOptions processing) + : this(audioSourceType, 0, 0, processing) { } // Sources that generate a fixed, known format (e.g. test signal generators) declare it // directly. Passing 0 for either value falls back to the device configuration. protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, uint channels) + : this(audioSourceType, sampleRate, channels, null) { } + + protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, uint channels, AudioProcessingOptions? processing) { _sourceType = audioSourceType; @@ -110,6 +134,19 @@ protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, ui _info = res.NewAudioSource.Source.Info; Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle); Utils.Debug($"{DebugTag} created handle={Handle.DangerousGetHandle()} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); + + if (processing is { } options && options.AnyProcessingEnabled) + { + try + { + _processor = new AudioProcessor(options, SendProcessedFrame); + } + catch (Exception e) + { + // Publish unprocessed rather than not at all. + Utils.Warning($"{DebugTag} audio processing unavailable, capturing unprocessed: {e.Message}"); + } + } } // Reads Unity's actual output audio configuration. The capture path delivers buffers at the @@ -150,6 +187,7 @@ public virtual void Start() { if (_started) return; AudioRead += OnAudioRead; + _processor?.Start(); _started = true; Utils.Debug($"{DebugTag} start"); } @@ -161,6 +199,7 @@ public virtual void Stop() { if (!_started) return; AudioRead -= OnAudioRead; + _processor?.Stop(); _started = false; var pendingCount = PendingFrameCount(); if (pendingCount > 0) @@ -174,45 +213,59 @@ private void OnAudioRead(float[] data, int channels, int sampleRate) if (_muted) return; if (_disposed) return; - var frameIndex = Interlocked.Increment(ref _audioReadCount); + var readIndex = Interlocked.Increment(ref _audioReadCount); if (channels <= 0) { - Utils.Warning($"{DebugTag} dropping audio frame #{frameIndex} because channels={channels}"); + Utils.Warning($"{DebugTag} dropping audio frame #{readIndex} because channels={channels}"); return; } if (data.Length == 0 || data.Length % channels != 0) { - Utils.Warning($"{DebugTag} audio frame #{frameIndex} has invalid shape samples={data.Length} channels={channels}"); + Utils.Warning($"{DebugTag} audio frame #{readIndex} has invalid shape samples={data.Length} channels={channels}"); return; } if ((uint)sampleRate != _expectedSampleRate || (uint)channels != _expectedChannels) { - Utils.Warning($"{DebugTag} audio frame #{frameIndex} metadata mismatch actualRate={sampleRate} actualChannels={channels} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); + Utils.Warning($"{DebugTag} audio frame #{readIndex} metadata mismatch actualRate={sampleRate} actualChannels={channels} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); } - var pendingBeforeSend = PendingFrameCount(); - if (frameIndex <= 3 || frameIndex % 100 == 0 || pendingBeforeSend >= 3) - { - Utils.Debug($"{DebugTag} capture frame #{frameIndex} samples={data.Length} channels={channels} sampleRate={sampleRate} pendingBeforeSend={pendingBeforeSend} thread={Thread.CurrentThread.ManagedThreadId}"); - } + // Optional processing stage: the block is re-chunked into 10 ms frames, run through the + // module and delivered to SendFrame one chunk at a time via SendProcessedFrame. + if (_processor != null && _processor.TryProcessCapture(data, channels, sampleRate)) + return; // Each captured frame gets its own backing buffer so the native encoder can safely // consume it asynchronously after request.Send() returns. var frameData = new NativeArray(data.Length, Allocator.Persistent); + for (int i = 0; i < data.Length; i++) + frameData[i] = PcmConvert.FloatToS16(data[i]); + + SendFrame(frameData, channels, sampleRate); + } - // Copy from the audio read buffer into the frame buffer, converting - // each sample to a 16-bit signed integer. - static short FloatToS16(float v) + // Audio thread, from the processing stage. Owns the frame from here on. + private void SendProcessedFrame(NativeArray frame, int channels, int sampleRate) + { + if (_disposed || _muted) { - v *= 32768f; - v = Math.Min(v, 32767f); - v = Math.Max(v, -32768f); - return (short)(v + Math.Sign(v) * 0.5f); + frame.Dispose(); + return; + } + SendFrame(frame, channels, sampleRate); + } + + // Hands one int16 frame to the native source. Takes ownership of frameData: it is released + // when the CaptureAudioFrame callback completes, is canceled, or the send fails. + private void SendFrame(NativeArray frameData, int channels, int sampleRate) + { + var frameIndex = Interlocked.Increment(ref _sentFrameCount); + var pendingBeforeSend = PendingFrameCount(); + if (frameIndex <= 3 || frameIndex % 100 == 0 || pendingBeforeSend >= 3) + { + Utils.Debug($"{DebugTag} capture frame #{frameIndex} samples={frameData.Length} channels={channels} sampleRate={sampleRate} pendingBeforeSend={pendingBeforeSend} thread={Thread.CurrentThread.ManagedThreadId}"); } - for (int i = 0; i < data.Length; i++) - frameData[i] = FloatToS16(data[i]); // Capture the frame. using var request = FFIBridge.Instance.NewRequest(); @@ -228,7 +281,7 @@ static short FloatToS16(float v) } pushFrame.Buffer.NumChannels = (uint)channels; pushFrame.Buffer.SampleRate = (uint)sampleRate; - pushFrame.Buffer.SamplesPerChannel = (uint)data.Length / (uint)channels; + pushFrame.Buffer.SamplesPerChannel = (uint)frameData.Length / (uint)channels; // Wait for async callback, log an error if the capture fails. The callback's AsyncId // echoes the RequestAsyncId that Unity wrote onto the request. @@ -239,7 +292,7 @@ static short FloatToS16(float v) FrameIndex = frameIndex, SampleRate = sampleRate, Channels = channels, - SampleCount = data.Length, + SampleCount = frameData.Length, StartedTimestamp = Stopwatch.GetTimestamp(), }; lock (_pendingFrameDataLock) @@ -292,6 +345,13 @@ void OnCanceled() } } + /// + /// Clears the audio processing stage's buffers. Call after the capture path restarts (e.g. a + /// microphone resume) so stale samples do not misalign the echo canceller. No-op without + /// processing. + /// + protected void ResetAudioProcessing() => _processor?.RequestReset(); + /// /// Mutes or unmutes the audio source. /// @@ -328,6 +388,7 @@ protected virtual void Dispose(bool disposing) } _pendingFrameData.Clear(); } + _processor?.Dispose(); Handle?.Dispose(); _disposed = true; Utils.Debug($"{DebugTag} disposed"); diff --git a/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs b/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs index b31f4dc0..fb9ed957 100644 --- a/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs +++ b/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs @@ -152,6 +152,19 @@ public static void Inject(this FfiRequest ffiRequest, T request) case RemixAndResampleRequest remixAndResampleRequest: ffiRequest.RemixAndResample = remixAndResampleRequest; break; + // Audio processing module + case NewApmRequest newApmRequest: + ffiRequest.NewApm = newApmRequest; + break; + case ApmProcessStreamRequest apmProcessStreamRequest: + ffiRequest.ApmProcessStream = apmProcessStreamRequest; + break; + case ApmProcessReverseStreamRequest apmProcessReverseStreamRequest: + ffiRequest.ApmProcessReverseStream = apmProcessReverseStreamRequest; + break; + case ApmSetStreamDelayRequest apmSetStreamDelayRequest: + ffiRequest.ApmSetStreamDelay = apmSetStreamDelayRequest; + break; // PlatformAudio case NewPlatformAudioRequest newPlatformAudioRequest: ffiRequest.NewPlatformAudio = newPlatformAudioRequest; diff --git a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs index 1ca6c412..0fb70078 100644 --- a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs +++ b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs @@ -13,6 +13,7 @@ public class MeetManagerEditor : Editor private SerializedProperty noiseSuppression; private SerializedProperty autoGainControl; private SerializedProperty preferHardwareProcessing; + private SerializedProperty remoteAudioGain; private void OnEnable() { @@ -25,6 +26,7 @@ private void OnEnable() noiseSuppression = serializedObject.FindProperty("noiseSuppression"); autoGainControl = serializedObject.FindProperty("autoGainControl"); preferHardwareProcessing = serializedObject.FindProperty("preferHardwareProcessing"); + remoteAudioGain = serializedObject.FindProperty("remoteAudioGain"); } public override void OnInspectorGUI() @@ -47,26 +49,40 @@ public override void OnInspectorGUI() "Provides AEC, AGC, and NS. Disable to use Unity's Microphone API instead.")); EditorGUILayout.Space(); - EditorGUILayout.LabelField("Audio Processing (PlatformAudio only)", EditorStyles.boldLabel); + EditorGUILayout.LabelField("Audio Processing", EditorStyles.boldLabel); - // Gray out audio processing options when PlatformAudio is disabled bool platformAudioEnabled = usePlatformAudio.boolValue; + EditorGUILayout.PropertyField(echoCancellation, new GUIContent("Echo Cancellation", + "Enable echo cancellation. PlatformAudio: WebRTC's ADM. Unity audio: libwebrtc's AEC3 over the " + + "Microphone capture, with the mix Unity plays as the reference.")); + EditorGUILayout.PropertyField(noiseSuppression, new GUIContent("Noise Suppression", + "Enable noise suppression to remove background noise.")); + EditorGUILayout.PropertyField(autoGainControl, new GUIContent("Auto Gain Control", + "Enable auto gain control to normalize audio levels.")); + + // Hardware processing is an ADM feature; gray it out when PlatformAudio is disabled. using (new EditorGUI.DisabledGroupScope(!platformAudioEnabled)) { - if (!platformAudioEnabled) - { - EditorGUILayout.HelpBox("Audio processing options are only available when 'Use Platform Audio' is enabled.", MessageType.Info); - } - - EditorGUILayout.PropertyField(echoCancellation, new GUIContent("Echo Cancellation", - "Enable echo cancellation to remove echo from speaker playback.")); - EditorGUILayout.PropertyField(noiseSuppression, new GUIContent("Noise Suppression", - "Enable noise suppression to remove background noise.")); - EditorGUILayout.PropertyField(autoGainControl, new GUIContent("Auto Gain Control", - "Enable auto gain control to normalize audio levels.")); EditorGUILayout.PropertyField(preferHardwareProcessing, new GUIContent("Prefer Hardware Processing", - "Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")); + "PlatformAudio only. Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have " + + "different quality characteristics.")); + } + + EditorGUILayout.Space(); + EditorGUILayout.LabelField("Unity Audio (PlatformAudio off)", EditorStyles.boldLabel); + + // Gray out Unity audio options when PlatformAudio is enabled + using (new EditorGUI.DisabledGroupScope(platformAudioEnabled)) + { + EditorGUILayout.HelpBox(platformAudioEnabled + ? "Unity audio options are only used when 'Use Platform Audio' is disabled." + : "Echo cancellation in this mode runs libwebrtc's AEC3 in the SDK; the reference is the mix on the " + + "AudioListener (a PlayoutReference component is attached automatically).", MessageType.Info); + + EditorGUILayout.PropertyField(remoteAudioGain, new GUIContent("Remote Audio Gain", + "Playback gain for every remote AudioSource. Below 1 keeps headroom so full-volume playout does not " + + "distort or overload the echo canceller. 0.7 is -3.1 dB.")); } serializedObject.ApplyModifiedProperties(); diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index 1dc27c71..29314dfa 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -13,7 +13,9 @@ /// - PlatformAudio (default): Uses WebRTC's ADM for microphone capture and automatic /// speaker playout. Provides echo cancellation (AEC), AGC, and noise suppression. /// - Unity Audio: Uses Unity's Microphone API and AudioStream for manual audio handling. -/// No AEC support but gives more control over audio processing. +/// Gives more control over audio processing. The same AEC/NS/AGC toggles apply: the SDK runs +/// libwebrtc's audio processing over the Microphone capture, with the mix Unity plays as the +/// echo reference (see ). /// [RequireComponent(typeof(TokenSourceComponent))] public class MeetManager : MonoBehaviour @@ -34,16 +36,24 @@ public class MeetManager : MonoBehaviour "Provides AEC, AGC, and NS. Disable to use Unity's Microphone API instead.")] [SerializeField] private bool usePlatformAudio = true; - [Header("Audio Processing (PlatformAudio only)")] - [Tooltip("Enable echo cancellation to remove echo from speaker playback.")] + [Header("Audio Processing")] + [Tooltip("Enable echo cancellation. PlatformAudio: WebRTC's ADM. Unity audio: libwebrtc's AEC3 over the " + + "Microphone capture, with the mix Unity plays as the reference.")] [SerializeField] private bool echoCancellation = true; [Tooltip("Enable noise suppression to remove background noise.")] [SerializeField] private bool noiseSuppression = true; [Tooltip("Enable auto gain control to normalize audio levels.")] [SerializeField] private bool autoGainControl = true; - [Tooltip("Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] + [Tooltip("PlatformAudio only. Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] [SerializeField] private bool preferHardwareProcessing = true; + [Header("Unity Audio (PlatformAudio off)")] + [Tooltip("Playback gain for every remote AudioSource in Unity audio mode. Kept below 1 so a device " + + "at full speaker volume keeps headroom: full-scale playout distorts on Android and feeds " + + "the echo canceller more echo than it can remove. Linear amplitude, 0.7 is -3.1 dB.")] + [Range(0f, 1f)] + [SerializeField] private float remoteAudioGain = 0.7f; + private const string PlaceholderTextureResourceName = "PlaceholderTileSquare"; private Texture _placeholderTexture; @@ -372,6 +382,7 @@ private void AddRemoteAudioTrack(RemoteAudioTrack audioTrack) audioObject.transform.SetParent(_audioTrackParent); var source = audioObject.AddComponent(); + source.volume = remoteAudioGain; var audiostream = new AudioStream(audioTrack, source); _audioStreams.Add(sid, audiostream); @@ -605,15 +616,25 @@ private IEnumerator PublishLocalMicrophonePlatform() private IEnumerator PublishLocalMicrophoneUnity() { - Debug.Log("Publishing microphone using Unity Microphone API"); + Debug.Log($"Publishing microphone using Unity Microphone API (AEC={echoCancellation}, NS={noiseSuppression}, AGC={autoGainControl})"); // Start the microphone here for early iOS permission request and android getting access to Microphone.devices Microphone.Start(null, true, 10, 44100); - + var audioObject = new GameObject($"My Microphone: {Microphone.devices[0]}"); audioObject.transform.SetParent(_audioTrackParent); - var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject); + // With options, MicrophoneSource runs libwebrtc's audio processing over the capture. Echo + // cancellation takes its reference from the mix Unity plays (the SDK attaches a + // PlayoutReference to the AudioListener), so it covers every remote AudioStream and the + // app's own audio. If the module cannot be created the source publishes the raw microphone. + var processing = new AudioProcessingOptions + { + EchoCancellation = echoCancellation, + NoiseSuppression = noiseSuppression, + AutoGainControl = autoGainControl + }; + var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject, processing); _localAudioTrack = LocalAudioTrack.CreateAudioTrack(LocalAudioTrackName, rtcSource, _room); @@ -628,6 +649,9 @@ private IEnumerator PublishLocalMicrophoneUnity() if (publish.IsError) { + // Dispose before destroying the host object so the source and its processing module + // are released now rather than by the finalizer. + rtcSource.Dispose(); Destroy(audioObject); _localAudioTrack = null; yield break; @@ -638,7 +662,24 @@ private IEnumerator PublishLocalMicrophoneUnity() _localRtcAudioSource = rtcSource; rtcSource.Start(); - Debug.Log("Microphone published via Unity Microphone API (no AEC)"); + Debug.Log(rtcSource.AudioProcessingEnabled + ? "Microphone published via Unity Microphone API (audio processing active)" + : "Microphone published via Unity Microphone API (no audio processing)"); + + if (rtcSource.AudioProcessingEnabled) + StartCoroutine(LogAudioProcessingStats(rtcSource)); + } + + // Periodic snapshot of the processing stage while the Unity microphone is published: confirms + // the playout reference is attached and that capture and reference chunks are flowing. + private IEnumerator LogAudioProcessingStats(RtcAudioSource source) + { + while (_microphoneActive && ReferenceEquals(_localRtcAudioSource, source)) + { + yield return new WaitForSeconds(5f); + if (!_microphoneActive || !ReferenceEquals(_localRtcAudioSource, source)) yield break; + Debug.Log($"Audio processing: {source.AudioProcessingStats}"); + } } private void UnpublishLocalMicrophone() diff --git a/Samples~/Meet/Assets/Scenes/MeetApp.unity b/Samples~/Meet/Assets/Scenes/MeetApp.unity index 91fc1e07..3e03e2d1 100644 --- a/Samples~/Meet/Assets/Scenes/MeetApp.unity +++ b/Samples~/Meet/Assets/Scenes/MeetApp.unity @@ -902,11 +902,12 @@ MonoBehaviour: videoTrackParent: {fileID: 2128321498} participantTilePrefab: {fileID: 4315784896331113596, guid: bec493bbc3d574c07b5bbf8dd2be26b3, type: 3} frameRate: 30 - usePlatformAudio: 1 + usePlatformAudio: 0 echoCancellation: 1 noiseSuppression: 1 autoGainControl: 1 preferHardwareProcessing: 1 + remoteAudioGain: 0.7 --- !u!114 &1478206705 MonoBehaviour: m_ObjectHideFlags: 0 @@ -919,7 +920,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: a498c208deeab40c39b4ba609d7d222c, type: 3} m_Name: m_EditorClassIdentifier: - _config: {fileID: 11400000, guid: 1a1b8efb5101449b280f574f853b7459, type: 2} + _config: {fileID: 11400000, guid: aae6c4b1158ca4c929c5d84962c95f91, type: 2} --- !u!1 &1651282853 GameObject: m_ObjectHideFlags: 0 diff --git a/Tests/EditMode/AudioProcessingTests.cs b/Tests/EditMode/AudioProcessingTests.cs new file mode 100644 index 00000000..4c29ba3b --- /dev/null +++ b/Tests/EditMode/AudioProcessingTests.cs @@ -0,0 +1,154 @@ +using NUnit.Framework; + +namespace LiveKit.EditModeTests +{ + /// + /// Pure-managed tests for the Unity-audio processing helpers: the 10 ms re-chunking, sample + /// conversion, rate rules and the delay hint. No FFI, no audio device, so they always run. + /// + public class AudioProcessingTests + { + [Test] + public void PcmRingBuffer_DrainsInWriteOrder() + { + var ring = new PcmRingBuffer(8); + ring.Write(new short[] { 1, 2, 3, 4, 5 }, 0, 5); + + var dest = new short[3]; + Assert.IsTrue(ring.TryDrain(dest, 3)); + Assert.AreEqual(new short[] { 1, 2, 3 }, dest); + Assert.AreEqual(2, ring.Available); + + var rest = new short[2]; + Assert.IsTrue(ring.TryDrain(rest, 2)); + Assert.AreEqual(new short[] { 4, 5 }, rest); + Assert.AreEqual(0, ring.Available); + Assert.AreEqual(0, ring.OverflowSamples); + } + + [Test] + public void PcmRingBuffer_TryDrain_WithoutEnoughSamples_ConsumesNothing() + { + var ring = new PcmRingBuffer(8); + ring.Write(new short[] { 1, 2 }, 0, 2); + + Assert.IsFalse(ring.TryDrain(new short[3], 3)); + Assert.AreEqual(2, ring.Available); + } + + [Test] + public void PcmRingBuffer_WhenFull_DropsOldest() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + ring.Write(new short[] { 4, 5 }, 0, 2); + + Assert.AreEqual(1, ring.OverflowSamples); + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 2, 3, 4, 5 }, dest); + } + + [Test] + public void PcmRingBuffer_WriteLargerThanCapacity_KeepsNewest() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2 }, 0, 2); + ring.Write(new short[] { 3, 4, 5, 6, 7 }, 0, 5); + + Assert.AreEqual(3, ring.OverflowSamples); + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 4, 5, 6, 7 }, dest); + } + + [Test] + public void PcmRingBuffer_WrapsAround() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + Assert.IsTrue(ring.TryDrain(new short[2], 2)); + ring.Write(new short[] { 4, 5, 6 }, 0, 3); + + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 3, 4, 5, 6 }, dest); + Assert.AreEqual(0, ring.OverflowSamples); + } + + [Test] + public void PcmRingBuffer_Clear_Empties() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + ring.Clear(); + + Assert.AreEqual(0, ring.Available); + Assert.IsFalse(ring.TryDrain(new short[1], 1)); + } + + [Test] + public void PcmConvert_FloatToS16_ClampsAndRounds() + { + Assert.AreEqual(0, PcmConvert.FloatToS16(0f)); + Assert.AreEqual(16384, PcmConvert.FloatToS16(0.5f)); + Assert.AreEqual(-16384, PcmConvert.FloatToS16(-0.5f)); + Assert.AreEqual(short.MaxValue, PcmConvert.FloatToS16(1f)); + Assert.AreEqual(short.MinValue, PcmConvert.FloatToS16(-1f)); + Assert.AreEqual(short.MaxValue, PcmConvert.FloatToS16(3f)); + Assert.AreEqual(short.MinValue, PcmConvert.FloatToS16(-3f)); + } + + [Test] + public void AudioProcessingModule_SupportedApiRates_NeedWholeSampleChunks() + { + // The Rust side asserts on a frame that is not a whole multiple of 10 ms, so a rate whose + // 10 ms chunk is fractional must be refused up front. + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(48000)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(44100)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(24000)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(16000)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(22050)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(11025)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(0)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(-48000)); + } + + [Test] + public void AudioProcessingModule_FrameSizeFor_IsTenMilliseconds() + { + Assert.AreEqual(480, AudioProcessingModule.FrameSizeFor(48000)); + Assert.AreEqual(441, AudioProcessingModule.FrameSizeFor(44100)); + Assert.AreEqual(240, AudioProcessingModule.FrameSizeFor(24000)); + Assert.IsTrue(AudioProcessingModule.IsNativeSampleRate(48000)); + Assert.IsFalse(AudioProcessingModule.IsNativeSampleRate(44100)); + } + + [Test] + public void AudioProcessingOptions_AnyProcessingEnabled_TracksTheProcessingStages() + { + Assert.IsTrue(AudioProcessingOptions.Default.AnyProcessingEnabled); + + // An all-false struct means "no processing"; PreferHardware alone is not a stage. + Assert.IsFalse(default(AudioProcessingOptions).AnyProcessingEnabled); + Assert.IsFalse(new AudioProcessingOptions { PreferHardware = true }.AnyProcessingEnabled); + Assert.IsTrue(new AudioProcessingOptions { NoiseSuppression = true }.AnyProcessingEnabled); + } + + [Test] + public void DelayHint_SumsQueueDeviceAndMicrophoneTerms() + { + // 1024 frames at 48 kHz = 21.33 ms per block; two queued blocks + 30 ms device + 50 ms + // microphone read-behind = 122.67 ms. + Assert.AreEqual(123, AudioProcessingDelayHint.EstimateMs(1024, 48000, 30)); + Assert.AreEqual(AudioProcessingDelayHint.MicrophoneReadBehindMs, AudioProcessingDelayHint.EstimateMs(0, 0, 0)); + } + + [Test] + public void DelayHint_ClampsToRange() + { + Assert.AreEqual(AudioProcessingDelayHint.MaxDelayMs, AudioProcessingDelayHint.EstimateMs(48000, 48000, 1000)); + Assert.AreEqual(AudioProcessingDelayHint.MinDelayMs, AudioProcessingDelayHint.EstimateMs(0, 0, -1000)); + } + } +} diff --git a/Tests/EditMode/AudioProcessingTests.cs.meta b/Tests/EditMode/AudioProcessingTests.cs.meta new file mode 100644 index 00000000..80c9b2e9 --- /dev/null +++ b/Tests/EditMode/AudioProcessingTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: e9ce682bd85ef402eb8ce57e87d70231 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/AudioProcessingTests.cs b/Tests/PlayMode/AudioProcessingTests.cs new file mode 100644 index 00000000..5e5f5c73 --- /dev/null +++ b/Tests/PlayMode/AudioProcessingTests.cs @@ -0,0 +1,241 @@ +using System; +using System.Collections; +using System.Runtime.InteropServices; +using NUnit.Framework; +using Unity.Collections; +using UnityEngine; +using UnityEngine.TestTools; + +namespace LiveKit.PlayModeTests +{ + /// + /// Tests for the Unity-audio processing stage. They need the FFI for the module and, for the + /// echo test, Unity's audio thread. No LiveKit server. + /// + class AudioProcessingTests + { + [Test] + public void AudioProcessingModule_AcceptsTenMillisecondChunks() + { + using var apm = new AudioProcessingModule( + echoCancellerEnabled: true, + gainControllerEnabled: true, + highPassFilterEnabled: true, + noiseSuppressionEnabled: true); + + const int rate = 48000; + const int channels = 2; + var chunk = new short[AudioProcessingModule.FrameSizeFor(rate) * channels]; + var pin = GCHandle.Alloc(chunk, GCHandleType.Pinned); + try + { + var ptr = pin.AddrOfPinnedObject(); + var bytes = chunk.Length * sizeof(short); + Assert.IsNull(apm.ProcessReverseStream(ptr, bytes, rate, channels)); + Assert.IsNull(apm.ProcessStream(ptr, bytes, rate, channels)); + Assert.IsNull(apm.SetStreamDelayMs(80)); + } + finally + { + pin.Free(); + } + } + + [UnityTest] + public IEnumerator RtcAudioSource_WithProcessing_CapturesInTenMillisecondChunks() + { + using var source = new PushAudioSource(AudioProcessingOptions.Default); + Assert.IsTrue(source.AudioProcessingEnabled, "module creation failed"); + + var rate = (int)source._expectedSampleRate; + var channels = (int)source._expectedChannels; + if (!AudioProcessingModule.IsSupportedApiRate(rate)) + Assert.Ignore($"output rate {rate} has no whole-sample 10 ms chunk"); + + source.Start(); + + const int blockFrames = 1024; + const int blocks = 10; + var block = new float[blockFrames * channels]; + for (int i = 0; i < block.Length; i++) + block[i] = 0.1f * Mathf.Sin(i * 0.05f); + for (int i = 0; i < blocks; i++) + source.Push(block, channels, rate); + + // Let the capture callbacks return before disposing. + yield return new WaitForSeconds(0.2f); + + var stats = source.AudioProcessingStats; + var expectedChunks = blockFrames * blocks / AudioProcessingModule.FrameSizeFor(rate); + Assert.IsTrue(stats.Active, stats.ToString()); + Assert.AreEqual(0, stats.FailedChunks, stats.LastError); + Assert.That(stats.CaptureChunks, Is.InRange(expectedChunks - 1, expectedChunks), stats.ToString()); + Assert.AreEqual(0, stats.DroppedCaptureSamples, stats.ToString()); + + source.Stop(); + } + + /// + /// End-to-end check of the canceller without hardware: the listener hears a noise source, + /// and a second source plays the same noise 120 ms later, probed as "microphone" and then + /// cleared so the mix contains only the far end. The capture is therefore a pure delayed + /// echo of the playout reference, which AEC3 must learn to remove. + /// + [UnityTest] + public IEnumerator AudioProcessor_CancelsDelayedEchoOfPlayout() + { + var rate = AudioSettings.outputSampleRate; + if (!AudioProcessingModule.IsSupportedApiRate(rate)) + Assert.Ignore($"output rate {rate} has no whole-sample 10 ms chunk"); + + var listenerGo = new GameObject("AecTestListener"); + listenerGo.AddComponent(); + + var clip = NoiseClip(rate, seconds: 2f, seed: 1234, amplitude: 0.3f); + + var farGo = new GameObject("AecTestFarEnd"); + var far = farGo.AddComponent(); + far.clip = clip; + far.loop = true; + + var nearGo = new GameObject("AecTestNearEnd"); + var near = nearGo.AddComponent(); + near.clip = clip; + near.loop = true; + var probe = nearGo.AddComponent(); + probe.ClearAfterInvocation(); + + var meter = new EchoMeter(); + var processor = new AudioProcessor( + new AudioProcessingOptions { EchoCancellation = true }, + meter.OnProcessed); + probe.AudioRead += (data, channels, sampleRate) => + { + meter.OnRaw(data); + processor.TryProcessCapture(data, channels, sampleRate); + }; + processor.Start(); + + var startTime = AudioSettings.dspTime + 0.2; + far.PlayScheduled(startTime); + near.PlayScheduled(startTime + 0.12); + + AudioProcessingStats stats; + float rawRms, processedRms; + try + { + yield return new WaitForSeconds(1f); + if (meter.RawBlocks == 0) + Assert.Ignore("Unity's audio thread delivered no capture callbacks (no audio device?)"); + Assert.IsTrue(PlayoutReference.IsAttached, "reference not attached to the listener"); + + // Convergence time, then a clean measurement window. + yield return new WaitForSeconds(3f); + meter.ResetWindow(); + yield return new WaitForSeconds(1.5f); + + (rawRms, processedRms) = meter.Window(); + stats = processor.GetStats(); + } + finally + { + processor.Dispose(); + UnityEngine.Object.Destroy(farGo); + UnityEngine.Object.Destroy(nearGo); + UnityEngine.Object.Destroy(listenerGo); + } + + Assert.Greater(rawRms, 0.01f, "near-end source produced no signal"); + Assert.AreEqual(0, stats.FailedChunks, stats.ToString()); + Assert.Greater(stats.ReferenceChunks, 0, stats.ToString()); + + var attenuationDb = 20f * Mathf.Log10(rawRms / Mathf.Max(processedRms, 1e-6f)); + Debug.Log($"AEC3 attenuated the synthetic echo by {attenuationDb:F1} dB ({stats})"); + Assert.GreaterOrEqual(attenuationDb, 6f, $"AEC3 attenuated the echo by only {attenuationDb:F1} dB; {stats}"); + } + + private static AudioClip NoiseClip(int sampleRate, float seconds, int seed, float amplitude) + { + var samples = (int)(sampleRate * seconds); + var data = new float[samples]; + var random = new System.Random(seed); + for (int i = 0; i < samples; i++) + data[i] = amplitude * (float)(random.NextDouble() * 2.0 - 1.0); + + var clip = AudioClip.Create("AecTestNoise", samples, 1, sampleRate, false); + clip.SetData(data, 0); + return clip; + } + + private sealed class PushAudioSource : RtcAudioSource + { + public override event Action AudioRead; + + public PushAudioSource(AudioProcessingOptions options) + : base(RtcAudioSourceType.AudioSourceMicrophone, options) { } + + public void Push(float[] data, int channels, int sampleRate) => AudioRead?.Invoke(data, channels, sampleRate); + } + + // Accumulates energy of the raw near end and of the processed output. Both callbacks run on + // the Unity audio thread; the sink owns and disposes the frames it is handed. + private sealed class EchoMeter + { + private readonly object _lock = new object(); + private double _rawSum; + private long _rawCount; + private double _processedSum; + private long _processedCount; + private int _rawBlocks; + + public int RawBlocks { get { lock (_lock) return _rawBlocks; } } + + public void OnRaw(float[] data) + { + double sum = 0; + for (int i = 0; i < data.Length; i++) sum += data[i] * data[i]; + lock (_lock) + { + _rawSum += sum; + _rawCount += data.Length; + _rawBlocks++; + } + } + + public void OnProcessed(NativeArray frame, int channels, int sampleRate) + { + double sum = 0; + var length = frame.Length; + for (int i = 0; i < length; i++) + { + var v = frame[i] / 32768.0; + sum += v * v; + } + frame.Dispose(); + lock (_lock) + { + _processedSum += sum; + _processedCount += length; + } + } + + public void ResetWindow() + { + lock (_lock) + { + _rawSum = 0; + _rawCount = 0; + _processedSum = 0; + _processedCount = 0; + } + } + + public (float raw, float processed) Window() + { + lock (_lock) return (Rms(_rawSum, _rawCount), Rms(_processedSum, _processedCount)); + } + + private static float Rms(double sum, long count) => count == 0 ? 0f : (float)Math.Sqrt(sum / count); + } + } +} diff --git a/Tests/PlayMode/AudioProcessingTests.cs.meta b/Tests/PlayMode/AudioProcessingTests.cs.meta new file mode 100644 index 00000000..4b6b1e59 --- /dev/null +++ b/Tests/PlayMode/AudioProcessingTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b22944e23e00341eb8cd913bce198b4e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: