diff --git a/README.md b/README.md index 284dee4c..5f4c5bd6 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,8 @@ IEnumerator ConnectToRoom() } ``` +Subscribe to `room.Disconnected` (or `room.DisconnectedWithReason`) for your teardown: it is raised for server-side disconnects and, with `DisconnectReason.ClientInitiated`, for your own `room.Disconnect()` or `Dispose()` as well, so one handler covers both. Handlers run before the room's handles are released. `room.ConnectionStateChanged` reports the same transitions, and `room.IsConnected` is true from the moment `Connected` is raised. One gap to know: a `Disconnect()` while the connect is still pending is a no-op, so disconnect again once `Connect` has completed. + ### Video #### Publishing a texture (e.g Unity Camera) @@ -330,8 +332,6 @@ void TrackSubscribed(IRemoteTrack track, RemoteTrackPublication publication, Rem With Platform Audio, the audio input and output are managed by the native ADM of WebRTC. This unlocks echo cancellation, noise suppression, auto gain control and hardware processing if available. There are some known issues with Platform Audio, that we are working on resolving: -- On iOS, disposing of Platform Audio object stops Unity audio output -- On iOS and Unity 6, backgrounding the app breaks Platform Audio - On MacOS with bluetooth headset, unmuting can break audio output #### Initialize Platform Audio @@ -356,11 +356,6 @@ void InitializePlatformAudio() foreach (var device in playout) Debug.Log($" [{device.Index}] {device.Name}"); - if (platformAudio.RecordingDeviceCount > 0) - platformAudio.SetRecordingDevice(0); - if (platformAudio.PlayoutDeviceCount > 0) - platformAudio.SetPlayoutDevice(0); - Debug.Log($"PlatformAudio ready. AEC={echoCancellation}, NS={noiseSuppression}, AGC={autoGainControl}, HW={preferHardwareProcessing}"); } catch (System.Exception e) @@ -417,6 +412,74 @@ IEnumerator PublishLocalMicrophonePlatform(PlatformAudio platformAudio, Room roo Using Platform Audio, for audio output of subscribed remote audio tracks you don't need any Unity handling. +#### Audio Output Routing + +On mobile, the OS decides where call audio plays (Bluetooth headset, wired headset, loudspeaker, earpiece). The route is duplex: the microphone follows whatever output route is active, so there is no separate microphone selection on mobile (`SetRecordingDevice` has no effect there and logs a warning). `PlatformAudio` exposes a routing policy on top of that: + +```cs +// Automatic policy: route to the best available output kind, most preferred first. +// The default ranking is Bluetooth > WiredHeadset > Speaker > Earpiece. +platformAudio.PlayoutPreference = new[] { AudioDeviceKind.Bluetooth, AudioDeviceKind.WiredHeadset, AudioDeviceKind.Speaker }; + +// Prefer the earpiece over the loudspeaker: the same list with the two swapped. +// A speakerphone toggle is just switching between these two rankings. +platformAudio.PlayoutPreference = new[] { AudioDeviceKind.Bluetooth, AudioDeviceKind.WiredHeadset, AudioDeviceKind.Earpiece, AudioDeviceKind.Speaker }; + +// Sticky override on an explicit user choice: audio stays routed to the device until +// the override is cleared or the device disappears (then the automatic policy resumes). +var (recording, playout) = platformAudio.GetDevices(); +platformAudio.SetPlayoutDevice(playout[0].Guid); +platformAudio.ClearPlayoutDeviceSelection(); + +// Observability: raised on the Unity main thread whenever the available devices or +// the active route change. AudioDevice.Kind and AudioDevice.IsSelected tell you what +// each entry is and which one is playing. +platformAudio.DevicesChanged += (playoutDevices, recordingDevices) => { /* refresh your device UI */ }; +``` + +##### The call audio session + +Routing is only asserted while a call is in progress, and the SDK decides that for you: `PlatformAudio` holds the platform's call audio session while at least one `Room` is connected and releases it when the last one disconnects. So the usual pattern — create `PlatformAudio` once at startup to keep a single ADM alive across calls — needs nothing else: + +```cs +var platformAudio = new PlatformAudio(); // no call session yet + +// ... a call starts: +yield return room.Connect(url, token, options); // session taken +yield return platformAudio.StartRecording(); + +// ... the call ends: +platformAudio.StopRecording(); +room.Disconnect(); // session released +``` + +While released, the SDK holds no call audio session: on iOS WebRTC's voice-processing unit is off and the session sits in a music-friendly idle state, and on Android 12+ the SDK requests neither `MODE_IN_COMMUNICATION` nor the output route pin, so the platform's normal routing applies. Constructing `PlatformAudio` outside a call issues no audio-mode traffic at all, and device enumeration and `DevicesChanged` keep working on both platforms, so a device picker can be populated before the first call. A `PlatformAudio` created while a room is already connected takes the session immediately; a room that carries no audio at all should simply not have a `PlatformAudio` alive. The session is taken and released as part of the room's own connection-state transitions, before `Room.Connected` and `Room.Disconnected` reach your handlers — on a local `Disconnect()` and a server-side disconnect alike. + + +Unity's own audio engine is a separate layer that the SDK does not touch, and it needs a little care from an app that plays its own audio (music, SFX) alongside calls. When an output device is added or removed, Unity reinitializes its engine, which **stops every `AudioSource`** — and it raises `AudioSettings.OnAudioConfigurationChanged` only afterwards, so by the time the app is notified there is nothing left playing to inspect. What should still be audible therefore has to be remembered from before the change and restarted in that callback. On Android the callback's `deviceWasChanged` argument is `false` even for a real device change, so it cannot be used to filter these events. This is Unity's own behavior — it reproduces in a plain Unity scene without the SDK — so restarting the app's sources is the app's responsibility; the Agents sample's `PlatformAudioController` shows one way to do it. + +**Known limitation — the platform's Bluetooth SCO state can get stuck.** Android brings a Bluetooth headset's *call* link up asynchronously, and its SCO state machine can be left in a pending state that never resolves. While it is, the platform accepts `setCommunicationDevice` but never applies it (`AS.BtHelper: requestScoState: failed to connect in state 1`, `preferredCommunicationDevice: null`), so a call's audio — and any media the app plays alongside it — stays on the loudspeaker for the whole call and returns to the headset when the call ends. It is platform state, not app state: it survives the app being restarted, and the SDK cannot clear it (the outstanding request belongs to another client in the process). The SDK logs a warning naming this and retries with backoff. + +Two things are known to provoke or reveal it, device-verified on a Pixel 8a (Android 16): + +- Unity's audio engine claims the call link itself through the deprecated `AudioManager.startBluetoothSco()` when it initializes with a headset already connected — about 3 s before this SDK creates its ADM, and not triggered by anything in the SDK or the samples. Present in 2022.3 and Unity 6 alike; neither version uses the Android 12 communication-device API, which is why the two collide. +- Once stuck, only the platform clears it: disconnecting and reconnecting the headset (which triggers the platform's own `resetBluetoothSco`), toggling Bluetooth, or restarting the phone. After that, routing works normally — the call link comes up in well under a second. + +The reliable workaround is to connect the headset *after* the app has started, or to reconnect it once if a call has landed on the loudspeaker. + +Do **not** call `AudioSettings.Reset` as part of that recovery on Android. Unity has already reopened its output by the time it notifies you, so a reset adds nothing — and reinitializing the engine makes Unity claim a Bluetooth headset's call link through the deprecated `AudioManager.startBluetoothSco()`, which evicts the `setCommunicationDevice` route pin the SDK holds and can leave the platform's SCO state machine unable to connect at all (`AS.BtHelper: requestScoState: failed to connect in state 1` on every subsequent attempt). Call audio and game audio then both stay on the loudspeaker for the rest of the session, no matter how often the route is re-pinned. Restarting the app's own `AudioSource`s is enough and stays out of the platform's way. + +One consequence to design around on Android: while a call session is active on a classic (BR/EDR) Bluetooth headset, the platform suspends the headset's A2DP media link and routes *all* output — the app's own media included — over the headset's call link. Observed on a Pixel 8a (Android 16) with `adb shell dumpsys audio`: `STREAM_MUSIC` moves to `bt_sco_hs` while the call is active and back to `bt_a2dp` afterwards. Game audio therefore keeps playing during a call, but at the call link's quality, and it returns to full quality once the room disconnects and the SDK releases the session. This is a platform property of classic Bluetooth, not something the routing API can override. + +Per-platform behavior: + +- **Android 12+ (API 31)**: the full `PlayoutPreference` ranking applies — the SDK routes to the highest-ranked available kind and re-routes on device changes; kinds missing from the list are never auto-selected (when nothing ranked is available, the OS default route applies). `SetPlayoutDevice` pins a device from `GetDevices().Playout` (by `Guid`) as the communication device; the pin is dropped once that device disappears. Pinning selects the call route, not only the output: Android pairs the microphone with the communication device — a Bluetooth headset's own mic, the built-in mic when the speaker is pinned (even with a wired headset plugged in), the headset mic for the earpiece or a wired headset — and moves a running capture along, so `SetRecordingDevice` has no effect (a warning is logged). While no room is connected, `SetPlayoutDevice` only records the choice — it is applied when the next room connects, and until then `GetDevices`/`DevicesChanged` keep reporting the platform's own route. There is deliberately no pending flag for that deferral: a pre-call device picker should treat its own last `SetPlayoutDevice` call as the pending choice and confirm application via the `IsSelected` flip in `GetDevices`/`DevicesChanged` once a room is connected; a deferred choice whose device disappears first is dropped for good (same rule as an active pin), observable as the device leaving the playout list. `DevicesChanged` is raised on communication-device changes and on device add/remove (via `AudioDeviceCallback`, bridged through the `LiveKitAudioDeviceMonitor` Java source plugin shipped in the package); there is no polling. Requires the `MODIFY_AUDIO_SETTINGS` permission in your `AndroidManifest.xml`. Routing is asserted only while a room is connected: the SDK then holds `MODE_IN_COMMUNICATION` with the route pinned, and clears the pin and restores the mode it replaced when the last room disconnects, while enumeration and `DevicesChanged` stay live either way. Note: since Android 13 the OS only honors the app's communication-mode request — and with it the route pin — while the app has an active voice-communication capture, so keep the mic capture running for the whole call, even while muted with the track unpublished (see `PlatformAudioController` in the Meet sample); an active capture outside a connected room hands routing back to the platform, so start it once the room is connected and stop it when the room disconnects. +- **Older Android**: no routing backend — `PlayoutPreference` is stored and round-trips but has no routing effect, and `SetPlayoutDevice` and `SetRecordingDevice` have no effect (a warning is logged). `DevicesChanged` is never raised. +- **iOS**: external devices (Bluetooth, wired) always take priority over the built-in outputs, so the Speaker/Earpiece relative order is the only part of the ranking with an effect. It decides where audio goes when no external device is connected, is applied through the audio session mode (never by overriding the output port), and takes effect immediately, including mid-call. `SetPlayoutDevice` has no effect (a warning is logged) — the OS owns route selection on iOS; present the system route picker (`AVRoutePickerView`) instead. `SetRecordingDevice` has no effect either (a warning is logged): the OS pairs the microphone with the active route. `GetDevices().Playout` is the audio session's current output route (iOS does not enumerate every reachable device), and `DevicesChanged` is raised when that route changes. +- **Desktop (Windows/macOS/Linux)**: output is selected per device with `SetPlayoutDevice` and input independently with `SetRecordingDevice`; the `PlayoutPreference` ranking has no routing effect. `DevicesChanged` is never raised (no hot-plug events yet). + +> **Upgrading from 2.0.x:** `SetPlayoutDevice` used to be a no-op on Android and iOS. On Android 12+ it now pins the device as a sticky override that shadows `PlayoutPreference` until `ClearPlayoutDeviceSelection` is called or the device disappears. Remove any "select playout device 0 at startup" call (the earlier sample did this): on Android 12+ it would pin whichever device the OS lists first for the whole session. Call `SetPlayoutDevice` only on an explicit user choice and let the ranking route otherwise. + ### RPC Perform your own predefined method calls from one participant to another. diff --git a/Runtime/Plugins/Android.meta b/Runtime/Plugins/Android.meta new file mode 100644 index 00000000..c3a37b55 --- /dev/null +++ b/Runtime/Plugins/Android.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 5383b5f9da47046b9a297c8d16ed0ee7 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Plugins/Android/LiveKitAudioDeviceMonitor.java b/Runtime/Plugins/Android/LiveKitAudioDeviceMonitor.java new file mode 100644 index 00000000..e2a13167 --- /dev/null +++ b/Runtime/Plugins/Android/LiveKitAudioDeviceMonitor.java @@ -0,0 +1,102 @@ +// Bridges android.media.AudioDeviceCallback to C#. +// +// AudioDeviceCallback is an abstract class and Unity's AndroidJavaProxy can only +// implement Java interfaces, so the subclass has to live in Java. This class does +// nothing but forward device add/remove notifications to the Listener interface, +// which AndroidRouteController implements on the C# side through an AndroidJavaProxy. +// Unity compiles this source as part of the Gradle build (Android plugin, source form). +package io.livekit.unity; + +import android.media.AudioDeviceCallback; +import android.media.AudioDeviceInfo; +import android.media.AudioManager; +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +public final class LiveKitAudioDeviceMonitor extends AudioDeviceCallback { + // Diagnostic logging for the on-device verification of this bridge: shows whether + // Android invokes the callback and whether the forward into C# returns. + private static final String TAG = "LiveKit"; + + /** Implemented on the C# side via AndroidJavaProxy. */ + public interface Listener { + /** + * Invoked on the main looper whenever audio output devices were added to or + * removed from the system. The counts only include sinks (output devices); a + * change that touches inputs alone is not reported, because input routing on + * Android follows the communication device and never needs a re-evaluation. + */ + void onAudioDevicesChanged(int addedSinks, int removedSinks); + } + + private final AudioManager audioManager; + private final Listener listener; + private boolean registered; + + public LiveKitAudioDeviceMonitor(AudioManager audioManager, Listener listener) { + this.audioManager = audioManager; + this.listener = listener; + } + + /** + * Starts receiving callbacks on the main looper. Android delivers one immediate + * onAudioDevicesAdded with the currently connected devices right after registering. + */ + public synchronized void register() { + if (registered) { + return; + } + audioManager.registerAudioDeviceCallback(this, new Handler(Looper.getMainLooper())); + registered = true; + Log.i(TAG, "AudioDeviceMonitor: registered; outputs currently enumerable: " + + audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS).length); + } + + public synchronized void unregister() { + if (!registered) { + return; + } + audioManager.unregisterAudioDeviceCallback(this); + registered = false; + } + + @Override + public void onAudioDevicesAdded(AudioDeviceInfo[] addedDevices) { + int sinks = countSinks(addedDevices); + Log.i(TAG, "AudioDeviceMonitor: onAudioDevicesAdded total=" + length(addedDevices) + + " sinks=" + sinks + " thread=" + Thread.currentThread().getName()); + if (sinks > 0) { + listener.onAudioDevicesChanged(sinks, 0); + Log.i(TAG, "AudioDeviceMonitor: forwarded added=" + sinks + " to C#"); + } + } + + @Override + public void onAudioDevicesRemoved(AudioDeviceInfo[] removedDevices) { + int sinks = countSinks(removedDevices); + Log.i(TAG, "AudioDeviceMonitor: onAudioDevicesRemoved total=" + length(removedDevices) + + " sinks=" + sinks + " thread=" + Thread.currentThread().getName()); + if (sinks > 0) { + listener.onAudioDevicesChanged(0, sinks); + Log.i(TAG, "AudioDeviceMonitor: forwarded removed=" + sinks + " to C#"); + } + } + + private static int length(AudioDeviceInfo[] devices) { + return devices == null ? 0 : devices.length; + } + + private static int countSinks(AudioDeviceInfo[] devices) { + if (devices == null) { + return 0; + } + int count = 0; + for (AudioDeviceInfo device : devices) { + if (device.isSink()) { + count++; + } + } + return count; + } +} diff --git a/Runtime/Plugins/Android/LiveKitAudioDeviceMonitor.java.meta b/Runtime/Plugins/Android/LiveKitAudioDeviceMonitor.java.meta new file mode 100644 index 00000000..12a291d8 --- /dev/null +++ b/Runtime/Plugins/Android/LiveKitAudioDeviceMonitor.java.meta @@ -0,0 +1,32 @@ +fileFormatVersion: 2 +guid: 8b574b2e5b3e048f39594bc3f9aaa7b9 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 1 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Android: Android + second: + enabled: 1 + settings: {} + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Plugins/iOS/LiveKitAudioSession.mm b/Runtime/Plugins/iOS/LiveKitAudioSession.mm index f341fc56..cf59aa15 100644 --- a/Runtime/Plugins/iOS/LiveKitAudioSession.mm +++ b/Runtime/Plugins/iOS/LiveKitAudioSession.mm @@ -15,55 +15,590 @@ */ #import +#import + +#include +#include + +// This plugin coordinates the single shared AVAudioSession with WebRTC's iOS +// Audio Device Module (ADM). WebRTC ships an RTCAudioSession proxy that, left in +// its default "automatic" mode, reconfigures the category/route and *deactivates* +// the session whenever a call's playout/recording starts and stops. That fights +// with Unity/FMOD: on join the app's other audio (e.g. ambient music) is rerouted +// to the earpiece and attenuated, and on hang-up the session is deactivated out +// from under Unity so its audio dies. +// +// To make playout stable and keep Unity audio alive across call state, we put +// RTCAudioSession into MANUAL mode and have the app own the session: +// * We hold exactly one permanent activation (setActive:YES). Because +// RTCAudioSession ref-counts activation, WebRTC's per-call setActive:YES/NO +// only cycles the count and never actually deactivates the hardware session. +// * The category/mode/options are derived from a session STATE machine driven +// from C# (PlatformAudio knows the call's recording state) plus a +// speaker-vs-earpiece preference (see the table below). Every apply is also +// mirrored into WebRTC's RTCAudioSessionConfiguration snapshot so the ADM +// re-applies the same config on its own restarts. +// * The speaker preference is expressed via the session MODE only (VideoChat +// routes to the loudspeaker by default, VoiceChat to the receiver), never via +// overrideOutputAudioPort, so connected wired/Bluetooth devices always win. +// * The VPIO voice-processing unit (hardware AEC/AGC/NS) is gated by +// isAudioEnabled. It defaults to YES so call audio works out of the box (the +// unit still only initializes once a call actually has an audio track, so +// pre-call audio is unaffected). Callers can toggle it via +// LiveKit_SetAudioEnabled -- e.g. OFF on hang-up so the unit stops between +// calls while our held activation keeps the session alive for Unity. +// * Backgrounding interrupts the session and WebRTC stops its audio unit. On +// foreground, RTCAudioSession's own recovery restarts the unit exactly once, +// with no retry -- and Unity/FMOD restarts *its* audio around the same moment, +// reconfiguring the shared session (observed with Unity 6). Whoever loses that +// race stays broken, so we observe foreground/interruption-end ourselves and, +// after Unity's restart has settled, re-assert the current state's config and +// cycle isAudioEnabled to force a clean rebuild of the audio unit. +// * Route changes (headset plug/unplug, Bluetooth connect, mode switches) are +// observed via AVAudioSessionRouteChangeNotification and forwarded to C# +// through a registered callback so the SDK can raise its DevicesChanged event. +// +// Session state table (state is set from C# via LiveKit_SetSessionState): +// +// state category mode options +// 0 idle PlayAndRecord Default BT | A2DP | MixWithOthers +// | DefaultToSpeaker* +// 1 playout-only PlayAndRecord Default same as idle +// 2 recording PlayAndRecord VideoChat (speaker) / BT | A2DP +// VoiceChat (earpiece) +// +// *DefaultToSpeaker only while the speaker is preferred. In the recording state +// the speaker preference is carried by the mode alone. Idle and playout-only +// share a config: PlayAndRecord stays because the ADM initializes its VPIO unit +// with input disabled for playout-only (InitPlayOrRecord(false)) but nothing +// guarantees VPIO under the Playback category; mode Default + MixWithOthers is +// the music-friendliest config the ADM demonstrably supports. The states stay +// distinct so the mapping can diverge without touching the C# driver. +// +// RTCAudioSession lives inside the statically-linked liblivekit_ffi; we reach it +// dynamically via NSClassFromString + a protocol-typed id so this file never +// creates a link-time dependency on the class. If the class can't be found we +// fall back to configuring AVAudioSession directly (legacy behavior). + +/// Minimal subset of WebRTC's RTCAudioSession that we message dynamically. +@protocol LiveKitRTCAudioSession +@property(nonatomic, assign) BOOL useManualAudio; +@property(nonatomic, assign) BOOL isAudioEnabled; +@property(nonatomic, readonly) int activationCount; +- (void)lockForConfiguration; +- (void)unlockForConfiguration; +- (BOOL)setActive:(BOOL)active error:(NSError**)outError; +- (BOOL)setCategory:(AVAudioSessionCategory)category + mode:(AVAudioSessionMode)mode + options:(AVAudioSessionCategoryOptions)options + error:(NSError**)outError; +@end + +/// Minimal subset of WebRTC's RTCAudioSessionConfiguration (the snapshot the ADM +/// re-applies on its own restarts), messaged dynamically like RTCAudioSession. +@protocol LiveKitRTCAudioSessionConfiguration +@property(nonatomic, strong) NSString* category; +@property(nonatomic, assign) AVAudioSessionCategoryOptions categoryOptions; +@property(nonatomic, strong) NSString* mode; +@end + +/// Session states, mirroring PlatformAudio's driver in C#. Do not renumber. +enum { + kLiveKitSessionStateIdle = 0, + kLiveKitSessionStatePlayoutOnly = 1, + kLiveKitSessionStateRecording = 2, +}; + +typedef void (*LiveKitRouteChangeCallback)(void); + +// Tracks whether *we* currently hold the one app-owned activation, so we add and +// release it exactly once regardless of how many times configure/restore run. +static BOOL s_liveKitHoldsActivation = NO; + +// Snapshot of the AVAudioSession configuration as it was the first time LiveKit +// touched the session (i.e. whatever Unity set up from its iOS Player Settings). +// Captured lazily in LiveKit_ConfigureAudioSessionForVoIP and re-applied by +// LiveKit_RestoreDefaultAudioSession when the last PlatformAudio is disposed. +static BOOL s_hasCachedState = NO; +static NSString* s_cachedCategory = nil; +static NSString* s_cachedMode = nil; +static AVAudioSessionCategoryOptions s_cachedCategoryOptions = 0; + +// YES between configure and restore: gates the foreground-recovery observers so +// they no-op once LiveKit has handed the session back to the app. +static BOOL s_liveKitConfigured = NO; +// The isAudioEnabled state the caller wants (updated by LiveKit_SetAudioEnabled), +// so recovery knows whether to restart the audio unit after re-asserting config. +static BOOL s_audioDesired = NO; +// Coalesces recovery requests (didBecomeActive and interruption-ended both fire +// on foreground) into one delayed pass. +static BOOL s_recoveryPending = NO; + +// The state machine inputs (see the table above). The defaults match what a fresh +// PlatformAudio pushes right after construction, so the config applied by +// configure is already the one the C# driver expects. +static int s_sessionState = kLiveKitSessionStatePlayoutOnly; +static BOOL s_speakerPreferred = YES; + +// Invoked (on the main queue) whenever the audio route changes, so the C# side +// can re-query the route and raise DevicesChanged. +static LiveKitRouteChangeCallback s_routeChangeCallback = NULL; + +// AllowBluetooth was renamed AllowBluetoothHFP in the iOS 26 SDK; same guard the +// WebRTC fork uses. The speaker preference never rides on these options. +#if defined(__IPHONE_26_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_26_0 +static const AVAudioSessionCategoryOptions kLiveKitBluetoothOptions = + AVAudioSessionCategoryOptionAllowBluetoothHFP | + AVAudioSessionCategoryOptionAllowBluetoothA2DP; +#else +static const AVAudioSessionCategoryOptions kLiveKitBluetoothOptions = + AVAudioSessionCategoryOptionAllowBluetooth | + AVAudioSessionCategoryOptionAllowBluetoothA2DP; +#endif + +/// Returns WebRTC's shared RTCAudioSession if it's present in the linked binary, +/// or nil if the class can't be found (in which case callers use AVAudioSession). +static id LiveKit_RTCSession() { + Class cls = NSClassFromString(@"RTCAudioSession"); + if (!cls || ![cls respondsToSelector:@selector(sharedInstance)]) { + return nil; + } +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Warc-performSelector-leaks" + return (id)[cls performSelector:@selector(sharedInstance)]; +#pragma clang diagnostic pop +} + +static NSString* LiveKit_DesiredMode() { + if (s_sessionState == kLiveKitSessionStateRecording) { + return s_speakerPreferred ? AVAudioSessionModeVideoChat + : AVAudioSessionModeVoiceChat; + } + return AVAudioSessionModeDefault; +} + +static AVAudioSessionCategoryOptions LiveKit_DesiredOptions() { + AVAudioSessionCategoryOptions options = kLiveKitBluetoothOptions; + if (s_sessionState != kLiveKitSessionStateRecording) { + options |= AVAudioSessionCategoryOptionMixWithOthers; + // Mode Default routes PlayAndRecord to the receiver; outside a call there + // is no mode that both prefers the speaker and leaves music processing + // alone, so here -- and only here -- the preference rides on an option. + if (s_speakerPreferred) { + options |= AVAudioSessionCategoryOptionDefaultToSpeaker; + } + } + return options; +} + +/// Mirrors our category/mode/options into WebRTC's RTCAudioSessionConfiguration +/// snapshot so the ADM re-applies the same config whenever it (re)configures the +/// session itself (audio unit init, interruption recovery). +static void LiveKit_MirrorWebRTCConfiguration(NSString* mode, + AVAudioSessionCategoryOptions options) { + Class cls = NSClassFromString(@"RTCAudioSessionConfiguration"); + if (!cls || ![cls respondsToSelector:@selector(webRTCConfiguration)] || + ![cls respondsToSelector:@selector(setWebRTCConfiguration:)]) { + return; + } +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Warc-performSelector-leaks" + id config = + (id)[cls performSelector:@selector(webRTCConfiguration)]; + if (config == nil) { + return; + } + config.category = AVAudioSessionCategoryPlayAndRecord; + config.mode = mode; + config.categoryOptions = options; + [cls performSelector:@selector(setWebRTCConfiguration:) withObject:config]; +#pragma clang diagnostic pop +} + +/// Applies the config derived from (s_sessionState, s_speakerPreferred) to the +/// session and mirrors it into the WebRTC snapshot. Logs expected vs. actual so +/// device tests can see who won when something else reconfigures the session. +/// +/// rebuildAudioUnitOnModeChange: the ADM does not rebuild its VPIO unit on a +/// route change that keeps the hardware sample rate (HandleValidRouteChange -> +/// HandleSampleRateChange no-ops when the audio parameters are intact), so a +/// live mode switch leaves the unit calibrated for the previous route -- +/// device-observed as an attenuated loudspeaker after an earpiece -> speaker +/// toggle. Passing YES cycles isAudioEnabled after a mode change to force a +/// clean rebuild against the new route, at the cost of a brief audio gap. +/// Callers that handle the rebuild themselves (foreground recovery) or run +/// before the unit exists (configure) pass NO. +static void LiveKit_ApplySessionConfig(NSString* reason, BOOL rebuildAudioUnitOnModeChange) { + NSString* mode = LiveKit_DesiredMode(); + AVAudioSessionCategoryOptions options = LiveKit_DesiredOptions(); + BOOL modeChanged = ![[AVAudioSession sharedInstance].mode isEqualToString:mode]; + + id rtc = LiveKit_RTCSession(); + NSError* error = nil; + if (rtc != nil) { + [rtc lockForConfiguration]; + if (![rtc setCategory:AVAudioSessionCategoryPlayAndRecord + mode:mode + options:options + error:&error] || error) { + NSLog(@"LiveKit: failed to apply session config (%@): %@", + reason, error.localizedDescription); + } + [rtc unlockForConfiguration]; + } else { + AVAudioSession* session = [AVAudioSession sharedInstance]; + if (![session setCategory:AVAudioSessionCategoryPlayAndRecord + mode:mode + options:options + error:&error] || error) { + NSLog(@"LiveKit: failed to apply session config (%@): %@", + reason, error.localizedDescription); + } + } + + LiveKit_MirrorWebRTCConfiguration(mode, options); + + AVAudioSession* current = [AVAudioSession sharedInstance]; + NSLog(@"LiveKit: session config (%@): state=%d speakerPreferred=%d expected mode=%@ options=%lu" + " -> actual category=%@ mode=%@ options=%lu", + reason, s_sessionState, s_speakerPreferred, mode, (unsigned long)options, + current.category, current.mode, (unsigned long)current.categoryOptions); + + if (rebuildAudioUnitOnModeChange && modeChanged && s_audioDesired && rtc != nil) { + rtc.isAudioEnabled = NO; + rtc.isAudioEnabled = YES; + NSLog(@"LiveKit: cycled isAudioEnabled to rebuild the audio unit after mode change (%@)", + reason); + } +} + +/// Re-applies the current state's config and reactivates the session, then +/// cycles isAudioEnabled to force WebRTC to rebuild its VPIO audio unit. Runs on +/// a delay so it lands after Unity/FMOD's own foreground audio restart (which is +/// itself delayed and can reconfigure the shared session underneath WebRTC's +/// one-shot, no-retry interruption recovery -- the Unity 6 focus race). +static void LiveKit_ScheduleSessionRecovery() { + if (!s_liveKitConfigured || s_recoveryPending) { + return; + } + s_recoveryPending = YES; + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.5 * NSEC_PER_SEC)), + dispatch_get_main_queue(), ^{ + s_recoveryPending = NO; + if (!s_liveKitConfigured) { + return; // restored/disposed while the recovery was pending + } + + AVAudioSession* session = [AVAudioSession sharedInstance]; + // "Who won" the focus race: what Unity/FMOD left the session as. + NSLog(@"LiveKit: foreground recovery; session before re-assert: category=%@ mode=%@ options=%lu", + session.category, session.mode, (unsigned long)session.categoryOptions); + + LiveKit_ApplySessionConfig(@"foreground recovery", NO); + + // Reactivate directly on AVAudioSession: the OS deactivated the hardware + // session during the interruption, but RTCAudioSession's activation + // ref-count still includes our held activation, so reactivating through + // the proxy would double-count it. + NSError* error = nil; + if (![session setActive:YES error:&error] || error) { + NSLog(@"LiveKit: recovery failed to reactivate session: %@", error.localizedDescription); + } + + // Cycle isAudioEnabled to rebuild the VPIO unit against the re-asserted + // session (WebRTC's own foreground restart may have failed, or been undone + // by Unity's). Harmless when no call audio is active: with playout and + // recording uninitialized WebRTC ignores the change. + id rtc = LiveKit_RTCSession(); + if (rtc != nil && s_audioDesired) { + rtc.isAudioEnabled = NO; + rtc.isAudioEnabled = YES; + } + + NSLog(@"LiveKit: foreground recovery done (audioDesired=%d, activationCount=%d)", + s_audioDesired, rtc != nil ? rtc.activationCount : -1); + }); +} + +/// Registers app-lifetime observers for foreground/interruption recovery and for +/// route-change forwarding. Registered once on first configure; the handlers +/// no-op while LiveKit is not configured. +static void LiveKit_RegisterLifecycleObserversIfNeeded() { + static BOOL s_observersRegistered = NO; + if (s_observersRegistered) { + return; + } + s_observersRegistered = YES; + + NSNotificationCenter* center = [NSNotificationCenter defaultCenter]; + [center addObserverForName:UIApplicationDidBecomeActiveNotification + object:nil + queue:[NSOperationQueue mainQueue] + usingBlock:^(NSNotification* note) { + LiveKit_ScheduleSessionRecovery(); + }]; + [center addObserverForName:AVAudioSessionInterruptionNotification + object:nil + queue:[NSOperationQueue mainQueue] + usingBlock:^(NSNotification* note) { + NSNumber* type = note.userInfo[AVAudioSessionInterruptionTypeKey]; + if (type.unsignedIntegerValue == AVAudioSessionInterruptionTypeEnded) { + LiveKit_ScheduleSessionRecovery(); + } + }]; + [center addObserverForName:AVAudioSessionRouteChangeNotification + object:nil + queue:[NSOperationQueue mainQueue] + usingBlock:^(NSNotification* note) { + if (!s_liveKitConfigured) { + return; + } + NSNumber* reason = note.userInfo[AVAudioSessionRouteChangeReasonKey]; + NSMutableArray* outputs = [NSMutableArray array]; + for (AVAudioSessionPortDescription* port in + [AVAudioSession sharedInstance].currentRoute.outputs) { + [outputs addObject:[NSString stringWithFormat:@"%@ (%@)", port.portName, port.portType]]; + } + NSLog(@"LiveKit: route changed (reason=%lu) outputs=%@", + (unsigned long)reason.unsignedIntegerValue, + [outputs componentsJoinedByString:@", "]); + LiveKitRouteChangeCallback callback = s_routeChangeCallback; + if (callback != NULL) { + callback(); + } + }]; +} + +/// Captures the current audio session category/mode/options exactly once, before +/// LiveKit reconfigures the session for VoIP. Subsequent calls are no-ops so the +/// snapshot always reflects the pristine, pre-LiveKit (Unity-configured) state. +static void LiveKit_CacheSessionStateIfNeeded() { + if (s_hasCachedState) { + return; + } + AVAudioSession* session = [AVAudioSession sharedInstance]; + // copy so the strings persist for the app lifetime regardless of ARC/MRC. + s_cachedCategory = [session.category copy]; + s_cachedMode = [session.mode copy]; + s_cachedCategoryOptions = session.categoryOptions; + s_hasCachedState = YES; + NSLog(@"LiveKit: cached audio session state: category=%@, mode=%@, options=%lu", + s_cachedCategory, s_cachedMode, (unsigned long)s_cachedCategoryOptions); +} + +/// Maps an AVAudioSessionPort type to the C# AudioDeviceKind numbering +/// (Unknown=0, Earpiece=1, Speaker=2, WiredHeadset=3, Bluetooth=4, Usb=5, +/// HearingAid=6). Do not renumber. AVAudioSession has no dedicated hearing-aid +/// port type, so 6 is never produced here; AirPlay/HDMI/CarAudio and other +/// unroutable-by-us ports map to Unknown. +static int LiveKit_OutputKindForPortType(NSString* portType) { + if ([portType isEqualToString:AVAudioSessionPortBuiltInReceiver]) return 1; + if ([portType isEqualToString:AVAudioSessionPortBuiltInSpeaker]) return 2; + if ([portType isEqualToString:AVAudioSessionPortHeadphones]) return 3; + if ([portType isEqualToString:AVAudioSessionPortBluetoothA2DP] || + [portType isEqualToString:AVAudioSessionPortBluetoothHFP] || + [portType isEqualToString:AVAudioSessionPortBluetoothLE]) return 4; + if ([portType isEqualToString:AVAudioSessionPortUSBAudio]) return 5; + return 0; +} extern "C" { -/// Configures the iOS audio session for VoIP/WebRTC use. -/// This sets AVAudioSessionCategoryPlayAndRecord with VoiceChat mode, -/// which enables the VPIO (Voice Processing IO) AudioUnit for: -/// - Hardware echo cancellation (AEC) -/// - Automatic gain control (AGC) -/// - Noise suppression (NS) +/// Configures the iOS audio session for VoIP/WebRTC use and takes app ownership +/// of the shared AVAudioSession. +/// +/// This applies the config for the current session state (playout-only for a +/// fresh PlatformAudio; see the state table at the top of this file), puts +/// RTCAudioSession into manual mode, and holds a single permanent activation so +/// WebRTC never deactivates the session on its own. /// -/// Call this before creating PlatformAudio to ensure WebRTC can -/// properly initialize the microphone and speaker. +/// Call this before creating PlatformAudio. Call audio is enabled by default, so +/// no further call is required for it to work; use LiveKit_SetAudioEnabled(false) +/// to stop the VPIO unit between calls (e.g. on hang-up). void LiveKit_ConfigureAudioSessionForVoIP() { - AVAudioSession* session = [AVAudioSession sharedInstance]; - NSError* error = nil; + // Snapshot the pristine (Unity Player Settings) session before we change it. + LiveKit_CacheSessionStateIfNeeded(); + + LiveKit_RegisterLifecycleObserversIfNeeded(); + s_liveKitConfigured = YES; + s_audioDesired = YES; // mirrors the isAudioEnabled default set below + + id rtc = LiveKit_RTCSession(); + + // Manual mode: WebRTC won't activate/deactivate the session on its own, and + // won't initialize the VPIO unit until we grant permission via isAudioEnabled + // (set below). This is what lets us own activation and gate the unit. Set + // before the first apply so the ADM never races the initial configuration. + if (rtc != nil) { + rtc.useManualAudio = YES; + } - // Configure for VoIP with echo cancellation - BOOL success = [session setCategory:AVAudioSessionCategoryPlayAndRecord - mode:AVAudioSessionModeVoiceChat - options:AVAudioSessionCategoryOptionDefaultToSpeaker | - AVAudioSessionCategoryOptionAllowBluetooth | - AVAudioSessionCategoryOptionAllowBluetoothA2DP - error:&error]; + LiveKit_ApplySessionConfig(@"configure", NO); - if (!success || error) { - NSLog(@"LiveKit: Failed to configure VoIP audio session: %@", error.localizedDescription); + if (rtc == nil) { + // RTCAudioSession unavailable: activate AVAudioSession directly (legacy). + AVAudioSession* session = [AVAudioSession sharedInstance]; + NSError* error = nil; + if (![session setActive:YES error:&error] || error) { + NSLog(@"LiveKit: Failed to activate audio session: %@", error.localizedDescription); + return; + } + NSLog(@"LiveKit: Audio session configured (AVAudioSession fallback)"); return; } - // Activate the audio session - success = [session setActive:YES error:&error]; - if (!success || error) { - NSLog(@"LiveKit: Failed to activate audio session: %@", error.localizedDescription); + // Hold exactly one app-owned activation. RTCAudioSession ref-counts activation, + // so WebRTC's balanced setActive:YES/NO during a call never drops the real + // session below active while we hold this. + if (!s_liveKitHoldsActivation) { + [rtc lockForConfiguration]; + NSError* error = nil; + if ([rtc setActive:YES error:&error] && !error) { + s_liveKitHoldsActivation = YES; + } else { + NSLog(@"LiveKit: Failed to activate audio session: %@", error.localizedDescription); + } + [rtc unlockForConfiguration]; + } + + // Grant WebRTC permission to initialize its audio unit by default so call audio + // works without an explicit LiveKit_SetAudioEnabled(true). The unit is only + // actually created once a call has an audio track, so pre-call audio is + // unaffected. Callers may still disable it (e.g. on hang-up) via + // LiveKit_SetAudioEnabled(false). + rtc.isAudioEnabled = YES; + + NSLog(@"LiveKit: Audio session configured for VoIP (manual mode, activationCount=%d)", + rtc.activationCount); +} + +/// Enables or disables WebRTC's VPIO audio unit while the app keeps ownership of +/// the session. Pass true when a call connects and false when it ends. +/// +/// This is only effective in manual mode (set up by LiveKit_ConfigureAudioSessionForVoIP). +/// Disabling on hang-up stops incoming/outgoing call audio and the VPIO processing, +/// but leaves the session active (via the app's held activation), so Unity audio +/// keeps playing. +void LiveKit_SetAudioEnabled(bool enabled) { + s_audioDesired = enabled ? YES : NO; + id rtc = LiveKit_RTCSession(); + if (rtc == nil) { return; } + rtc.isAudioEnabled = enabled ? YES : NO; + NSLog(@"LiveKit: isAudioEnabled=%@ (activationCount=%d)", enabled ? @"YES" : @"NO", rtc.activationCount); +} + +/// Sets whether the loudspeaker is preferred over the earpiece for the built-in +/// outputs and live-applies the resulting config (see the state table). External +/// devices (wired, Bluetooth) always take priority over both; this only decides +/// where audio goes when no external device is connected. +void LiveKit_SetSpeakerPreferred(bool preferred) { + BOOL value = preferred ? YES : NO; + if (s_speakerPreferred == value) { + return; + } + s_speakerPreferred = value; + if (s_liveKitConfigured) { + LiveKit_ApplySessionConfig(@"speaker preference", YES); + } +} - NSLog(@"LiveKit: Audio session configured for VoIP (PlayAndRecord + VoiceChat mode)"); +/// Sets the session state (0 idle, 1 playout-only, 2 recording; see the state +/// table) and live-applies the resulting config. Driven from C#: PlatformAudio +/// knows whether recording is active and whether call audio is wanted. +void LiveKit_SetSessionState(int state) { + if (state < kLiveKitSessionStateIdle || state > kLiveKitSessionStateRecording) { + NSLog(@"LiveKit: ignoring unknown session state %d", state); + return; + } + if (s_sessionState == state) { + return; + } + s_sessionState = state; + if (s_liveKitConfigured) { + LiveKit_ApplySessionConfig(@"session state", YES); + } } -/// Restores the audio session to the default ambient category. -/// Call this when PlatformAudio is disposed if you want to restore -/// the original audio behavior. +/// Registers (or clears, with NULL) the callback invoked on the main queue +/// whenever the audio route changes. The callback carries no payload; the C# +/// side re-queries LiveKit_GetCurrentOutputRoutes. +void LiveKit_SetRouteChangeCallback(LiveKitRouteChangeCallback callback) { + s_routeChangeCallback = callback; +} + +/// Returns the current output route as newline-separated "kind\tname\tuid" +/// entries (kind per LiveKit_OutputKindForPortType). The caller must release the +/// returned buffer with LiveKit_FreeRouteString. +char* LiveKit_GetCurrentOutputRoutes() { + NSMutableString* result = [NSMutableString string]; + for (AVAudioSessionPortDescription* port in + [AVAudioSession sharedInstance].currentRoute.outputs) { + [result appendFormat:@"%d\t%@\t%@\n", + LiveKit_OutputKindForPortType(port.portType), + port.portName ?: @"", + port.UID ?: @""]; + } + return strdup(result.UTF8String); +} + +/// Frees a buffer returned by LiveKit_GetCurrentOutputRoutes. +void LiveKit_FreeRouteString(char* str) { + free(str); +} + +/// Restores the audio session Unity had before LiveKit touched it (or the ambient +/// category if LiveKit never configured it), relinquishes the app-owned activation +/// and manual mode, and reactivates the session so Unity audio output resumes. +/// Call this when the last PlatformAudio is disposed. void LiveKit_RestoreDefaultAudioSession() { + // Stand down the foreground-recovery observers before touching the session. + s_liveKitConfigured = NO; + s_audioDesired = NO; + // Reset the state machine to the defaults a fresh PlatformAudio expects, so a + // later reconfigure starts from the same config it will be driven to. + s_sessionState = kLiveKitSessionStatePlayoutOnly; + s_speakerPreferred = YES; + + id rtc = LiveKit_RTCSession(); + + if (rtc != nil) { + // Stop the VPIO unit and release our activation before handing control back. + rtc.isAudioEnabled = NO; + if (s_liveKitHoldsActivation) { + NSError* error = nil; + if (![rtc setActive:NO error:&error] || error) { + NSLog(@"LiveKit: Failed to deactivate audio session: %@", error.localizedDescription); + } + s_liveKitHoldsActivation = NO; + } + rtc.useManualAudio = NO; + } + AVAudioSession* session = [AVAudioSession sharedInstance]; NSError* error = nil; + if (s_hasCachedState) { + if (![session setCategory:s_cachedCategory + mode:s_cachedMode + options:s_cachedCategoryOptions + error:&error] || error) { + NSLog(@"LiveKit: Failed to restore cached audio session (category=%@, mode=%@): %@", + s_cachedCategory, s_cachedMode, error.localizedDescription); + } + } else { + // Configure was never called, so we have nothing to restore to; fall back + // to the ambient category. + [session setCategory:AVAudioSessionCategoryAmbient error:&error]; + if (error) { + NSLog(@"LiveKit: Failed to restore default audio session: %@", error.localizedDescription); + } + } - [session setCategory:AVAudioSessionCategoryAmbient error:&error]; - if (error) { - NSLog(@"LiveKit: Failed to restore default audio session: %@", error.localizedDescription); + // Hand an active session back to Unity so its audio output resumes. + error = nil; + if (![session setActive:YES error:&error] || error) { + NSLog(@"LiveKit: Failed to reactivate audio session: %@", error.localizedDescription); } } diff --git a/Runtime/Scripts/Audio/AndroidRouteController.cs b/Runtime/Scripts/Audio/AndroidRouteController.cs new file mode 100644 index 00000000..3e0b9834 --- /dev/null +++ b/Runtime/Scripts/Audio/AndroidRouteController.cs @@ -0,0 +1,1020 @@ +#if UNITY_ANDROID && !UNITY_EDITOR +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading; +using LiveKit.Internal; +using UnityEngine; + +namespace LiveKit +{ + /// + /// Android routing backend for , built on the + /// communication-device APIs introduced in Android 12 (API 31): + /// AudioManager.getAvailableCommunicationDevices / + /// setCommunicationDevice / clearCommunicationDevice. + /// + /// The controller owns the voice-communication audio session while it holds it — + /// session audio enabled (, which + /// drives from Room connections — i.e. a call is in + /// progress) and acquired by a first trigger, see the lazy-acquisition paragraph: + /// it enters MODE_IN_COMMUNICATION (saving and restoring the prior mode) and + /// keeps the output route pinned to the best device — the sticky + /// override while its device is still available, otherwise + /// the highest-ranked available kind per the current + /// . Owning the mode is what makes the + /// pin authoritative: without it the platform periodically reasserts its own default + /// route (observed on Pixel 8a: Telecom flipped playout back to the earpiece every + /// ~6 s after a Bluetooth session ended). Note that since Android 13 the mode request + /// is only honored while the app has active voice-communication capture, so + /// re-asserts the policy when capture + /// (re)starts — through , which like every other + /// re-evaluation path pins nothing while session audio is disabled. + /// + /// The session is acquired lazily: construction issues no setMode and no pin + /// even though session audio starts out enabled. The first trigger that needs the + /// session while it is enabled acquires it — an explicit + /// call with true, an + /// (which includes the + /// re-assert) or a + /// . disables session + /// audio right after construction unless a Room is already connected, so creating + /// an instance outside a call causes no audio-mode traffic at all; the eager + /// constructor acquisition produced a take → pin → clear transient there, and with + /// a Bluetooth headset connected it started an asynchronous SCO activation only to + /// clear it mid-negotiation. The explicit enable PlatformAudio issues when the + /// first Room connects acquires the session even when nothing else has needed it + /// yet, so a receive-only app that never records and never touches routing still + /// gets the mode and pin for its call. + /// + /// While the session is not held — session audio disabled, or enabled but nothing + /// has needed it yet — it belongs to the platform (communication device cleared, + /// prior mode restored on release), so the mode request and the route pin cover the + /// call rather than the lifetime of the instance. Enumeration and both OS listeners + /// stay registered regardless, so and + /// keep reporting the platform's own routing while + /// idle. + /// + /// Route changes are detected by two OS callbacks: + /// - OnCommunicationDeviceChangedListener — fires when the OS changes or + /// clears the pin (e.g. the pinned device disconnected). + /// - AudioDeviceCallback, via the LiveKitAudioDeviceMonitor Java plugin + /// in Runtime/Plugins/Android — fires when output devices are added to or removed + /// from the system, which the communication-device listener does not report: a + /// device added while a pin is active, and the trace-verified teardown where a + /// powered-off Bluetooth headset stays in the available list up to ~10 s after the + /// route already fell back to the earpiece, then leaves the list without another + /// communication-device change. Device-verified on a Pixel 8a: with this callback + /// the route recovers to the speaker; without it, it stays on the earpiece (the + /// 1.5 s poll this callback replaced used to catch it). + /// + /// There is no periodic poll. The one thing no callback delivers is time, so a + /// one-shot retry timer, armed only while work is outstanding, re-runs the pass + /// when a pin has been issued but the platform has not applied it yet (the + /// Bluetooth settle window and its backoff) or a session enter/leave failed and + /// must be retried. A pass that finds nothing pending disarms it. + /// + /// Threading: re-evaluation runs on whichever thread triggered it (Unity main, + /// the Android main executor/looper, or a thread-pool thread the retry timer + /// attaches to the JVM for the pass) behind one lock. + /// may therefore be raised from any of them; + /// marshals it to the Unity main thread. + /// + internal sealed class AndroidRouteController : IRouteController + { + // android.media.AudioManager / AudioAttributes constants. + private const int ModeInCommunication = 3; // AudioManager.MODE_IN_COMMUNICATION + private const int AudioFocusGain = 1; // AudioManager.AUDIOFOCUS_GAIN + private const int AudioFocusRequestGranted = 1; // AudioManager.AUDIOFOCUS_REQUEST_GRANTED + private const int UsageVoiceCommunication = 2; // AudioAttributes.USAGE_VOICE_COMMUNICATION + private const int ContentTypeSpeech = 1; // AudioAttributes.CONTENT_TYPE_SPEECH + + private const int MinSupportedApiLevel = 31; + // Delay before the retry timer re-runs the pass for work with no settle window of + // its own: a failed session enter/leave, or a non-Bluetooth pin the platform did + // not take. Same cadence as the poll it replaced. + private static readonly TimeSpan RetryInterval = TimeSpan.FromSeconds(1.5); + // Floor for a computed remaining settle time (Timer.Change rejects negative values). + private static readonly TimeSpan MinRetryDelay = TimeSpan.FromMilliseconds(1); + // How long a pin is given to take effect before it is issued again. Selecting a + // Bluetooth device starts an asynchronous SCO negotiation, and until it completes + // the platform keeps reporting the previous communication device — so without this + // a re-evaluation re-issues the pin into its own pending activation, which the platform + // refuses ("BtHelper: requestScoState: failed to connect in state 1", device-verified + // on a Pixel 8a / Android 16) and the route never arrives at all. Real route changes + // come through the change listener, so this only slows down recovering from a pin the + // platform dropped silently. + private static readonly TimeSpan PinSettleTimeout = TimeSpan.FromSeconds(6); + // Ceiling for the backoff applied when the platform keeps taking the pin without + // acting on it — a state this SDK cannot clear (see the Bluetooth note in README). + private static readonly TimeSpan PinSettleTimeoutMax = TimeSpan.FromSeconds(30); + + private readonly PlatformAudio _owner; + private readonly object _gate = new object(); + private readonly List _recordingSnapshot; + // One-shot retry timer (see the class doc): armed by Reevaluate only while a pin + // is outstanding or a session transition failed, disarmed once nothing is pending. + // Fires on a thread-pool thread, which the callback attaches to the JVM. + private readonly System.Threading.Timer _retryTimer; + + private List _ranked; + private int _stickyDeviceId = -1; + private int _pinnedDeviceId = -1; + // When the outstanding pin was last issued — Stopwatch ticks, monotonic, so a + // wall-clock step can neither cut the settle window short nor stretch it — to + // give it _pinSettleTimeout to take effect, and whether the platform has been + // seen honoring it since. + private long _pinIssuedAtTimestamp; + private bool _pinApplied; + private TimeSpan _pinSettleTimeout = PinSettleTimeout; + // Starts enabled; PlatformAudio sets the real state — whether a Room is + // connected — right after construction (uniform with iOS). + private bool _sessionAudioEnabled = true; + // Whether this controller currently holds the call session (mode entered, pin + // allowed). Never true while _sessionAudioEnabled is false. Acquisition is + // lazy: despite the enabled default, nothing is taken until the first trigger + // that needs the session — see AcquireSessionIfNeeded and the class doc. + private bool _sessionAcquired; + private int _savedAudioMode; + private bool _audioModeSaved; + // Set when an enter/leave transition failed (JNI unavailable, platform error) so + // Reevaluate retries it: without the retry, one transient failure on disable + // would leave the platform in MODE_IN_COMMUNICATION with the route pinned until + // the next call boundary, while this controller reports the session released. + private bool _sessionTransitionPending; + private CommunicationDeviceListener _listener; + // The Java AudioDeviceCallback subclass and the C# proxy it forwards to; both + // null when the plugin could not be registered (add/remove then goes unreported). + private AndroidJavaObject _deviceMonitor; + private AudioDeviceMonitorListener _deviceMonitorListener; + private AndroidJavaObject _audioFocusRequest; + private bool _audioFocusEnabled; + private List<(int Id, AudioDeviceKind Kind, bool IsSelected)> _lastSignature; + private bool _disposed; + + public event Action, IReadOnlyList> DevicesChanged; + + /// + /// Creates the Android backend, or an on + /// Android versions below 12 (API 31), which lack the communication-device APIs + /// this backend is built on. On those versions the routing verbs are documented + /// no-ops/throws, matching the gate the sample hotfix carried. + /// + internal static IRouteController Create(PlatformAudio owner, IReadOnlyList initialPreference) + { + int sdkInt; + try + { + sdkInt = AndroidSdkInt(); + } + catch (Exception e) + { + Utils.Warning($"AndroidRouteController: failed to read Build.VERSION.SDK_INT, routing disabled: {e.Message}"); + return new UnsupportedRouteController(owner, "this Android device"); + } + + if (sdkInt < MinSupportedApiLevel) + return new UnsupportedRouteController(owner, $"Android API {sdkInt} (routing requires API {MinSupportedApiLevel})"); + + return new AndroidRouteController(owner, initialPreference); + } + + private AndroidRouteController(PlatformAudio owner, IReadOnlyList initialPreference) + { + _owner = owner; + _ranked = new List(initialPreference); + + // The FFI exposes a single placeholder entry for the OS default input on + // Android; input routing follows the communication device, so this list is + // static and can back every DevicesChanged payload. Fetched before any + // session state is touched so a failure here has no side effects. + _recordingSnapshot = owner.GetDevicesViaFfi().Recording; + + // Session audio defaults to enabled, but the call session is NOT taken + // here: acquisition waits for the first trigger that needs it (see the + // class doc), and the prior audio mode is saved at that acquisition, where + // it reflects the state actually being replaced. This initial Reevaluate is + // therefore observation-only — it seeds the device signature and reports + // the platform's own route. + // The timer exists before anything can run a pass: a listener callback may + // fire as soon as it is registered, and every pass ends in ScheduleRetry. + _retryTimer = new System.Threading.Timer(OnRetryTimer, null, Timeout.Infinite, Timeout.Infinite); + RegisterListener(); + RegisterDeviceMonitor(); + Reevaluate(); + } + + public (List Recording, List Playout) GetDevices() + { + var recording = _owner.GetDevicesViaFfi().Recording; + var playout = new List(); + try + { + using var audioManager = GetAudioManager(); + using var current = audioManager.Call("getCommunicationDevice"); + var currentId = current != null ? current.Call("getId") : -1; + + using var available = audioManager.Call("getAvailableCommunicationDevices"); + var count = available.Call("size"); + for (var i = 0; i < count; i++) + { + using var device = available.Call("get", i); + playout.Add(ToAudioDevice(device, (uint)i, currentId)); + } + } + catch (Exception e) + { + Utils.Warning($"AndroidRouteController: device enumeration failed: {e.Message}"); + } + return (recording, playout); + } + + public void ApplyPlayoutPreference(IReadOnlyList ranked) + { + lock (_gate) + { + _ranked = new List(ranked); + AcquireSessionIfNeeded(); + } + Reevaluate(); + } + + public void SetPlayoutDevice(string deviceId) + { + // Validated against the live communication-device list so an unknown id fails + // per the public contract instead of being parked as a sticky id that + // Reevaluate would silently drop as "disappeared". + var found = false; + var id = -1; + foreach (var candidate in GetDevices().Playout) + { + if (candidate.Guid != deviceId) continue; + found = int.TryParse(candidate.Guid, NumberStyles.Integer, CultureInfo.InvariantCulture, out id); + break; + } + if (!found) + throw new InvalidOperationException( + $"Playout device '{deviceId}' is not a current playout device; " + + "pass the Guid of an entry from GetDevices().Playout"); + + lock (_gate) + { + _stickyDeviceId = id; + AcquireSessionIfNeeded(); + } + Reevaluate(); + } + + public void ClearPlayoutDeviceSelection() + { + lock (_gate) + { + if (_stickyDeviceId == -1) + return; + _stickyDeviceId = -1; + } + Reevaluate(); + } + + /// + /// Takes or hands back the call audio session: enabling enters + /// MODE_IN_COMMUNICATION and lets the policy pin the route, disabling + /// clears the pin and restores the mode this controller replaced. An explicit + /// enable acquires the session even when the state was already enabled — the + /// lazy default means "enabled but nothing has needed the session yet" is a real + /// state, and the enable issues when the first Room + /// connects is what takes the session for a receive-only app that never records + /// or touches routing. Disabling before anything acquired the session + /// releases nothing: there is nothing to release, and issuing a clear/restore + /// there would be exactly the startup transient lazy acquisition removes. + /// The ranked preference survives the transition unconditionally. The sticky + /// override survives it only while its device stays available: the + /// drop-on-disappear bookkeeping keeps running while the session is disabled, so + /// a device that leaves the list between calls (a headset powered off) clears + /// the override for good, and the next call routes by the ranked preference. + /// + public void SetSessionAudioEnabled(bool enabled) + { + lock (_gate) + { + if (_disposed || (_sessionAudioEnabled == enabled && _sessionAcquired == enabled)) + return; + _sessionAudioEnabled = enabled; + if (enabled) + { + AcquireSessionIfNeeded(); + } + else if (_sessionAcquired) + { + _sessionAcquired = false; + LeaveCommunicationMode(); + } + } + + // Re-evaluate outside the lock (Reevaluate takes it): pin the policy's target + // on enable, report the platform's own route on disable. + Reevaluate(); + } + + // Called under _gate. The first routing trigger while session audio is enabled + // takes the call session (lazy acquisition — see the class doc); every later + // call is a no-op. Routing verbs express the intent to route, which is what the + // session exists for, so all of them funnel through here: an explicit enable, + // ApplyPlayoutPreference (including the StartRecording re-assert) and + // SetPlayoutDevice. + private void AcquireSessionIfNeeded() + { + if (_disposed || !_sessionAudioEnabled || _sessionAcquired) + return; + _sessionAcquired = true; + EnterCommunicationMode(); + } + + /// + /// Optional audio-focus request (AUDIOFOCUS_GAIN with voice-communication + /// attributes) held while enabled. Off by default. Not exposed on the public + /// API surface (PAR-019 defines it once); flip it here when embedding scenarios + /// need focus, until a supported knob exists. + /// + internal bool AudioFocusEnabled + { + get + { + lock (_gate) return _audioFocusEnabled; + } + set + { + lock (_gate) + { + if (_disposed || _audioFocusEnabled == value) + return; + _audioFocusEnabled = value; + if (value) + RequestAudioFocus(); + else + AbandonAudioFocus(); + } + } + } + + public void Dispose() + { + lock (_gate) + { + if (_disposed) + return; + _disposed = true; + } + + // Cancel any pending retry. A callback already running cannot overlap the + // teardown below: the pass and the teardown both run under _gate, and the pass + // bails out on _disposed, which was set under that lock above. Nothing arms the + // timer after this point either (ScheduleRetry checks _disposed under _gate). + _retryTimer.Dispose(); + + // Unregister BEFORE clearing the pin: clearCommunicationDevice fires the + // change event, and a still-registered listener would immediately re-pin. + UnregisterListener(); + UnregisterDeviceMonitor(); + + lock (_gate) + { + AbandonAudioFocus(); + // Same idempotent release as a session-audio disable: only a session + // this controller holds (or a transition still pending retry) is handed + // back — a never-acquired session leaves the platform untouched, so + // creating and disposing an instance without a call issues no audio + // traffic at all. + if (_sessionAcquired || _sessionTransitionPending) + LeaveCommunicationMode(); + _sessionAcquired = false; + _sessionAudioEnabled = false; + } + } + + /// + /// Single policy pass: picks the target device (sticky override while its device + /// is still available — dropped for good once it disappears — else the best + /// available kind by rank), pins it when it differs from the active route, and + /// raises when the observable list (ids, kinds, + /// selection) changed since the last pass. Re-pinning is skipped when the target + /// is already active: our own setCommunicationDevice fires the change listener, + /// and that no-op check is what stops the feedback loop. When nothing sticky or + /// ranked is available, an existing pin is released so the OS default applies; + /// kinds missing from the ranking are never auto-selected. Ends by arming the + /// retry timer when the platform has not applied the target yet or a session + /// transition still fails, and by disarming it otherwise. + /// + /// While the session is not held — session audio disabled, or enabled but not + /// yet acquired — the pass is observation-only: it enumerates, keeps the sticky + /// bookkeeping current and still raises , but issues + /// no setCommunicationDevice / clearCommunicationDevice and reports the + /// platform's own communication device as the selected one. The OS callbacks + /// and the retry timer run through here without acquiring anything, so none of + /// them can resurrect a released session nor take a lazily-deferred one; the + /// re-assert acquires first (in + /// ) and then runs through here like the + /// rest. + /// + private void Reevaluate() + { + List playout = null; + lock (_gate) + { + if (_disposed) + return; + // A failed enter/leave transition is retried from here: every trigger — + // the OS callbacks, the retry timer, the StartRecording re-assert — + // funnels through this pass, so a transient JNI failure cannot leave the + // platform holding (or missing) the call session until the next call + // boundary; a transition that fails again arms the retry timer below. + // Retries the transition for the CURRENT desired state, so a flip that + // happened in between is never undone. + if (_sessionTransitionPending) + { + if (_sessionAcquired) + EnterCommunicationMode(); + else + LeaveCommunicationMode(); + } + // Set when this pass leaves work the platform still has to finish or a + // retry has to redo; null disarms the timer. + TimeSpan? retryIn = null; + try + { + using var audioManager = GetAudioManager(); + using var current = audioManager.Call("getCommunicationDevice"); + var currentId = current != null ? current.Call("getId") : -1; + + using var available = audioManager.Call("getAvailableCommunicationDevices"); + var count = available.Call("size"); + var devices = new List<(AndroidJavaObject Device, int Id, AudioDeviceKind Kind)>(count); + try + { + for (var i = 0; i < count; i++) + { + var device = available.Call("get", i); + devices.Add((device, device.Call("getId"), KindFromDeviceType(device.Call("getType")))); + } + + var targetIndex = -1; + if (_stickyDeviceId != -1) + { + targetIndex = devices.FindIndex(d => d.Id == _stickyDeviceId); + if (targetIndex < 0) + { + Utils.Debug("AndroidRouteController: sticky output device disappeared; reverting to automatic policy"); + _stickyDeviceId = -1; + } + } + + if (targetIndex < 0) + { + var bestRank = int.MaxValue; + for (var i = 0; i < devices.Count; i++) + { + var rank = _ranked.IndexOf(devices[i].Kind); + if (rank >= 0 && rank < bestRank) + { + bestRank = rank; + targetIndex = i; + } + } + } + + int selectedId; + if (!_sessionAcquired) + { + // No session held (no call in progress, or nothing has + // needed the session yet): report which device the platform + // would use for communication audio, and touch nothing. The + // target computed above is still worth running — it keeps + // the sticky override's "dropped once the device disappears" + // bookkeeping alive while idle — but it is only applied once + // the session is acquired. + selectedId = currentId; + } + else if (targetIndex >= 0) + { + var target = devices[targetIndex]; + if (currentId == _pinnedDeviceId) + _pinApplied = true; + if (target.Id != currentId) + { + // A Bluetooth pin that has not taken effect yet is left to + // finish: setCommunicationDevice starts an asynchronous SCO + // negotiation there, and re-issuing lands in the platform's + // own pending activation and gets refused, so hammering it + // keeps the route from ever arriving. Once the pin has been + // seen applied, a later divergence is the platform dropping + // it (reported by the change listener) and is re-pinned at once. + // The other kinds apply without a negotiation, so a + // divergence there is always a dropped or ignored pin and + // is re-issued immediately, as before the settle window + // existed. See PinSettleTimeout, and _pinSettleTimeout for + // the backoff applied when the platform takes a Bluetooth + // pin but never acts on it. + var retry = _pinnedDeviceId == target.Id && !_pinApplied + && target.Kind == AudioDeviceKind.Bluetooth; + var settling = retry && ElapsedSincePinIssued() < _pinSettleTimeout; + if (settling) + { + // Waiting on the negotiation: report the device the + // platform still has, never the one merely requested. + // The change listener re-runs this pass the moment the + // pin lands, and the selection flip raises the + // DevicesChanged for the real arrival. + selectedId = currentId; + } + else + { + var ok = audioManager.Call("setCommunicationDevice", target.Device); + Utils.Debug($"AndroidRouteController: setCommunicationDevice(kind={target.Kind}) -> {ok}"); + // Stamped on every attempt, not only on success: + // measured from a stale issue time the settle window + // expires for good after one refused re-issue, and the + // backoff decays into a warn+re-issue every retry. + _pinIssuedAtTimestamp = System.Diagnostics.Stopwatch.GetTimestamp(); + if (ok) + { + _pinnedDeviceId = target.Id; + _pinApplied = false; + } + if (retry) + { + // The platform is taking the request and not acting on + // it. Back off rather than keep asking: retrying into + // an activation the platform will not start achieves + // nothing, and the cause is usually outside this SDK + // (see the Bluetooth note in the README). + Utils.Warning( + $"AndroidRouteController: the platform is not applying the route pin for " + + $"{target.Kind} after {_pinSettleTimeout.TotalSeconds:0}s. If this is a " + + "Bluetooth headset, another component in this process (Unity's audio engine " + + "does this when it initializes with a headset connected) may hold an " + + "outstanding startBluetoothSco request, which blocks the call link until it " + + "resolves. Call audio stays on the previous output until then."); + var next = TimeSpan.FromTicks(_pinSettleTimeout.Ticks * 2); + _pinSettleTimeout = next > PinSettleTimeoutMax ? PinSettleTimeoutMax : next; + } + else + { + _pinSettleTimeout = PinSettleTimeout; + } + // Report the platform's answer, not the request: the + // synchronous kinds are visible in this re-read right + // away, while a pending Bluetooth pin must not be + // announced as selected before it lands — GetDevices() + // reads the same truth, and a premature "selected" + // would also swallow the arrival event, because the + // signature would never change again. + using var applied = audioManager.Call("getCommunicationDevice"); + selectedId = applied != null ? applied.Call("getId") : -1; + if (ok && selectedId == target.Id) + _pinApplied = true; + } + } + else + { + selectedId = currentId; + } + } + else + { + if (_pinnedDeviceId != -1) + { + audioManager.Call("clearCommunicationDevice"); + ResetPinTracking(); + Utils.Debug("AndroidRouteController: no ranked device available; cleared pin, OS default applies"); + using var fallback = audioManager.Call("getCommunicationDevice"); + selectedId = fallback != null ? fallback.Call("getId") : -1; + } + else + { + selectedId = currentId; + } + } + + if (_sessionAcquired && targetIndex >= 0 && devices[targetIndex].Id != selectedId) + { + // The platform is not on our target yet: a Bluetooth pin still + // negotiating, or a pin it dropped, refused or never acted on. + // No OS event announces "still nothing", so time has to: come + // back when the settle window closes (to warn and re-issue with + // backoff), or after one retry interval for the kinds that + // apply synchronously. The arrival itself still comes through + // the change listener; this only bounds how long silence lasts. + var settlingBluetooth = _pinnedDeviceId == devices[targetIndex].Id && !_pinApplied + && devices[targetIndex].Kind == AudioDeviceKind.Bluetooth; + var remaining = _pinSettleTimeout - ElapsedSincePinIssued(); + retryIn = settlingBluetooth && remaining > TimeSpan.Zero ? remaining : RetryInterval; + } + + var signature = new List<(int Id, AudioDeviceKind Kind, bool IsSelected)>(devices.Count); + foreach (var d in devices) + signature.Add((d.Id, d.Kind, d.Id == selectedId)); + + if (SignatureChanged(signature)) + { + _lastSignature = signature; + playout = new List(devices.Count); + for (var i = 0; i < devices.Count; i++) + playout.Add(ToAudioDevice(devices[i].Device, (uint)i, selectedId)); + } + } + finally + { + foreach (var d in devices) + d.Device.Dispose(); + } + } + catch (Exception e) + { + Utils.Warning($"AndroidRouteController: route evaluation failed: {e.Message}"); + // The live state could not be read; fall back to the tracked state to + // decide whether anything is worth coming back for. + if (_sessionAcquired && _pinnedDeviceId != -1 && !_pinApplied) + retryIn = RetryInterval; + } + if (_sessionTransitionPending) + retryIn = RetryInterval; + ScheduleRetry(retryIn); + } + + // Raised outside the lock; PlatformAudio marshals to the Unity main thread. + if (playout != null) + DevicesChanged?.Invoke(playout, new List(_recordingSnapshot)); + } + + // Called under _gate from both pin-release sites. Clears everything the + // settle/backoff logic keys on; anything left behind resurfaces on the next + // pin as a spurious settle-skip or backoff warning. + private void ResetPinTracking() + { + _pinnedDeviceId = -1; + _pinApplied = false; + _pinSettleTimeout = PinSettleTimeout; + } + + private TimeSpan ElapsedSincePinIssued() + { + var elapsedTicks = System.Diagnostics.Stopwatch.GetTimestamp() - _pinIssuedAtTimestamp; + return TimeSpan.FromSeconds((double)elapsedTicks / System.Diagnostics.Stopwatch.Frequency); + } + + private bool SignatureChanged(List<(int Id, AudioDeviceKind Kind, bool IsSelected)> signature) + { + if (_lastSignature == null || _lastSignature.Count != signature.Count) + return true; + for (var i = 0; i < signature.Count; i++) + { + if (!_lastSignature[i].Equals(signature[i])) + return true; + } + return false; + } + + // Called under _gate at the end of every pass: arms the one-shot retry timer for + // the given delay, or disarms it when the pass left nothing pending. Dispose sets + // _disposed under the same lock before it disposes the timer, so the timer is + // alive whenever the check below passes. + private void ScheduleRetry(TimeSpan? delay) + { + if (_disposed) + return; + if (delay == null) + { + _retryTimer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + return; + } + var due = delay.Value < MinRetryDelay ? MinRetryDelay : delay.Value; + _retryTimer.Change(due, Timeout.InfiniteTimeSpan); + } + + // Timer callback, on a thread-pool thread: attach it to the JVM for the pass + // (AndroidJavaObject needs an attached thread), run the pass — which re-arms the + // timer itself if the work is still outstanding — and detach again. + private void OnRetryTimer(object _) + { + if (AndroidJNI.AttachCurrentThread() != 0) + { + Utils.Warning("AndroidRouteController: failed to attach the retry timer thread to the JVM; the pending re-evaluation waits for the next OS event or routing call"); + return; + } + Utils.Debug("AndroidRouteController: retry timer fired"); + try + { + Reevaluate(); + } + catch (Exception e) + { + Utils.Warning($"AndroidRouteController: retry re-evaluation failed: {e.Message}"); + } + finally + { + AndroidJNI.DetachCurrentThread(); + } + } + + // Both mode methods are called under _gate. The save/restore pairs up per + // acquire -> release transition and is idempotent in both directions: the prior + // mode is only captured when we do not already hold one, and it is only restored + // when it was actually read from the platform — a failed read must never turn + // into an unconditional MODE_NORMAL, which would stomp a mode this app does not + // own (the rule the PAR-000 hotfix established). A failure marks the transition + // pending, and Reevaluate retries it (warned once, retries silent) — both + // methods are safe to re-run partially completed. + private void EnterCommunicationMode() + { + try + { + using var audioManager = GetAudioManager(); + if (!_audioModeSaved) + { + _savedAudioMode = audioManager.Call("getMode"); + _audioModeSaved = true; + } + audioManager.Call("setMode", ModeInCommunication); + _sessionTransitionPending = false; + Utils.Debug($"AndroidRouteController: audio mode -> MODE_IN_COMMUNICATION (was {_savedAudioMode})"); + } + catch (Exception e) + { + if (!_sessionTransitionPending) + Utils.Warning($"AndroidRouteController: failed to enter communication mode (will retry): {e.Message}"); + _sessionTransitionPending = true; + } + } + + private void LeaveCommunicationMode() + { + try + { + using var audioManager = GetAudioManager(); + audioManager.Call("clearCommunicationDevice"); + ResetPinTracking(); + if (_audioModeSaved) + { + audioManager.Call("setMode", _savedAudioMode); + _audioModeSaved = false; + Utils.Debug($"AndroidRouteController: route cleared, audio mode restored ({_savedAudioMode})"); + } + else + { + Utils.Debug("AndroidRouteController: route cleared, no saved audio mode to restore"); + } + _sessionTransitionPending = false; + } + catch (Exception e) + { + if (!_sessionTransitionPending) + Utils.Warning($"AndroidRouteController: failed to release the audio session (will retry): {e.Message}"); + _sessionTransitionPending = true; + } + } + + private void RegisterListener() + { + try + { + using var unityPlayer = new AndroidJavaClass("com.unity3d.player.UnityPlayer"); + using var activity = unityPlayer.GetStatic("currentActivity"); + using var audioManager = activity.Call("getSystemService", "audio"); + using var executor = activity.Call("getMainExecutor"); + + _listener = new CommunicationDeviceListener(this); + audioManager.Call("addOnCommunicationDeviceChangedListener", executor, _listener); + } + catch (Exception e) + { + _listener = null; + Utils.Warning($"AndroidRouteController: failed to register the communication-device listener; route changes the platform makes on its own will go unreported: {e.Message}"); + } + } + + private void UnregisterListener() + { + if (_listener == null) + return; + try + { + using var audioManager = GetAudioManager(); + audioManager.Call("removeOnCommunicationDeviceChangedListener", _listener); + } + catch (Exception e) + { + Utils.Warning($"AndroidRouteController: failed to unregister device listener: {e.Message}"); + } + _listener = null; + } + + // Both focus methods are called under _gate. + private void RequestAudioFocus() + { + try + { + using var attributesBuilder = new AndroidJavaObject("android.media.AudioAttributes$Builder"); + using var withUsage = attributesBuilder.Call("setUsage", UsageVoiceCommunication); + using var withContentType = withUsage.Call("setContentType", ContentTypeSpeech); + using var attributes = withContentType.Call("build"); + using var focusBuilder = new AndroidJavaObject("android.media.AudioFocusRequest$Builder", AudioFocusGain); + using var withAttributes = focusBuilder.Call("setAudioAttributes", attributes); + _audioFocusRequest = withAttributes.Call("build"); + + using var audioManager = GetAudioManager(); + var result = audioManager.Call("requestAudioFocus", _audioFocusRequest); + Utils.Debug($"AndroidRouteController: requestAudioFocus -> {(result == AudioFocusRequestGranted ? "granted" : result.ToString())}"); + } + catch (Exception e) + { + _audioFocusRequest?.Dispose(); + _audioFocusRequest = null; + Utils.Warning($"AndroidRouteController: audio focus request failed: {e.Message}"); + } + } + + private void AbandonAudioFocus() + { + if (_audioFocusRequest == null) + return; + try + { + using var audioManager = GetAudioManager(); + audioManager.Call("abandonAudioFocusRequest", _audioFocusRequest); + } + catch (Exception e) + { + Utils.Warning($"AndroidRouteController: failed to abandon audio focus: {e.Message}"); + } + _audioFocusRequest.Dispose(); + _audioFocusRequest = null; + } + + private void OnCommunicationDeviceChangedFromJava() + { + try + { + Reevaluate(); + } + catch (Exception e) + { + Utils.Warning($"AndroidRouteController: listener re-evaluation failed: {e.Message}"); + } + } + + // Registers the Java AudioDeviceCallback subclass. Failure is non-fatal but leaves + // device add/remove unreported: only communication-device changes re-route then. + private void RegisterDeviceMonitor() + { + try + { + using var audioManager = GetAudioManager(); + _deviceMonitorListener = new AudioDeviceMonitorListener(this); + _deviceMonitor = new AndroidJavaObject( + "io.livekit.unity.LiveKitAudioDeviceMonitor", audioManager, _deviceMonitorListener); + _deviceMonitor.Call("register"); + Utils.Debug("AndroidRouteController: AudioDeviceCallback registered"); + } + catch (Exception e) + { + _deviceMonitor?.Dispose(); + _deviceMonitor = null; + _deviceMonitorListener = null; + Utils.Warning("AndroidRouteController: failed to register AudioDeviceCallback (is the " + + "LiveKitAudioDeviceMonitor Java plugin in the build?); device add/remove will go " + + $"unreported and only communication-device changes re-route: {e.Message}"); + } + } + + private void UnregisterDeviceMonitor() + { + if (_deviceMonitor == null) + return; + try + { + _deviceMonitor.Call("unregister"); + Utils.Debug("AndroidRouteController: AudioDeviceCallback unregistered"); + } + catch (Exception e) + { + Utils.Warning($"AndroidRouteController: failed to unregister AudioDeviceCallback: {e.Message}"); + } + _deviceMonitor.Dispose(); + _deviceMonitor = null; + _deviceMonitorListener = null; + } + + private void OnAudioDevicesChangedFromJava(int addedSinks, int removedSinks) + { + Utils.Debug($"AndroidRouteController: AudioDeviceCallback (added={addedSinks}, removed={removedSinks})"); + try + { + Reevaluate(); + } + catch (Exception e) + { + Utils.Warning($"AndroidRouteController: AudioDeviceCallback re-evaluation failed: {e.Message}"); + } + } + + private static AudioDevice ToAudioDevice(AndroidJavaObject device, uint index, int selectedId) + { + var id = device.Call("getId"); + using var productName = device.Call("getProductName"); + return new AudioDevice + { + Index = index, + Name = productName?.Call("toString") ?? string.Empty, + Guid = id.ToString(CultureInfo.InvariantCulture), + Kind = KindFromDeviceType(device.Call("getType")), + IsSelected = id == selectedId, + }; + } + + // AudioDeviceInfo.TYPE_* to AudioDeviceKind, mirroring the planned FFI mapping. + private static AudioDeviceKind KindFromDeviceType(int deviceType) + { + switch (deviceType) + { + case 1: // TYPE_BUILTIN_EARPIECE + return AudioDeviceKind.Earpiece; + case 2: // TYPE_BUILTIN_SPEAKER + return AudioDeviceKind.Speaker; + case 3: // TYPE_WIRED_HEADSET + case 4: // TYPE_WIRED_HEADPHONES + return AudioDeviceKind.WiredHeadset; + case 7: // TYPE_BLUETOOTH_SCO + case 26: // TYPE_BLE_HEADSET + case 27: // TYPE_BLE_SPEAKER + return AudioDeviceKind.Bluetooth; + case 22: // TYPE_USB_HEADSET + return AudioDeviceKind.Usb; + case 23: // TYPE_HEARING_AID + return AudioDeviceKind.HearingAid; + default: + return AudioDeviceKind.Unknown; + } + } + + private static int AndroidSdkInt() + { + using var version = new AndroidJavaClass("android.os.Build$VERSION"); + return version.GetStatic("SDK_INT"); + } + + // Caller owns the returned object (wrap it in `using var`). + private static AndroidJavaObject GetAudioManager() + { + using var unityPlayer = new AndroidJavaClass("com.unity3d.player.UnityPlayer"); + using var activity = unityPlayer.GetStatic("currentActivity"); + return activity.Call("getSystemService", "audio"); + } + + // C#-side implementation of the Java callback interface for communication-device + // changes (the OS changing or clearing the pin). List add/remove transitions fire + // no communication-device event; they arrive through AudioDeviceMonitorListener + // below. + private sealed class CommunicationDeviceListener : AndroidJavaProxy + { + private readonly AndroidRouteController _controller; + + public CommunicationDeviceListener(AndroidRouteController controller) + : base("android.media.AudioManager$OnCommunicationDeviceChangedListener") + { + _controller = controller; + } + + // Invoked by Android on the activity's main executor — a JVM-attached + // thread, but not the Unity main thread. + public void onCommunicationDeviceChanged(AndroidJavaObject device) + { + device?.Dispose(); + _controller.OnCommunicationDeviceChangedFromJava(); + } + } + + // C#-side implementation of LiveKitAudioDeviceMonitor.Listener, the interface the + // Java plugin forwards android.media.AudioDeviceCallback to. AndroidJavaProxy can + // only implement interfaces and AudioDeviceCallback is an abstract class, so the + // subclass lives in Java (Runtime/Plugins/Android/LiveKitAudioDeviceMonitor.java). + private sealed class AudioDeviceMonitorListener : AndroidJavaProxy + { + private readonly AndroidRouteController _controller; + + public AudioDeviceMonitorListener(AndroidRouteController controller) + : base("io.livekit.unity.LiveKitAudioDeviceMonitor$Listener") + { + _controller = controller; + } + + // Invoked by Android on the main looper — a JVM-attached thread, but not the + // Unity main thread. Registration delivers one immediate "added" callback + // with the current device set, which runs an observation-only pass. + public void onAudioDevicesChanged(int addedSinks, int removedSinks) + { + _controller.OnAudioDevicesChangedFromJava(addedSinks, removedSinks); + } + } + } +} +#endif diff --git a/Runtime/Scripts/Audio/AndroidRouteController.cs.meta b/Runtime/Scripts/Audio/AndroidRouteController.cs.meta new file mode 100644 index 00000000..1f1eb67b --- /dev/null +++ b/Runtime/Scripts/Audio/AndroidRouteController.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: f96269970c2ac4b4ea77f794848cafae +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/IosRouteController.cs b/Runtime/Scripts/Audio/IosRouteController.cs new file mode 100644 index 00000000..a304c1c3 --- /dev/null +++ b/Runtime/Scripts/Audio/IosRouteController.cs @@ -0,0 +1,223 @@ +#if UNITY_IOS && !UNITY_EDITOR +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; +using LiveKit.Internal; + +namespace LiveKit +{ + /// + /// iOS routing backend over the LiveKitAudioSession.mm plugin. The OS owns output + /// route selection on iOS, so this backend does not pick devices: it reduces + /// to the speaker-vs-earpiece relative + /// order (applied as the audio session mode by the plugin; external devices always + /// take priority over both built-ins), reports the session's current output route + /// as the playout device list, and raises from the + /// plugin's route-change observation. is ignored with + /// a warning: apps that want explicit device picking should present the system route + /// picker (AVRoutePickerView). + /// + /// All plugin P/Invoke for route observation stays inside this class; the session + /// state machine itself is driven by (which knows the + /// recording state) through . + /// + internal sealed class IosRouteController : IRouteController + { + private delegate void RouteChangeDelegate(); + + [DllImport("__Internal")] + private static extern void LiveKit_SetRouteChangeCallback(RouteChangeDelegate callback); + + [DllImport("__Internal")] + private static extern void LiveKit_SetSpeakerPreferred([MarshalAs(UnmanagedType.I1)] bool preferred); + + [DllImport("__Internal")] + private static extern IntPtr LiveKit_GetCurrentOutputRoutes(); + + [DllImport("__Internal")] + private static extern void LiveKit_FreeRouteString(IntPtr routes); + + // The native callback slot is registered once for the app lifetime (matching + // the plugin's app-lifetime notification observers) and fans out to the live + // controllers; keeping the delegate in a static field pins it for the native + // side. Instances add and remove themselves under StaticGate. + private static readonly object StaticGate = new object(); + private static readonly List LiveControllers = new List(); + private static readonly RouteChangeDelegate NativeRouteChanged = OnNativeRouteChanged; + private static bool _callbackRegistered; + + private readonly object _gate = new object(); + // The FFI recording list (a single placeholder for the OS default input), + // captured once: route changes never affect it and re-querying the FFI from + // the route callback would be wasted work. + private readonly List _recordingSnapshot; + private string _lastSignature; + private bool _disposed; + + public event Action, IReadOnlyList> DevicesChanged; + + internal IosRouteController(PlatformAudio owner, IReadOnlyList initialPreference) + { + _recordingSnapshot = owner.GetDevicesViaFfi().Recording; + + ApplyPlayoutPreference(initialPreference); + _lastSignature = Signature(QueryCurrentOutputs()); + + lock (StaticGate) + { + LiveControllers.Add(this); + if (!_callbackRegistered) + { + LiveKit_SetRouteChangeCallback(NativeRouteChanged); + _callbackRegistered = true; + } + } + } + + public (List Recording, List Playout) GetDevices() + { + return (new List(_recordingSnapshot), QueryCurrentOutputs()); + } + + public void ApplyPlayoutPreference(IReadOnlyList ranked) + { + // Reduce the ranked list per the PAR-019 precedence rule: the only part of + // the ranking iOS can express is whether Speaker outranks Earpiece. + var speaker = -1; + var earpiece = -1; + for (var i = 0; i < ranked.Count; i++) + { + if (ranked[i] == AudioDeviceKind.Speaker) speaker = i; + else if (ranked[i] == AudioDeviceKind.Earpiece) earpiece = i; + } + var speakerPreferred = speaker >= 0 && (earpiece < 0 || speaker < earpiece); + LiveKit_SetSpeakerPreferred(speakerPreferred); + } + + public void SetPlayoutDevice(string deviceId) + { + Utils.Warning( + "PlatformAudio.SetPlayoutDevice has no effect on iOS: the OS owns output route " + + "selection. Present the system route picker (AVRoutePickerView) instead, or use " + + "PlayoutPreference for the built-in outputs."); + } + + public void ClearPlayoutDeviceSelection() + { + // No override can exist on iOS: SetPlayoutDevice is ignored. + } + + public void SetSessionAudioEnabled(bool enabled) + { + // Handled by PlatformAudio itself on iOS: it drives the session state machine + // (which needs the recording state this backend does not know) through + // IOSAudioSessionHelper. Nothing route-specific to gate here. + } + + public void Dispose() + { + lock (StaticGate) + { + LiveControllers.Remove(this); + } + lock (_gate) + { + _disposed = true; + } + } + + /// + /// Native route-change entry point, invoked by the plugin on the iOS main + /// queue (not the Unity main thread; marshals the + /// public event). + /// + [AOT.MonoPInvokeCallback(typeof(RouteChangeDelegate))] + private static void OnNativeRouteChanged() + { + IosRouteController[] controllers; + lock (StaticGate) + { + controllers = LiveControllers.ToArray(); + } + foreach (var controller in controllers) + controller.HandleRouteChanged(); + } + + private void HandleRouteChanged() + { + List playout; + lock (_gate) + { + if (_disposed) return; + + playout = QueryCurrentOutputs(); + var signature = Signature(playout); + if (signature == _lastSignature) return; + _lastSignature = signature; + } + + DevicesChanged?.Invoke(playout, new List(_recordingSnapshot)); + } + + /// + /// The current output route reported by the audio session. On iOS this is the + /// active route (usually one device), not an enumeration of every reachable + /// device — AVAudioSession exposes no such list for outputs. + /// + private static List QueryCurrentOutputs() + { + var devices = new List(); + + var routesPtr = LiveKit_GetCurrentOutputRoutes(); + if (routesPtr == IntPtr.Zero) return devices; + + string routes; + try + { + routes = Marshal.PtrToStringUTF8(routesPtr); + } + finally + { + LiveKit_FreeRouteString(routesPtr); + } + if (string.IsNullOrEmpty(routes)) return devices; + + foreach (var line in routes.Split('\n')) + { + if (line.Length == 0) continue; + var fields = line.Split('\t'); + if (fields.Length != 3) + { + Utils.Warning($"IosRouteController: malformed route entry '{line}'"); + continue; + } + + var kind = int.TryParse(fields[0], out var rawKind) + && Enum.IsDefined(typeof(AudioDeviceKind), rawKind) + ? (AudioDeviceKind)rawKind + : AudioDeviceKind.Unknown; + devices.Add(new AudioDevice + { + Index = (uint)devices.Count, + Name = fields[1], + Guid = fields[2], + Kind = kind, + // Everything in the current route is live output by definition. + IsSelected = true, + }); + } + + return devices; + } + + private static string Signature(List playout) + { + var builder = new StringBuilder(); + foreach (var device in playout) + builder.Append(device.Guid).Append('\u001f').Append((int)device.Kind).Append('\u001e'); + return builder.ToString(); + } + } +} +#endif diff --git a/Runtime/Scripts/Audio/IosRouteController.cs.meta b/Runtime/Scripts/Audio/IosRouteController.cs.meta new file mode 100644 index 00000000..069ce527 --- /dev/null +++ b/Runtime/Scripts/Audio/IosRouteController.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2b7a83f0ed8eb4ce1bbd824fdf80c424 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/PlatformAudio.cs b/Runtime/Scripts/Audio/PlatformAudio.cs index 7c113e20..94a5a403 100644 --- a/Runtime/Scripts/Audio/PlatformAudio.cs +++ b/Runtime/Scripts/Audio/PlatformAudio.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Threading; using LiveKit.Proto; using LiveKit.Internal; using LiveKit.Internal.FFI.Requests; @@ -27,13 +28,59 @@ internal static class IOSAudioSessionHelper internal static extern void LiveKit_ConfigureAudioSessionForVoIP(); /// - /// Restores the iOS audio session to ambient mode. + /// Restores the audio session Unity had before LiveKit configured it + /// (or the ambient category as a fallback) and reactivates it so Unity + /// audio output resumes. Called when the last PlatformAudio is disposed. /// [DllImport("__Internal")] internal static extern void LiveKit_RestoreDefaultAudioSession(); + + /// + /// Enables or disables WebRTC's VPIO audio unit while the app keeps + /// ownership of the audio session. Enable when a call connects, disable + /// when it ends. Disabling on hang-up stops call audio without + /// deactivating the session, so other app audio keeps playing. + /// + [DllImport("__Internal")] + internal static extern void LiveKit_SetAudioEnabled([MarshalAs(UnmanagedType.I1)] bool enabled); + + /// + /// Sets the audio session state (0 idle, 1 playout-only, 2 recording) so the + /// plugin can apply the matching category/mode/options (see the state table in + /// LiveKitAudioSession.mm). Driven by PlatformAudio, which knows whether + /// recording is active and whether call audio is wanted. + /// + [DllImport("__Internal")] + internal static extern void LiveKit_SetSessionState(int state); } #endif + /// + /// The kind of audio device (). Reported for playout + /// devices on mobile platforms and used to rank the automatic routing policy there + /// (see ). + /// + /// The numeric values mirror the planned FFI protocol enum of the same name one-to-one + /// so a future FFI-backed implementation maps without translation. Do not renumber. + /// + public enum AudioDeviceKind + { + /// The platform did not report a device type. + Unknown = 0, + /// The phone's built-in earpiece (receiver). + Earpiece = 1, + /// The built-in loudspeaker. + Speaker = 2, + /// A wired headset or headphones. + WiredHeadset = 3, + /// A Bluetooth audio device. + Bluetooth = 4, + /// A USB audio device. + Usb = 5, + /// A hearing aid. + HearingAid = 6, + } + /// /// Information about an audio device (microphone or speaker). /// @@ -49,6 +96,21 @@ public struct AudioDevice /// over index for device selection. /// public string Guid; + /// + /// The kind of device this entry represents. Classified by the routing backend + /// for playout devices — on iOS from the audio session's current route, on + /// Android 12 (API 31) and newer from the communication-device list; where the platform does not report a type + /// (recording devices, desktop, older Android). + /// + public AudioDeviceKind Kind; + /// + /// Whether this device is the active output route. Reported by the routing + /// backend for playout devices on iOS and on Android 12 (API 31) and newer; + /// always false where no backend reports selection state (recording devices, + /// desktop, older Android). + /// + public bool IsSelected; } /// @@ -73,17 +135,76 @@ public sealed class PlatformAudio : IDisposable { internal readonly FfiHandle Handle; private readonly PlatformAudioInfo _info; + private readonly IRouteController _routeController; + private readonly SynchronizationContext _syncContext; + private List _playoutPreference = new List(DefaultPlayoutPreference); private bool _disposed = false; + // Whether we last asked the ADM to record: set after a successful StartRecording, + // cleared by StopRecording and Dispose. It mirrors our requests, not the ADM's own + // state, and can diverge from it when the platform stops the capture on its own + // (e.g. an iOS interruption). Not exposed and not used to gate StartRecording / + // StopRecording: the native ADM is the authority on redundant calls. + private bool _isRecording; +#if UNITY_IOS && !UNITY_EDITOR + // Tracks live PlatformAudio instances so the iOS audio session is restored + // only when the last one is disposed (aligned with the native ADM ref-count). + private static int _instanceCount; + + // Inputs of the iOS session-state machine (see the state table in + // LiveKitAudioSession.mm). PlatformAudio is the driver because it is the one + // that knows both: whether recording is active (_isRecording) and whether call + // audio is wanted (a Room is connected). + private const int IosSessionStateIdle = 0; + private const int IosSessionStatePlayoutOnly = 1; + private const int IosSessionStateRecording = 2; + private bool _iosSessionAudioEnabled; + + private void UpdateIosSessionState() + { + var state = !_iosSessionAudioEnabled ? IosSessionStateIdle + : _isRecording ? IosSessionStateRecording + : IosSessionStatePlayoutOnly; + IOSAudioSessionHelper.LiveKit_SetSessionState(state); + } +#endif + + private static readonly AudioDeviceKind[] DefaultPlayoutPreference = + { + AudioDeviceKind.Bluetooth, + AudioDeviceKind.WiredHeadset, + AudioDeviceKind.Speaker, + AudioDeviceKind.Earpiece, + }; /// /// Number of available recording (microphone) devices. /// - public int RecordingDeviceCount => _info.RecordingDeviceCount; + public int RecordingDeviceCount + { + get + { + ThrowIfDisposed(); + return _info.RecordingDeviceCount; + } + } /// /// Number of available playout (speaker) devices. /// - public int PlayoutDeviceCount => _info.PlayoutDeviceCount; + public int PlayoutDeviceCount + { + get + { + ThrowIfDisposed(); + return _info.PlayoutDeviceCount; + } + } + + private void ThrowIfDisposed() + { + if (_disposed) + throw new ObjectDisposedException(nameof(PlatformAudio)); + } /// /// Creates a new PlatformAudio instance, enabling the platform ADM. @@ -91,9 +212,21 @@ public sealed class PlatformAudio : IDisposable /// This must be called before creating any PlatformAudioSource or connecting /// to a room if you want automatic speaker playout for remote audio. /// - /// On iOS, this automatically configures the audio session for VoIP mode - /// (PlayAndRecord category with VoiceChat mode) to enable hardware echo - /// cancellation and microphone input. + /// On iOS, this automatically configures the audio session for VoIP use and + /// takes app ownership of it. The session's mode follows the call state: a + /// voice/video-chat mode (enabling hardware echo cancellation) while recording + /// is active, a music-friendly default mode while a call is connected without + /// recording, and an idle state — WebRTC's voice-processing unit off — outside + /// a call (see / ). + /// + /// The platform's call audio session follows connections + /// automatically: it is held while at least one Room is connected and released + /// when the last one disconnects, so an instance created at app start — the + /// usual pattern, to keep a single ADM alive across calls — holds no call + /// session until a call actually starts. On Android 12+ that session is + /// MODE_IN_COMMUNICATION plus the output route pin per + /// ; on iOS it is WebRTC's voice-processing + /// unit. Construction itself changes no audio mode and pins no route. /// /// /// Thrown if the platform ADM could not be initialized (e.g., no audio devices, @@ -103,7 +236,7 @@ public PlatformAudio() { #if UNITY_IOS && !UNITY_EDITOR // Configure iOS audio session for VoIP before initializing WebRTC ADM. - // This sets PlayAndRecord category with VoiceChat mode for hardware AEC. + // This sets PlayAndRecord category with VideoChat mode for hardware AEC. IOSAudioSessionHelper.LiveKit_ConfigureAudioSessionForVoIP(); #endif @@ -118,7 +251,49 @@ public PlatformAudio() Handle = FfiHandle.FromOwnedHandle(platformAudio.Handle); _info = platformAudio.Info; + try + { + _syncContext = SynchronizationContext.Current; + _routeController = CreateRouteController(); + _routeController.DevicesChanged += OnRouteControllerDevicesChanged; + + // The call audio session follows Room connections from here on. Applied + // unconditionally: an instance created while a room is already connected + // takes the session right away (on Android the explicit take is what + // acquires the lazily-held session), and one created outside a call + // drops to the idle state — on iOS out of the plugin's post-configure + // "audio enabled" default. + Room.ConnectedRoomCountChanged += OnConnectedRoomCountChanged; + ApplySessionAudio(Room.ConnectedRoomCount > 0); + } + catch + { + // Without this, a route-controller failure would leak the FFI handle + // until the SafeHandle finalizer eventually reclaims it. + Room.ConnectedRoomCountChanged -= OnConnectedRoomCountChanged; + _routeController?.Dispose(); + Handle.Dispose(); + throw; + } + Utils.Debug($"PlatformAudio created: {RecordingDeviceCount} recording devices, {PlayoutDeviceCount} playout devices"); + +#if UNITY_IOS && !UNITY_EDITOR + // Count this instance only after successful construction so a failed + // ctor never leaves the counter stuck above zero. + System.Threading.Interlocked.Increment(ref _instanceCount); +#endif + } + + private IRouteController CreateRouteController() + { +#if UNITY_ANDROID && !UNITY_EDITOR + return AndroidRouteController.Create(this, _playoutPreference); +#elif UNITY_IOS && !UNITY_EDITOR + return new IosRouteController(this, _playoutPreference); +#else + return new DesktopRouteController(this); +#endif } /// @@ -128,24 +303,47 @@ public PlatformAudio() /// - Desktop (Windows/macOS/Linux): returns the full list of microphones and /// speakers reported by the OS. Devices can be selected with /// / . - /// - iOS and Android: returns a single placeholder entry at index 0 for each - /// list, representing the system's currently selected default input/output. - /// The OS owns audio routing on these platforms (AVAudioSession on iOS, - /// AudioManager on Android), so individual devices are not enumerated and - /// selecting one is a no-op (see / + /// - iOS: the playout list is the audio session's current output route (usually + /// one device, with and + /// set) — iOS does not enumerate every + /// reachable output device, and has no + /// effect there. The recording list is a single placeholder entry for the OS + /// default input. + /// - Android 12 (API 31) and newer: the playout list contains the available + /// communication devices with and + /// set; entries can be routed to with + /// . The recording list stays a single placeholder + /// entry for the OS default input — input routing follows the selected + /// communication device. + /// - Older Android: returns a single placeholder entry at index 0 for each list, + /// representing the system's currently selected default input/output. The OS + /// owns audio routing (AudioManager), so individual devices are not enumerated + /// and selecting one is a no-op (see / /// ). /// /// /// A tuple containing: /// - Recording: List of available microphones (on iOS/Android, a single /// placeholder for the OS default input) - /// - Playout: List of available speakers/headphones (on iOS/Android, a single - /// placeholder for the OS default output) + /// - Playout: List of available speakers/headphones (on iOS, the current output + /// route; on pre-API-31 Android, a single placeholder for the OS default + /// output) /// /// /// Thrown if device enumeration failed. /// public (List Recording, List Playout) GetDevices() + { + ThrowIfDisposed(); + return _routeController.GetDevices(); + } + + /// + /// Device enumeration through the FFI, shared by the route controllers. + /// and are not + /// reported by the FFI and stay at their defaults (Unknown / false). + /// + internal (List Recording, List Playout) GetDevicesViaFfi() { using var request = FFIBridge.Instance.NewRequest(); request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); @@ -179,6 +377,111 @@ public PlatformAudio() return (recording, playout); } + /// + /// Ranked automatic output routing policy, most preferred first. When no explicit + /// output override is active (), the platform + /// routes to the highest-ranked kind that has a connected device. + /// + /// Default: Bluetooth > WiredHeadset > Speaker > Earpiece. + /// + /// Platform notes: on iOS, external devices (Bluetooth, wired) always take priority + /// over the built-in outputs, so the Speaker/Earpiece relative order is the only part + /// of the ranking with an effect; it is applied through the audio session mode and + /// takes effect immediately, including mid-call. On Android the full ranking applies: + /// the backend routes to the highest-ranked available kind on Android 12 (API 31) + /// and newer, and kinds missing from the list are never auto-selected (when + /// nothing ranked is available the OS default route applies). The selected route + /// is duplex on both mobile platforms: the OS pairs the microphone with it (see + /// ). On desktop, output is selected per + /// device () and the ranking has no routing + /// effect. + /// On older Android versions (routing backend not implemented there) the value + /// is stored and round-trips, but has no routing effect either. + /// + /// Thrown if set to null. + /// + /// Thrown if the list contains or duplicates. + /// + public IReadOnlyList PlayoutPreference + { + get + { + ThrowIfDisposed(); + return _playoutPreference.AsReadOnly(); + } + set + { + ThrowIfDisposed(); + if (value == null) + throw new ArgumentNullException(nameof(value)); + + var ranked = new List(value.Count); + foreach (var kind in value) + { + if (kind == AudioDeviceKind.Unknown) + throw new ArgumentException( + "PlayoutPreference cannot contain AudioDeviceKind.Unknown", nameof(value)); + if (ranked.Contains(kind)) + throw new ArgumentException( + $"PlayoutPreference contains {kind} more than once", nameof(value)); + ranked.Add(kind); + } + + _playoutPreference = ranked; + _routeController.ApplyPlayoutPreference(_playoutPreference.AsReadOnly()); + } + } + + /// + /// Clears the sticky override set by so the + /// automatic policy applies again. + /// + /// Platform notes: on desktop there is no automatic policy to fall back to yet, so + /// clearing keeps the currently selected device (no-op). On Android 12 (API 31) + /// and newer the automatic policy re-routes immediately. On older Android + /// versions and on iOS no override can exist ( + /// is ignored there), so this is a no-op too. + /// + public void ClearPlayoutDeviceSelection() + { + ThrowIfDisposed(); + _routeController.ClearPlayoutDeviceSelection(); + } + + /// + /// Raised when the set of available audio devices changes, with the current playout + /// and recording device lists. Raised on the Unity main thread. + /// + /// On iOS this fires when the audio session's output route changes (headset + /// plugged/unplugged, Bluetooth connected, speaker/earpiece switches); the playout + /// list is the new route. On Android it is raised by the routing backend + /// (Android 12/API 31 and newer) when the available communication devices or the + /// active route change, driven by the OS device add/remove and + /// communication-device callbacks (no polling). Desktop hot-plug events are not + /// implemented yet in + /// this version, so the event is never raised there. Subscribing and + /// unsubscribing is safe at any time, including after . + /// + public event Action, IReadOnlyList> DevicesChanged; + + private void OnRouteControllerDevicesChanged( + IReadOnlyList playout, IReadOnlyList recording) + { + if (_disposed) return; + + if (_syncContext != null && _syncContext != SynchronizationContext.Current) + { + _syncContext.Post(_ => + { + if (!_disposed) + DevicesChanged?.Invoke(playout, recording); + }, null); + return; + } + + DevicesChanged?.Invoke(playout, recording); + } + /// /// Sets the recording device (microphone) by index. /// @@ -192,6 +495,7 @@ public PlatformAudio() /// public void SetRecordingDevice(uint index) { + ThrowIfDisposed(); var (recording, _) = GetDevices(); if (index >= recording.Count) throw new InvalidOperationException($"Recording device index {index} out of range (max: {recording.Count - 1})"); @@ -202,12 +506,15 @@ public void SetRecordingDevice(uint index) /// /// Sets the recording device (microphone) by device ID (GUID). /// - /// On Android and iOS this is a no-op in the native ADM: input routing is - /// governed by the OS (AVAudioSession on iOS, AudioManager on Android) and - /// the call is acknowledged but ignored. The method is still safe to call, - /// and the response carries no error. only exposes a - /// single placeholder entry (index 0) for the OS default input on these - /// platforms, so there is nothing else to select. + /// Platform notes: + /// - Desktop (Windows/macOS/Linux): selects the ADM recording device, independently + /// of the playout device. + /// - Android and iOS: no effect (a warning is logged) — the OS pairs the microphone + /// with the call route, so the mic follows / + /// on Android and the active audio session route + /// on iOS. only exposes a single placeholder entry + /// (index 0) for the OS default input there, so there is nothing else to select; + /// this overload never throws on these platforms. /// /// Device ID/GUID from GetDevices().Recording[i].Guid /// @@ -215,6 +522,18 @@ public void SetRecordingDevice(uint index) /// public void SetRecordingDevice(string deviceId) { + ThrowIfDisposed(); +#if UNITY_IOS && !UNITY_EDITOR + // The native ADM would acknowledge and ignore the request; warn instead so an + // unsupported selection is as visible as SetPlayoutDevice's no-op on iOS. + Utils.Warning( + "PlatformAudio.SetRecordingDevice has no effect on iOS: the OS pairs the microphone " + + "with the active audio route."); +#elif UNITY_ANDROID && !UNITY_EDITOR + Utils.Warning( + "PlatformAudio.SetRecordingDevice has no effect on Android: the OS pairs the microphone " + + "with the call route selected by SetPlayoutDevice / PlayoutPreference."); +#else using var request = FFIBridge.Instance.NewRequest(); request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); request.request.DeviceId = deviceId; @@ -226,14 +545,16 @@ public void SetRecordingDevice(string deviceId) throw new InvalidOperationException($"Failed to set recording device: {res.SetRecordingDevice.Error}"); Utils.Debug($"PlatformAudio: set recording device to {deviceId}"); +#endif } /// /// Sets the playout device (speaker/headphones) by index. /// /// Convenience wrapper around that looks - /// up the GUID from . Prefer the GUID overload for code - /// that persists a selection — indices can shift when devices are added/removed. + /// up the GUID from ; see that overload for the + /// per-platform behavior. Prefer the GUID overload for code that persists a + /// selection — indices can shift when devices are added/removed. /// /// Device index from GetDevices().Playout /// @@ -241,6 +562,7 @@ public void SetRecordingDevice(string deviceId) /// public void SetPlayoutDevice(uint index) { + ThrowIfDisposed(); var (_, playout) = GetDevices(); if (index >= playout.Count) throw new InvalidOperationException($"Playout device index {index} out of range (max: {playout.Count - 1})"); @@ -249,20 +571,56 @@ public void SetPlayoutDevice(uint index) } /// - /// Sets the playout device (speaker/headphones) by device ID (GUID). + /// Routes audio output to the playout device with the given ID + /// ( from ) as a sticky + /// override of the automatic policy: the route stays + /// on the device until is called. /// - /// On Android and iOS this is a no-op in the native ADM: output routing is - /// governed by the OS (AVAudioSession on iOS, AudioManager on Android) and - /// the call is acknowledged but ignored. The method is still safe to call, - /// and the response carries no error. only exposes a - /// single placeholder entry (index 0) for the OS default output on these - /// platforms, so there is nothing else to select. + /// Platform notes: + /// - Desktop (Windows/macOS/Linux): selects the ADM playout device. + /// - Android 12 (API 31) and newer: pins the device as the communication device. + /// This selects the call route, not only the output: Android pairs the microphone + /// with the communication device (a Bluetooth headset's own mic; the built-in mic + /// when the speaker is pinned, even with a wired headset plugged in; the headset + /// mic for the earpiece or a wired headset) and moves a running capture along. + /// The override is dropped once the device disappears from the playout list + /// (automatic policy resumes). While no is connected the + /// choice is only recorded — no pin is issued, and / + /// keep reporting the platform's own route — until + /// a room connects and the SDK takes the call session. There is deliberately no + /// pending flag for that deferral: the app holds both inputs (its own + /// SetPlayoutDevice call and its own room connection), so a pre-call device + /// picker should treat its last selection as the pending choice and confirm + /// application through the existing surface — once the room is connected and + /// the pin lands, the device's + /// flips in / . A deferred + /// choice is dropped for good when its device disappears before the session is + /// enabled (the same drop-on-disappear rule as an active pin), observable as the + /// device leaving the playout list in the same events. Because the override + /// shadows until cleared, do not call this at + /// startup to "pick the default": let the ranking route, and call this only on + /// an explicit user choice. + /// - iOS: no effect (a warning is logged) — the OS owns output route selection. + /// Present the system route picker (AVRoutePickerView) instead, or use + /// for the built-in outputs. + /// - Older Android: no effect (a warning is logged) — no routing backend there. /// /// Device ID/GUID from GetDevices().Playout[i].Guid /// - /// Thrown if the device is not found or the operation failed. + /// Thrown if the ID does not match a current playout device (desktop, Android 12+) + /// or the selection failed. /// public void SetPlayoutDevice(string deviceId) + { + ThrowIfDisposed(); + _routeController.SetPlayoutDevice(deviceId); + } + + /// + /// Playout device selection through the FFI (the ADM's own device list), used by + /// the desktop route controller. + /// + internal void SetPlayoutDeviceViaFfi(string deviceId) { using var request = FFIBridge.Instance.NewRequest(); request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); @@ -280,15 +638,25 @@ public void SetPlayoutDevice(string deviceId) /// /// Starts recording from the microphone. /// - /// Recording is started automatically when PlatformAudio is created. - /// Use this to resume recording after calling StopRecording. + /// Recording does not start on its own when PlatformAudio is created — call + /// this to start capturing, and again to resume after . + /// On Android and iOS the coroutine first awaits the OS microphone-permission + /// dialog when the permission has not been granted yet, and only then opens the + /// capture — a capture opened while the prompt is pending would record silence. /// This turns on the system's recording privacy indicator (e.g., on macOS/iOS). + /// On iOS this also switches the audio session to its recording state + /// (voice/video-chat mode per , enabling + /// hardware echo cancellation). /// /// /// Thrown if the operation failed. /// public IEnumerator StartRecording() { + // Iterator method: this throws on the first MoveNext, like the other + // exceptions below — Unity's StartCoroutine runs that synchronously. + ThrowIfDisposed(); + #if PLATFORM_ANDROID if (!Permission.HasUserAuthorizedPermission(Permission.Microphone)) { @@ -313,6 +681,22 @@ public IEnumerator StartRecording() } #endif +#if UNITY_IOS && !UNITY_EDITOR + if (!UnityEngine.Application.HasUserAuthorization(UnityEngine.UserAuthorization.Microphone)) + { + // Ask for the record permission BEFORE the ADM opens the input unit. The + // system prompt is asynchronous: a capture opened while it is still + // pending records silence, and nothing reopens the input after the user + // grants — so without this gate the first run of an app publishes a + // silent microphone track. + yield return UnityEngine.Application.RequestUserAuthorization( + UnityEngine.UserAuthorization.Microphone); + if (!UnityEngine.Application.HasUserAuthorization(UnityEngine.UserAuthorization.Microphone)) + throw new InvalidOperationException( + "Microphone permission denied by user; cannot start recording."); + } +#endif + using var request = FFIBridge.Instance.NewRequest(); request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); @@ -322,8 +706,20 @@ public IEnumerator StartRecording() if (res.StartRecording.HasError && !string.IsNullOrEmpty(res.StartRecording.Error)) throw new InvalidOperationException($"Failed to start recording: {res.StartRecording.Error}"); + _isRecording = true; +#if UNITY_IOS && !UNITY_EDITOR + UpdateIosSessionState(); +#endif + Utils.Debug("PlatformAudio: started recording"); + // Re-assert the routing policy now that capture is active. Since Android 13 + // the app's MODE_IN_COMMUNICATION request — and with it the + // communication-device pin — is only honored while the app has active + // voice-communication capture, so the platform may have moved the route + // while it was un-owned. No-op on the other backends. + _routeController.ApplyPlayoutPreference(_playoutPreference.AsReadOnly()); + // Ensures this method is always a valid iterator even when the PLATFORM_ANDROID // branch is compiled out (no `yield return` would otherwise be reachable on // non-Android builds, which is a compile error for IEnumerator-returning methods). @@ -336,12 +732,15 @@ public IEnumerator StartRecording() /// Use this to temporarily stop recording without disposing PlatformAudio. /// This turns off the system's recording privacy indicator (e.g., on macOS/iOS). /// Call StartRecording to resume recording. + /// On iOS this also switches the audio session back to its playout-only state + /// (music-friendly default mode). /// /// /// Thrown if the operation failed. /// public void StopRecording() { + ThrowIfDisposed(); using var request = FFIBridge.Instance.NewRequest(); request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); @@ -351,20 +750,111 @@ public void StopRecording() if (res.StopRecording.HasError && !string.IsNullOrEmpty(res.StopRecording.Error)) throw new InvalidOperationException($"Failed to stop recording: {res.StopRecording.Error}"); + _isRecording = false; +#if UNITY_IOS && !UNITY_EDITOR + UpdateIosSessionState(); +#endif + Utils.Debug("PlatformAudio: stopped recording"); } + /// + /// Whether the platform's call audio session is currently held: true while at + /// least one is connected. Exposed for tests. + /// + internal bool SessionAudioEnabled { get; private set; } + + // Room raises this on the Unity main thread whenever a room's connection state + // crosses into or out of ConnDisconnected, before its public events. The count + // is re-read at apply time, so a stale argument from a posted call can never win + // over a later change. + private void OnConnectedRoomCountChanged(int connectedRooms) + { + if (_disposed) return; + + if (_syncContext != null && _syncContext != SynchronizationContext.Current) + { + _syncContext.Post(_ => OnConnectedRoomCountChanged(Room.ConnectedRoomCount), null); + return; + } + + var enabled = Room.ConnectedRoomCount > 0; + if (enabled == SessionAudioEnabled) return; + try + { + ApplySessionAudio(enabled); + } + catch (Exception e) + { + // A platform hiccup here must not surface in Room's connect or + // disconnect path. + Utils.Warning($"PlatformAudio: failed to {(enabled ? "take" : "release")} the call audio session: {e.Message}"); + } + } + + // Takes (true) or releases (false) the platform's call audio session. + // + // On iOS this gates WebRTC's VPIO audio unit while the app retains ownership of + // the shared AVAudioSession. Releasing stops the microphone/remote audio path and + // the hardware voice processing and drops the session to its idle state + // (music-friendly default mode), but keeps the audio session active so other + // Unity audio (e.g. background music) is not interrupted — which is why Unity + // audio survives a hang-up. + // + // On Android 12 (API 31) and newer this gates the voice-communication audio + // session the routing backend holds: while taken the SDK requests + // MODE_IN_COMMUNICATION and keeps the output route pinned per PlayoutPreference; + // while released it holds neither, so the OS applies its normal routing and the + // call session covers the call rather than the lifetime of this instance. Device + // enumeration and DevicesChanged keep working either way. Releasing does not + // stop the ADM or the capture: StopRecording stays the app's call at the end of + // a call — an active capture without the session is what lets the platform take + // routing back (see StartRecording). + // + // On the remaining platforms this is a no-op: the OS/ADM manages the session + // directly. + private void ApplySessionAudio(bool enabled) + { + SessionAudioEnabled = enabled; +#if UNITY_IOS && !UNITY_EDITOR + IOSAudioSessionHelper.LiveKit_SetAudioEnabled(enabled); + _iosSessionAudioEnabled = enabled; + UpdateIosSessionState(); +#endif + _routeController.SetSessionAudioEnabled(enabled); + Utils.Debug($"PlatformAudio: call audio session {(enabled ? "taken" : "released")} ({Room.ConnectedRoomCount} connected room(s))"); + } + /// /// Releases the PlatformAudio resources. /// /// When disposed, the platform ADM may be disabled if this was the last /// PlatformAudio instance. + /// + /// Disposing is idempotent. After disposal every public member throws + /// , except subscribing to / + /// unsubscribing from , which stays safe. /// public void Dispose() { if (_disposed) return; - Handle.Dispose(); _disposed = true; + Room.ConnectedRoomCountChanged -= OnConnectedRoomCountChanged; + _routeController.DevicesChanged -= OnRouteControllerDevicesChanged; + _routeController.Dispose(); + Handle.Dispose(); + _isRecording = false; + +#if UNITY_IOS && !UNITY_EDITOR + // Once the last instance is gone, relinquish the app-owned audio session: + // disable call audio, release our activation, leave manual mode, restore + // the session Unity had before LiveKit touched it, and reactivate it so + // Unity audio output resumes. Balances LiveKit_ConfigureAudioSessionForVoIP() + // in the constructor so the session isn't left stuck in PlayAndRecord. + if (System.Threading.Interlocked.Decrement(ref _instanceCount) == 0) + IOSAudioSessionHelper.LiveKit_RestoreDefaultAudioSession(); +#endif + Utils.Debug("PlatformAudio disposed"); } } diff --git a/Runtime/Scripts/Audio/RouteController.cs b/Runtime/Scripts/Audio/RouteController.cs new file mode 100644 index 00000000..4a7e14c9 --- /dev/null +++ b/Runtime/Scripts/Audio/RouteController.cs @@ -0,0 +1,160 @@ +using System; +using System.Collections.Generic; +using LiveKit.Internal; + +namespace LiveKit +{ + /// + /// Backend seam for audio output routing. registers one + /// implementation per platform and forwards its public routing API + /// (, + /// , + /// , , + /// ) through it, so the plumbing can be swapped + /// per platform — and later wholesale for an FFI-backed implementation — without changing + /// a public signature. + /// + internal interface IRouteController : IDisposable + { + /// Snapshot of the current recording and playout device lists. + (List Recording, List Playout) GetDevices(); + + /// Applies the ranked automatic output policy, most preferred first. + void ApplyPlayoutPreference(IReadOnlyList ranked); + + /// + /// Routes output to the device with the given id ( + /// from ) as a sticky override of the automatic policy. + /// Validation is the backend's job: desktop hands the id to the FFI, which checks + /// it against the ADM's device list, Android checks it against the live + /// communication-device list, and the backends without device selection ignore it + /// with a warning. + /// + void SetPlayoutDevice(string deviceId); + + /// Clears the sticky override so the automatic policy applies again. + void ClearPlayoutDeviceSelection(); + + /// + /// Signals whether a call is in progress, i.e. whether the backend may hold the + /// platform's voice-communication audio session. Device enumeration and + /// must keep working while disabled. + /// + void SetSessionAudioEnabled(bool enabled); + + /// + /// Raised when the available devices change, with the current (playout, recording) + /// lists. May be raised from any thread; marshals it to + /// the Unity main thread before re-raising publicly. + /// + event Action, IReadOnlyList> DevicesChanged; + } + + /// + /// Desktop routing backend: wraps the FFI device enumeration and per-device GUID + /// selection. Ranked-kind policy is not implemented on desktop (output is chosen per + /// device), and no desktop hot-plug events exist yet, so + /// is never raised. + /// + internal sealed class DesktopRouteController : IRouteController + { + private readonly PlatformAudio _owner; + + public DesktopRouteController(PlatformAudio owner) + { + _owner = owner; + } + + public (List Recording, List Playout) GetDevices() + { + return _owner.GetDevicesViaFfi(); + } + + public void ApplyPlayoutPreference(IReadOnlyList ranked) + { + // No routing effect on desktop: output is selected per device, not by kind. + } + + public void SetPlayoutDevice(string deviceId) + { + // Straight to the FFI, as before the routing backends existed; it validates the + // id against the ADM's device list (unknown id -> "Device not found"). + _owner.SetPlayoutDeviceViaFfi(deviceId); + } + + public void ClearPlayoutDeviceSelection() + { + // No automatic policy to fall back to on desktop; the selected device stays. + } + + public void SetSessionAudioEnabled(bool enabled) + { + // No call session to hold on desktop: the ADM owns the devices directly. + } + + public event Action, IReadOnlyList> DevicesChanged + { + add { } + remove { } + } + + public void Dispose() + { + } + } + + /// + /// Placeholder backend for platforms without a routing implementation: Android below + /// API 31 (which lacks the communication-device APIs the Android backend is built + /// on). Device snapshots still work through the FFI (a single placeholder entry for + /// the OS default input/output); the routing verbs no-op as documented on the public + /// API. + /// + internal sealed class UnsupportedRouteController : IRouteController + { + private readonly PlatformAudio _owner; + private readonly string _platform; + + public UnsupportedRouteController(PlatformAudio owner, string platform) + { + _owner = owner; + _platform = platform; + } + + public (List Recording, List Playout) GetDevices() + { + return _owner.GetDevicesViaFfi(); + } + + public void ApplyPlayoutPreference(IReadOnlyList ranked) + { + // Stored by PlatformAudio; no routing effect until this platform's backend lands. + } + + public void SetPlayoutDevice(string deviceId) + { + Utils.Warning( + $"PlatformAudio.SetPlayoutDevice has no effect on {_platform}: the OS owns output routing."); + } + + public void ClearPlayoutDeviceSelection() + { + // No override can exist on this platform: SetPlayoutDevice is ignored. + } + + public void SetSessionAudioEnabled(bool enabled) + { + // Nothing to gate: this platform has no routing backend holding a session. + } + + public event Action, IReadOnlyList> DevicesChanged + { + add { } + remove { } + } + + public void Dispose() + { + } + } +} diff --git a/Runtime/Scripts/Audio/RouteController.cs.meta b/Runtime/Scripts/Audio/RouteController.cs.meta new file mode 100644 index 00000000..82c5747b --- /dev/null +++ b/Runtime/Scripts/Audio/RouteController.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 3a99955b361ca4e5aa765a7e6dfc9e73 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Core/Room.cs b/Runtime/Scripts/Core/Room.cs index 44d52302..31bf33fc 100644 --- a/Runtime/Scripts/Core/Room.cs +++ b/Runtime/Scripts/Core/Room.cs @@ -112,10 +112,21 @@ public Proto.RoomOptions ToProto() public class Room : IDisposable { internal FfiHandle RoomHandle = null; + // Set at the start of Teardown, before the disconnect is reported, so a handler + // that calls Disconnect() re-entrantly is a no-op; reset by OnConnect so a Room + // can be connected again after a disconnect. private bool _disposed = false; private readonly Dictionary _participants = new(); private StreamHandlerRegistry _streamHandlers = new(); + // How many rooms are connected right now, across all instances. SDK components + // whose platform state follows "a call is in progress" — PlatformAudio's call + // audio session — subscribe to ConnectedRoomCountChanged instead of asking the + // app to signal its call boundaries. Maintained by SetConnectionState, the one + // place every connection-state transition goes through, on the Unity main thread. + internal static int ConnectedRoomCount { get; private set; } + internal static event Action ConnectedRoomCountChanged; + public delegate void MetaDelegate(string metaData); public delegate void ParticipantDelegate(Participant participant); public delegate void RemoteParticipantDelegate(RemoteParticipant participant); @@ -190,20 +201,22 @@ public ConnectInstruction Connect(string url, string token, RoomOptions options) return instruction; } + /// + /// Disconnects from the room. A local disconnect is reported the way a + /// server-side one is — , + /// and with + /// , synchronously and with the + /// room's handles still live — so one teardown path covers both. + /// goes through here too. Calling this from a handler of + /// those events, or on a room that already disconnected, is a no-op. So is a + /// call while is still pending: that connect completes, + /// and the app has to disconnect again once it has. + /// public void Disconnect() { if (_disposed || RoomHandle == null) return; - var (response, _) = FFIBridge.Instance.SendDisconnectRequest(this); - using (response) - { - Utils.Debug($"Disconnect.... {RoomHandle}"); - Utils.Debug($"Disconnect response.... {response}"); - } - // Release the Rust-side room synchronously. Without this the FfiRoom - // (peer connection, signaling client, libwebrtc state) lingers in the - // FFI handle table until the SafeHandle finalizer runs. - Cleanup(); + Teardown(DisconnectReason.ClientInitiated, closeFfiRoom: true); } public void Dispose() @@ -212,20 +225,67 @@ public void Dispose() GC.SuppressFinalize(this); } - private void Cleanup() + // The single exit for every disconnect path — a local Disconnect(), the core's + // Disconnected event, a panic. Reports the disconnect once, then releases the + // room: handlers run first, with the handles still live, as they do on a + // server-side disconnect, and the release runs in a finally so a throwing + // handler cannot leave the Rust-side room alive. closeFfiRoom asks the Rust side + // to close the room (local path); on the other paths the core already has. + private void Teardown(DisconnectReason reason, bool closeFfiRoom) { if (_disposed) return; _disposed = true; + // Unsubscribed first: nothing still queued for this room — including the + // core's own Disconnected event for a local close — is processed from here on. FfiClient.Instance.RoomEventReceived -= OnEventReceived; FfiClient.Instance.RpcMethodInvocationReceived -= OnRpcMethodInvocationReceived; FfiClient.Instance.DisconnectReceived -= OnDisconnectReceived; FfiClient.Instance.PanicReceived -= OnPanicReceived; - // Participant + track + publication FFI handles are independent entries in the - // Rust handle table — dropping the room handle alone does not cascade to them, so - // they would otherwise linger until C# GC finalizes each SafeHandle. + try + { + DisconnectReason = reason; + SetConnectionState(ConnectionState.ConnDisconnected); + Disconnected?.Invoke(this); + DisconnectedWithReason?.Invoke(this, reason); + } + finally + { + try + { + if (closeFfiRoom) + CloseFfiRoom(); + } + finally + { + ReleaseHandles(); + } + } + } + + // Sent after the handlers on purpose: the Rust-side close replaces the remote + // track handles on its worker thread right away, so a handler touching tracks + // would otherwise race it. + private void CloseFfiRoom() + { + var (response, _) = FFIBridge.Instance.SendDisconnectRequest(this); + using (response) + { + Utils.Debug($"Disconnect.... {RoomHandle}"); + Utils.Debug($"Disconnect response.... {response}"); + } + } + + // Released synchronously. Without this the FfiRoom (peer connection, signaling + // client, libwebrtc state) lingers in the FFI handle table until the SafeHandle + // finalizer runs. Participant + track + publication FFI handles are independent + // entries in the Rust handle table — dropping the room handle alone does not + // cascade to them, so they would otherwise linger until C# GC finalizes each + // SafeHandle. + private void ReleaseHandles() + { LocalParticipant?.DisposeHandles(); foreach (var p in _participants.Values) p.DisposeHandles(); @@ -235,6 +295,29 @@ private void Cleanup() RoomHandle = null; } + // Records a connection-state transition and raises ConnectionStateChanged for it. + // A repeat of the current state is dropped: the core's own ConnectionStateChanged + // (Connected) is queued behind the connect callback and drains one event pass + // after OnConnect recorded the transition, so this check is what keeps every + // connect to a single Connected report. The connected-room count is kept here + // too: a room counts while it is anything but ConnDisconnected, so a reconnect + // in progress still counts as a call, and the count moves before the public + // event so SDK components have settled by the time app handlers run. + private void SetConnectionState(ConnectionState state) + { + if (ConnectionState == state) + return; + var wasConnected = ConnectionState != ConnectionState.ConnDisconnected; + ConnectionState = state; + var isConnected = state != ConnectionState.ConnDisconnected; + if (isConnected != wasConnected) + { + ConnectedRoomCount += isConnected ? 1 : -1; + ConnectedRoomCountChanged?.Invoke(ConnectedRoomCount); + } + ConnectionStateChanged?.Invoke(state); + } + /// /// Registers a handler for incoming text streams matching the given topic. /// @@ -308,7 +391,7 @@ internal void OnRpcMethodInvocationReceived(RpcMethodInvocationEvent e) internal void OnEventReceived(RoomEvent e) { - // After Cleanup() the handle is null but late events may still flow + // After Teardown the handle is null but late events may still flow // through the FfiClient before the unsubscribe fully takes effect. if (RoomHandle == null) return; @@ -542,14 +625,17 @@ internal void OnEventReceived(RoomEvent e) } break; case RoomEvent.MessageOneofCase.ConnectionStateChanged: - ConnectionState = e.ConnectionStateChanged.State; - ConnectionStateChanged?.Invoke(e.ConnectionStateChanged.State); + // The core reports a disconnect as ConnectionStateChanged(Disconnected) + // followed by Disconnected{reason}. The transition is recorded from + // the latter, once the reason is known, so handlers of either event + // see the same DisconnectReason on every path, and a handler that + // disconnects in between cannot replace the core's reason with + // ClientInitiated. + if (e.ConnectionStateChanged.State != ConnectionState.ConnDisconnected) + SetConnectionState(e.ConnectionStateChanged.State); break; case RoomEvent.MessageOneofCase.Disconnected: - DisconnectReason = e.Disconnected.Reason; - Disconnected?.Invoke(this); - DisconnectedWithReason?.Invoke(this, DisconnectReason); - OnDisconnect(); + Teardown(e.Disconnected.Reason, closeFfiRoom: false); break; case RoomEvent.MessageOneofCase.Reconnecting: Reconnecting?.Invoke(this); @@ -590,6 +676,9 @@ internal void OnEventReceived(RoomEvent e) internal void OnConnect(ConnectCallback info) { RoomHandle = FfiHandle.FromOwnedHandle(info.Result.Room.Handle); + // A Room may be connected again after a disconnect: each connect starts a + // fresh lifecycle. + _disposed = false; UpdateFromInfo(info.Result.Room.Info); LocalParticipant = new LocalParticipant(info.Result.LocalParticipant, this); @@ -604,7 +693,7 @@ internal void OnConnect(ConnectCallback info) FfiClient.Instance.RpcMethodInvocationReceived += OnRpcMethodInvocationReceived; // Signal Rust that listeners are installed and it can start forwarding room events. - // Without this the FFI side parks for 1s after ConnectCallback and then drops the room + // Without this the FFI side parks for 15s after ConnectCallback and then drops the room // with ConnectionTimeout. Must run after the FfiClient.RoomEventReceived subscription // above so no event can race ahead of OnEventReceived. using (var readyRequest = FFIBridge.Instance.NewRequest()) @@ -613,6 +702,15 @@ internal void OnConnect(ConnectCallback info) using var readyResponse = readyRequest.Send(); } + // The core recorded this transition during connect, before this room was + // subscribed to its events (its own copy drains later and is dropped as a + // repeat, see SetConnectionState); recording it here makes IsConnected true + // from the moment Connected is raised. + SetConnectionState(ConnectionState.ConnConnected); + // A ConnectionStateChanged handler may already have disconnected the room; + // Connected is not raised on a torn-down room. + if (_disposed) + return; Connected?.Invoke(this); } @@ -628,15 +726,7 @@ private void OnPanicReceived(Panic e) // room could silently stop receiving events (including Disconnected // itself), so the panic is surfaced through the disconnect path apps // already handle. - DisconnectReason = DisconnectReason.UnknownReason; - Disconnected?.Invoke(this); - DisconnectedWithReason?.Invoke(this, DisconnectReason); - OnDisconnect(); - } - - private void OnDisconnect() - { - Cleanup(); + Teardown(DisconnectReason.UnknownReason, closeFfiRoom: false); } internal RemoteParticipant CreateRemoteParticipantWithTracks(ConnectCallback.Types.ParticipantWithTracks item) @@ -694,18 +784,26 @@ void OnConnect(ConnectCallback e) return; bool success = string.IsNullOrEmpty(e.Error); - if (success) + try { - if (_roomOptions.E2EE != null) + if (success) { - _room.E2EEManager = new E2EEManager(_room.RoomHandle, _roomOptions.E2EE); - } + if (_roomOptions.E2EE != null) + { + _room.E2EEManager = new E2EEManager(_room.RoomHandle, _roomOptions.E2EE); + } - _room.OnConnect(e); + _room.OnConnect(e); + } + } + finally + { + // Completed even when a ConnectionStateChanged or Connected handler throws + // inside OnConnect, so the code awaiting the connect resumes instead of + // hanging forever. + IsError = !success; + IsDone = true; } - - IsError = !success; - IsDone = true; } void OnCanceled() diff --git a/Samples~/Agents/Assets/Plugins/Android/AndroidManifest.xml b/Samples~/Agents/Assets/Plugins/Android/AndroidManifest.xml index abc4bbc0..de1869eb 100644 --- a/Samples~/Agents/Assets/Plugins/Android/AndroidManifest.xml +++ b/Samples~/Agents/Assets/Plugins/Android/AndroidManifest.xml @@ -7,6 +7,7 @@ + wired headset > speaker > +// earpiece) and keeps the route pinned across device changes while a call is in +// progress. This controller only demonstrates the observability side by logging +// DevicesChanged. On Android an active mic capture is what keeps the SDK's route +// authoritative (since Android 13 the OS only honors an app's communication-mode request +// while it has active voice-communication capture), so start the capture with +// StartCapture when the call begins (even when joining muted) — it then stays open +// across mute cycles until StopCapture when the call ends. See StartCapture and +// Unpublish. +// +// The ADM is created once at app start and kept alive across calls; the platform's call +// audio session itself is held by the SDK only while a Room is connected, so nothing +// here has to track call boundaries. public sealed class PlatformAudioController : IDisposable { - const string MicTrackName = "player-mic"; + readonly string _trackName; + readonly AudioProcessingOptions _audioOptions; PlatformAudio _platformAudio; PlatformAudioSource _source; LocalAudioTrack _track; Room _room; + // What should be audible after an output device change, remembered from before it. + readonly Dictionary _audibleSources = new Dictionary(); + // Whether a remember pass has ever swept the scene; gates the adopt-loops fallback + // in RestartAudibleSources to the very first switch. + bool _sceneSwept; + public bool IsInitialized => _platformAudio != null; public bool IsPublished { get; private set; } + public PlatformAudioController(string trackName, AudioProcessingOptions audioOptions) + { + _trackName = trackName; + _audioOptions = audioOptions; + } + // Creates the WebRTC ADM. This MUST run before Room.Connect so the SDK wires automatic - // speaker playout for remote tracks to this ADM — otherwise remote (agent) audio is never + // speaker playout for remote tracks to this ADM — otherwise remote audio is never // routed to an output and stays silent. Returns false if the ADM could not be created. public bool Initialize() { - return InitializePlatformAudio(); + if (!InitializePlatformAudio()) + return false; + + // The SDK routes output automatically from here on; the default + // PlatformAudio.PlayoutPreference ranking is already what a call app wants. + // A custom ranking would be a one-liner: + // _platformAudio.PlayoutPreference = new[] { AudioDeviceKind.WiredHeadset, AudioDeviceKind.Speaker }; + _platformAudio.DevicesChanged += OnDevicesChanged; + AudioSettings.OnAudioConfigurationChanged += OnUnityAudioConfigurationChanged; + return true; } - // Starts recording and publishes the mic track into the room. Initialize() must have been - // called (before the room connected) first. On any failure it disposes whatever was - // constructed and leaves IsPublished false; the caller should tear the rest down. + // Starts recording and publishes the mic track into the room. Initialize() must have + // been called (before the room connected) first. On any failure it unpublishes whatever + // was constructed and leaves IsPublished false; the ADM stays alive so a later Publish + // can retry. public IEnumerator Publish(Room room) { _room = room; @@ -39,19 +80,18 @@ public IEnumerator Publish(Room room) Debug.LogError("[PlatformAudioController] Publish called before Initialize(); aborting."); yield break; } + if (IsPublished) + yield break; - // Begin capturing from the default microphone. On macOS/iOS this turns on the - // recording privacy indicator and triggers the OS permission prompt; on Android - // it awaits the RECORD_AUDIO runtime permission dialog. - Debug.Log("[PlatformAudioController] Starting platform recording."); - yield return _platformAudio.StartRecording(); + // Harmless when StartCapture already ran at call start (the normal case on + // Android) or when the capture was kept running across a mute cycle (see + // Unpublish): the ADM ignores a start while it is already recording. + yield return StartCapture(); - // AudioProcessingOptions.Default enables AEC, noise suppression, auto gain control - // and prefers hardware processing. - _source = new PlatformAudioSource(_platformAudio, AudioProcessingOptions.Default); - _track = LocalAudioTrack.CreateAudioTrack(MicTrackName, _source, _room); + _source = new PlatformAudioSource(_platformAudio, _audioOptions); + _track = LocalAudioTrack.CreateAudioTrack(_trackName, _source, _room); - Debug.Log($"[PlatformAudioController] Publishing mic track '{MicTrackName}'..."); + Debug.Log($"[PlatformAudioController] Publishing mic track '{_trackName}'..."); var options = new TrackPublishOptions { AudioEncoding = new AudioEncoding { MaxBitrate = 64000 }, @@ -62,15 +102,88 @@ public IEnumerator Publish(Room room) if (publish.IsError) { Debug.LogError("[PlatformAudioController] Failed to publish microphone track."); - Dispose(); + Unpublish(); yield break; } IsPublished = true; - Debug.Log("[PlatformAudioController] Microphone track published (AEC enabled)."); + Debug.Log($"[PlatformAudioController] Microphone track '{_trackName}' published."); + } + + // Starts the microphone capture without publishing a track; Publish() reuses the + // running capture. On macOS this turns on the recording privacy indicator; on iOS + // and Android the coroutine first awaits the OS microphone-permission dialog and + // only then opens the capture. On Android call this as soon as the call starts, even when + // joining muted: since Android 13 the app's communication-mode request — and with + // it the SDK's output route pin — is only honored while the app has ACTIVE + // voice-communication capture or playback, and the ADM's playout stream does not + // register as active, only the recorder does. The SDK re-asserts its routing policy + // whenever the capture (re)starts. + public IEnumerator StartCapture() + { + if (_platformAudio == null) + { + Debug.LogError("[PlatformAudioController] StartCapture called before Initialize(); aborting."); + yield break; + } + + Debug.Log("[PlatformAudioController] Starting platform recording (no-op if already running)."); + yield return _platformAudio.StartRecording(); + } + + // Tears down the mic capture and track but keeps the ADM alive: remote playout + // continues and a later Publish() reuses it (e.g. a mute/unmute toggle). + public void Unpublish() + { + IsPublished = false; + + if (_track != null && _room != null) + { + Debug.Log("[PlatformAudioController] Unpublishing microphone track."); + _room.LocalParticipant.UnpublishTrack(_track, stopOnUnpublish: false); + } + _track = null; + +#if UNITY_ANDROID && !UNITY_EDITOR + // Keep the capture stream open while muted. Since Android 13, AudioService only + // honors this app's communication-mode request — and with it the SDK's output + // route pin — while the app has ACTIVE voice-communication capture or playback: + // with the recorder stopped, the mode drops back to MODE_NORMAL and the platform + // re-asserts the earpiece route. The track is unpublished and its source + // disposed below, so no audio reaches the room, but the OS mic-in-use indicator + // stays on while muted — same as other conferencing apps. Recording stops in + // StopCapture (call end) or Dispose. +#else + StopCapture(); +#endif + + _source?.Dispose(); + _source = null; + } + + // Stops the microphone capture; a stop while idle is ignored by the ADM. Only call + // this once the call has ended (after Unpublish): on Android, stopping the capture + // while still in a call hands routing authority back to the platform — see + // StartCapture. The next StartCapture (or Publish) restarts it. + public void StopCapture() + { + if (_platformAudio == null) + return; + try + { + _platformAudio.StopRecording(); + } + catch (Exception e) + { + Debug.LogWarning($"[PlatformAudioController] Failed to stop recording: {e.Message}"); + } } - // Sets up PlatformAudio with the default recording/playout devices. + // Creates PlatformAudio and logs the device lists. No device is selected here: the ADM + // starts on the OS default microphone and output, and output routing is left to the + // SDK's PlayoutPreference (SetPlayoutDevice is a sticky override on Android 12+ that + // would shadow the ranking for the whole session). Both selection verbs are reserved + // for an explicit user choice. bool InitializePlatformAudio() { try @@ -80,10 +193,8 @@ bool InitializePlatformAudio() $"[PlatformAudioController] PlatformAudio initialized " + $"({_platformAudio.RecordingDeviceCount} mic(s), {_platformAudio.PlayoutDeviceCount} speaker(s))."); - if (_platformAudio.RecordingDeviceCount > 0) - _platformAudio.SetRecordingDevice(0); - if (_platformAudio.PlayoutDeviceCount > 0) - _platformAudio.SetPlayoutDevice(0); + var (recording, playout) = _platformAudio.GetDevices(); + Debug.Log(FormatDeviceLists(playout, recording)); return true; } @@ -96,34 +207,163 @@ bool InitializePlatformAudio() } } - public void Dispose() + // Demonstrates the SDK's routing observability: the routing backend raises + // DevicesChanged (on the Unity main thread) whenever the available devices or the + // active route change — headset plugged/unplugged, Bluetooth connected, the route + // re-pinned after a device disappeared. An app would refresh its device picker here. + // This sample also uses it as the early warning for the Unity-audio recovery below. + void OnDevicesChanged(IReadOnlyList playout, IReadOnlyList recording) { - IsPublished = false; + Debug.Log("[PlatformAudioController] Audio devices changed.\n" + + FormatDeviceLists(playout, recording)); - if (_track != null && _room != null) + // Note what is audible while the engine is still healthy, but do not touch it: + // this event arrives early in a device switch (device-verified on a Pixel 8a / + // Android 16, where a Bluetooth headset's call profile appears ~650 ms before the + // media profile takes over), so reopening the engine here would reopen it onto the + // output the platform is about to leave. + RememberAudibleSources(forgetStopped: true); + } + + // Unity's audio engine opens an output device when the app starts. When that device + // goes away or another one takes over (Bluetooth connect or disconnect, wired + // plug/unplug), Unity reinitializes the engine, which stops every AudioSource, and + // raises this callback afterwards — device-verified on a Pixel 8a (Android 16): + // + // AudioTrack stop(11092): called with 92104 frames delivered <- sources stopped + // [PlatformAudioController] Unity audio configuration changed <- 25 ms later + // + // So the app has to restart its audio here, and it cannot learn what to restart from + // the scene at this point: everything is already stopped. What should be audible has + // to be remembered from before the switch (RememberAudibleSources) and put back now. + // Leaving that out is exactly how game audio ends up silent on the new device. + // + // deviceWasChanged is false even for a real device change on Android, so it cannot be + // used to filter these callbacks; the recovery reacts to all of them and stays safe + // through idempotence instead (a source already playing is left alone). + void OnUnityAudioConfigurationChanged(bool deviceWasChanged) + { + Debug.Log("[PlatformAudioController] Unity audio configuration changed " + + $"(deviceWasChanged={deviceWasChanged}, outputSampleRate={AudioSettings.outputSampleRate}, " + + $"speakerMode={AudioSettings.speakerMode})."); + + // Restore FIRST: the engine reinit has already stopped every source, so a + // remember pass at this point would see nothing playing and forget the very + // sources — one-shots above all — that it is supposed to put back. The refresh + // afterwards carries the restarted positions into the next switch without + // evicting anything, so a Play() the engine rejected mid-teardown keeps its + // slot for the next callback of the same switch. + RestartAudibleSources(); + RememberAudibleSources(forgetStopped: false); + } + + // Records what this sample intends to keep audible, so a device change can put it + // back. With forgetStopped, a source that is not playing is dropped from the set — + // loops included: that pass runs while the engine is healthy (OnDevicesChanged fires + // before the engine reinit), so a stopped source there was stopped by the app or has + // finished, and a deliberate Stop() must not be undone by the next device change. + // Without it, the pass only refreshes positions and adopts survivors — used right + // after a restore, when a Play() the engine rejected must not cost a source its + // slot. An app would consult its own audio state here instead of sweeping the scene. + void RememberAudibleSources(bool forgetStopped) + { + foreach (var source in UnityEngine.Object.FindObjectsByType(FindObjectsSortMode.None)) { - Debug.Log("[PlatformAudioController] Unpublishing microphone track."); - _room.LocalParticipant.UnpublishTrack(_track, stopOnUnpublish: false); + if (source.isPlaying) + _audibleSources[source] = source.time; + else if (forgetStopped) + _audibleSources.Remove(source); } - _track = null; + _sceneSwept = true; + } - if (_platformAudio != null) + // Puts the remembered audio back on the reopened engine. Note what this does NOT do: + // it never calls AudioSettings.Reset. Unity has already reopened its output by the + // time it raises the callback, so a reset adds nothing — and on Android it does real + // harm. Device-verified on a Pixel 8a (Android 16): reinitializing the engine makes + // Unity claim the headset's call link through the deprecated + // AudioManager.startBluetoothSco(), which evicts the SDK's setCommunicationDevice pin + // and leaves the platform's SCO state machine unable to connect — + // + // AS.AudioDeviceBroker: setCommunicationRouteForClient … type:bt_sco addr: + // … from API: startBluetoothSco()) from u/pid:… <- evicts our pinned device + // AS.BtHelper: requestScoState: failed to connect in state 1 <- every retry after + // + // after which call audio and game audio are both stuck on the loudspeaker for the + // rest of the session, however often the SDK re-pins the route. + void RestartAudibleSources() + { + // First-switch safety net: when no remember pass has ever swept the scene, a + // stopped looping source is taken to have been stopped by the engine reinit and + // is adopted rather than left silent. Once a sweep has run, an absent loop is + // one the app stopped (or never started), and adopting it would undo the app's + // intent. + if (_audibleSources.Count == 0 && !_sceneSwept) + foreach (var source in UnityEngine.Object.FindObjectsByType(FindObjectsSortMode.None)) + if (source.loop) + _audibleSources[source] = 0f; + + var restarted = 0; + // Copied because the loop drops destroyed sources from the dictionary. + foreach (var entry in new List>(_audibleSources)) { - try + var source = entry.Key; + if (source == null) { - _platformAudio.StopRecording(); + _audibleSources.Remove(source); + continue; } - catch (Exception e) + // A source the app deactivated is treated like one it stopped: forgotten, + // not retried. Play() on a disabled source only logs a warning on every + // callback, and force-playing it after a reactivation would undo the app's + // intent. + if (!source.isActiveAndEnabled) { - Debug.LogWarning($"[PlatformAudioController] Failed to stop recording: {e.Message}"); + _audibleSources.Remove(source); + continue; } + // Idempotent on purpose: a device switch raises several of these callbacks, + // and anything Unity left running must be left alone. + if (source.isPlaying) continue; + + if (source.clip != null) + source.time = Mathf.Clamp(entry.Value, 0f, Mathf.Max(0f, source.clip.length - 0.05f)); + source.Play(); + if (source.isPlaying) restarted++; } - _source?.Dispose(); - _source = null; + Debug.Log($"[PlatformAudioController] Restarted {restarted} of {_audibleSources.Count} remembered " + + $"source(s) on {AudioSettings.speakerMode} @ {AudioSettings.outputSampleRate} Hz."); + } + + static string FormatDeviceLists(IReadOnlyList playout, IReadOnlyList recording) + { + var sb = new StringBuilder("Playout devices:"); + foreach (var device in playout) + { + sb.Append($"\n [{device.Index}] {device.Name} (kind={device.Kind}"); + if (device.IsSelected) + sb.Append(", selected"); + sb.Append(')'); + } + sb.Append("\nRecording devices:"); + foreach (var device in recording) + sb.Append($"\n [{device.Index}] {device.Name}"); + return sb.ToString(); + } + + public void Dispose() + { + Unpublish(); + StopCapture(); - _platformAudio?.Dispose(); - _platformAudio = null; + if (_platformAudio != null) + { + AudioSettings.OnAudioConfigurationChanged -= OnUnityAudioConfigurationChanged; + _platformAudio.DevicesChanged -= OnDevicesChanged; + _platformAudio.Dispose(); + _platformAudio = null; + } _room = null; } diff --git a/Samples~/Meet/Assets/Plugins/Android/AndroidManifest.xml b/Samples~/Meet/Assets/Plugins/Android/AndroidManifest.xml index 94ea9440..e5cca6ac 100644 --- a/Samples~/Meet/Assets/Plugins/Android/AndroidManifest.xml +++ b/Samples~/Meet/Assets/Plugins/Android/AndroidManifest.xml @@ -7,6 +7,7 @@ + 0) - _platformAudio.SetRecordingDevice(0); - if (_platformAudio.PlayoutDeviceCount > 0) - _platformAudio.SetPlayoutDevice(0); + EchoCancellation = echoCancellation, + NoiseSuppression = noiseSuppression, + AutoGainControl = autoGainControl, + PreferHardware = preferHardwareProcessing + }; - Debug.Log($"PlatformAudio ready. AEC={echoCancellation}, NS={noiseSuppression}, AGC={autoGainControl}, HW={preferHardwareProcessing}"); - } - catch (System.Exception e) + _platformAudioController = new PlatformAudioController(LocalAudioTrackName, audioOptions); + if (!_platformAudioController.Initialize()) { - Debug.LogError($"Failed to initialize PlatformAudio, falling back to Unity audio: {e.Message}"); + Debug.LogError("Failed to initialize PlatformAudio, falling back to Unity audio"); usePlatformAudio = false; - _platformAudio = null; + _platformAudioController = null; + return; } + + Debug.Log($"PlatformAudio ready. AEC={echoCancellation}, NS={noiseSuppression}, AGC={autoGainControl}, HW={preferHardwareProcessing}"); } private void OnApplicationPause(bool pause) @@ -142,17 +131,13 @@ private void Update() private void OnDestroy() { // Without this, scene change / app quit while connected leaves all tracks, - // streams, and their backing GPU/native resources allocated. - if (_room != null) - { - _room.Disconnect(); - _room = null; - } + // streams, and their backing GPU/native resources allocated. Disconnect reports + // back through OnDisconnected, which tears the call down; CleanUpAllTracks after + // it covers anything created without a room. + _room?.Disconnect(); CleanUpAllTracks(); _webCamTexture?.Stop(); - _platformAudioSource?.Dispose(); - _platformAudio?.Dispose(); - _room?.Disconnect(); + _platformAudioController?.Dispose(); } #endregion @@ -166,13 +151,9 @@ private void OnStartCall() private void OnEndCall() { - if (_room == null) return; - - _room.Disconnect(); - CleanUpAllTracks(); - _room = null; - _localId = null; - buttonBar.SetConnected(false); + // The SDK reports a local disconnect through OnDisconnected (with + // DisconnectReason.ClientInitiated), which owns the teardown. + _room?.Disconnect(); } private void OnToggleCamera() @@ -249,6 +230,15 @@ private IEnumerator ConnectToRoom() _localId = _room.LocalParticipant.Identity; buttonBar.SetConnected(true); +#if UNITY_ANDROID && !UNITY_EDITOR + // Keep the mic capture running for the whole call, even while muted: without + // an active capture Android treats the communication-mode request as inactive + // and the SDK's output route pin is not honored after a Bluetooth episode — see + // PlatformAudioController.StartCapture. Publishing (unmuting) reuses the capture. + if (usePlatformAudio && _platformAudioController != null) + StartCoroutine(_platformAudioController.StartCapture()); +#endif + EnsureParticipantTile(_localId); foreach (var remote in _room.RemoteParticipants.Values) EnsureParticipantTile(remote.Identity); @@ -360,7 +350,7 @@ private void AddRemoteAudioTrack(RemoteAudioTrack audioTrack) { var sid = audioTrack.Sid; - if (usePlatformAudio && _platformAudio != null) + if (usePlatformAudio && _platformAudioController != null) { // PlatformAudio mode: ADM handles speaker playback automatically. // No AudioStream / GameObject needed. @@ -432,8 +422,25 @@ private void OnParticipantDisconnected(Participant participant, DisconnectReason DestroyParticipantTile(participant.Identity); } + // The one teardown path: raised for the hang-up button, a server-side disconnect + // (kick, room deleted, token expiry) and the scene going away (OnDestroy) alike, + // synchronously and with the room's handles still live. private void OnDisconnected(Room room) - => Debug.Log($"Disconnected from room: {room.DisconnectReason}"); + { + Debug.Log($"Disconnected from room: {room.DisconnectReason}"); + + // Stopping the capture here matters: it is deliberately kept running across + // mute cycles, so without this teardown a server-side disconnect would leave + // the microphone recording — indicator on — with no call to feed. The SDK has + // already released the platform's call audio session by the time this runs; + // Unity's own audio keeps playing. + CleanUpAllTracks(); + _room = null; + _localId = null; + // Destroyed already when this runs from OnDestroy during a scene unload. + if (buttonBar != null) + buttonBar.SetConnected(false); + } private void OnTrackMuted(TrackPublication publication, Participant participant) { @@ -549,7 +556,7 @@ private IEnumerator PublishLocalMicrophone() { if (_microphoneActive) yield break; - if (usePlatformAudio && _platformAudio != null) + if (usePlatformAudio && _platformAudioController != null) yield return PublishLocalMicrophonePlatform(); else yield return PublishLocalMicrophoneUnity(); @@ -562,45 +569,8 @@ private IEnumerator PublishLocalMicrophonePlatform() { Debug.Log("Publishing microphone using PlatformAudio (ADM)"); - // Start recording (in case it was stopped by a previous mute). - // This turns on the privacy indicator on macOS/iOS. On Android this also - // awaits the RECORD_AUDIO runtime permission dialog if not yet granted. - if (_platformAudio != null) - { - yield return _platformAudio.StartRecording(); - } - - var audioOptions = new AudioProcessingOptions - { - EchoCancellation = echoCancellation, - NoiseSuppression = noiseSuppression, - AutoGainControl = autoGainControl, - PreferHardware = preferHardwareProcessing - }; - - _platformAudioSource = new PlatformAudioSource(_platformAudio, audioOptions); - _localAudioTrack = LocalAudioTrack.CreateAudioTrack(LocalAudioTrackName, _platformAudioSource, _room); - - var options = new TrackPublishOptions - { - AudioEncoding = new AudioEncoding { MaxBitrate = 64000 }, - Source = TrackSource.SourceMicrophone - }; - - var publish = _room.LocalParticipant.PublishTrack(_localAudioTrack, options); - yield return publish; - - if (publish.IsError) - { - Debug.LogError("Failed to publish microphone track"); - _platformAudioSource?.Dispose(); - _platformAudioSource = null; - _localAudioTrack = null; - yield break; - } - - _microphoneActive = true; - Debug.Log("Microphone published via PlatformAudio (AEC enabled)"); + yield return _platformAudioController.Publish(_room); + _microphoneActive = _platformAudioController.IsPublished; } private IEnumerator PublishLocalMicrophoneUnity() @@ -643,19 +613,11 @@ private IEnumerator PublishLocalMicrophoneUnity() private void UnpublishLocalMicrophone() { - if (usePlatformAudio && _platformAudioSource != null) + if (usePlatformAudio && _platformAudioController != null) { - try - { - _platformAudio?.StopRecording(); - } - catch (System.Exception e) - { - Debug.LogWarning($"Failed to stop recording: {e.Message}"); - } - - _platformAudioSource.Dispose(); - _platformAudioSource = null; + // The controller owns the platform track: this stops recording and + // unpublishes while keeping the ADM alive for the next unmute. + _platformAudioController.Unpublish(); } else { @@ -670,10 +632,11 @@ private void UnpublishLocalMicrophone() } _audioObjects.Remove(LocalAudioTrackName); } + + _room.LocalParticipant.UnpublishTrack(_localAudioTrack, false); + _localAudioTrack = null; } - _room.LocalParticipant.UnpublishTrack(_localAudioTrack, false); - _localAudioTrack = null; if (_participantTiles.TryGetValue(_localId, out var tile)) tile.SetMicMuted(true); _microphoneActive = false; @@ -743,8 +706,11 @@ private void CleanUpAllTracks() DisposeSource(ref _localRtcAudioSource); DisposeSource(ref _localRtcVideoSource); - _platformAudioSource?.Dispose(); - _platformAudioSource = null; + // Keep the ADM itself alive so the next call can reuse it; only the mic + // capture and track go away here (ConnectToRoom restarts the capture on the + // next call). + _platformAudioController?.Unpublish(); + _platformAudioController?.StopCapture(); foreach (var obj in _audioObjects.Values) { diff --git a/Samples~/Meet/Assets/Runtime/PlatformAudioController.cs b/Samples~/Meet/Assets/Runtime/PlatformAudioController.cs new file mode 100644 index 00000000..4dee224f --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/PlatformAudioController.cs @@ -0,0 +1,244 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Text; +using LiveKit; +using LiveKit.Proto; +using UnityEngine; + +// Drives the duplex platform audio (WebRTC ADM): captures the default microphone with +// the configured audio processing (AEC/NS/AGC) and publishes it as a LiveKit track; remote +// tracks play back through the SDK-routed output automatically. Publish/Unpublish can be +// cycled (e.g. a mute toggle) while the ADM stays alive; Dispose tears everything down in +// dependency order. +// +// Output routing is owned by the SDK: PlatformAudio routes to the best available output +// per its ranked PlayoutPreference (default: Bluetooth > wired headset > speaker > +// earpiece) and keeps the route pinned across device changes while a call is in +// progress. This controller only demonstrates the observability side by logging +// DevicesChanged. On Android an active mic capture is what keeps the SDK's route +// authoritative (since Android 13 the OS only honors an app's communication-mode request +// while it has active voice-communication capture), so start the capture with +// StartCapture when the call begins (even when joining muted) — it then stays open +// across mute cycles until StopCapture when the call ends. See StartCapture and +// Unpublish. +// +// The ADM is created once at app start and kept alive across calls; the platform's call +// audio session itself is held by the SDK only while a Room is connected, so nothing +// here has to track call boundaries. +public sealed class PlatformAudioController : IDisposable +{ + readonly string _trackName; + readonly AudioProcessingOptions _audioOptions; + + PlatformAudio _platformAudio; + PlatformAudioSource _source; + LocalAudioTrack _track; + Room _room; + + public bool IsInitialized => _platformAudio != null; + public bool IsPublished { get; private set; } + + public PlatformAudioController(string trackName, AudioProcessingOptions audioOptions) + { + _trackName = trackName; + _audioOptions = audioOptions; + } + + // Creates the WebRTC ADM. This MUST run before Room.Connect so the SDK wires automatic + // speaker playout for remote tracks to this ADM — otherwise remote audio is never + // routed to an output and stays silent. Returns false if the ADM could not be created. + public bool Initialize() + { + if (!InitializePlatformAudio()) + return false; + + // The SDK routes output automatically from here on; the default + // PlatformAudio.PlayoutPreference ranking is already what a call app wants. + // A custom ranking would be a one-liner: + // _platformAudio.PlayoutPreference = new[] { AudioDeviceKind.WiredHeadset, AudioDeviceKind.Speaker }; + _platformAudio.DevicesChanged += OnDevicesChanged; + return true; + } + + // Starts recording and publishes the mic track into the room. Initialize() must have + // been called (before the room connected) first. On any failure it unpublishes whatever + // was constructed and leaves IsPublished false; the ADM stays alive so a later Publish + // can retry. + public IEnumerator Publish(Room room) + { + _room = room; + + if (_platformAudio == null) + { + Debug.LogError("[PlatformAudioController] Publish called before Initialize(); aborting."); + yield break; + } + if (IsPublished) + yield break; + + // Harmless when StartCapture already ran at call start (the normal case on + // Android) or when the capture was kept running across a mute cycle (see + // Unpublish): the ADM ignores a start while it is already recording. + yield return StartCapture(); + + _source = new PlatformAudioSource(_platformAudio, _audioOptions); + _track = LocalAudioTrack.CreateAudioTrack(_trackName, _source, _room); + + Debug.Log($"[PlatformAudioController] Publishing mic track '{_trackName}'..."); + var options = new TrackPublishOptions + { + AudioEncoding = new AudioEncoding { MaxBitrate = 64000 }, + Source = TrackSource.SourceMicrophone + }; + var publish = _room.LocalParticipant.PublishTrack(_track, options); + yield return publish; + if (publish.IsError) + { + Debug.LogError("[PlatformAudioController] Failed to publish microphone track."); + Unpublish(); + yield break; + } + + IsPublished = true; + Debug.Log($"[PlatformAudioController] Microphone track '{_trackName}' published."); + } + + // Starts the microphone capture without publishing a track; Publish() reuses the + // running capture. On macOS this turns on the recording privacy indicator; on iOS + // and Android the coroutine first awaits the OS microphone-permission dialog and + // only then opens the capture. On Android call this as soon as the call starts, even when + // joining muted: since Android 13 the app's communication-mode request — and with + // it the SDK's output route pin — is only honored while the app has ACTIVE + // voice-communication capture or playback, and the ADM's playout stream does not + // register as active, only the recorder does. The SDK re-asserts its routing policy + // whenever the capture (re)starts. + public IEnumerator StartCapture() + { + if (_platformAudio == null) + { + Debug.LogError("[PlatformAudioController] StartCapture called before Initialize(); aborting."); + yield break; + } + + Debug.Log("[PlatformAudioController] Starting platform recording (no-op if already running)."); + yield return _platformAudio.StartRecording(); + } + + // Tears down the mic capture and track but keeps the ADM alive: remote playout + // continues and a later Publish() reuses it (e.g. a mute/unmute toggle). + public void Unpublish() + { + IsPublished = false; + + if (_track != null && _room != null) + { + Debug.Log("[PlatformAudioController] Unpublishing microphone track."); + _room.LocalParticipant.UnpublishTrack(_track, stopOnUnpublish: false); + } + _track = null; + +#if UNITY_ANDROID && !UNITY_EDITOR + // Keep the capture stream open while muted. Since Android 13, AudioService only + // honors this app's communication-mode request — and with it the SDK's output + // route pin — while the app has ACTIVE voice-communication capture or playback: + // with the recorder stopped, the mode drops back to MODE_NORMAL and the platform + // re-asserts the earpiece route. The track is unpublished and its source + // disposed below, so no audio reaches the room, but the OS mic-in-use indicator + // stays on while muted — same as other conferencing apps. Recording stops in + // StopCapture (call end) or Dispose. +#else + StopCapture(); +#endif + + _source?.Dispose(); + _source = null; + } + + // Stops the microphone capture; a stop while idle is ignored by the ADM. Only call + // this once the call has ended (after Unpublish): on Android, stopping the capture + // while still in a call hands routing authority back to the platform — see + // StartCapture. The next StartCapture (or Publish) restarts it. + public void StopCapture() + { + if (_platformAudio == null) + return; + try + { + _platformAudio.StopRecording(); + } + catch (Exception e) + { + Debug.LogWarning($"[PlatformAudioController] Failed to stop recording: {e.Message}"); + } + } + + // Creates PlatformAudio and logs the device lists. No device is selected here: the ADM + // starts on the OS default microphone and output, and output routing is left to the + // SDK's PlayoutPreference (SetPlayoutDevice is a sticky override on Android 12+ that + // would shadow the ranking for the whole session). Both selection verbs are reserved + // for an explicit user choice. + bool InitializePlatformAudio() + { + try + { + _platformAudio = new PlatformAudio(); + Debug.Log( + $"[PlatformAudioController] PlatformAudio initialized " + + $"({_platformAudio.RecordingDeviceCount} mic(s), {_platformAudio.PlayoutDeviceCount} speaker(s))."); + + var (recording, playout) = _platformAudio.GetDevices(); + Debug.Log(FormatDeviceLists(playout, recording)); + + return true; + } + catch (Exception e) + { + Debug.LogError($"[PlatformAudioController] Failed to initialize PlatformAudio: {e.Message}"); + _platformAudio?.Dispose(); + _platformAudio = null; + return false; + } + } + + // Demonstrates the SDK's routing observability: the routing backend raises + // DevicesChanged (on the Unity main thread) whenever the available devices or the + // active route change — headset plugged/unplugged, Bluetooth connected, the route + // re-pinned after a device disappeared. An app would refresh its device picker here. + void OnDevicesChanged(IReadOnlyList playout, IReadOnlyList recording) + { + Debug.Log("[PlatformAudioController] Audio devices changed.\n" + + FormatDeviceLists(playout, recording)); + } + + static string FormatDeviceLists(IReadOnlyList playout, IReadOnlyList recording) + { + var sb = new StringBuilder("Playout devices:"); + foreach (var device in playout) + { + sb.Append($"\n [{device.Index}] {device.Name} (kind={device.Kind}"); + if (device.IsSelected) + sb.Append(", selected"); + sb.Append(')'); + } + sb.Append("\nRecording devices:"); + foreach (var device in recording) + sb.Append($"\n [{device.Index}] {device.Name}"); + return sb.ToString(); + } + + public void Dispose() + { + Unpublish(); + StopCapture(); + + if (_platformAudio != null) + { + _platformAudio.DevicesChanged -= OnDevicesChanged; + _platformAudio.Dispose(); + _platformAudio = null; + } + + _room = null; + } +} diff --git a/Samples~/Meet/Assets/Runtime/PlatformAudioController.cs.meta b/Samples~/Meet/Assets/Runtime/PlatformAudioController.cs.meta new file mode 100644 index 00000000..217b7816 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/PlatformAudioController.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: e60c87b3bbd504941ae86b78548a89d5 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Scenes/MeetApp.unity b/Samples~/Meet/Assets/Scenes/MeetApp.unity index 91fc1e07..f95de4b6 100644 --- a/Samples~/Meet/Assets/Scenes/MeetApp.unity +++ b/Samples~/Meet/Assets/Scenes/MeetApp.unity @@ -853,6 +853,134 @@ Canvas: m_SortingLayerID: 0 m_SortingOrder: 0 m_TargetDisplay: 0 +--- !u!1 &1060469597 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1060469599} + - component: {fileID: 1060469598} + m_Layer: 0 + m_Name: Audio Source + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!82 &1060469598 +AudioSource: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1060469597} + m_Enabled: 1 + serializedVersion: 4 + OutputAudioMixerGroup: {fileID: 0} + m_audioClip: {fileID: 8300000, guid: 5502fe8358d2e474bb1a61cc34a21592, type: 3} + m_PlayOnAwake: 1 + m_Volume: 1 + m_Pitch: 1 + Loop: 1 + Mute: 0 + Spatialize: 0 + SpatializePostEffects: 0 + Priority: 128 + DopplerLevel: 1 + MinDistance: 1 + MaxDistance: 500 + Pan2D: 0 + rolloffMode: 0 + BypassEffects: 0 + BypassListenerEffects: 0 + BypassReverbZones: 0 + rolloffCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 1 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + - serializedVersion: 3 + time: 1 + value: 0 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 + panLevelCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 0 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 + spreadCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 0 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 + reverbZoneMixCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 1 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 +--- !u!4 &1060469599 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1060469597} + serializedVersion: 2 + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 472.96027, y: 1137.1221, z: -12.568851} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 0} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} --- !u!1 &1478206700 GameObject: m_ObjectHideFlags: 0 @@ -919,7 +1047,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: a498c208deeab40c39b4ba609d7d222c, type: 3} m_Name: m_EditorClassIdentifier: - _config: {fileID: 11400000, guid: 1a1b8efb5101449b280f574f853b7459, type: 2} + _config: {fileID: 11400000, guid: aae6c4b1158ca4c929c5d84962c95f91, type: 2} --- !u!1 &1651282853 GameObject: m_ObjectHideFlags: 0 @@ -2734,3 +2862,4 @@ SceneRoots: - {fileID: 1759245559} - {fileID: 1478206702} - {fileID: 782480977} + - {fileID: 1060469599} diff --git a/Samples~/Meet/README.md b/Samples~/Meet/README.md index 864d185b..14d63a7d 100644 --- a/Samples~/Meet/README.md +++ b/Samples~/Meet/README.md @@ -18,6 +18,8 @@ In order to connect to your LiveKit server, configure the token source component The LiveKit Unity SDK offers two audio systems. The Unity audio path uses the Unity APIs for audio input and output. Platform Audio is the alternative, where the native LiveKit plugin manages audio input and output. You can select which path to use on the MeetManager component. +With Platform Audio, output routing (Bluetooth/wired headset/speaker/earpiece on mobile) is handled by the SDK's `PlatformAudio.PlayoutPreference` policy — the sample contains no routing code of its own and only logs the SDK's `DevicesChanged` events (see `PlatformAudioController`). The audio-routing section of the [SDK README](https://github.com/livekit/client-sdk-unity#audio-output-routing) documents the API and per-platform behavior. + ### Common sample package In order to get access to common sample functions like the on device scrolling log, make sure to import the [Common](https://github.com/livekit/client-sdk-unity/tree/main/Samples~/Common) sample from the LiveKit Unity Package in the package manager. \ No newline at end of file diff --git a/Tests/EditMode/RoomDisconnectReasonTests.cs b/Tests/EditMode/RoomDisconnectReasonTests.cs index 11ed1ebc..046eed53 100644 --- a/Tests/EditMode/RoomDisconnectReasonTests.cs +++ b/Tests/EditMode/RoomDisconnectReasonTests.cs @@ -84,5 +84,54 @@ public void ParticipantDisconnected_SurfacesReasonOnEvent() Assert.IsNotNull(eventParticipant); Assert.AreEqual(identity, eventParticipant.Identity); } + + [Test] + public void Disconnected_ReentrantDisconnectFromHandlers_KeepsServerReason() + { + var room = new Room(); + room.RoomHandle = new FfiHandle(IntPtr.Zero); + // A fresh Room already reads ConnDisconnected (the enum default), so record a + // connected state first, as the core's own event would after a connect. + room.OnEventReceived(new RoomEvent + { + RoomHandle = 0, + ConnectionStateChanged = new ConnectionStateChanged { State = ConnectionState.ConnConnected } + }); + Assert.AreEqual(ConnectionState.ConnConnected, room.ConnectionState); + + var reports = 0; + DisconnectReason? reasonSeenByStateHandler = null; + room.ConnectionStateChanged += state => + { + if (state != ConnectionState.ConnDisconnected) return; + reasonSeenByStateHandler = room.DisconnectReason; + // An app that disconnects "to be sure" from its state handler. + room.Disconnect(); + }; + room.DisconnectedWithReason += (_, __) => + { + reports++; + room.Disconnect(); + }; + + // The core reports a server-side disconnect as two queued events. + room.OnEventReceived(new RoomEvent + { + RoomHandle = 0, + ConnectionStateChanged = new ConnectionStateChanged { State = ConnectionState.ConnDisconnected } + }); + room.OnEventReceived(new RoomEvent + { + RoomHandle = 0, + Disconnected = new Disconnected { Reason = DisconnectReason.ServerShutdown } + }); + + Assert.AreEqual(1, reports, "the disconnect is reported once"); + Assert.AreEqual(DisconnectReason.ServerShutdown, room.DisconnectReason, + "a re-entrant Disconnect() must not replace the server's reason with ClientInitiated"); + Assert.AreEqual(DisconnectReason.ServerShutdown, reasonSeenByStateHandler, + "the ConnectionStateChanged handler sees the reason already recorded"); + Assert.AreEqual(ConnectionState.ConnDisconnected, room.ConnectionState); + } } } diff --git a/Tests/PlayMode/PlatformAudioIntegrationTests.cs b/Tests/PlayMode/PlatformAudioIntegrationTests.cs index d521f3c7..5abf3232 100644 --- a/Tests/PlayMode/PlatformAudioIntegrationTests.cs +++ b/Tests/PlayMode/PlatformAudioIntegrationTests.cs @@ -209,5 +209,43 @@ public IEnumerator PlatformAudioFramesReachRemote_ViaStats() Assert.IsNotNull(inboundRtp, "expected an InboundRtp stat for the platform audio track"); Assert.AreEqual("audio", inboundRtp.Stream.Kind); } + + [UnityTest, Category("E2E")] + public IEnumerator SessionAudio_FollowsRoomConnection() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + Assert.AreEqual(0, Room.ConnectedRoomCount, + "a previous test left a room connected; the call session cannot start idle"); + Assert.IsFalse(platformAudio.SessionAudioEnabled, "call session must be released outside a call"); + + using var context = new TestRoomContext(); + yield return context.ConnectAll(); + Assert.IsNull(context.ConnectionError, context.ConnectionError); + + Assert.AreEqual(1, Room.ConnectedRoomCount); + Assert.IsTrue(platformAudio.SessionAudioEnabled, "call session must be taken while a room is connected"); + + context.Rooms[0].Disconnect(); + Assert.AreEqual(0, Room.ConnectedRoomCount); + Assert.IsFalse(platformAudio.SessionAudioEnabled, "call session must be released on disconnect"); + + // A second Disconnect (TestRoomContext.Dispose issues one too) must not + // drive the count negative. + context.Rooms[0].Disconnect(); + Assert.AreEqual(0, Room.ConnectedRoomCount); + } + + [UnityTest, Category("E2E")] + public IEnumerator SessionAudio_TakenWhenCreatedDuringCall() + { + using var context = new TestRoomContext(); + yield return context.ConnectAll(); + Assert.IsNull(context.ConnectionError, context.ConnectionError); + + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + Assert.IsTrue(platformAudio.SessionAudioEnabled, + "an instance created while a room is connected must take the call session"); + } + } } diff --git a/Tests/PlayMode/PlatformAudioTests.cs b/Tests/PlayMode/PlatformAudioTests.cs index 28b0ed7b..7090c1e8 100644 --- a/Tests/PlayMode/PlatformAudioTests.cs +++ b/Tests/PlayMode/PlatformAudioTests.cs @@ -102,6 +102,157 @@ public IEnumerator SetRecordingDeviceByIndex_OutOfRange_Throws() yield break; } + [UnityTest] + public IEnumerator PlayoutPreference_DefaultsAndRoundtrips() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + // Documented default ranking. + CollectionAssert.AreEqual( + new[] + { + AudioDeviceKind.Bluetooth, + AudioDeviceKind.WiredHeadset, + AudioDeviceKind.Speaker, + AudioDeviceKind.Earpiece, + }, + platformAudio.PlayoutPreference); + + // Set/get roundtrip preserves order and content. + var ranked = new[] { AudioDeviceKind.Usb, AudioDeviceKind.Speaker, AudioDeviceKind.Bluetooth }; + platformAudio.PlayoutPreference = ranked; + CollectionAssert.AreEqual(ranked, platformAudio.PlayoutPreference); + + yield break; + } + + [UnityTest] + public IEnumerator PlayoutPreference_RejectsInvalidLists() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + Assert.Throws(() => platformAudio.PlayoutPreference = null); + Assert.Throws(() => + platformAudio.PlayoutPreference = new[] { AudioDeviceKind.Unknown }); + Assert.Throws(() => + platformAudio.PlayoutPreference = new[] { AudioDeviceKind.Speaker, AudioDeviceKind.Speaker }); + + // A rejected assignment leaves the stored preference untouched. + CollectionAssert.AreEqual( + new[] + { + AudioDeviceKind.Bluetooth, + AudioDeviceKind.WiredHeadset, + AudioDeviceKind.Speaker, + AudioDeviceKind.Earpiece, + }, + platformAudio.PlayoutPreference); + + yield break; + } + + [UnityTest] + public IEnumerator SetPlayoutDevice_UnknownGuid() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + +#if UNITY_IOS && !UNITY_EDITOR + // iOS ignores the call (with a warning): the OS owns route selection. + Assert.DoesNotThrow(() => platformAudio.SetPlayoutDevice("no-such-guid")); +#elif UNITY_ANDROID && !UNITY_EDITOR + // Android 12+ validates against the communication-device list; older Android has + // no routing backend and ignores the call. + var sdkInt = new UnityEngine.AndroidJavaClass("android.os.Build$VERSION").GetStatic("SDK_INT"); + if (sdkInt >= 31) + Assert.Throws(() => platformAudio.SetPlayoutDevice("no-such-guid")); + else + Assert.DoesNotThrow(() => platformAudio.SetPlayoutDevice("no-such-guid")); +#else + // Desktop: the FFI validates the id against the ADM's device list. + Assert.Throws(() => platformAudio.SetPlayoutDevice("no-such-guid")); +#endif + + // Clearing is always safe, whether or not an override exists. + Assert.DoesNotThrow(() => platformAudio.ClearPlayoutDeviceSelection()); + + yield break; + } + + [UnityTest] + public IEnumerator DevicesChanged_SubscribeUnsubscribe_SafeAcrossDispose() + { + var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + Action, IReadOnlyList> handler = (playout, recording) => { }; + platformAudio.DevicesChanged += handler; + platformAudio.Dispose(); + + Assert.DoesNotThrow(() => platformAudio.DevicesChanged -= handler); + Assert.DoesNotThrow(() => platformAudio.DevicesChanged += handler); + Assert.DoesNotThrow(() => platformAudio.Dispose()); + + yield break; + } + + [UnityTest] + public IEnumerator CreateDisposeCreate_OneSession_Works() + { + // The native ADM is ref-counted across PlatformAudio instances; after a full + // dispose the count must have returned to zero cleanly so a later instance in + // the same session comes up working (an app's second call after tearing the + // first one down). + var first = PlatformAudioTestHelper.TryCreateOrIgnore(); + first.PlayoutPreference = new[] { AudioDeviceKind.Usb }; + first.Dispose(); + + using var second = new PlatformAudio(); + Assert.DoesNotThrow(() => second.GetDevices()); + + // Preference state is per instance: the first instance's mutation must not + // leak into the fresh one. + CollectionAssert.AreEqual( + new[] + { + AudioDeviceKind.Bluetooth, + AudioDeviceKind.WiredHeadset, + AudioDeviceKind.Speaker, + AudioDeviceKind.Earpiece, + }, + second.PlayoutPreference); + + yield break; + } + + [UnityTest] + public IEnumerator PublicMembers_AfterDispose_ThrowObjectDisposed() + { + var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + platformAudio.Dispose(); + + Assert.Throws(() => _ = platformAudio.RecordingDeviceCount); + Assert.Throws(() => _ = platformAudio.PlayoutDeviceCount); + Assert.Throws(() => platformAudio.GetDevices()); + Assert.Throws(() => _ = platformAudio.PlayoutPreference); + Assert.Throws(() => + platformAudio.PlayoutPreference = new[] { AudioDeviceKind.Speaker }); + Assert.Throws(() => platformAudio.ClearPlayoutDeviceSelection()); + Assert.Throws(() => platformAudio.SetRecordingDevice((uint)0)); + Assert.Throws(() => platformAudio.SetRecordingDevice("")); + Assert.Throws(() => platformAudio.SetPlayoutDevice((uint)0)); + Assert.Throws(() => platformAudio.SetPlayoutDevice("")); + Assert.Throws(() => platformAudio.StopRecording()); + + // StartRecording is an iterator method: the guard throws on the first MoveNext. + var start = platformAudio.StartRecording(); + Assert.Throws(() => start.MoveNext()); + + // The guards must not break dispose idempotency or event safety + // (DevicesChanged_SubscribeUnsubscribe_SafeAcrossDispose covers the rest). + Assert.DoesNotThrow(() => platformAudio.Dispose()); + + yield break; + } + [UnityTest] public IEnumerator StartThenStopRecording_DoesNotThrow() { diff --git a/Tests/PlayMode/RoomTests.cs b/Tests/PlayMode/RoomTests.cs index 669b0dc6..5cafba35 100644 --- a/Tests/PlayMode/RoomTests.cs +++ b/Tests/PlayMode/RoomTests.cs @@ -129,7 +129,7 @@ public IEnumerator RoomSid_StartsWithRM() StringAssert.StartsWith("RM_", context.Rooms[0].Sid); } - [UnityTest, Category("E2E"), Ignore("Known issue")] + [UnityTest, Category("E2E")] public IEnumerator ConnectionState_IsConnected() { using var context = new TestRoomContext(); @@ -232,7 +232,7 @@ public IEnumerator ParticipantDisconnect_TriggersEvent() if (expectation.Error != null) Assert.Fail(expectation.Error); } - [UnityTest, Category("E2E"), Ignore("Known issue")] + [UnityTest, Category("E2E")] public IEnumerator Disconnect_TriggersEvent() { using var context = new TestRoomContext(); @@ -267,5 +267,86 @@ public IEnumerator Disconnect_TriggersEvent() yield return expectation.Wait(); if (expectation.Error != null) Assert.Fail(expectation.Error); } + + [UnityTest, Category("E2E")] + public IEnumerator Connect_ReportsConnectedOnce() + { + using var context = new TestRoomContext(); + var room = context.Rooms[0]; + var connectedReports = 0; + room.ConnectionStateChanged += s => + { + if (s == ConnectionState.ConnConnected) connectedReports++; + }; + + yield return context.ConnectAll(); + Assert.IsNull(context.ConnectionError, context.ConnectionError); + Assert.AreEqual(1, connectedReports, "recorded from the connect callback"); + + // The core's own ConnectionStateChanged(Connected) is queued behind the connect + // callback and drains on a later frame; it must be absorbed as a repeat. + for (var i = 0; i < 5; i++) yield return null; + Assert.AreEqual(1, connectedReports, "the core's copy of the transition must not be reported again"); + } + + [UnityTest, Category("E2E")] + public IEnumerator Disconnect_ReportsClientInitiated_OnceWithRoomIntact() + { + using var context = new TestRoomContext(); + yield return context.ConnectAll(); + Assert.IsNull(context.ConnectionError, context.ConnectionError); + var room = context.Rooms[0]; + + var disconnected = 0; + var disconnectedWithReason = 0; + DisconnectReason? reason = null; + room.Disconnected += r => + { + disconnected++; + // Handlers run before the release, with the state already recorded. + Assert.IsNotNull(r.RoomHandle, "the room handle must still be live in the handler"); + Assert.IsFalse(r.LocalParticipant.Handle.IsClosed, "participant handles must still be live in the handler"); + Assert.AreEqual(ConnectionState.ConnDisconnected, r.ConnectionState); + Assert.AreEqual(DisconnectReason.ClientInitiated, r.DisconnectReason); + // A teardown handler that hangs up "to be sure" must not re-report. + r.Disconnect(); + }; + room.DisconnectedWithReason += (_, r) => + { + disconnectedWithReason++; + reason = r; + }; + + // Dispose is a disconnect too. Reported synchronously: no frame has to pass + // for a hang-up to be observable. + room.Dispose(); + + Assert.AreEqual(1, disconnected); + Assert.AreEqual(1, disconnectedWithReason); + Assert.AreEqual(DisconnectReason.ClientInitiated, reason); + Assert.AreEqual(ConnectionState.ConnDisconnected, room.ConnectionState); + Assert.IsNull(room.RoomHandle, "the room is released once the handlers ran"); + + // A later Disconnect (TestRoomContext.Dispose issues one too) reports nothing. + room.Disconnect(); + Assert.AreEqual(1, disconnected); + } + + [UnityTest, Category("E2E")] + public IEnumerator Disconnect_ThrowingHandler_StillReleasesRoom() + { + using var context = new TestRoomContext(); + yield return context.ConnectAll(); + Assert.IsNull(context.ConnectionError, context.ConnectionError); + var room = context.Rooms[0]; + + room.Disconnected += _ => throw new System.InvalidOperationException("handler failed"); + + Assert.Throws(() => room.Disconnect()); + + // The handler's exception surfaces, but the room is still released. + Assert.IsNull(room.RoomHandle, "the release must run even when a Disconnected handler throws"); + Assert.DoesNotThrow(() => room.Dispose()); + } } } \ No newline at end of file