From 5f9dd50568a801451d3054fd637bc3ad4d521c3e Mon Sep 17 00:00:00 2001 From: Seto Elkahfi <1797197+setoelkahfi@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:42:37 +0200 Subject: [PATCH] Improve ControlKit UI introspection and tvOS diagnostics Co-Authored-By: siGit Code <297239231+sigitc@users.noreply.github.com> --- crates/xcrs/src/mcp.rs | 28 ++++++--- crates/xcrs/src/xcrs.rs | 129 ++++++++++++++++++++++++++++++++++++---- docs/controlkit.md | 9 ++- 3 files changed, 147 insertions(+), 19 deletions(-) diff --git a/crates/xcrs/src/mcp.rs b/crates/xcrs/src/mcp.rs index 3bae78a..bc1a249 100644 --- a/crates/xcrs/src/mcp.rs +++ b/crates/xcrs/src/mcp.rs @@ -584,6 +584,8 @@ pub struct UiTargetArgs { /// Local ControlKit JSON-RPC port. Defaults to 12004. #[serde(default)] pub controlkit_port: Option, + /// Bundle identifier of the app whose accessibility hierarchy should be read. + pub bundle_id: String, } #[derive(Debug, Deserialize, JsonSchema)] @@ -957,9 +959,9 @@ macro_rules! xcrs_mcp_tools { #[::rmcp::tool( name = $app_install_launch_name, - title = "Install and launch iOS app", - annotations(title = "Install and launch iOS app", read_only_hint = false, destructive_hint = true, idempotent_hint = false), - description = "Purpose: one-shot end-to-end setup for a freshly built iOS app: boot the simulator, install the .app bundle, optionally terminate a previous instance, launch it, and return its app container path. When to use vs siblings: use this once to get a build under test onto a simulator; use app_launch/app_terminate afterwards for an already-installed app, and screen_capture/ui_describe to inspect it. Behavior: boots the target simulator if it is not already booted, installs app_path, optionally force-terminates bundle_id first, launches bundle_id, then reads back the app's container directory. Prerequisites: Xcode command line tools installed; app_path must point to an existing .app bundle built for the simulator (not a device) architecture. Failure modes: errors if neither simulator_name nor simulator_udid is given, if the simulator cannot be found, or if any underlying `simctl` step fails. Limitations: iOS-simulator-only; there is no Android or physical-device equivalent in this tool." + title = "Install and launch Apple simulator app", + annotations(title = "Install and launch Apple simulator app", read_only_hint = false, destructive_hint = true, idempotent_hint = false), + description = "Purpose: one-shot end-to-end setup for a freshly built Apple-platform app: boot an iOS, tvOS, watchOS, or visionOS simulator, install the .app bundle, optionally terminate a previous instance, launch it, and return its app container path. When to use vs siblings: use this once to get a build under test onto a simulator; use app_launch/app_terminate afterwards for an already-installed app, and screen_capture/ui_describe to inspect it. Behavior: boots the target simulator if it is not already booted, installs app_path, optionally force-terminates bundle_id first, launches bundle_id, then reads back the app's container directory. Prerequisites: Xcode command line tools installed; app_path must point to an existing .app bundle built for the selected simulator platform and architecture. Failure modes: errors if neither simulator_name nor simulator_udid is given, if the simulator cannot be found, if the app bundle targets a different platform, or if any underlying `simctl` step fails. Limitations: simulator-only; there is no Android or physical-device equivalent in this tool." )] async fn app_install_launch( &self, @@ -1317,7 +1319,7 @@ macro_rules! xcrs_mcp_tools { name = $ui_describe_name, title = "Describe UI", annotations(title = "Describe UI", read_only_hint = true, idempotent_hint = true), - description = "Purpose: return the full accessibility hierarchy of the foreground app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: calls the resolved target's ControlKit `device.dump.ui` method and returns the raw hierarchy alongside the resolved simulator (if any). Prerequisites: a reachable ControlKit endpoint. Failure modes: errors if no target can be resolved, or if the resolved target is Android (Apple-only tool; ControlKit UI introspection has no Android equivalent), or if the ControlKit call fails." + description = "Purpose: return the full accessibility hierarchy of an Apple app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method and returns the raw hierarchy alongside the resolved simulator (if any). Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool; ControlKit UI introspection has no Android equivalent), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable." )] async fn ui_describe( &self, @@ -1339,7 +1341,13 @@ macro_rules! xcrs_mcp_tools { )?; let (simulator, controlkit) = Self::controlkit_for_target(&target)?; let result = controlkit - .call("device.dump.ui", ::serde_json::json!({ "format": "json" })) + .call( + "device.dump.ui", + ::serde_json::json!({ + "format": "json", + "bundleId": args.bundle_id, + }), + ) .await .map_err(|error| { ::rmcp::model::ErrorData::internal_error(error.to_string(), None) @@ -1356,7 +1364,7 @@ macro_rules! xcrs_mcp_tools { name = $ui_element_list_name, title = "List UI elements", annotations(title = "List UI elements", read_only_hint = true, idempotent_hint = true), - description = "Purpose: list just the actionable accessibility elements of the foreground app with their labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: calls the resolved target's ControlKit `device.dump.ui` method, then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: a reachable ControlKit endpoint. Failure modes: errors if no target can be resolved, or if the resolved target is Android (Apple-only tool), or if the ControlKit call fails." + description = "Purpose: list just the actionable accessibility elements of an Apple app with their labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method, then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable." )] async fn ui_element_list( &self, @@ -1378,7 +1386,13 @@ macro_rules! xcrs_mcp_tools { )?; let (simulator, controlkit) = Self::controlkit_for_target(&target)?; let ui = controlkit - .call("device.dump.ui", ::serde_json::json!({ "format": "json" })) + .call( + "device.dump.ui", + ::serde_json::json!({ + "format": "json", + "bundleId": args.bundle_id, + }), + ) .await .map_err(|error| { ::rmcp::model::ErrorData::internal_error(error.to_string(), None) diff --git a/crates/xcrs/src/xcrs.rs b/crates/xcrs/src/xcrs.rs index 0ac8cea..6c9039d 100644 --- a/crates/xcrs/src/xcrs.rs +++ b/crates/xcrs/src/xcrs.rs @@ -10,6 +10,7 @@ use std::time::Duration; pub mod mcp; const IOS_SIMULATOR_DESTINATION_PREFIX: &str = "platform=iOS Simulator,id="; +const CONTROLKIT_METHOD_NOT_FOUND: i64 = -32601; pub fn encode_base64(data: impl AsRef<[u8]>) -> String { use base64::{engine::general_purpose::STANDARD, Engine}; @@ -46,7 +47,7 @@ fn collect_controlkit_elements(element: &serde_json::Value, elements: &mut Vec Result { + let body = self.send(method, params).await?; + + if let Some(error) = body.get("error") { + let runner_info = if error.get("code").and_then(serde_json::Value::as_i64) + == Some(CONTROLKIT_METHOD_NOT_FOUND) + && method != "device.info" + { + self.send("device.info", serde_json::json!({})) + .await + .ok() + .and_then(|body| body.get("result").cloned()) + } else { + None + }; + return Err(anyhow!(controlkit_rpc_error( + method, + error, + runner_info.as_ref() + ))); + } + + body.get("result") + .cloned() + .ok_or_else(|| anyhow!("ControlKit response for '{method}' did not contain a result")) + } + + async fn send(&self, method: &str, params: serde_json::Value) -> Result { let response = self .client .post(format!("{}/rpc", self.base_url)) @@ -150,16 +182,43 @@ impl ControlKit { return Err(anyhow!("ControlKit returned HTTP {}: {}", status, body)); } - if let Some(error) = body.get("error") { - return Err(anyhow!("ControlKit method '{method}' failed: {error}")); - } - - body.get("result") - .cloned() - .ok_or_else(|| anyhow!("ControlKit response for '{method}' did not contain a result")) + Ok(body) } } +fn controlkit_rpc_error( + method: &str, + error: &serde_json::Value, + runner_info: Option<&serde_json::Value>, +) -> String { + let code = error + .get("code") + .and_then(serde_json::Value::as_i64) + .unwrap_or_default(); + let message = error + .get("message") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown JSON-RPC error"); + + if code != CONTROLKIT_METHOD_NOT_FOUND { + return format!("ControlKit method '{method}' failed ({code}): {message}"); + } + + let runner = runner_info + .and_then(|info| info.get("runner")) + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown ControlKit runner"); + let protocol = runner_info + .and_then(|info| info.get("protocolVersion")) + .and_then(serde_json::Value::as_u64) + .map(|version| format!(" protocol {version}")) + .unwrap_or_default(); + + format!( + "{runner}{protocol} does not implement ControlKit method '{method}'. Rebuild or upgrade xcrs-controlkit, restart its runner, and retry." + ) +} + #[derive(Debug, Clone)] pub struct XcodeCommandLineTools { xcrun_path: PathBuf, @@ -297,14 +356,14 @@ impl Simctl<'_> { self.list_simulators()? .into_iter() .find(|simulator| simulator.name == name) - .ok_or_else(|| anyhow!("iOS simulator named '{name}' was not found")) + .ok_or_else(|| anyhow!("Apple simulator named '{name}' was not found")) } pub fn find_simulator_by_udid(&self, udid: &str) -> Result { self.list_simulators()? .into_iter() .find(|simulator| simulator.udid == udid) - .ok_or_else(|| anyhow!("iOS simulator with UDID '{udid}' was not found")) + .ok_or_else(|| anyhow!("Apple simulator with UDID '{udid}' was not found")) } pub fn boot(&self, udid: &str) -> Result<()> { @@ -1273,6 +1332,56 @@ mod tests { assert_eq!(controlkit.base_url, "http://[fdb4:e020:7377::1]:12006"); } + #[test] + fn reports_actionable_controlkit_method_mismatch() { + let error = serde_json::json!({ + "code": -32601, + "message": "Method not found" + }); + let runner_info = serde_json::json!({ + "runner": "XCRSControlKit", + "protocolVersion": 1 + }); + + let message = controlkit_rpc_error("device.dump.ui", &error, Some(&runner_info)); + + assert!(message.contains("XCRSControlKit protocol 1")); + assert!(message.contains("device.dump.ui")); + assert!(message.contains("Rebuild or upgrade")); + } + + #[test] + fn extracts_identified_visible_controlkit_elements() { + let hierarchy = serde_json::json!({ + "type": "Application", + "rect": { "x": 0, "y": 0, "width": 1920, "height": 1080 }, + "children": [ + { + "type": "Button", + "label": "Play", + "rawIdentifier": "play-button", + "rect": { "x": 100, "y": 200, "width": 80, "height": 40 }, + "enabled": true, + "selected": false, + "hittable": true, + "children": [] + }, + { + "type": "Image", + "label": "", + "rect": { "x": 0, "y": 0, "width": 40, "height": 40 }, + "children": [] + } + ] + }); + + let elements = extract_controlkit_elements(&hierarchy); + + assert_eq!(elements.len(), 1); + assert_eq!(elements[0]["label"], serde_json::json!("Play")); + assert_eq!(elements[0]["hittable"], serde_json::json!(true)); + } + #[test] fn parses_device_tunnel_address() { let output = "• Device Name: iPhone\n• Tunnel IP Address: fd55:33ce:ad87::1\n"; diff --git a/docs/controlkit.md b/docs/controlkit.md index cabfeb2..b1a82b2 100644 --- a/docs/controlkit.md +++ b/docs/controlkit.md @@ -117,8 +117,13 @@ expose the same unprefixed tool names. | `input_button` | iOS / tvOS, simulator or physical | Press a Home or tvOS remote button. | | `app_launch`/`app_terminate` | simulator or physical | Launch/terminate an app by bundle ID. On a physical device this calls the runner's `device.apps.launch`/`device.apps.terminate` RPC methods instead of `simctl`. | | `screen_capture` | Simulator or physical | Capture a PNG screenshot. Uses `simctl` for a simulator, or `devicectl device capture screenshot` when a `device` identifier is given. | -| `ui_describe` | All UI-test runners | Read the accessibility hierarchy. | -| `ui_element_list` | All UI-test runners | Read only actionable elements with tap coordinates. | +| `ui_describe` | All UI-test runners | Read the accessibility hierarchy for the supplied `bundle_id`. | +| `ui_element_list` | All UI-test runners | Read actionable elements with tap coordinates for the supplied `bundle_id`. | + +UI introspection requires a recent ControlKit runner that implements +`device.dump.ui`. Both UI tools require the target app's `bundle_id`; this +lets XCTest attach to an already-running app instead of accidentally reading +the ControlKit host app. For a local macOS runner, omit simulator fields and pass `host` and `controlkit_port` when needed: