From 9cc52659d6f4eb4f2021ae61a5888953ada63050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 20 Sep 2026 22:41:53 +0200 Subject: [PATCH 1/2] feat(fold): pose a foldable iPhone simulator through Device Hub and verify it with CoreDevice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `agent-device fold ` and `client.command.fold({ pose })`. ADR 0025 recorded that no official host API sets the hinge pose and left it to the operator. The operator's own control is the pose action bar in the Xcode Device Hub window, whose controls are ordinary accessibility buttons; the macOS helper gains a `device-hub pose` subcommand that finds Device Hub in the process table (LaunchServices registers the trampolined app with pid -1, which is why System Events saw no nodes), reopens its window with a reopen event when it shows none, selects the simulator through the sidebar row keyed `TableRow.Device.`, and presses the control. The press is not the evidence. The Apple owner reads the hinge angle back through `devicectl device motion hinge-angle` until it agrees with the request: closed is 0°, open is 180°, and half-open is any angle between them reported only once two consecutive reads agree the hinge stopped moving, because a hinge sweeps through half-open angles on its way to either stop (the live run read 175.1° one stream after pressing Book, 130° after it settled). The stream never ends on its own, so each read is bounded by devicectl's smallest `--timeout` of five seconds. The response carries the verified pose, the angle, and the lit panel's point size, so an agent sees its refs are stale without another capture. `setFoldPose` is a host-side device-runtime operation rather than an interactor method: it needs no XCUITest runner, and the leaf fact admits iPhone/iPad simulators while the operation itself refuses a single-panel simulator from CoreDevice's display table. Every other platform and provider states its own refusal cell. Verified through the built CLI on an iOS 27.1 iPhone Duo simulator: open → 180° on LCD-1 (669x951pt, screenshot 951x669), half-open → 130°, closed → 0° on LCD (466x678pt, screenshot 466x678), from a Device Hub that showed no window for the device; an iPhone 17 refuses with UNSUPPORTED_OPERATION before anything is pressed. Co-Authored-By: Claude Fable 5.1 --- .../attempt-1/replay-timing.ndjson | 12 +- .../01-flow.yaml/attempt-1/result.txt | 2 +- .../attempt-1/replay-timing.ndjson | 12 +- .../02-flow.yaml/attempt-1/result.txt | 2 +- .../attempt-1/replay-timing.ndjson | 12 +- .../01-flow.yaml/attempt-1/result.txt | 2 +- .../attempt-1/replay-timing.ndjson | 12 +- .../02-flow.yaml/attempt-1/result.txt | 2 +- CHANGELOG.md | 13 + apple/macos-helper/Package.swift | 9 +- .../DeviceHubPose.swift | 47 +++ .../DeviceHubPose.swift | 320 ++++++++++++++++++ .../Sources/AgentDeviceMacOSHelper/main.swift | 2 + .../DeviceHubPoseTests.swift | 30 ++ docs/adr/0025-foldable-apple-panels.md | 69 +++- docs/agents/device-verification.md | 9 +- .../src/__tests__/command-result.test.ts | 1 + .../command-registry/src/command-result.ts | 2 + packages/command-registry/src/registry.ts | 14 + .../command-registry/src/timeout-policy.ts | 13 + packages/contracts/package.json | 4 + packages/contracts/src/client-system.ts | 6 +- .../src/device-rotation-fold-pose.test.ts | 37 ++ packages/contracts/src/device-rotation.ts | 60 ++++ packages/contracts/src/facades/client.ts | 1 + packages/contracts/src/facades/device.ts | 6 +- packages/contracts/src/fold-runtime.ts | 45 +++ packages/contracts/src/navigation.ts | 19 +- .../src/platform-runtime-operations.ts | 3 + .../src/platform-runtime-unavailable.test.ts | 1 + .../src/platform-runtime-unavailable.ts | 5 + .../contracts/src/runtime-operation-names.ts | 1 + packages/platform-android/src/runtime.test.ts | 20 ++ packages/platform-android/src/runtime.ts | 11 + .../src/core/__tests__/hinge-angle.test.ts | 71 ++++ packages/platform-apple/src/core/config.ts | 10 + .../src/core/display-inventory.ts | 2 +- .../platform-apple/src/core/hinge-angle.ts | 75 ++++ .../platform-apple/src/foldable/pose.test.ts | 208 ++++++++++++ packages/platform-apple/src/foldable/pose.ts | 160 +++++++++ .../platform-apple/src/foldable/runtime.ts | 50 +++ .../platform-apple/src/os/macos/helper.ts | 35 ++ packages/platform-apple/src/runner-demand.ts | 1 + packages/platform-apple/src/runtime.test.ts | 27 ++ packages/platform-apple/src/runtime.ts | 6 + .../platform-harmonyos/src/runtime.test.ts | 2 + packages/platform-harmonyos/src/runtime.ts | 2 + packages/platform-linux/src/runtime.test.ts | 2 + packages/platform-linux/src/runtime.ts | 1 + packages/platform-vega/src/runtime.ts | 5 + packages/platform-web/src/runtime.test.ts | 2 + packages/platform-web/src/runtime.ts | 1 + .../provider-limrun/src/app-log-runtime.ts | 6 +- packages/provider-limrun/src/facts-runtime.ts | 3 + .../src/interaction-operations.ts | 12 + .../src/platform-runtime.ts | 10 + .../src/session-event-action-presentation.ts | 11 +- scripts/help-conformance-cases.mjs | 15 +- .../test-utils/property-arbitraries.ts | 1 + .../test-utils/runtime-operation-facts.ts | 1 + src/agent-device-client.ts | 1 + src/client/client-types.ts | 2 + src/commands/schema/cli-help-overview.ts | 2 +- src/commands/schema/cli-help.ts | 10 +- src/commands/system/index.test.ts | 24 ++ src/commands/system/index.ts | 50 ++- src/commands/system/output.ts | 1 + src/daemon/__tests__/fold-runtime.test.ts | 157 +++++++++ src/daemon/fold-runtime.ts | 62 ++++ src/daemon/generic-runtime-execution.ts | 8 + .../handlers/__tests__/install-source.test.ts | 1 + .../handlers/__tests__/session-state.test.ts | 2 + src/mcp/command-output-schemas.ts | 19 +- src/platform-runtime-gateway.test.ts | 2 + .../provider-device-runtime.fixtures.ts | 2 + .../provider-ios-runner-transport.test.ts | 21 +- website/docs/docs/client-api.md | 2 + website/docs/docs/commands.md | 8 +- 78 files changed, 1830 insertions(+), 67 deletions(-) create mode 100644 apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubPose.swift create mode 100644 apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift create mode 100644 apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubPoseTests.swift create mode 100644 packages/contracts/src/device-rotation-fold-pose.test.ts create mode 100644 packages/contracts/src/fold-runtime.ts create mode 100644 packages/platform-apple/src/core/__tests__/hinge-angle.test.ts create mode 100644 packages/platform-apple/src/core/hinge-angle.ts create mode 100644 packages/platform-apple/src/foldable/pose.test.ts create mode 100644 packages/platform-apple/src/foldable/pose.ts create mode 100644 packages/platform-apple/src/foldable/runtime.ts create mode 100644 src/daemon/__tests__/fold-runtime.test.ts create mode 100644 src/daemon/fold-runtime.ts diff --git a/.agent-device/test-artifacts/req-maestro-test-wire-true/01-flow.yaml/attempt-1/replay-timing.ndjson b/.agent-device/test-artifacts/req-maestro-test-wire-true/01-flow.yaml/attempt-1/replay-timing.ndjson index c3de3c93c6..aa2cf96361 100644 --- a/.agent-device/test-artifacts/req-maestro-test-wire-true/01-flow.yaml/attempt-1/replay-timing.ndjson +++ b/.agent-device/test-artifacts/req-maestro-test-wire-true/01-flow.yaml/attempt-1/replay-timing.ndjson @@ -1,6 +1,6 @@ -{"type":"replay_test_attempt_start","ts":"2026-09-19T14:32:46.818Z","replayPath":"/tmp/agent-device-test-run-75313-0YH0p7/agent-device-maestro-remote-test-pzCEKx/01-flow.yaml","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1","requestId":"req-maestro-test-wire-true:test:1:01-flow.yaml:attempt:1"} -{"type":"replay_test_attempt_stop","ts":"2026-09-19T14:32:46.819Z","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1","ok":true,"timedOut":false,"durationMs":2} -{"type":"replay_test_finalize_start","ts":"2026-09-19T14:32:46.819Z","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1"} -{"type":"replay_test_finalize_stop","ts":"2026-09-19T14:32:46.820Z","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1","ok":true,"durationMs":1} -{"type":"replay_test_cleanup_start","ts":"2026-09-19T14:32:46.820Z","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1"} -{"type":"replay_test_cleanup_stop","ts":"2026-09-19T14:32:46.820Z","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1","ok":true,"durationMs":0} +{"type":"replay_test_attempt_start","ts":"2026-09-20T20:37:45.965Z","replayPath":"/tmp/agent-device-test-run-94227-XxlCsl/agent-device-maestro-remote-test-IAQdpt/01-flow.yaml","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1","requestId":"req-maestro-test-wire-true:test:1:01-flow.yaml:attempt:1"} +{"type":"replay_test_attempt_stop","ts":"2026-09-20T20:37:45.967Z","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1","ok":true,"timedOut":false,"durationMs":2} +{"type":"replay_test_finalize_start","ts":"2026-09-20T20:37:45.967Z","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1"} +{"type":"replay_test_finalize_stop","ts":"2026-09-20T20:37:45.967Z","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1","ok":true,"durationMs":0} +{"type":"replay_test_cleanup_start","ts":"2026-09-20T20:37:45.967Z","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1"} +{"type":"replay_test_cleanup_stop","ts":"2026-09-20T20:37:45.967Z","session":"default:test:req-maestro-test-wire-true:1-01-flow:attempt-1","ok":true,"durationMs":0} diff --git a/.agent-device/test-artifacts/req-maestro-test-wire-true/01-flow.yaml/attempt-1/result.txt b/.agent-device/test-artifacts/req-maestro-test-wire-true/01-flow.yaml/attempt-1/result.txt index 6cb55621a0..26c5d97e3b 100644 --- a/.agent-device/test-artifacts/req-maestro-test-wire-true/01-flow.yaml/attempt-1/result.txt +++ b/.agent-device/test-artifacts/req-maestro-test-wire-true/01-flow.yaml/attempt-1/result.txt @@ -1,4 +1,4 @@ -file: /tmp/agent-device-test-run-75313-0YH0p7/agent-device-maestro-remote-test-pzCEKx/01-flow.yaml +file: /tmp/agent-device-test-run-94227-XxlCsl/agent-device-maestro-remote-test-IAQdpt/01-flow.yaml session: default:test:req-maestro-test-wire-true:1-01-flow:attempt-1 attempt: 1/1 status: passed diff --git a/.agent-device/test-artifacts/req-maestro-test-wire-true/02-flow.yaml/attempt-1/replay-timing.ndjson b/.agent-device/test-artifacts/req-maestro-test-wire-true/02-flow.yaml/attempt-1/replay-timing.ndjson index 5d0e648b70..5a01b6d043 100644 --- a/.agent-device/test-artifacts/req-maestro-test-wire-true/02-flow.yaml/attempt-1/replay-timing.ndjson +++ b/.agent-device/test-artifacts/req-maestro-test-wire-true/02-flow.yaml/attempt-1/replay-timing.ndjson @@ -1,6 +1,6 @@ -{"type":"replay_test_attempt_start","ts":"2026-09-19T14:32:46.820Z","replayPath":"/tmp/agent-device-test-run-75313-0YH0p7/agent-device-maestro-remote-test-pzCEKx/02-flow.yaml","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1","requestId":"req-maestro-test-wire-true:test:2:02-flow.yaml:attempt:1"} -{"type":"replay_test_attempt_stop","ts":"2026-09-19T14:32:46.821Z","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1","ok":true,"timedOut":false,"durationMs":1} -{"type":"replay_test_finalize_start","ts":"2026-09-19T14:32:46.821Z","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1"} -{"type":"replay_test_finalize_stop","ts":"2026-09-19T14:32:46.821Z","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1","ok":true,"durationMs":0} -{"type":"replay_test_cleanup_start","ts":"2026-09-19T14:32:46.821Z","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1"} -{"type":"replay_test_cleanup_stop","ts":"2026-09-19T14:32:46.821Z","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1","ok":true,"durationMs":0} +{"type":"replay_test_attempt_start","ts":"2026-09-20T20:37:45.968Z","replayPath":"/tmp/agent-device-test-run-94227-XxlCsl/agent-device-maestro-remote-test-IAQdpt/02-flow.yaml","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1","requestId":"req-maestro-test-wire-true:test:2:02-flow.yaml:attempt:1"} +{"type":"replay_test_attempt_stop","ts":"2026-09-20T20:37:45.968Z","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1","ok":true,"timedOut":false,"durationMs":0} +{"type":"replay_test_finalize_start","ts":"2026-09-20T20:37:45.969Z","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1"} +{"type":"replay_test_finalize_stop","ts":"2026-09-20T20:37:45.969Z","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1","ok":true,"durationMs":0} +{"type":"replay_test_cleanup_start","ts":"2026-09-20T20:37:45.969Z","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1"} +{"type":"replay_test_cleanup_stop","ts":"2026-09-20T20:37:45.969Z","session":"default:test:req-maestro-test-wire-true:2-02-flow:attempt-1","ok":true,"durationMs":0} diff --git a/.agent-device/test-artifacts/req-maestro-test-wire-true/02-flow.yaml/attempt-1/result.txt b/.agent-device/test-artifacts/req-maestro-test-wire-true/02-flow.yaml/attempt-1/result.txt index e634dd45f4..b90929e840 100644 --- a/.agent-device/test-artifacts/req-maestro-test-wire-true/02-flow.yaml/attempt-1/result.txt +++ b/.agent-device/test-artifacts/req-maestro-test-wire-true/02-flow.yaml/attempt-1/result.txt @@ -1,4 +1,4 @@ -file: /tmp/agent-device-test-run-75313-0YH0p7/agent-device-maestro-remote-test-pzCEKx/02-flow.yaml +file: /tmp/agent-device-test-run-94227-XxlCsl/agent-device-maestro-remote-test-IAQdpt/02-flow.yaml session: default:test:req-maestro-test-wire-true:2-02-flow:attempt-1 attempt: 1/1 status: passed diff --git a/.agent-device/test-artifacts/req-maestro-test-wire-undefined/01-flow.yaml/attempt-1/replay-timing.ndjson b/.agent-device/test-artifacts/req-maestro-test-wire-undefined/01-flow.yaml/attempt-1/replay-timing.ndjson index 4ae708cda6..9a7df766f9 100644 --- a/.agent-device/test-artifacts/req-maestro-test-wire-undefined/01-flow.yaml/attempt-1/replay-timing.ndjson +++ b/.agent-device/test-artifacts/req-maestro-test-wire-undefined/01-flow.yaml/attempt-1/replay-timing.ndjson @@ -1,6 +1,6 @@ -{"type":"replay_test_attempt_start","ts":"2026-09-19T14:32:46.823Z","replayPath":"/tmp/agent-device-test-run-75313-0YH0p7/agent-device-maestro-remote-test-Wnbhbp/01-flow.yaml","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1","requestId":"req-maestro-test-wire-undefined:test:1:01-flow.yaml:attempt:1"} -{"type":"replay_test_attempt_stop","ts":"2026-09-19T14:32:46.823Z","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1","ok":true,"timedOut":false,"durationMs":0} -{"type":"replay_test_finalize_start","ts":"2026-09-19T14:32:46.823Z","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1"} -{"type":"replay_test_finalize_stop","ts":"2026-09-19T14:32:46.823Z","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1","ok":true,"durationMs":0} -{"type":"replay_test_cleanup_start","ts":"2026-09-19T14:32:46.823Z","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1"} -{"type":"replay_test_cleanup_stop","ts":"2026-09-19T14:32:46.823Z","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1","ok":true,"durationMs":0} +{"type":"replay_test_attempt_start","ts":"2026-09-20T20:37:45.971Z","replayPath":"/tmp/agent-device-test-run-94227-XxlCsl/agent-device-maestro-remote-test-XmcOFG/01-flow.yaml","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1","requestId":"req-maestro-test-wire-undefined:test:1:01-flow.yaml:attempt:1"} +{"type":"replay_test_attempt_stop","ts":"2026-09-20T20:37:45.971Z","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1","ok":true,"timedOut":false,"durationMs":0} +{"type":"replay_test_finalize_start","ts":"2026-09-20T20:37:45.971Z","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1"} +{"type":"replay_test_finalize_stop","ts":"2026-09-20T20:37:45.971Z","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1","ok":true,"durationMs":0} +{"type":"replay_test_cleanup_start","ts":"2026-09-20T20:37:45.971Z","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1"} +{"type":"replay_test_cleanup_stop","ts":"2026-09-20T20:37:45.971Z","session":"default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1","ok":true,"durationMs":0} diff --git a/.agent-device/test-artifacts/req-maestro-test-wire-undefined/01-flow.yaml/attempt-1/result.txt b/.agent-device/test-artifacts/req-maestro-test-wire-undefined/01-flow.yaml/attempt-1/result.txt index 69b4b78571..4f721cd2cd 100644 --- a/.agent-device/test-artifacts/req-maestro-test-wire-undefined/01-flow.yaml/attempt-1/result.txt +++ b/.agent-device/test-artifacts/req-maestro-test-wire-undefined/01-flow.yaml/attempt-1/result.txt @@ -1,4 +1,4 @@ -file: /tmp/agent-device-test-run-75313-0YH0p7/agent-device-maestro-remote-test-Wnbhbp/01-flow.yaml +file: /tmp/agent-device-test-run-94227-XxlCsl/agent-device-maestro-remote-test-XmcOFG/01-flow.yaml session: default:test:req-maestro-test-wire-undefined:1-01-flow:attempt-1 attempt: 1/1 status: passed diff --git a/.agent-device/test-artifacts/req-maestro-test-wire-undefined/02-flow.yaml/attempt-1/replay-timing.ndjson b/.agent-device/test-artifacts/req-maestro-test-wire-undefined/02-flow.yaml/attempt-1/replay-timing.ndjson index 9107c923bb..88da636057 100644 --- a/.agent-device/test-artifacts/req-maestro-test-wire-undefined/02-flow.yaml/attempt-1/replay-timing.ndjson +++ b/.agent-device/test-artifacts/req-maestro-test-wire-undefined/02-flow.yaml/attempt-1/replay-timing.ndjson @@ -1,6 +1,6 @@ -{"type":"replay_test_attempt_start","ts":"2026-09-19T14:32:46.824Z","replayPath":"/tmp/agent-device-test-run-75313-0YH0p7/agent-device-maestro-remote-test-Wnbhbp/02-flow.yaml","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1","requestId":"req-maestro-test-wire-undefined:test:2:02-flow.yaml:attempt:1"} -{"type":"replay_test_attempt_stop","ts":"2026-09-19T14:32:46.824Z","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1","ok":true,"timedOut":false,"durationMs":0} -{"type":"replay_test_finalize_start","ts":"2026-09-19T14:32:46.824Z","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1"} -{"type":"replay_test_finalize_stop","ts":"2026-09-19T14:32:46.824Z","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1","ok":true,"durationMs":0} -{"type":"replay_test_cleanup_start","ts":"2026-09-19T14:32:46.824Z","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1"} -{"type":"replay_test_cleanup_stop","ts":"2026-09-19T14:32:46.824Z","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1","ok":true,"durationMs":0} +{"type":"replay_test_attempt_start","ts":"2026-09-20T20:37:45.972Z","replayPath":"/tmp/agent-device-test-run-94227-XxlCsl/agent-device-maestro-remote-test-XmcOFG/02-flow.yaml","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1","requestId":"req-maestro-test-wire-undefined:test:2:02-flow.yaml:attempt:1"} +{"type":"replay_test_attempt_stop","ts":"2026-09-20T20:37:45.972Z","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1","ok":true,"timedOut":false,"durationMs":0} +{"type":"replay_test_finalize_start","ts":"2026-09-20T20:37:45.972Z","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1"} +{"type":"replay_test_finalize_stop","ts":"2026-09-20T20:37:45.973Z","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1","ok":true,"durationMs":1} +{"type":"replay_test_cleanup_start","ts":"2026-09-20T20:37:45.973Z","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1"} +{"type":"replay_test_cleanup_stop","ts":"2026-09-20T20:37:45.973Z","session":"default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1","ok":true,"durationMs":0} diff --git a/.agent-device/test-artifacts/req-maestro-test-wire-undefined/02-flow.yaml/attempt-1/result.txt b/.agent-device/test-artifacts/req-maestro-test-wire-undefined/02-flow.yaml/attempt-1/result.txt index a3bc90a611..d529a1fe8e 100644 --- a/.agent-device/test-artifacts/req-maestro-test-wire-undefined/02-flow.yaml/attempt-1/result.txt +++ b/.agent-device/test-artifacts/req-maestro-test-wire-undefined/02-flow.yaml/attempt-1/result.txt @@ -1,4 +1,4 @@ -file: /tmp/agent-device-test-run-75313-0YH0p7/agent-device-maestro-remote-test-Wnbhbp/02-flow.yaml +file: /tmp/agent-device-test-run-94227-XxlCsl/agent-device-maestro-remote-test-XmcOFG/02-flow.yaml session: default:test:req-maestro-test-wire-undefined:2-02-flow:attempt-1 attempt: 1/1 status: passed diff --git a/CHANGELOG.md b/CHANGELOG.md index 213f7fd9a9..ce848903a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +- Added (ios): `fold ` and `client.command.fold({ pose })` put a foldable + iPhone simulator (iPhone Duo) into a hinge pose. ADR 0025 recorded that no official host API sets + the pose and left it to the operator; the pose is now set the way the operator did it, by pressing + the pose control in the Xcode Device Hub window through macOS accessibility, and it is confirmed + the way ADR 0025 asked for, by reading the hinge angle back from CoreDevice + (`devicectl device motion hinge-angle`) until it agrees with the request. The response reports + the verified pose, the hinge angle, and the panel the device now lights with its point size, so an + agent can see that its refs and coordinates are stale without another capture. The macOS helper + gained a `device-hub pose` subcommand that finds Device Hub in the process table (LaunchServices + registers the trampolined app with no pid), reopens its window when it shows none, and selects the + simulator through the sidebar row keyed by its UDID so two simulators sharing a name cannot be + confused. Simulator-only; a single-panel simulator refuses with `UNSUPPORTED_OPERATION`, and every + other platform states its own refusal cell. - Added (diff): `diff screenshot` accepts a JPEG baseline or current image. Both inputs had to be PNG, so a capture exported by another tool had to be converted first and a HarmonyOS capture — which the platform serves as JPEG under whatever name the command was given — could never be compared. Each diff --git a/apple/macos-helper/Package.swift b/apple/macos-helper/Package.swift index 3a472a3ca2..a287ecdb48 100644 --- a/apple/macos-helper/Package.swift +++ b/apple/macos-helper/Package.swift @@ -14,13 +14,20 @@ let package = Package( .target( name: "AgentDeviceMacOSInput" ), + .target( + name: "AgentDeviceMacOSDeviceHub" + ), .executableTarget( name: "AgentDeviceMacOSHelper", - dependencies: ["AgentDeviceMacOSInput"] + dependencies: ["AgentDeviceMacOSInput", "AgentDeviceMacOSDeviceHub"] ), .testTarget( name: "AgentDeviceMacOSInputTests", dependencies: ["AgentDeviceMacOSInput"] ), + .testTarget( + name: "AgentDeviceMacOSDeviceHubTests", + dependencies: ["AgentDeviceMacOSDeviceHub"] + ), ] ) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubPose.swift b/apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubPose.swift new file mode 100644 index 0000000000..0cde6aa94f --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubPose.swift @@ -0,0 +1,47 @@ +import Foundation + +/// The decisions the Device Hub pose press makes without a window server: which control a pose +/// names, which window belongs to a device, and which sidebar row identifies it. They live apart +/// from the helper executable so the tests can exercise them without an accessibility session. + +/// Device Hub's three pose presets for a foldable simulator, named as the helper's `--pose` +/// argument spells them. Each maps to the accessibility description of its action-bar button. +public enum DeviceHubPose: String, CaseIterable, Sendable { + case closed + case book + case open + + public init?(argument: String) { + self.init(rawValue: argument.lowercased()) + } + + /// The `AXDescription` Device Hub gives the button for this pose. + public var controlDescription: String { + switch self { + case .closed: return "Closed" + case .book: return "Book" + case .open: return "Open" + } + } +} + +/// Device Hub is launched through a trampoline, so LaunchServices registers it with no process +/// identifier; the process table is matched on the executable path instead. +public let deviceHubExecutableSuffix = "/DeviceHub.app/Contents/MacOS/DeviceHub" + +/// The separator Device Hub puts between the device name and the OS in a window title. +public let deviceHubWindowTitleSeparator = " – " + +/// Device Hub titles a device window ` – `, so a window shows the +/// device when its title is the name or the name followed by that exact separator. A bare +/// space would also match a simulator whose name merely extends this one ("iPhone Duo Lab"). +public func deviceHubWindowShows(deviceName: String, title: String?) -> Bool { + guard let title else { return false } + return title == deviceName || title.hasPrefix(deviceName + deviceHubWindowTitleSeparator) +} + +/// The accessibility identifier of the sidebar row for one simulator. The UDID is the one device +/// identity Device Hub exposes that two simulators sharing a name cannot confuse. +public func deviceHubDeviceRowIdentifier(udid: String) -> String { + return "TableRow.Device.\(udid)" +} diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift new file mode 100644 index 0000000000..c33f4a69a4 --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift @@ -0,0 +1,320 @@ +import AgentDeviceMacOSDeviceHub +import AppKit +import ApplicationServices +import Darwin +import Foundation + +/// The Xcode Device Hub application. Its device windows carry an action bar whose pose controls +/// are the only public seam onto the private channel that folds a simulator. +private let deviceHubBundleId = "com.apple.dt.Devices" + +/// How long a reopen gets to restore a window, and a sidebar selection to switch the window's +/// device, before the press is refused. +private let deviceHubSettleDeadline: TimeInterval = 8 +private let deviceHubPoll: TimeInterval = 0.25 + +struct DeviceHubPoseResponse: Encodable { + let pose: String + let control: String + let windowTitle: String + /** Whether Device Hub had to be asked to reopen a window before the device could be selected. */ + let reopened: Bool + /** Whether the window was switched to the device through its sidebar row. */ + let selected: Bool +} + +func handleDeviceHub(arguments: [String]) throws -> any Encodable { + guard arguments.first == "pose" else { + throw HelperError.invalidArgs("device-hub requires pose") + } + let rest = Array(arguments.dropFirst()) + guard let udid = helperOptionValue(arguments: rest, name: "--udid")? + .trimmingCharacters(in: .whitespacesAndNewlines), + !udid.isEmpty + else { + throw HelperError.invalidArgs("device-hub pose requires --udid ") + } + guard let deviceName = helperOptionValue(arguments: rest, name: "--device-name")? + .trimmingCharacters(in: .whitespacesAndNewlines), + !deviceName.isEmpty + else { + throw HelperError.invalidArgs("device-hub pose requires --device-name ") + } + guard let pose = helperOptionValue(arguments: rest, name: "--pose").flatMap(DeviceHubPose.init(argument:)) + else { + throw HelperError.invalidArgs("device-hub pose requires --pose ") + } + let control = pose.controlDescription + guard AXIsProcessTrusted() else { + throw HelperError.commandFailed( + "fold needs Accessibility permission to press the Device Hub pose control", + details: ["reason": "accessibility-permission", "permission": "accessibility"] + ) + } + guard let pid = deviceHubProcessIdentifier() else { + throw HelperError.commandFailed( + "Xcode Device Hub is not running", + details: ["reason": "device-hub-not-running", "bundleId": deviceHubBundleId] + ) + } + + let appElement = AXUIElementCreateApplication(pid) + let (window, reopened) = try deviceHubWindow(in: appElement, pid: pid, deviceName: deviceName) + let selected = try selectDevice(udid: udid, deviceName: deviceName, in: window, appElement: appElement) + let windowTitle = stringAttribute(window, attribute: kAXTitleAttribute as String) ?? "" + guard let button = awaitPoseButton(in: window, description: control) else { + throw HelperError.commandFailed( + "Device Hub shows no \(control) pose control for \(deviceName)", + details: [ + "reason": "device-hub-pose-control-missing", + "windowTitle": windowTitle, + "control": control, + ] + ) + } + let status = AXUIElementPerformAction(button, kAXPressAction as CFString) + guard status == .success else { + throw HelperError.commandFailed( + "Device Hub refused the \(control) pose press", + details: ["reason": "device-hub-press-failed", "status": "\(status.rawValue)"] + ) + } + return SuccessEnvelope( + data: DeviceHubPoseResponse( + pose: pose.rawValue, + control: control, + windowTitle: windowTitle, + reopened: reopened, + selected: selected + ) + ) +} + +/// Device Hub is launched through a trampoline, so LaunchServices registers it with no process +/// identifier (`NSRunningApplication.processIdentifier` is -1) and an accessibility element built +/// from that identifier is invalid. The process table is the only place its real pid appears. +private func deviceHubProcessIdentifier() -> pid_t? { + let executableSuffix = deviceHubExecutableSuffix + let byteCount = proc_listpids(UInt32(PROC_ALL_PIDS), 0, nil, 0) + guard byteCount > 0 else { return nil } + var pids = [pid_t](repeating: 0, count: Int(byteCount) / MemoryLayout.size + 64) + let filled = proc_listpids( + UInt32(PROC_ALL_PIDS), 0, &pids, Int32(pids.count * MemoryLayout.size) + ) + guard filled > 0 else { return nil } + var path = [CChar](repeating: 0, count: 4096) + for pid in pids.prefix(Int(filled) / MemoryLayout.size) where pid > 0 { + guard proc_pidpath(pid, &path, UInt32(path.count)) > 0 else { continue } + if String(cString: path).hasSuffix(executableSuffix) { + return pid + } + } + return nil +} + +/// A device window to drive: the one already showing this device when there is one, otherwise +/// any device window, since its sidebar can switch it to the device. A Device Hub with no window +/// at all — a simulator booted headlessly leaves it that way — is asked to reopen one. +private func deviceHubWindow( + in appElement: AXUIElement, + pid: pid_t, + deviceName: String +) throws -> (window: AXUIElement, reopened: Bool) { + if let window = preferredWindow(in: appElement, deviceName: deviceName) { + return (window, false) + } + try sendReopenEvent(to: pid) + let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) + while Date() < deadline { + Thread.sleep(forTimeInterval: deviceHubPoll) + if let window = preferredWindow(in: appElement, deviceName: deviceName) { + return (window, true) + } + } + throw HelperError.commandFailed( + "Device Hub shows no device window to drive", + details: ["reason": "device-hub-window-missing", "deviceName": deviceName] + ) +} + +private func preferredWindow(in appElement: AXUIElement, deviceName: String) -> AXUIElement? { + let candidates = windows(of: appElement) + return candidates.first { windowShows(deviceName: deviceName, $0) } ?? candidates.first +} + +private func windowShows(deviceName: String, _ window: AXUIElement) -> Bool { + return deviceHubWindowShows( + deviceName: deviceName, + title: stringAttribute(window, attribute: kAXTitleAttribute as String) + ) +} + +/// `kAEReopenApplication` is what the Dock sends when an app with no open windows is clicked, and +/// it is the one event Device Hub answers by restoring the device window. Sent to the process +/// directly, because LaunchServices cannot address a trampolined app by bundle identifier. +private func sendReopenEvent(to pid: pid_t) throws { + let target = NSAppleEventDescriptor(processIdentifier: pid) + let event = NSAppleEventDescriptor( + eventClass: AEEventClass(kCoreEventClass), + eventID: AEEventID(kAEReopenApplication), + targetDescriptor: target, + returnID: AEReturnID(kAutoGenerateReturnID), + transactionID: AETransactionID(kAnyTransactionID) + ) + do { + _ = try event.sendEvent(options: [.noReply], timeout: 5) + } catch { + let code = (error as NSError).code + throw HelperError.commandFailed( + "fold could not ask Device Hub to reopen its device window", + details: [ + "reason": code == -1743 ? "automation-permission" : "device-hub-reopen-failed", + "error": String(describing: error), + ] + ) + } +} + +/// Switches the window to the device through its sidebar row, whose accessibility identifier is +/// `TableRow.Device.` — the one place Device Hub exposes a device identity that two +/// simulators sharing a name cannot confuse. A window already titled with the device still gets +/// the selection when the row is visible, because the title alone cannot tell such twins apart. +private func selectDevice( + udid: String, + deviceName: String, + in window: AXUIElement, + appElement: AXUIElement +) throws -> Bool { + var shownSidebar = false + var row = deviceRow(udid: udid, in: window) + if row == nil, showSidebar(appElement: appElement) { + shownSidebar = true + let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) + while row == nil, Date() < deadline { + Thread.sleep(forTimeInterval: deviceHubPoll) + row = deviceRow(udid: udid, in: window) + } + } + defer { + if shownSidebar { _ = pressMenuItem(appElement: appElement, menu: "View", item: "Hide Sidebar") } + } + guard let row else { + if windowShows(deviceName: deviceName, window) { + return false + } + throw HelperError.commandFailed( + "Device Hub lists no device \(udid) in its sidebar", + details: ["reason": "device-hub-device-missing", "udid": udid, "deviceName": deviceName] + ) + } + let status = AXUIElementSetAttributeValue(row, kAXSelectedAttribute as CFString, kCFBooleanTrue) + guard status == .success else { + throw HelperError.commandFailed( + "Device Hub refused to select \(deviceName) in its sidebar", + details: ["reason": "device-hub-select-failed", "status": "\(status.rawValue)"] + ) + } + let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) + while !windowShows(deviceName: deviceName, window), Date() < deadline { + Thread.sleep(forTimeInterval: deviceHubPoll) + } + guard windowShows(deviceName: deviceName, window) else { + throw HelperError.commandFailed( + "Device Hub did not switch its window to \(deviceName)", + details: [ + "reason": "device-hub-select-unconfirmed", + "windowTitle": stringAttribute(window, attribute: kAXTitleAttribute as String) ?? "", + ] + ) + } + return true +} + +private func deviceRow(udid: String, in window: AXUIElement) -> AXUIElement? { + guard let label = findElement(root: window, depth: 0, where: { + stringAttribute($0, attribute: "AXIdentifier") == deviceHubDeviceRowIdentifier(udid: udid) + }) else { + return nil + } + var current: AXUIElement? = label + while let element = current { + if stringAttribute(element, attribute: kAXRoleAttribute as String) == "AXRow" { + return element + } + current = elementAttribute(element, attribute: kAXParentAttribute as String) + } + return nil +} + +private func showSidebar(appElement: AXUIElement) -> Bool { + return pressMenuItem(appElement: appElement, menu: "View", item: "Show Sidebar") +} + +private func pressMenuItem(appElement: AXUIElement, menu: String, item: String) -> Bool { + guard let menuBar = elementAttribute(appElement, attribute: kAXMenuBarAttribute as String) else { + return false + } + for menuBarItem in children(of: menuBar) + where stringAttribute(menuBarItem, attribute: kAXTitleAttribute as String) == menu { + for submenu in children(of: menuBarItem) { + for menuItem in children(of: submenu) + where stringAttribute(menuItem, attribute: kAXTitleAttribute as String) == item { + return AXUIElementPerformAction(menuItem, kAXPressAction as CFString) == .success + } + } + } + return false +} + +/// The action bar is rebuilt for the device the window shows, so right after a sidebar selection +/// the window already carries the new title while the pose controls are still being laid out. +/// The controls are therefore awaited, not looked up once. +private func awaitPoseButton(in window: AXUIElement, description: String) -> AXUIElement? { + let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) + while true { + if let button = poseButton(in: window, description: description) { + return button + } + guard Date() < deadline else { return nil } + Thread.sleep(forTimeInterval: deviceHubPoll) + } +} + +/// The pose controls sit in the window's action bar as `AXButton`s described by their preset +/// name; the simulated screen inside the same window is an iOS content group whose own buttons +/// carry app labels, never these three. +private func poseButton(in window: AXUIElement, description: String) -> AXUIElement? { + return findElement(root: window, depth: 0) { + stringAttribute($0, attribute: kAXRoleAttribute as String) == "AXButton" + && stringAttribute($0, attribute: kAXDescriptionAttribute as String) == description + } +} + +private func findElement( + root: AXUIElement, + depth: Int, + where matches: (AXUIElement) -> Bool +) -> AXUIElement? { + if depth > 14 { + return nil + } + for child in children(of: root) { + if matches(child) { + return child + } + if stringAttribute(child, attribute: kAXSubroleAttribute as String) == "iOSContentGroup" { + continue + } + if let nested = findElement(root: child, depth: depth + 1, where: matches) { + return nested + } + } + return nil +} + +func helperOptionValue(arguments: [String], name: String) -> String? { + guard let index = arguments.firstIndex(of: name), arguments.indices.contains(index + 1) else { + return nil + } + return arguments[index + 1] +} diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift index cc9be1260a..21b10d0bff 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift @@ -127,6 +127,8 @@ struct AgentDeviceMacOSHelper { return try handleScreenshot(arguments: Array(arguments.dropFirst())) case "audio-probe": return try handleAudioProbe(arguments: Array(arguments.dropFirst())) + case "device-hub": + return try handleDeviceHub(arguments: Array(arguments.dropFirst())) default: throw HelperError.invalidArgs("unknown command: \(command)") } diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubPoseTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubPoseTests.swift new file mode 100644 index 0000000000..c6ef1b49e5 --- /dev/null +++ b/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubPoseTests.swift @@ -0,0 +1,30 @@ +import XCTest + +@testable import AgentDeviceMacOSDeviceHub + +final class DeviceHubPoseTests: XCTestCase { + func testEveryPoseNamesTheActionBarControlDeviceHubDescribes() { + XCTAssertEqual(DeviceHubPose(argument: "closed")?.controlDescription, "Closed") + XCTAssertEqual(DeviceHubPose(argument: "book")?.controlDescription, "Book") + XCTAssertEqual(DeviceHubPose(argument: "OPEN")?.controlDescription, "Open") + XCTAssertNil(DeviceHubPose(argument: "half-open"), "the helper takes Device Hub's names, not the CLI's") + XCTAssertEqual(Set(DeviceHubPose.allCases.map(\.controlDescription)), ["Closed", "Book", "Open"]) + } + + // Device Hub titles the window ` – iOS 27.1`; a simulator whose name merely starts + // with another's must not claim that window. + func testWindowTitleMatchesTheDeviceNameAsAPrefixWord() { + XCTAssertTrue(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo – iOS 27.1")) + XCTAssertTrue(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo")) + XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo Lab – iOS 27.1")) + XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: "bench-golden-v1 – iOS 27.0")) + XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: nil)) + } + + func testSidebarRowIsKeyedByUdid() { + XCTAssertEqual( + deviceHubDeviceRowIdentifier(udid: "4F879835-4AB3-4046-B033-5AB769209DD4"), + "TableRow.Device.4F879835-4AB3-4046-B033-5AB769209DD4" + ) + } +} diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index e2c8577676..26e2393899 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -19,8 +19,8 @@ device is in**. The first is answered by an official host API; the second only b | Panel power is ambiguous (zero or several lit panels) | Still name a panel — the `primary` one — emit `apple_display_capture_ambiguous`, and report the pose as `unknown` | | Device has one integrated panel | Keep the pre-panel behavior exactly: no display flag, no pose, unchanged scale probe | | Density normalization | Use the captured panel's own `pointScale`; a runner-fallback capture keeps the scale probe because `XCUIScreen.main` may be a different panel | -| Pose must be reported | Report `closed`, `fully-open`, or `unknown` from panel power, and never narrower | -| A pose change is requested | Refuse in guidance, not by inventing a command: no official host API sets pose | +| Pose must be reported | From panel power alone, report `closed`, `fully-open`, or `unknown`, and never narrower; `fold` reports the exact pose because it reads the hinge angle | +| A pose change is requested | `agent-device fold `: press the pose control in the Device Hub window through macOS accessibility, then read the hinge angle back from CoreDevice until it agrees; refuse the pose if it never does | | An external display is attached | It is not a panel: it never makes the device multi-screen and never produces a pose | | CoreDevice cannot answer | Return an unresolved inventory and keep the single-panel capture path; a missing host feature is not a capture failure | @@ -85,7 +85,37 @@ fix, and no screen-selection flag is warranted. `SIMULATOR_MAINSCREEN_SCALE` is likewise fixed to one panel while the captured panel can be the other, so density normalization now takes `pointScale` from the panel that was captured. -## Pose is derived, and official control does not exist +## Pose control: Device Hub's control, CoreDevice's verdict + +`agent-device fold` sets the pose, and the split above still holds: the press is not evidence, +the read-back is. The pieces, each of which was checked on the shipping 27.1 toolchain: + +| Piece | Finding | +| --- | --- | +| Who sets the pose | Device Hub's `CoreDevicePopDeviceKitExtension` (the V68 device view with its `poses` action bar) hands a "vendor defined" orientation-control payload to `CoreDevicePopCoreDeviceExtension`, which sends it through CoreDevice's private HID channel. No CLI, `simctl`, `devicectl`, or XCUITest surface reaches that channel | +| The public seam | The action bar's pose controls are ordinary `AXButton`s described `Closed`, `Book`, and `Open` in the Device Hub window; the simulated screen inside the same window is an `iOSContentGroup` with the app's own nodes. The earlier finding that the device surface exposes "zero accessibility nodes" was an artifact of System Events, which sees Device Hub with pid 0 because the app is launched through a trampoline; an `AXUIElement` built from the real pid works | +| Reading the pose | `xcrun devicectl device motion hinge-angle --device ` streams the hinge angle for the Duo simulator (`Range:0-180°`): Closed 0°, Book 130°, Open 180°. The stream does not end when `--session-timeout` elapses, so one read is bounded by devicectl's own `--timeout`, whose smallest accepted value is 5 seconds; the sample it printed before aborting itself is the reading | +| Device identity | Device Hub titles the window ` – iOS 27.1`, which two simulators sharing a name cannot distinguish. Its sidebar rows carry `AXIdentifier` `TableRow.Device.`, and setting `AXSelected` on a row switches the window to that device, so `fold` selects by UDID and only then presses | +| No window | A simulator booted headlessly leaves Device Hub running with no window. LaunchServices cannot address the trampolined process by bundle id (`open -b`, `NSRunningApplication.activate` do nothing), but a `kAEReopenApplication` event sent to the pid restores the device window, the same event a Dock click sends | + +The rule this yields: `closed` and `open` are the hinge's end stops, so one read at the stop is +the pose; every other angle is `half-open`, including the ones the hinge sweeps through on its way +somewhere else, so `half-open` is reported once two consecutive reads agree within 0.5°, or when +the four-read budget ends while the hinge still reads `half-open` — a refusal never names the pose +that was asked for. Each read takes the last sample the five-second stream printed, so a moving +hinge is reported where it is now. The live run that fixed the settle rule read 175.1° one stream +after pressing Book and 130° two streams later. A hinge whose last reading is some other pose is +refused as `fold-pose-unverified` with the angle CoreDevice still reports; the response of a verified pose carries the angle and the lit +panel's point size, because the point size is what tells an agent its refs are stale. + +Requirements the command states in its own errors: Accessibility permission for the host +(`accessibility-permission`), a running Device Hub (`fold` launches it in the background the way +`open` does), a device window it can reopen (`device-hub-window-missing`), and a sidebar row for +the UDID (`device-hub-device-missing`). A single-panel simulator is refused before anything is +pressed (`single-panel-device`), and the leaf fact refuses physical devices and every non-iPhone +simulator OS. + +## Pose is derived from panel power, and official control does not exist Apple ships fold state as an **app-side, read-only** API: `UIHinge.status` (`.closed`/`.partiallyOpen`/`.fullyOpen`) observed through `UIHingeInteraction`, and SwiftUI @@ -117,22 +147,25 @@ claims: usable: it is still not an API, and the Device Hub device surface exposes zero accessibility nodes. -Consequences that are now policy: no `fold`/`unfold`/`half-unfold` command, because a command that -cannot do the thing is worse than no command, and because a private per-guest XPC channel is exactly -the kind of undocumented hook that breaks without notice. Guidance in `agent-device help foldable` -tells agents pose is operator-controlled, and a derived `fully-open` verdict is documented to cover -Apple's `fullyOpen` **and** `partiallyOpen` — panel power cannot separate them, so only an in-app -`UIHinge.status` read can. +Consequences that are now policy: no private per-guest XPC channel is driven, because an undocumented +hook is exactly the kind that breaks without notice; the one host control that exists is Device +Hub's own, and `fold` drives it through the accessibility API and trusts only the CoreDevice +read-back (see the section above). A pose derived from panel power alone is still documented to +cover Apple's `fullyOpen` **and** `partiallyOpen` — panel power cannot separate them, so only the +hinge angle or an in-app `UIHinge.status` read can. ## Refuted alternatives - **Pose commands backed by `simctl io screenConfig power`.** Rejected: it does not move `UIHinge.status`, so the app under test would not behave as folded. It would produce green tests of a state the device is not in. -- **Pose commands backed by macOS UI automation of Device Hub.** Rejected: the Device Hub device - surface reports zero accessibility nodes, so it is coordinate-only and permission-bound - (Screen Recording), and no Xcode framework exposes the control it would press. Kept as a - documented possibility in help, not as shipped automation. +- **Pose commands backed by coordinate clicks on Device Hub.** Rejected: a coordinate press is + blind to which window and which device it lands on and needs Screen Recording to aim. The + accessibility press `fold` uses names the control, the window, and the device row, needs only + Accessibility permission, and is still not trusted on its own: the hinge read-back is. +- **A `fold` that reports the pose it requested.** Rejected: the press is dispatched to whatever + Device Hub window is frontmost for that device, and a press on the wrong window succeeds + silently. Only the CoreDevice hinge angle says the device moved. - **Luma or content heuristics to pick the lit panel.** Rejected: a black screenshot is legitimate content elsewhere, and the repo already forbids deciding on pixels when a typed fact exists. CoreDevice reports `active`/`backlightState` directly. @@ -145,8 +178,8 @@ Apple's `fullyOpen` **and** `partiallyOpen` — panel power cannot separate them ## Consequences for agents A pose change moves the app to a different panel with different point size, so refs and -coordinates do not survive it. `agent-device help foldable` states this and states that agents must -report which poses remain unverified rather than assume a pose was set. +coordinates do not survive it. `agent-device help foldable` states this, `fold` says so in its own +message, and agents fold to each pose a task names and re-snapshot rather than assume one. `simctl io recordVideo` has the same implicit-display default as `screenshot`, so recording names the lit panel through the same resolver. On an open Duo the 27.1 toolchain accepts the panel name @@ -197,6 +230,8 @@ replayed at the new point size, which is the pose-change rule working as designe - **Quarter-turn detection.** Both Duo panels report `currentOrientation: rot90`, and no available path rotates a foldable, so the orientation half of the inventory is carried but never exercised against a changed value. -- **Pose control.** No official host API sets the hinge angle, so both poses above depend on an - operator opening or closing the device in Device Hub. +- **Pose control on a second Device Hub instance.** `fold` drives the first `DeviceHub` process in + the process table. Two Xcodes each running a Device Hub is not a state this was verified in. +- **Physical foldables.** Device Hub poses simulators only; the leaf fact refuses a physical device, + and the hinge stream on one was not exercised. diff --git a/docs/agents/device-verification.md b/docs/agents/device-verification.md index 92194a7d8a..e802cba382 100644 --- a/docs/agents/device-verification.md +++ b/docs/agents/device-verification.md @@ -68,8 +68,13 @@ toolchain per command: - Panels: that command lists each integrated panel with `backlightState`. Only the lit panel is capturable — a capture of the dark panel exits 0 and writes an all-black PNG. -- Pose is not scriptable. Ask the operator to fold or open the device in Device Hub, then - re-snapshot; refs and coordinates do not survive the pose change. +- Pose: `agent-device fold closed|half-open|open` presses the Device Hub pose control and reads the + hinge back through `devicectl device motion hinge-angle`; re-snapshot afterwards, because refs + and coordinates do not survive the pose change. Expect 10-16s per fold. The host needs + Accessibility permission, and the command reopens Device Hub's window and selects the simulator + by UDID itself. To read the angle by hand: + `xcrun devicectl device motion hinge-angle --device --session-timeout 1 --timeout 5` + (the stream never ends on its own; 5 is the smallest timeout devicectl accepts). - When a recording must show touches, assume it cannot. The touch-overlay exporter loses the track geometry whenever it has touch events to draw — `220x480` on a plain iPhone 17 as well as on the inner panel — and returns all-black frames on long clips, always with exit 0. Record with diff --git a/packages/command-registry/src/__tests__/command-result.test.ts b/packages/command-registry/src/__tests__/command-result.test.ts index ed08432314..a576bd6ba3 100644 --- a/packages/command-registry/src/__tests__/command-result.test.ts +++ b/packages/command-registry/src/__tests__/command-result.test.ts @@ -151,6 +151,7 @@ test('CommandResultMap is seeded only from already-existing contract result type | 'orientation' | 'app-switcher' | 'action-button' + | 'fold' | 'clipboard' | 'appstate' | 'keyboard' diff --git a/packages/command-registry/src/command-result.ts b/packages/command-registry/src/command-result.ts index 2b857980b8..11f9d271e9 100644 --- a/packages/command-registry/src/command-result.ts +++ b/packages/command-registry/src/command-result.ts @@ -13,6 +13,7 @@ import type { ActionButtonCommandResult, AppSwitcherCommandResult, BackCommandResult, + FoldCommandResult, HomeCommandResult, OrientationCommandResult, TvRemoteCommandResult, @@ -68,6 +69,7 @@ export interface CommandResultMap { doctor: DoctorCommandResult; fill: FillCommandResponseData; find: FindCommandResponseData; + fold: FoldCommandResult; home: HomeCommandResult; hover: HoverCommandResponseData; keyboard: KeyboardCommandResult; diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index bf16a91a79..bf406ce22f 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -15,6 +15,7 @@ import { import { resolveWaitBudgetMs } from './wait-positionals.ts'; import { DEFAULT_TIMEOUT_POLICY, + FOLD_TIMEOUT_POLICY, INSTALL_REQUEST_TIMEOUT_MS, LEASE_ALLOCATE_REQUEST_TIMEOUT_MS, PREPARE_REQUEST_TIMEOUT_MS, @@ -40,6 +41,7 @@ import { inventoryUse } from '@agent-device/contracts/platform-module'; import { alertRuntimePlanUses, actionButtonRuntimeUse, + foldRuntimeUse, appEventRuntimeUse, appStateRuntimeUses, appSwitcherRuntimeUse, @@ -1428,6 +1430,18 @@ export const RAW_COMMAND_DESCRIPTORS = [ ...GENERIC_MUTATING_COMMAND_TRAITS, platformExecution: { kind: 'device-runtime', uses: [orientationRuntimeUse] }, }, + { + name: 'fold', + ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/system/index.ts'] as const } : {}), + catalog: { group: 'public' }, + frameworkTier: 'extended', + // Admission is the owner's `setFoldPose` fact, the same ADR 0019 §9 shape as `orientation`. + // A pose change moves the app to a different panel with a different point size, so the + // generic mutating traits' ref-frame invalidation is load-bearing here (ADR 0025). + ...GENERIC_MUTATING_COMMAND_TRAITS, + timeoutPolicy: FOLD_TIMEOUT_POLICY, + platformExecution: { kind: 'device-runtime', uses: [foldRuntimeUse] }, + }, { name: 'scroll', ...(ownerFilesEnabled ? { ownerFiles: ['src/daemon/scroll-runtime.ts'] as const } : {}), diff --git a/packages/command-registry/src/timeout-policy.ts b/packages/command-registry/src/timeout-policy.ts index 9cfb3383a6..366c731fb5 100644 --- a/packages/command-registry/src/timeout-policy.ts +++ b/packages/command-registry/src/timeout-policy.ts @@ -42,6 +42,19 @@ export const DEFAULT_TIMEOUT_POLICY: CommandTimeoutPolicy = { onTimeout: 'reset-daemon', }; +/** + * `fold` spends up to four bounded CoreDevice hinge reads (`IOS_HINGE_ANGLE_TIMEOUT_MS` each on a + * wedged host) after one macOS helper press with its own 30s deadline, which can sum past the + * standard envelope; the envelope covers that worst case with the usual margin. + */ +const FOLD_REQUEST_TIMEOUT_MS = 150_000; + +export const FOLD_TIMEOUT_POLICY: CommandTimeoutPolicy = { + budget: { source: 'none' }, + envelopeMs: FOLD_REQUEST_TIMEOUT_MS, + onTimeout: 'reset-daemon', +}; + type BoundedTimeoutPolicy = CommandTimeoutPolicy & { envelopeMs: number }; type FlagTimeoutBudget = Extract; type RequestTimeoutInput = Readonly<{ diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 2ce9fa24f5..38a33e18aa 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -188,6 +188,10 @@ "types": "./src/element-text-runtime.ts", "default": "./src/element-text-runtime.ts" }, + "./fold-runtime": { + "types": "./src/fold-runtime.ts", + "default": "./src/fold-runtime.ts" + }, "./focus-runtime": { "types": "./src/focus-runtime.ts", "default": "./src/focus-runtime.ts" diff --git a/packages/contracts/src/client-system.ts b/packages/contracts/src/client-system.ts index aa2acd1805..de53f8a736 100644 --- a/packages/contracts/src/client-system.ts +++ b/packages/contracts/src/client-system.ts @@ -5,7 +5,7 @@ import type { BackMode } from './back-mode.ts'; import type { SelectorSnapshotCommandOptions } from './client-capture.ts'; import type { DeviceCommandBaseOptions } from './client-connection.ts'; import type { SettleCommandOptions } from './client-gesture.ts'; -import type { DeviceRotation } from './device-rotation.ts'; +import type { DeviceRotation, FoldPose } from './device-rotation.ts'; import type { TvRemoteButton } from './tv-remote.ts'; export type WaitCommandTarget = @@ -90,6 +90,10 @@ export type OrientationCommandOptions = DeviceCommandBaseOptions & { orientation: DeviceRotation; }; +export type FoldCommandOptions = DeviceCommandBaseOptions & { + pose: FoldPose; +}; + export type AppSwitcherCommandOptions = DeviceCommandBaseOptions; export type ActionButtonCommandOptions = DeviceCommandBaseOptions; diff --git a/packages/contracts/src/device-rotation-fold-pose.test.ts b/packages/contracts/src/device-rotation-fold-pose.test.ts new file mode 100644 index 0000000000..3702fc1d27 --- /dev/null +++ b/packages/contracts/src/device-rotation-fold-pose.test.ts @@ -0,0 +1,37 @@ +import { expect, test } from 'vitest'; +import { FOLD_POSES, foldPoseForHingeAngle, parseFoldPose } from './device-rotation.ts'; + +test('parses the three poses and the aliases an agent is likely to type', () => { + expect(parseFoldPose('closed')).toBe('closed'); + expect(parseFoldPose('Folded')).toBe('closed'); + expect(parseFoldPose('half-open')).toBe('half-open'); + expect(parseFoldPose('book')).toBe('half-open'); + expect(parseFoldPose('half-unfolded')).toBe('half-open'); + expect(parseFoldPose('open')).toBe('open'); + expect(parseFoldPose('unfolded')).toBe('open'); + expect(parseFoldPose(' OPEN ')).toBe('open'); +}); + +test('refuses a missing or unknown pose with the usage the CLI prints', () => { + expect(() => parseFoldPose(undefined)).toThrow( + expect.objectContaining({ + code: 'INVALID_ARGS', + message: expect.stringContaining('closed|half-open|open'), + }), + ); + expect(() => parseFoldPose('sideways')).toThrow( + expect.objectContaining({ code: 'INVALID_ARGS', message: expect.stringContaining('sideways') }), + ); +}); + +test('reads a pose from the hinge angle with closed and open as the two fixed points', () => { + // The Device Hub presets measured on the iOS 27.1 Duo: Closed 0°, Book 130°, Open 180°. + expect(foldPoseForHingeAngle(0)).toBe('closed'); + expect(foldPoseForHingeAngle(130)).toBe('half-open'); + expect(foldPoseForHingeAngle(180)).toBe('open'); + // A hinge caught mid-animation is partially open, which is exactly why the verifier polls. + expect(foldPoseForHingeAngle(95.7)).toBe('half-open'); + expect(foldPoseForHingeAngle(166)).toBe('half-open'); + expect(foldPoseForHingeAngle(Number.NaN)).toBeUndefined(); + for (const pose of FOLD_POSES) expect(FOLD_POSES).toContain(pose); +}); diff --git a/packages/contracts/src/device-rotation.ts b/packages/contracts/src/device-rotation.ts index c32ad4ab1b..e913d4b13d 100644 --- a/packages/contracts/src/device-rotation.ts +++ b/packages/contracts/src/device-rotation.ts @@ -59,3 +59,63 @@ export function parseDeviceRotation(input: string | undefined): DeviceRotation { ); } } + +// ---- Hinge pose ------------------------------------------------------------------------------ +// Lives beside the rotation vocabulary because every entry that reads one device pose already +// evaluates this module; a module of its own would join the eager closure of entries that can +// never pose a hinge (the eager-closure budgets in scripts/__tests__/eager-closure-budgets.ts). + +/** + * The three hinge poses a foldable Apple device can be put in, named after what an agent sees + * rather than after Apple's `UIHinge.Status` cases: `closed` lights the outer panel only, + * `half-open` and `open` light the inner panel. Device Hub calls them Closed, Book, and Open; + * `UIHinge.Status` calls them `.closed`, `.partiallyOpen`, and `.fullyOpen`. + */ +export const FOLD_POSES = ['closed', 'half-open', 'open'] as const; +export type FoldPose = (typeof FOLD_POSES)[number]; + +export const FOLD_POSE_USAGE = 'closed|half-open|open'; + +/** + * `half-open` is the only pose whose hinge angle is not a fixed point: Device Hub's Book preset + * measured 130° on the iOS 27.1 Duo, and Apple's own status calls every angle strictly between + * closed and fully open `partiallyOpen`. The verifier therefore reads the pose from the angle + * with the same open interval rather than pinning one preset value. + */ +export function foldPoseForHingeAngle(angleDegrees: number): FoldPose | undefined { + if (!Number.isFinite(angleDegrees)) return undefined; + if (angleDegrees <= FOLD_CLOSED_MAX_DEGREES) return 'closed'; + if (angleDegrees >= FOLD_OPEN_MIN_DEGREES) return 'open'; + return 'half-open'; +} + +const FOLD_CLOSED_MAX_DEGREES = 1; +const FOLD_OPEN_MIN_DEGREES = 179; + +export function parseFoldPose(input: string | undefined): FoldPose { + if (input === undefined) { + throw new AppError('INVALID_ARGS', `fold requires a pose argument. Use ${FOLD_POSE_USAGE}.`); + } + const normalized = input.trim().toLowerCase(); + switch (normalized) { + case 'closed': + case 'close': + case 'fold': + case 'folded': + return 'closed'; + case 'half-open': + case 'half': + case 'half-unfolded': + case 'partially-open': + case 'book': + return 'half-open'; + case 'open': + case 'unfold': + case 'unfolded': + case 'fully-open': + case 'flat': + return 'open'; + default: + throw new AppError('INVALID_ARGS', `Invalid fold pose: ${input}. Use ${FOLD_POSE_USAGE}.`); + } +} diff --git a/packages/contracts/src/facades/client.ts b/packages/contracts/src/facades/client.ts index a74d01cdf1..196eec8c94 100644 --- a/packages/contracts/src/facades/client.ts +++ b/packages/contracts/src/facades/client.ts @@ -116,6 +116,7 @@ export type { AppSwitcherCommandOptions, BackCommandOptions, ClipboardCommandOptions, + FoldCommandOptions, DoctorCommandOptions, HomeCommandOptions, KeyboardCommandOptions, diff --git a/packages/contracts/src/facades/device.ts b/packages/contracts/src/facades/device.ts index 88c5044c08..23b1ce5caa 100644 --- a/packages/contracts/src/facades/device.ts +++ b/packages/contracts/src/facades/device.ts @@ -31,11 +31,15 @@ export type { export { DEVICE_ROTATIONS, DEVICE_ROTATION_SURFACE_INDEX, + FOLD_POSES, + FOLD_POSE_USAGE, deviceRotationOrientation, deviceRotationSurfaceDegrees, + foldPoseForHingeAngle, parseDeviceRotation, + parseFoldPose, } from '../device-rotation.ts'; -export type { DeviceRotation } from '../device-rotation.ts'; +export type { DeviceRotation, FoldPose } from '../device-rotation.ts'; export type { BootCommandResult, ShutdownCommandResult } from '../device.ts'; export type { ProviderDeviceInstallOptions, diff --git a/packages/contracts/src/fold-runtime.ts b/packages/contracts/src/fold-runtime.ts new file mode 100644 index 0000000000..2aeab8c1ea --- /dev/null +++ b/packages/contracts/src/fold-runtime.ts @@ -0,0 +1,45 @@ +import type { FoldPose } from './device-rotation.ts'; +import type { RuntimeOperationFact } from './platform-runtime.ts'; + +/** + * Neutral intent for one hinge pose change. `pose` is already parsed by the caller + * (`parseFoldPose`); the operation names no command, request, session, or CLI flag, and it + * carries no runner metadata because no runner takes part in a fold. + */ +export type SetFoldPoseInput = Readonly<{ pose: FoldPose }>; + +/** + * The lit panel after the pose settled, in the points the next snapshot will use. Reported so an + * agent can see that the coordinate space changed without a second capture. + */ +export type FoldScreenReport = Readonly<{ + /** The CoreDevice display name of the panel the device now lights. */ + display: string; + widthPt: number; + heightPt: number; +}>; + +/** + * The owner's own closed result: the pose it verified on the device, the hinge angle that + * verification read, and the panel that ended up lit. An owner reports a pose only after reading + * it back, so there is no unconfirmed variant here — a pose the owner could not verify is an error. + */ +export type SetFoldPoseResult = Readonly<{ + pose: FoldPose; + hingeAngleDegrees: number; + screen?: FoldScreenReport; +}>; + +export type FoldRuntimeOperations = Readonly<{ + setFoldPose(input: SetFoldPoseInput): Promise; +}>; + +export type FoldRuntimeOperationFacts = Readonly<{ + setFoldPose: RuntimeOperationFact; +}>; + +export function foldRuntimeOperationFacts( + input: Readonly<{ fold: RuntimeOperationFact }>, +): FoldRuntimeOperationFacts { + return Object.freeze({ setFoldPose: input.fold }); +} diff --git a/packages/contracts/src/navigation.ts b/packages/contracts/src/navigation.ts index 1044dcddac..861efb3cce 100644 --- a/packages/contracts/src/navigation.ts +++ b/packages/contracts/src/navigation.ts @@ -1,5 +1,6 @@ import type { BackMode } from './back-mode.ts'; -import type { DeviceRotation } from './device-rotation.ts'; +import type { DeviceRotation, FoldPose } from './device-rotation.ts'; +import type { FoldScreenReport } from './fold-runtime.ts'; import type { SettleObservation } from './interaction.ts'; import type { TvRemoteButton } from './tv-remote.ts'; @@ -47,6 +48,22 @@ export type OrientationCommandResult = { warning?: string; }; +/** + * `fold` — `{ action: 'fold', pose, hingeAngleDegrees, screen?, message }`. + * + * Unlike `orientation`, there is no unconfirmed variant: the Apple owner reads the hinge angle + * back from CoreDevice after pressing the Device Hub pose control, and reports a pose only when + * that reading agrees with the request. `screen` names the panel the device lights afterwards, in + * points, because a pose change moves the app to a different coordinate space (ADR 0025). + */ +export type FoldCommandResult = { + action: 'fold'; + pose: FoldPose; + hingeAngleDegrees: number; + screen?: FoldScreenReport; + message: string; +}; + /** `app-switcher` — `{ action: 'app-switcher', message: 'Opened app switcher' }`. */ export type AppSwitcherCommandResult = { action: 'app-switcher'; diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index f3627735c1..acc8ea394d 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -22,6 +22,7 @@ import type { TypeTextRuntimeOperations } from './type-text-runtime.ts'; import type { ElementTextRuntimeOperations } from './element-text-runtime.ts'; import type { BackRuntimeOperations } from './back-runtime.ts'; import type { OrientationRuntimeOperations } from './orientation-runtime.ts'; +import type { FoldRuntimeOperations } from './fold-runtime.ts'; import type { TvRemoteRuntimeOperations } from './tv-remote-runtime.ts'; import type { KeyboardRuntimeOperations } from './keyboard-runtime.ts'; import type { ClipboardRuntimeOperations } from './clipboard-runtime.ts'; @@ -74,6 +75,7 @@ export type PlatformRuntimeOperations = AppLogRuntimeOperations & ElementTextRuntimeOperations & BackRuntimeOperations & OrientationRuntimeOperations & + FoldRuntimeOperations & TvRemoteRuntimeOperations & KeyboardRuntimeOperations & ClipboardRuntimeOperations & @@ -108,6 +110,7 @@ export const typeTextRuntimeUse = defineUse({ required: ['typeText'] }); export const backRuntimeUse = defineUse({ required: ['back'] }); export const homeRuntimeUse = defineUse({ required: ['home'] }); export const orientationRuntimeUse = defineUse({ required: ['setOrientation'] }); +export const foldRuntimeUse = defineUse({ required: ['setFoldPose'] }); export const tvRemoteRuntimeUse = defineUse({ required: ['tvRemote'] }); export const keyboardStatusUse = defineUse({ required: ['keyboardStatus'] }); export const keyboardDismissUse = defineUse({ required: ['keyboardDismiss'] }); diff --git a/packages/contracts/src/platform-runtime-unavailable.test.ts b/packages/contracts/src/platform-runtime-unavailable.test.ts index b5745a1e2d..3d146a4357 100644 --- a/packages/contracts/src/platform-runtime-unavailable.test.ts +++ b/packages/contracts/src/platform-runtime-unavailable.test.ts @@ -49,6 +49,7 @@ const UNAVAILABLE_FACTS: UnavailablePlatformRuntimeFacts = { keyboard: { available: false, reason: 'unsupported-provider-mode' }, clipboard: { available: false, reason: 'unsupported-provider-mode' }, systemButton: { available: false, reason: 'unsupported-provider-mode' }, + fold: { available: false, reason: 'unsupported-provider-mode' }, triggerAppEvent: { available: false, reason: 'unsupported-provider-mode' }, setSetting: { available: false, reason: 'unsupported-provider-mode' }, readAlert: { available: false, reason: 'unsupported-provider-mode' }, diff --git a/packages/contracts/src/platform-runtime-unavailable.ts b/packages/contracts/src/platform-runtime-unavailable.ts index aa57d5ded9..53b6dba509 100644 --- a/packages/contracts/src/platform-runtime-unavailable.ts +++ b/packages/contracts/src/platform-runtime-unavailable.ts @@ -58,6 +58,7 @@ export type UnavailablePlatformRuntimeFacts = Readonly<{ elementText: RuntimeOperationUnavailability; back: RuntimeOperationUnavailability; orientation: RuntimeOperationUnavailability; + fold: RuntimeOperationUnavailability; tvRemote: RuntimeOperationUnavailability; keyboard: RuntimeOperationUnavailability; clipboard: RuntimeOperationUnavailability; @@ -112,6 +113,7 @@ const UNAVAILABLE_CELLS = { elementText: true, back: true, orientation: true, + fold: true, tvRemote: true, keyboard: true, clipboard: true, @@ -231,6 +233,9 @@ export function createUnavailablePlatformRuntimeFacts( ...elementTextRuntimeOperationFacts({ readTextAtPoint: frozen.elementText }), ...backRuntimeOperationFacts({ back: frozen.back }), ...orientationRuntimeOperationFacts({ orientation: frozen.orientation }), + // Stated directly rather than through `foldRuntimeOperationFacts`, so this hub does not + // evaluate the fold contract for owners that never pose a hinge (eager-closure budgets). + setFoldPose: frozen.fold, ...tvRemoteRuntimeOperationFacts({ tvRemote: frozen.tvRemote }), ...keyboardRuntimeOperationFacts({ unsupported: frozen.keyboard }), ...clipboardRuntimeOperationFacts({ unsupported: frozen.clipboard }), diff --git a/packages/contracts/src/runtime-operation-names.ts b/packages/contracts/src/runtime-operation-names.ts index 76b9d921d3..7c4d4bbb74 100644 --- a/packages/contracts/src/runtime-operation-names.ts +++ b/packages/contracts/src/runtime-operation-names.ts @@ -74,6 +74,7 @@ export const RUNTIME_OPERATION_NAMES = [ 'screenRecordingStart', 'scrollDirection', 'sendPushNotification', + 'setFoldPose', 'setOrientation', 'setSetting', 'setViewport', diff --git a/packages/platform-android/src/runtime.test.ts b/packages/platform-android/src/runtime.test.ts index 138bc23cf5..7ad613a3bd 100644 --- a/packages/platform-android/src/runtime.test.ts +++ b/packages/platform-android/src/runtime.test.ts @@ -215,6 +215,26 @@ test('Android refuses the action-button fact on every kind', async () => { } }); +test('Android refuses the fold fact on every kind', async () => { + for (const runtimeDevice of [ + ANDROID_EMULATOR, + { ...ANDROID_EMULATOR, kind: 'device' as const }, + UNKNOWN_KIND_DEVICE, + ]) { + const binding = await bindOrdinary( + createAndroidPlatformRuntime(androidNavigationHost()), + runtimeDevice, + ); + // A foldable hinge is posed through Xcode Device Hub; no adb surface poses one. + expect(binding.facts.operations.setFoldPose).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'fold drives the hinge of a foldable iPhone simulator; the Android emulator posture control is not driven by agent-device yet.', + }); + expect(binding.operations.setFoldPose).toBeUndefined(); + } +}); + // R55 parity: the retired `clipboard` bucket was `ANDROID_ALL` (emulator/device/unknown) with no // Android admission closure, so `cmd clipboard get/set text` is admitted on every real kind and // refused only on the synthetic `simulator` row the bucket never listed. (`unknown` is the diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index e1905a5e14..74ecd1bbf5 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -96,6 +96,16 @@ const systemButtonUnavailable = Object.freeze({ reason: 'unsupported-platform-leaf', hint: 'Android has no key event for this system button.', } as const); +/** + * Foldable Android emulators do carry a posture control (the emulator console's `fold` and + * `posture` commands), but nothing in this project drives it yet, so the cell refuses on every + * kind rather than advertising a pose it cannot set. + */ +const foldUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'fold drives the hinge of a foldable iPhone simulator; the Android emulator posture control is not driven by agent-device yet.', +} as const); const headlessUnavailable = Object.freeze({ available: false, reason: 'unsupported-device-kind', @@ -363,6 +373,7 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor home: androidTouchFact(device), appSwitcher: androidTouchFact(device), }), + setFoldPose: foldUnavailable, // The deep link opens through `am start`, admitted wherever the retired `ANDROID_ALL` // bucket admitted it. ...appEventRuntimeOperationFacts({ triggerAppEvent: androidTouchFact(device) }), diff --git a/packages/platform-apple/src/core/__tests__/hinge-angle.test.ts b/packages/platform-apple/src/core/__tests__/hinge-angle.test.ts new file mode 100644 index 0000000000..109a5af3aa --- /dev/null +++ b/packages/platform-apple/src/core/__tests__/hinge-angle.test.ts @@ -0,0 +1,71 @@ +import { beforeEach, expect, test, vi } from 'vitest'; + +vi.mock('../tool-provider.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, runXcrun: vi.fn(async () => ({ exitCode: 0, stdout: '', stderr: '' })) }; +}); + +import { parseHingeAngleSample, readAppleHingeAngle } from '../hinge-angle.ts'; +import { runXcrun } from '../tool-provider.ts'; +import { IOS_TEST_SIMULATOR } from './apple-core-stub-helpers.ts'; + +const mockRunXcrun = vi.mocked(runXcrun); + +/** Verbatim stream output from a comma-decimal host, including the self-abort devicectl ends on. */ +const STREAM_OUTPUT = `Hinge angle monitoring started. 1 seconds remaining: +• +0,000s : Angle:180,0° Mech:180,0° Velocity:+0,0°/s AngleValid:Y VelocityValid:N Range:0-180° +ERROR: Command timeout of 5.0 seconds exceeded. Assuming command got stuck and aborting. +`; + +beforeEach(() => { + mockRunXcrun.mockReset(); +}); + +test('parses the human sample line under comma and dot decimal locales', () => { + expect(parseHingeAngleSample(STREAM_OUTPUT)).toBe(180); + // A hinge moving during the stream prints several samples; the last one is where it is now. + expect( + parseHingeAngleSample( + '• +0,000s : Angle:175,1° Mech:175,1°\n• +2,100s : Angle:140,0° Mech:140,0°\n• +4,000s : Angle:130,0° Mech:130,0°\n', + ), + ).toBe(130); + expect(parseHingeAngleSample('• +0.000s : Angle: 0.0° Mech: 0.0°')).toBe(0); + expect(parseHingeAngleSample('• +0,000s : Angle: 95,7° Mech: 95,7°')).toBe(95.7); + expect(parseHingeAngleSample('Hinge angle monitoring started.')).toBeUndefined(); +}); + +test('bounds the stream with the smallest devicectl timeout and reads the sample it printed', async () => { + // Exit code 2 is the deadline devicectl set for itself, not a failure of the read. + mockRunXcrun.mockResolvedValueOnce({ exitCode: 2, stdout: STREAM_OUTPUT, stderr: '' }); + + await expect(readAppleHingeAngle(IOS_TEST_SIMULATOR)).resolves.toBe(180); + + expect(mockRunXcrun).toHaveBeenCalledWith( + [ + 'devicectl', + 'device', + 'motion', + 'hinge-angle', + '--device', + IOS_TEST_SIMULATOR.id, + '--session-timeout', + '1', + '--timeout', + '5', + ], + expect.objectContaining({ allowFailure: true, timeoutMs: 20_000 }), + ); +}); + +test('refuses a stream that printed no sample, naming the toolchain gap', async () => { + mockRunXcrun.mockResolvedValueOnce({ + exitCode: 1, + stdout: '', + stderr: 'ERROR: Hinge angle monitoring is not available on this device.', + }); + + await expect(readAppleHingeAngle(IOS_TEST_SIMULATOR)).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: expect.objectContaining({ hint: expect.stringContaining('hinge-angle') }), + }); +}); diff --git a/packages/platform-apple/src/core/config.ts b/packages/platform-apple/src/core/config.ts index bdb870e699..efd7f58418 100644 --- a/packages/platform-apple/src/core/config.ts +++ b/packages/platform-apple/src/core/config.ts @@ -19,6 +19,16 @@ export const IOS_SIMULATOR_SCREENSHOT_TIMEOUT_MS = 20_000; // wedged CoreDevice must not spend the screenshot's own time and trip the // request-level daemon reset. Measured probe cost is ~0.2s. export const IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS = 5_000; +/** + * The smallest `--timeout` devicectl accepts. The hinge-angle stream never ends on its own, so + * one read costs exactly this long and the exec deadline below only guards a wedged CoreDevice. + */ +export const IOS_HINGE_ANGLE_STREAM_SECONDS = 5; +export const IOS_HINGE_ANGLE_TIMEOUT_MS = 20_000; +/** How many hinge reads a pressed Device Hub pose control gets to reach its pose before the pose is refused. */ +export const IOS_FOLD_POSE_SETTLE_ATTEMPTS = 4; +/** Two consecutive reads this close together mean the hinge has stopped moving. */ +export const IOS_FOLD_POSE_STABLE_DEGREES = 0.5; // CoreSimulator can briefly stall while it services the scale lookup immediately // after a keyboard transition. Keep this bounded below the full capture budget. diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index 3c983068bb..3e72731e74 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -91,7 +91,7 @@ const DISPLAYS_UNSUPPORTED_HINT = * every caller has a correct pre-inventory path, and a missing host feature must * not fail the user's capture. */ -async function queryAppleDisplayInventory( +export async function queryAppleDisplayInventory( device: DeviceInfo, options: { timeoutMs?: number; signal?: AbortSignal }, ): Promise { diff --git a/packages/platform-apple/src/core/hinge-angle.ts b/packages/platform-apple/src/core/hinge-angle.ts new file mode 100644 index 0000000000..d7d8a1531b --- /dev/null +++ b/packages/platform-apple/src/core/hinge-angle.ts @@ -0,0 +1,75 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { execFailureDetails } from '@agent-device/host-kit/command'; + +import { IOS_HINGE_ANGLE_STREAM_SECONDS, IOS_HINGE_ANGLE_TIMEOUT_MS } from './config.ts'; +import { runXcrun } from './tool-provider.ts'; + +/** + * `devicectl device motion hinge-angle` prints one sample line per reading, formatted for humans + * in the host locale (`Angle:180,0°` under a comma-decimal locale, `Angle: 0.0°` under a dot one). + * The JSON output carries no samples, so the human line is the only wire form there is. + */ +const HINGE_ANGLE_SAMPLE = /Angle:\s*(-?\d+(?:[.,]\d+)?)\s*°/g; + +const HINGE_ANGLE_UNSUPPORTED_HINT = + "This Xcode/CoreDevice toolchain does not stream 'devicectl device motion hinge-angle' for this device. Foldable poses can be verified only through that stream; update Xcode to a version that ships it."; + +/** + * Reads the hinge angle CoreDevice currently reports for one Apple device, in degrees: 0 is + * closed, 180 is fully open. + * + * The stream in the shipping toolchain does not end when its `--session-timeout` elapses, so the + * read is bounded by devicectl's own `--timeout`, which is the smallest value it accepts. The + * command therefore exits non-zero by design once the deadline passes; the last sample it + * printed before that is the answer. + */ +export async function readAppleHingeAngle( + device: DeviceInfo, + options: { signal?: AbortSignal } = {}, +): Promise { + const args = [ + 'devicectl', + 'device', + 'motion', + 'hinge-angle', + '--device', + device.id, + '--session-timeout', + '1', + '--timeout', + String(IOS_HINGE_ANGLE_STREAM_SECONDS), + ]; + const result = await runXcrun(args, { + allowFailure: true, + signal: options.signal, + timeoutMs: IOS_HINGE_ANGLE_TIMEOUT_MS, + }); + const angle = parseHingeAngleSample(result.stdout) ?? parseHingeAngleSample(result.stderr); + if (angle !== undefined) return angle; + throw new AppError( + 'COMMAND_FAILED', + 'CoreDevice reported no hinge angle sample', + execFailureDetails(result, { + cmd: 'xcrun', + args, + stdout: result.stdout, + stderr: result.stderr, + deviceId: device.id, + hint: HINGE_ANGLE_UNSUPPORTED_HINT, + }), + ); +} + +/** + * The freshest reading the stream printed before its deadline: the last sample line, not the + * first, so a hinge that moved during the five-second stream is reported where it is now. + */ +export function parseHingeAngleSample(output: string): number | undefined { + let angle: number | undefined; + for (const match of output.matchAll(HINGE_ANGLE_SAMPLE)) { + const parsed = Number.parseFloat(match[1]!.replace(',', '.')); + if (Number.isFinite(parsed)) angle = parsed; + } + return angle; +} diff --git a/packages/platform-apple/src/foldable/pose.test.ts b/packages/platform-apple/src/foldable/pose.test.ts new file mode 100644 index 0000000000..738a53da34 --- /dev/null +++ b/packages/platform-apple/src/foldable/pose.test.ts @@ -0,0 +1,208 @@ +import { beforeEach, expect, test, vi } from 'vitest'; + +vi.mock('../core/display-inventory.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, queryAppleDisplayInventory: vi.fn() }; +}); +vi.mock('../core/hinge-angle.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, readAppleHingeAngle: vi.fn() }; +}); +vi.mock('../core/simulator.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, openIosSimulatorApp: vi.fn(async () => {}) }; +}); +vi.mock('../os/macos/helper.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, runMacOsDeviceHubPoseAction: vi.fn() }; +}); +vi.mock('@agent-device/host-kit/diagnostics', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, emitDiagnostic: vi.fn() }; +}); + +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { + buildInventory, + queryAppleDisplayInventory, + type AppleDeviceDisplay, +} from '../core/display-inventory.ts'; +import { readAppleHingeAngle } from '../core/hinge-angle.ts'; +import { runMacOsDeviceHubPoseAction } from '../os/macos/helper.ts'; +import { setAppleFoldPose } from './pose.ts'; + +const mockInventory = vi.mocked(queryAppleDisplayInventory); +const mockHinge = vi.mocked(readAppleHingeAngle); +const mockPress = vi.mocked(runMacOsDeviceHubPoseAction); + +const duo: DeviceInfo = { + platform: 'apple', + appleOs: 'ios', + id: '4F879835-4AB3-4046-B033-5AB769209DD4', + name: 'iPhone Duo', + kind: 'simulator', + target: 'mobile', + booted: true, +}; + +function panel(overrides: Partial): AppleDeviceDisplay { + return { + name: 'LCD', + displayId: 1, + power: 'lit', + primary: true, + widthPx: 1398, + heightPx: 2034, + pointScale: 3, + currentOrientation: 'rot0', + integrated: true, + ...overrides, + }; +} + +/** The two Duo panels as CoreDevice reports them, with `lit` naming the panel the pose lights. */ +function duoInventory(lit: 'outer' | 'inner') { + return buildInventory([ + panel({ power: lit === 'outer' ? 'lit' : 'dark' }), + panel({ + name: 'LCD-1', + displayId: 3, + primary: false, + power: lit === 'inner' ? 'lit' : 'dark', + widthPx: 2007, + heightPx: 2853, + currentOrientation: 'rot90', + }), + ]); +} + +beforeEach(() => { + mockInventory.mockReset(); + mockHinge.mockReset(); + mockPress.mockReset(); + mockPress.mockResolvedValue({ + pose: 'open', + control: 'Open', + windowTitle: 'iPhone Duo – iOS 27.1', + reopened: false, + selected: true, + }); +}); + +test('presses the Device Hub control for the pose and reports the pose CoreDevice read back', async () => { + mockInventory + .mockResolvedValueOnce(duoInventory('outer')) + .mockResolvedValueOnce(duoInventory('inner')); + // The first read catches the hinge mid-animation; the verifier polls until it settles. + mockHinge.mockResolvedValueOnce(95.7).mockResolvedValueOnce(180); + + await expect(setAppleFoldPose(duo, 'open')).resolves.toEqual({ + pose: 'open', + hingeAngleDegrees: 180, + screen: { display: 'LCD-1', widthPt: 669, heightPt: 951 }, + }); + + expect(mockPress).toHaveBeenCalledWith({ + udid: duo.id, + deviceName: 'iPhone Duo', + pose: 'open', + signal: undefined, + }); + expect(mockHinge).toHaveBeenCalledTimes(2); +}); + +test('maps half-open onto the Book preset and reports it only once the hinge has stopped', async () => { + mockInventory + .mockResolvedValueOnce(duoInventory('inner')) + .mockResolvedValueOnce(duoInventory('inner')); + // A hinge on its way from open to Book sweeps through half-open angles; the live run read 175.1° + // one stream after the press. Only the repeated 130° is the preset. + mockHinge.mockResolvedValueOnce(175.1).mockResolvedValueOnce(130).mockResolvedValueOnce(130); + + await expect(setAppleFoldPose(duo, 'half-open')).resolves.toMatchObject({ + pose: 'half-open', + hingeAngleDegrees: 130, + }); + expect(mockPress).toHaveBeenCalledWith(expect.objectContaining({ pose: 'book' })); + expect(mockHinge).toHaveBeenCalledTimes(3); +}); + +test('reports half-open when the budget ends on it even though the hinge never came to rest', async () => { + mockInventory + .mockResolvedValueOnce(duoInventory('inner')) + .mockResolvedValueOnce(duoInventory('inner')); + mockHinge + .mockResolvedValueOnce(170) + .mockResolvedValueOnce(150) + .mockResolvedValueOnce(120) + .mockResolvedValueOnce(90); + + // A refusal must never name the pose that was requested: every one of these reads is + // half-open, so the last one is the answer, not a failure that cites it. + await expect(setAppleFoldPose(duo, 'half-open')).resolves.toMatchObject({ + pose: 'half-open', + hingeAngleDegrees: 90, + }); + expect(mockHinge).toHaveBeenCalledTimes(4); +}); + +test('refuses half-open only when the last reading is some other pose', async () => { + mockInventory.mockResolvedValueOnce(duoInventory('inner')); + mockHinge + .mockResolvedValueOnce(170) + .mockResolvedValueOnce(150) + .mockResolvedValueOnce(120) + .mockResolvedValueOnce(180); + + await expect(setAppleFoldPose(duo, 'half-open')).rejects.toMatchObject({ + details: expect.objectContaining({ + reason: 'fold-pose-unverified', + observedPose: 'open', + hingeAngleDegrees: 180, + }), + }); +}); + +test('refuses the pose when the hinge never reaches it, naming what CoreDevice still reports', async () => { + mockInventory.mockResolvedValueOnce(duoInventory('inner')); + mockHinge.mockResolvedValue(180); + + await expect(setAppleFoldPose(duo, 'closed')).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: expect.objectContaining({ + reason: 'fold-pose-unverified', + requestedPose: 'closed', + observedPose: 'open', + hingeAngleDegrees: 180, + }), + }); + expect(mockHinge).toHaveBeenCalledTimes(4); +}); + +test('refuses a single-panel simulator before pressing anything', async () => { + mockInventory.mockResolvedValueOnce(buildInventory([panel({})])); + + await expect(setAppleFoldPose({ ...duo, name: 'iPhone 17' }, 'open')).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: expect.objectContaining({ reason: 'single-panel-device' }), + }); + expect(mockPress).not.toHaveBeenCalled(); + expect(mockHinge).not.toHaveBeenCalled(); +}); + +test('refuses a physical device and an unreadable display table before pressing anything', async () => { + await expect(setAppleFoldPose({ ...duo, kind: 'device' }, 'open')).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + }); + mockInventory.mockResolvedValueOnce({ + displays: [], + multiScreen: false, + ambiguous: false, + unresolved: true, + }); + await expect(setAppleFoldPose(duo, 'open')).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: expect.objectContaining({ hint: expect.stringContaining('displays') }), + }); + expect(mockPress).not.toHaveBeenCalled(); +}); diff --git a/packages/platform-apple/src/foldable/pose.ts b/packages/platform-apple/src/foldable/pose.ts new file mode 100644 index 0000000000..2f75de6452 --- /dev/null +++ b/packages/platform-apple/src/foldable/pose.ts @@ -0,0 +1,160 @@ +import { foldPoseForHingeAngle, type FoldPose } from '@agent-device/contracts/device'; +import type { FoldScreenReport, SetFoldPoseResult } from '@agent-device/contracts/fold-runtime'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; + +import { IOS_FOLD_POSE_SETTLE_ATTEMPTS, IOS_FOLD_POSE_STABLE_DEGREES } from '../core/config.ts'; +import { + queryAppleDisplayInventory, + type AppleDeviceDisplay, + type AppleDisplayInventory, +} from '../core/display-inventory.ts'; +import { readAppleHingeAngle } from '../core/hinge-angle.ts'; +import { openIosSimulatorApp, requireSimulatorDevice } from '../core/simulator.ts'; +import { runMacOsDeviceHubPoseAction, type MacOsDeviceHubPose } from '../os/macos/helper.ts'; + +/** + * The Device Hub action-bar control each pose maps to. Device Hub labels its presets after the + * shape of the device (Closed, Book, Open); the command names them after what the app sees. + */ +const DEVICE_HUB_POSE_CONTROLS = { + closed: 'closed', + 'half-open': 'book', + open: 'open', +} as const satisfies Record; + +const FOLDABLE_REQUIRED_HINT = + 'fold drives the pose controls Xcode Device Hub shows for a foldable simulator such as iPhone Duo; this simulator reports one integrated panel, so it has no hinge to pose.'; + +const INVENTORY_REQUIRED_HINT = + "fold needs 'devicectl device info displays' to tell a foldable from a single-panel simulator; update Xcode to a version that ships the display-information feature."; + +/** + * Puts a foldable simulator into `pose` and verifies it did get there. + * + * No official host API sets a hinge pose (ADR 0025): Device Hub sends it to the simulator through + * a private CoreDevice channel, and the only public seam onto that channel is the pose control in + * Device Hub's own window. The press is therefore a macOS accessibility action on that control, + * and the truth of the outcome comes from CoreDevice, not from the press: the hinge angle is read + * back until it agrees with the request, and the pose is refused if it never does. + */ +export async function setAppleFoldPose( + device: DeviceInfo, + pose: FoldPose, + options: { signal?: AbortSignal } = {}, +): Promise { + requireSimulatorDevice(device, 'fold'); + const inventory = await queryAppleDisplayInventory(device, { signal: options.signal }); + requireFoldableInventory(device, inventory); + + // A headless boot leaves Device Hub unlaunched; the same launch `open` performs brings it up + // in the background, and the helper then drives whichever window it shows. + await openIosSimulatorApp({ deviceHub: true, background: true, signal: options.signal }); + const pressed = await runMacOsDeviceHubPoseAction({ + udid: device.id, + deviceName: device.name, + pose: DEVICE_HUB_POSE_CONTROLS[pose], + signal: options.signal, + }); + emitDiagnostic({ + level: 'info', + phase: 'apple_fold_pose_pressed', + data: { + deviceId: device.id, + pose, + control: pressed.control, + windowTitle: pressed.windowTitle, + reopened: pressed.reopened, + selected: pressed.selected, + }, + }); + + const hingeAngleDegrees = await awaitHingePose(device, pose, options.signal); + const litPanel = await readLitPanel(device, options.signal); + return { + pose, + hingeAngleDegrees, + ...(litPanel ? { screen: screenReport(litPanel) } : {}), + }; +} + +function requireFoldableInventory(device: DeviceInfo, inventory: AppleDisplayInventory): void { + if (inventory.unresolved) { + throw new AppError( + 'COMMAND_FAILED', + 'CoreDevice reported no display table for this simulator', + { + deviceId: device.id, + hint: INVENTORY_REQUIRED_HINT, + }, + ); + } + if (!inventory.multiScreen) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + `${device.name} is not a foldable simulator: fold requires more than one integrated panel`, + { deviceId: device.id, reason: 'single-panel-device', hint: FOLDABLE_REQUIRED_HINT }, + ); + } +} + +/** + * Reads the hinge until it reports the requested pose. Each read costs one bounded devicectl + * stream, so the attempt count is the whole settle budget: the Device Hub press animates the + * hinge, and a press that landed on some other device's window never moves this one. + * + * `closed` and `open` are the hinge's two end stops, so one read at the stop is the pose. Every + * other angle is `half-open`, including the ones a hinge sweeps through on its way somewhere + * else, so that pose is reported once two consecutive reads agree the hinge has stopped. The rule + * a refusal obeys is that it never names the pose that was asked for: when the budget ends while + * the hinge still reads `half-open`, that is the pose, settled or not, and it is reported. + */ +async function awaitHingePose( + device: DeviceInfo, + pose: FoldPose, + signal: AbortSignal | undefined, +): Promise { + let observed: number | undefined; + let previous: number | undefined; + for (let attempt = 1; attempt <= IOS_FOLD_POSE_SETTLE_ATTEMPTS; attempt += 1) { + signal?.throwIfAborted(); + previous = observed; + observed = await readAppleHingeAngle(device, { signal }); + if (foldPoseForHingeAngle(observed) !== pose) continue; + if (pose !== 'half-open') return observed; + if (previous !== undefined && Math.abs(observed - previous) <= IOS_FOLD_POSE_STABLE_DEGREES) { + return observed; + } + } + if (observed !== undefined && foldPoseForHingeAngle(observed) === pose) return observed; + throw new AppError( + 'COMMAND_FAILED', + `${device.name} did not reach the ${pose} pose: CoreDevice still reports a hinge angle of ${observed}°`, + { + deviceId: device.id, + reason: 'fold-pose-unverified', + requestedPose: pose, + observedPose: observed === undefined ? undefined : foldPoseForHingeAngle(observed), + hingeAngleDegrees: observed, + hint: 'The Device Hub pose control was pressed, but the hinge did not follow. If several Device Hub windows are titled with this device name, close the ones for other simulators so the press reaches this one, then retry.', + }, + ); +} + +async function readLitPanel( + device: DeviceInfo, + signal: AbortSignal | undefined, +): Promise { + const inventory = await queryAppleDisplayInventory(device, { signal }); + if (inventory.unresolved || inventory.ambiguous) return undefined; + return inventory.activeDisplay; +} + +function screenReport(display: AppleDeviceDisplay): FoldScreenReport { + return { + display: display.name, + widthPt: Math.round(display.widthPx / display.pointScale), + heightPt: Math.round(display.heightPx / display.pointScale), + }; +} diff --git a/packages/platform-apple/src/foldable/runtime.ts b/packages/platform-apple/src/foldable/runtime.ts new file mode 100644 index 0000000000..8067e8c01c --- /dev/null +++ b/packages/platform-apple/src/foldable/runtime.ts @@ -0,0 +1,50 @@ +import { + foldRuntimeOperationFacts, + type SetFoldPoseInput, +} from '@agent-device/contracts/fold-runtime'; +import type { RuntimeOperationFact } from '@agent-device/contracts/platform-runtime'; +import { whenAdmitted } from '@agent-device/contracts/platform-runtime'; +import { resolveDeviceAppleOs, type DeviceInfo } from '@agent-device/kernel/device'; + +import { setAppleFoldPose } from './pose.ts'; + +const available = Object.freeze({ available: true } as const); + +const foldKindUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', + hint: 'fold is supported on foldable iPhone simulators driven by Xcode Device Hub; a physical device is folded by hand.', +} as const); +const foldOsUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'fold poses the hinge of a foldable iPhone; tvOS, macOS, watchOS and visionOS simulators have no hinge.', +} as const); + +/** + * The simulator leaf that can carry a hinge: iPhone and iPad. Which *model* inside it actually + * folds is not in `DeviceInfo`, so the operation answers that from CoreDevice's display table and + * refuses a single-panel simulator with a typed `UNSUPPORTED_OPERATION`, the way the runner + * answers for the Action Button hardware. + */ +function appleFoldFact(device: DeviceInfo): RuntimeOperationFact { + if (device.kind !== 'simulator') return foldKindUnavailable; + const os = resolveDeviceAppleOs(device); + return os === 'ios' || os === 'ipados' ? available : foldOsUnavailable; +} + +/** The foldable cell: `setFoldPose`. */ +export function appleFoldableFacts(device: DeviceInfo) { + return foldRuntimeOperationFacts({ fold: appleFoldFact(device) }); +} + +/** Binds `setFoldPose` when {@link appleFoldableFacts} admitted it. */ +export function createAppleFoldableOperations(params: { device: DeviceInfo; signal: AbortSignal }) { + const { device, signal } = params; + return whenAdmitted(appleFoldableFacts(device).setFoldPose, () => ({ + setFoldPose: async (input: SetFoldPoseInput) => { + signal.throwIfAborted(); + return await setAppleFoldPose(device, input.pose, { signal }); + }, + })); +} diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index 1729e88513..922ab61ee4 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -462,6 +462,41 @@ export async function runMacOsPressAction( }); } +export type MacOsDeviceHubPose = 'closed' | 'book' | 'open'; + +/** + * Presses one pose control in the Xcode Device Hub window for a foldable simulator. The helper + * asks Device Hub to reopen a window when it shows none, switches that window to the device + * through the sidebar row keyed by the simulator's UDID, and only then presses the control; it + * reports which of those steps it had to take. + */ +export async function runMacOsDeviceHubPoseAction(options: { + udid: string; + deviceName: string; + pose: MacOsDeviceHubPose; + signal?: AbortSignal; +}): Promise<{ + pose: MacOsDeviceHubPose; + control: string; + windowTitle: string; + reopened: boolean; + selected: boolean; +}> { + return await runMacOsHelper( + [ + 'device-hub', + 'pose', + '--udid', + options.udid, + '--device-name', + options.deviceName, + '--pose', + options.pose, + ], + { signal: options.signal }, + ); +} + export async function runMacOsScreenshotAction( outPath: string, options: { surface?: SessionSurface; fullscreen?: boolean } = {}, diff --git a/packages/platform-apple/src/runner-demand.ts b/packages/platform-apple/src/runner-demand.ts index b3c8664352..ee2f8ac4f1 100644 --- a/packages/platform-apple/src/runner-demand.ts +++ b/packages/platform-apple/src/runner-demand.ts @@ -49,6 +49,7 @@ const APPLE_SIMULATOR_OPERATION_HOSTS: Readonly< readClipboard: 'simulator', writeClipboard: 'simulator', setSetting: 'simulator', + setFoldPose: 'simulator', // Observation: the AX bridge presents regular and raw trees; custom actions need XCTest. captureSnapshot: 'simulator', captureSnapshotWithoutActiveApp: 'simulator', diff --git a/packages/platform-apple/src/runtime.test.ts b/packages/platform-apple/src/runtime.test.ts index 56084bf141..fc717d51d9 100644 --- a/packages/platform-apple/src/runtime.test.ts +++ b/packages/platform-apple/src/runtime.test.ts @@ -255,6 +255,33 @@ test.each(Object.entries(leaves))( }, ); +test.each(Object.entries(leaves))( + 'classifies the fold fact for the %s leaf', + async (_name, device) => { + const binding = await createApplePlatformRuntime(platformRuntimeHostFixture()).bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + // A hinge can exist on the iPhone/iPad simulator leaf only: the macOS host is not a simulator, + // and no other simulator OS ships a foldable. Whether this simulator is actually a foldable is + // answered by the operation from CoreDevice's display table, not by the leaf fact. + const available = + device.kind === 'simulator' && (device.appleOs === 'ios' || device.appleOs === 'ipados'); + expectOperationAvailability(binding, 'setFoldPose', available); + if (!available) { + expect(binding.facts.operations.setFoldPose).toHaveProperty( + 'reason', + device.kind === 'simulator' ? 'unsupported-platform-leaf' : 'unsupported-device-kind', + ); + } + }, +); + /** * The Action Button is a physical control on iPhone and iPad leaves only. visionOS is the leaf that * separates this from `orientation`'s mobile-input reading: a headset has a Digital Crown and no diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 7be614a5d0..c427038f3f 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -66,6 +66,7 @@ import { } from './deployment/runtime.ts'; import { appleNavigationFacts, createAppleNavigationOperations } from './navigation/runtime.ts'; import { appleSystemFacts, createAppleSystemOperations } from './system/runtime.ts'; +import { appleFoldableFacts, createAppleFoldableOperations } from './foldable/runtime.ts'; import { bindAppleFindTextRuntime, bindAppleSnapshotRuntime } from './runtime-snapshot.ts'; import { createAppleSnapshotRoute } from './snapshot-route.ts'; @@ -309,6 +310,7 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR }), ...elementTextRuntimeOperationFacts({ readTextAtPoint: appleElementTextFact(device) }), ...appleNavigationFacts(device), + ...appleFoldableFacts(device), ...appleSystemFacts(device), ...audioProbeRuntimeOperationFacts({ capture: appleAudioProbeCaptureFact(device), @@ -440,6 +442,10 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR device: request.device, signal: request.scope.signal, }), + ...createAppleFoldableOperations({ + device: request.device, + signal: request.scope.signal, + }), ...whenAdmitted(facts.operations.ensureReady, () => ({ ensureReady: async () => await ensureAppleReady(host, request.device, request.scope.signal), diff --git a/packages/platform-harmonyos/src/runtime.test.ts b/packages/platform-harmonyos/src/runtime.test.ts index a214cfb3c5..1b7d0fc303 100644 --- a/packages/platform-harmonyos/src/runtime.test.ts +++ b/packages/platform-harmonyos/src/runtime.test.ts @@ -99,6 +99,8 @@ test.each([ // even on the kinds the hdc-driven navigation gate admits. expect(facts.operations.actionButton).toMatchObject({ available: false }); expect(binding.operations.actionButton).toBeUndefined(); + expect(facts.operations.setFoldPose).toMatchObject({ available: false }); + expect(binding.operations.setFoldPose).toBeUndefined(); // Public orientation and TV-remote operations remain unavailable unconditionally. expect(facts.operations.setOrientation).toEqual({ available: false, diff --git a/packages/platform-harmonyos/src/runtime.ts b/packages/platform-harmonyos/src/runtime.ts index cad66c125e..ebe524b5fa 100644 --- a/packages/platform-harmonyos/src/runtime.ts +++ b/packages/platform-harmonyos/src/runtime.ts @@ -278,6 +278,8 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor home: harmonyFocusFact(device), appSwitcher: harmonyFocusFact(device), }), + // HarmonyOS devices have no foldable hinge control for HDC to pose. + setFoldPose: harmonyPlatformLeafUnavailable, // HarmonyOS has no trigger-app-event implementation. ...appEventRuntimeOperationFacts({ triggerAppEvent: harmonyPlatformLeafUnavailable }), // The HDC-driven settings surface shares the interaction kind gate. diff --git a/packages/platform-linux/src/runtime.test.ts b/packages/platform-linux/src/runtime.test.ts index 4fc06649f8..9da74e6353 100644 --- a/packages/platform-linux/src/runtime.test.ts +++ b/packages/platform-linux/src/runtime.test.ts @@ -220,6 +220,8 @@ function expectLinuxNavigationAndKeyboardFacts( 'appSwitcher', // The Action Button is iPhone/iPad hardware; the Linux desktop has no equivalent control. 'actionButton', + // Nor does it have a foldable hinge to pose. + 'setFoldPose', // R57: the retired `trigger-app-event` descriptor declared `linux: {}` too. 'triggerAppEvent', // R58/R59: and so did `settings` and `alert`. diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index 3809d120a9..b17388c75b 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -199,6 +199,7 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts // `home` is the one system button with a desktop cell, declared below; the Linux interactor's // own `appSwitcher` throws unsupported, and no Linux leaf has hardware buttons. systemButton: linuxPlatformLeafUnavailable, + fold: linuxPlatformLeafUnavailable, // The retired `trigger-app-event` descriptor declared `linux: {}`. triggerAppEvent: linuxPlatformLeafUnavailable, // The retired `settings` descriptor declared `linux: {}` too. diff --git a/packages/platform-vega/src/runtime.ts b/packages/platform-vega/src/runtime.ts index e5ab36d2c9..5d6273b5d1 100644 --- a/packages/platform-vega/src/runtime.ts +++ b/packages/platform-vega/src/runtime.ts @@ -147,6 +147,10 @@ const systemButtonUnavailable = vegaUnavailable( 'unsupported-platform-leaf', 'System buttons other than home are not supported on Vega OS.', ); +const foldUnavailable = vegaUnavailable( + 'unsupported-platform-leaf', + 'fold is not supported on Vega OS.', +); const clipboardUnavailable = vegaUnavailable( 'unsupported-platform-leaf', 'clipboard is not supported on Vega OS.', @@ -192,6 +196,7 @@ function vegaFacts(device: DeviceInfo): RuntimeFacts tvRemote: tvRemoteUnavailable, clipboard: clipboardUnavailable, systemButton: systemButtonUnavailable, + fold: foldUnavailable, triggerAppEvent: appEventUnavailable, setSetting: settingsUnavailable, readAlert: alertUnavailable, diff --git a/packages/platform-web/src/runtime.test.ts b/packages/platform-web/src/runtime.test.ts index 96235b3d52..52ac3d8d8f 100644 --- a/packages/platform-web/src/runtime.test.ts +++ b/packages/platform-web/src/runtime.test.ts @@ -213,6 +213,8 @@ test('clipboard, the app switcher, app events, settings and alerts carry no web 'appSwitcher', // The Action Button is iPhone/iPad hardware with no web analogue at all. 'actionButton', + // A foldable hinge is posed through Xcode Device Hub; the web target has none. + 'setFoldPose', 'triggerAppEvent', // R58/R59: the retired `settings` and `alert` descriptors declared no web leaf either. 'setSetting', diff --git a/packages/platform-web/src/runtime.ts b/packages/platform-web/src/runtime.ts index 83853f75b9..e774f4285d 100644 --- a/packages/platform-web/src/runtime.ts +++ b/packages/platform-web/src/runtime.ts @@ -423,6 +423,7 @@ function webRuntimeFacts( profileReport: navigationUnavailable, }), ...systemButtonRuntimeOperationFacts({ unsupported: navigationUnavailable }), + setFoldPose: navigationUnavailable, ...appEventRuntimeOperationFacts({ triggerAppEvent: navigationUnavailable }), ...settingsRuntimeOperationFacts({ setSetting: navigationUnavailable }), ...alertRuntimeOperationFacts({ diff --git a/packages/provider-limrun/src/app-log-runtime.ts b/packages/provider-limrun/src/app-log-runtime.ts index defb1b8a34..eb070fb642 100644 --- a/packages/provider-limrun/src/app-log-runtime.ts +++ b/packages/provider-limrun/src/app-log-runtime.ts @@ -1,7 +1,10 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; import type { AppsFilter, ProviderPortReverseOptions } from '@agent-device/contracts/device'; import type { Interactor, RunnerContext } from '@agent-device/contracts/interactor-types'; -import { bindLimrunInteractionOperations } from './interaction-operations.ts'; +import { + LIMRUN_FOLD_UNAVAILABLE, + bindLimrunInteractionOperations, +} from './interaction-operations.ts'; import { bindAdmittedProviderInteractorOperations } from '@agent-device/contracts/interactor-operation-catalog'; import { AppError } from '@agent-device/kernel/errors'; import { isSupportedLimrunAppLogDevice, parseLimrunDeviceId } from './device.ts'; @@ -115,6 +118,7 @@ export function createLimrunPlatformRuntimeOwner( keyboard: liveSessionUnavailable, clipboard: liveSessionUnavailable, systemButton: liveSessionUnavailable, + fold: LIMRUN_FOLD_UNAVAILABLE, triggerAppEvent: liveSessionUnavailable, setSetting: liveSessionUnavailable, readAlert: liveSessionUnavailable, diff --git a/packages/provider-limrun/src/facts-runtime.ts b/packages/provider-limrun/src/facts-runtime.ts index ac4bc6a281..a68d22c991 100644 --- a/packages/provider-limrun/src/facts-runtime.ts +++ b/packages/provider-limrun/src/facts-runtime.ts @@ -22,6 +22,7 @@ import { limrunSettingsOperationFacts, limrunAlertOperationFacts, limrunSystemButtonOperationFacts, + limrunFoldOperationFacts, limrunClipboardOperationFacts, limrunNavigationOperationFacts, } from './interaction-operations.ts'; @@ -194,6 +195,7 @@ export function limrunAppLogFacts( ...limrunKeyboardOperationFacts(device), ...limrunClipboardOperationFacts(device), ...limrunSystemButtonOperationFacts(device), + ...limrunFoldOperationFacts(), ...limrunAppEventOperationFacts(device), ...limrunSettingsOperationFacts(device), ...limrunAlertOperationFacts(device), @@ -256,6 +258,7 @@ export function limrunAppLogRecoveryFacts( ...limrunKeyboardOperationFacts(device, liveSessionUnavailable), ...limrunClipboardOperationFacts(device, liveSessionUnavailable), ...limrunSystemButtonOperationFacts(device, liveSessionUnavailable), + ...limrunFoldOperationFacts(), ...limrunAppEventOperationFacts(device, liveSessionUnavailable), ...limrunSettingsOperationFacts(device, liveSessionUnavailable), ...limrunAlertOperationFacts(device, liveSessionUnavailable), diff --git a/packages/provider-limrun/src/interaction-operations.ts b/packages/provider-limrun/src/interaction-operations.ts index 411d2eb232..3d0e7cf5b9 100644 --- a/packages/provider-limrun/src/interaction-operations.ts +++ b/packages/provider-limrun/src/interaction-operations.ts @@ -341,6 +341,18 @@ export function limrunSystemButtonOperationFacts( }); } +/** A foldable hinge is posed through the host's Xcode Device Hub, which no Limrun session has. */ +export const LIMRUN_FOLD_UNAVAILABLE = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'fold poses a foldable iPhone simulator through Xcode Device Hub on the host, which no Limrun session exposes.', +} as const); + +/** The fold refusal both Limrun legs share. */ +export function limrunFoldOperationFacts() { + return Object.freeze({ setFoldPose: LIMRUN_FOLD_UNAVAILABLE }); +} + /** * `trigger-app-event` is the one system leaf both direct-session legs genuinely serve: each * implements `open`, and a deep link is exactly what that method routes (`openUrl` on iOS, the diff --git a/packages/provider-webdriver/src/platform-runtime.ts b/packages/provider-webdriver/src/platform-runtime.ts index 74020d5f01..bebc8ae902 100644 --- a/packages/provider-webdriver/src/platform-runtime.ts +++ b/packages/provider-webdriver/src/platform-runtime.ts @@ -241,6 +241,13 @@ const systemButtonUnavailable = Object.freeze({ hint: 'No WebDriver backend presses this system button.', } as const); +/** No WebDriver `mobile:` script poses a foldable hinge; the refusal is unconditional like the Action Button's. */ +const foldUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'fold poses a foldable iPhone simulator through Xcode Device Hub, which no WebDriver backend exposes.', +} as const); + /** * The WebDriver interactor's own `setSetting` always throws unsupported (its capability map * declares `settings: unsupported`), so this cell is unavailable unconditionally rather than @@ -531,6 +538,7 @@ function webDriverFacts( keyboard: inactiveSession, clipboard: inactiveSession, systemButton: inactiveSession, + fold: foldUnavailable, triggerAppEvent: inactiveSession, setSetting: inactiveSession, readAlert: inactiveSession, @@ -573,6 +581,7 @@ function webDriverFacts( keyboard: keyboardUnavailable, clipboard: clipboardUnavailable, systemButton: systemButtonUnavailable, + fold: foldUnavailable, triggerAppEvent: appEventUnavailable, setSetting: settingsUnavailable, readAlert: alertUnavailable, @@ -668,6 +677,7 @@ function webDriverFacts( home: declared('home', homeUnavailable), appSwitcher: declared('appSwitcher', appSwitcherUnavailable), }), + setFoldPose: foldUnavailable, // The deep link opens through the same reachable interactor `open` every lifecycle command // drives on this provider. ...appEventRuntimeOperationFacts({ diff --git a/packages/session-journal/src/session-event-action-presentation.ts b/packages/session-journal/src/session-event-action-presentation.ts index 17e165d157..ee196cf3b7 100644 --- a/packages/session-journal/src/session-event-action-presentation.ts +++ b/packages/session-journal/src/session-event-action-presentation.ts @@ -1,5 +1,5 @@ import type { SessionAction } from '@agent-device/contracts/session'; -import { DEVICE_ROTATIONS } from '@agent-device/contracts/device'; +import { DEVICE_ROTATIONS, FOLD_POSES } from '@agent-device/contracts/device'; import { BACK_MODES } from '@agent-device/contracts/back-mode'; import { TV_REMOTE_BUTTONS } from '@agent-device/contracts/tv-remote'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; @@ -41,6 +41,8 @@ export function buildStructuredActionSummary(action: SessionAction): string | un return 'Pressed Action Button'; case PUBLIC_COMMANDS.orientation: return buildOrientationActionSummary(result); + case PUBLIC_COMMANDS.fold: + return buildFoldActionSummary(result); case PUBLIC_COMMANDS.viewport: return buildViewportActionSummary(result); case PUBLIC_COMMANDS.clipboard: @@ -86,6 +88,8 @@ export function buildStructuredActionDetails(action: SessionAction): Record): string { return mode === 'system' ? 'Went back using system navigation' : 'Went back'; } +function buildFoldActionSummary(result: Record): string { + const pose = readEnum(result.pose, FOLD_POSES); + return pose ? `Folded to ${pose}` : 'Changed fold pose'; +} + function buildOrientationActionSummary(result: Record): string { const orientation = readEnum(result.orientation, DEVICE_ROTATIONS); if (!orientation) return 'Changed orientation'; diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs index d8105f43e4..cb7d1d8396 100644 --- a/scripts/help-conformance-cases.mjs +++ b/scripts/help-conformance-cases.mjs @@ -762,9 +762,9 @@ Use the output already shown to determine whether the feed-search UI is present, ], }, { - id: 'foldable-pose-is-not-scriptable', + id: 'foldable-pose-is-verified-by-fold', docs: ['--help:first30', 'foldable'], - task: 'Plan commands for the installed app com.example.notes on a foldable iPhone Duo simulator that is currently closed. Open the session, capture the screen the device is actually showing, snapshot it, press the visible Compose control, and close. The final report must state which fold poses the run did and did not cover.', + task: 'Plan commands for the installed app com.example.notes on a foldable iPhone Duo simulator that is currently closed. Open the session, snapshot the closed pose, unfold the device fully, snapshot again, press the visible Compose control, and close. The final report must state which fold poses the run covered.', expectations: [ 'validPlanCommands', 'fullPrefix', @@ -772,11 +772,18 @@ Use the output already shown to determine whether the feed-search UI is present, 'usesSettleOnMutations', 'opensAndCloses', ], - matchers: [{ id: 'capturesPanelEvidence', pattern: /\bagent-device\s+screenshot\b/i }], + matchers: [ + { id: 'foldsToOpen', pattern: /(?:^|\n)agent-device\s+fold\s+open\b/i }, + { + id: 'resnapshotsAfterFold', + pattern: /agent-device\s+fold\s+open\b[\s\S]*agent-device\s+snapshot\b[^\n]*\s-i\b/i, + }, + ], forbidden: [ { id: 'noInventedPoseCommand', - pattern: /\bagent-device\b[^\n]*(?:\bfold\b|\bunfold\b|half-unfold|\bhinge\b|\bpose\b)/i, + pattern: + /\bagent-device\b(?!\s+fold\b)[^\n]*(?:\bunfold\b|half-unfold|\bhinge\b|\bpose\b)/i, }, { id: 'noScreenSelectionFlag', pattern: /\s--(?:screen|display)\b/i }, ], diff --git a/src/__tests__/test-utils/property-arbitraries.ts b/src/__tests__/test-utils/property-arbitraries.ts index 46fd7741f4..4767f11f81 100644 --- a/src/__tests__/test-utils/property-arbitraries.ts +++ b/src/__tests__/test-utils/property-arbitraries.ts @@ -282,6 +282,7 @@ const REPLAY_SCRIPT_LINE_PLANS = { alert: GENERIC_REPLAY_LINE, 'app-switcher': GENERIC_REPLAY_LINE, 'action-button': GENERIC_REPLAY_LINE, + fold: GENERIC_REPLAY_LINE, apps: GENERIC_REPLAY_LINE, appstate: GENERIC_REPLAY_LINE, artifacts: GENERIC_REPLAY_LINE, diff --git a/src/__tests__/test-utils/runtime-operation-facts.ts b/src/__tests__/test-utils/runtime-operation-facts.ts index 763a6ac1cb..ef5f94781a 100644 --- a/src/__tests__/test-utils/runtime-operation-facts.ts +++ b/src/__tests__/test-utils/runtime-operation-facts.ts @@ -66,6 +66,7 @@ export const unavailableDeploymentSnapshotAndShutdownOperationFacts = Object.fre ...keyboardRuntimeOperationFacts({ unsupported: unavailable }), ...clipboardRuntimeOperationFacts({ unsupported: unavailable }), ...systemButtonRuntimeOperationFacts({ unsupported: unavailable }), + setFoldPose: unavailable, triggerAppEvent: unavailable, setSetting: unavailable, readAlert: unavailable, diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index bc6dcb2e87..89d0b19617 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -145,6 +145,7 @@ export function createAgentDeviceClient( home: async (options = {}) => await executeCommand>('home', options), orientation: async (options) => await executeCommand>('orientation', options), + fold: async (options) => await executeCommand>('fold', options), appSwitcher: async (options = {}) => await executeCommand>('app-switcher', options), actionButton: async (options = {}) => diff --git a/src/client/client-types.ts b/src/client/client-types.ts index eccbcbb5fc..25b493df43 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -68,6 +68,7 @@ import type { AppOpenResult, AppPushOptions, ActionButtonCommandOptions, + FoldCommandOptions, AppStateCommandOptions, AppSwitcherCommandOptions, AppTriggerEventOptions, @@ -165,6 +166,7 @@ export type AgentDeviceCommandClient = { back: (options?: BackCommandOptions) => Promise>; home: (options?: HomeCommandOptions) => Promise>; orientation: (options: OrientationCommandOptions) => Promise>; + fold: (options: FoldCommandOptions) => Promise>; appSwitcher: (options?: AppSwitcherCommandOptions) => Promise>; actionButton: (options?: ActionButtonCommandOptions) => Promise>; tvRemote: (options: TvRemoteCommandOptions) => Promise>; diff --git a/src/commands/schema/cli-help-overview.ts b/src/commands/schema/cli-help-overview.ts index eb3bc6529f..f9bd11775a 100644 --- a/src/commands/schema/cli-help-overview.ts +++ b/src/commands/schema/cli-help-overview.ts @@ -56,6 +56,6 @@ Guides (agent-device help ): manual-qa / dogfood / validate / debugging / scripting / gestures react-native / react-devtools / cdp / tv / web / macos / remote physical-device / ios-system-ui / maestro - foldable iPhone Duo panels, pose, and why refs die on a pose change + foldable iPhone Duo panels, the fold command, and why refs die on a pose change `; } diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index b2addd9db2..9ef67a70f1 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -744,10 +744,12 @@ Screens are handled for you: The two panels are different sizes and different coordinate spaces (iPhone Duo: 466x678 points closed on the outer panel, 669x951 open on the inner). A pose change therefore invalidates every ref and coordinate. Re-snapshot after any pose change and never carry coordinates or refs across one. Check which panel is lit before trusting a geometry claim: agent-device screenshot reports its point size, and 466x678 versus 669x951 says which panel you captured. -Pose cannot be scripted: - iOS exposes fold state only to the app under test, as UIHinge.status (.closed/.partiallyOpen/.fullyOpen through UIHingeInteraction, or SwiftUI .onHingeChange). Nothing on the host sets it: simctl has no hinge/fold/pose subcommand, XCUITest has no hinge API, and devicectl only reports panel state (xcrun devicectl device info displays shows each panel's active/backlight state, which is how a closed device is detected). Do not write a step that changes pose, and do not claim a pose was set. - To exercise another pose, ask the operator to change it in Device Hub, then re-snapshot the iOS session. Driving Device Hub from a macOS session is possible in principle but its device surface exposes no accessibility nodes, so it is coordinate-only and needs Screen Recording permission; prefer asking the operator. - If a task asserts behavior for more than one pose, say which pose the current device is in, and state which poses remain unverified instead of assuming the device was folded.`, +Changing the pose: + agent-device fold closed | half-open | open + fold presses the pose control in the Xcode Device Hub window for this simulator (Closed, Book, Open) and then reads the hinge angle back from CoreDevice until it agrees: closed is 0 degrees, open is 180, and half-open is any angle between them (Device Hub's Book preset, 130 degrees on iOS 27.1), reported once the hinge stops moving or when the read budget ends while it still reads half-open. The response reports the verified pose, the hinge angle, and the panel the device now lights with its point size. A refusal never names the pose that was asked for: only a hinge whose last reading is some other pose fails, with COMMAND_FAILED and reason fold-pose-unverified. A single-panel simulator fails with UNSUPPORTED_OPERATION. + Expect a fold to take 10-16 seconds: each hinge read is a five-second devicectl stream, and half-open waits for the hinge to stop moving. Re-snapshot after every fold; refs and coordinates from before it are stale, and the command's message says so. + Requirements: an iOS simulator session on a foldable device, Xcode 27.1 or newer with Device Hub, and Accessibility permission for the host (agent-device settings permission grant accessibility --platform macos). The command launches Device Hub if needed, reopens its window when it shows none, and selects the simulator through its sidebar by UDID, so no operator step is needed. No official host API sets the pose; the app under test still reads it as UIHinge.status. + If a task asserts behavior for more than one pose, fold to each pose and re-snapshot, and report which poses the run covered.`, }, remote: { summary: 'Direct proxy, cloud profiles, and remote config', diff --git a/src/commands/system/index.test.ts b/src/commands/system/index.test.ts index 574db5a094..408c7690a9 100644 --- a/src/commands/system/index.test.ts +++ b/src/commands/system/index.test.ts @@ -4,6 +4,7 @@ import type { AgentDeviceCommandClient, AppSwitcherCommandOptions, BackCommandOptions, + FoldCommandOptions, HomeCommandOptions, OrientationCommandOptions, TvRemoteCommandOptions, @@ -16,6 +17,8 @@ import { backDaemonWriter, clipboardCliReader, clipboardDaemonWriter, + foldCliReader, + foldDaemonWriter, keyboardCliReader, keyboardDaemonWriter, orientationCliReader, @@ -50,6 +53,9 @@ describe('system command interface', () => { expectTypeOf().toEqualTypeOf< (options: OrientationCommandOptions) => Promise> >(); + expectTypeOf().toEqualTypeOf< + (options: FoldCommandOptions) => Promise> + >(); expectTypeOf().toEqualTypeOf< (options?: AppSwitcherCommandOptions) => Promise> >(); @@ -160,6 +166,24 @@ describe('system command interface', () => { expectInvalidArgs(() => orientationDaemonWriter({}), 'orientation requires orientation'); }); + test('fold reader and writer normalize the pose', () => { + expect(foldCliReader(['book'], flags())).toMatchObject({ pose: 'half-open' }); + expect(foldCliReader(['Unfolded'], flags({ platform: 'ios' }))).toMatchObject({ + platform: 'ios', + pose: 'open', + }); + expect(foldDaemonWriter({ pose: 'closed' })).toMatchObject({ + command: 'fold', + positionals: ['closed'], + }); + }); + + test('fold reader and writer reject a missing or unknown pose', () => { + expectInvalidArgs(() => foldCliReader([], flags()), 'fold requires a pose'); + expectInvalidArgs(() => foldCliReader(['sideways'], flags()), 'Invalid fold pose'); + expectInvalidArgs(() => foldDaemonWriter({}), 'fold requires pose'); + }); + test('keyboard reader maps aliases and validates arguments', () => { expect(keyboardCliReader(['get'], flags())).toMatchObject({ action: 'status' }); expect(keyboardCliReader([], flags())).not.toHaveProperty('action'); diff --git a/src/commands/system/index.ts b/src/commands/system/index.ts index 45cecd3895..e2ba1eb557 100644 --- a/src/commands/system/index.ts +++ b/src/commands/system/index.ts @@ -1,5 +1,11 @@ import type { ClipboardCommandOptions } from '@agent-device/contracts/client'; -import { DEVICE_ROTATIONS, parseDeviceRotation } from '@agent-device/contracts/device'; +import { + DEVICE_ROTATIONS, + FOLD_POSES, + FOLD_POSE_USAGE, + parseDeviceRotation, + parseFoldPose, +} from '@agent-device/contracts/device'; import { type BackMode, BACK_MODES } from '@agent-device/contracts/back-mode'; import { TV_REMOTE_BUTTONS, @@ -35,6 +41,7 @@ const APPSTATE_COMMAND_NAME = 'appstate'; const BACK_COMMAND_NAME = 'back'; const HOME_COMMAND_NAME = 'home'; const ORIENTATION_COMMAND_NAME = 'orientation'; +const FOLD_COMMAND_NAME = 'fold'; const APP_SWITCHER_COMMAND_NAME = 'app-switcher'; const ACTION_BUTTON_COMMAND_NAME = 'action-button'; const KEYBOARD_COMMAND_NAME = 'keyboard'; @@ -52,6 +59,8 @@ const backCommandDescription = const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = 'Set device orientation on iOS and Android'; +const foldCommandDescription = + 'Fold or unfold a foldable iPhone simulator (iPhone Duo) into the closed, half-open, or open pose by pressing the pose control in Xcode Device Hub, then read the hinge angle back from CoreDevice to confirm it. A pose change moves the app to a different panel with a different point size, so every ref and coordinate from before it is stale: re-snapshot after this command. Simulator-only; the device window must be open in Device Hub and the host needs Accessibility permission.'; const appSwitcherCommandDescription = 'Open the device app switcher to inspect or change foreground apps. This changes the visible system UI and may move focus away from the current app.'; const keyboardCommandDescription = @@ -76,6 +85,15 @@ const orientationCommandMetadata = defineFieldCommandMetadata( }, ); +const foldCommandMetadata = defineFieldCommandMetadata(FOLD_COMMAND_NAME, foldCommandDescription, { + pose: requiredField( + enumField( + FOLD_POSES, + 'The hinge pose to reach: closed lights the outer panel; half-open (Device Hub Book) and open light the inner panel.', + ), + ), +}); + const keyboardCommandMetadata = defineFieldCommandMetadata( KEYBOARD_COMMAND_NAME, keyboardCommandDescription, @@ -120,6 +138,11 @@ const orientationCliSchema = { positionalArgs: ['orientation'], } as const satisfies CommandSchemaOverride; +const foldCliSchema = { + usageOverride: `fold <${FOLD_POSE_USAGE}>`, + positionalArgs: ['pose'], +} as const satisfies CommandSchemaOverride; + const keyboardCliSchema = { usageOverride: 'keyboard [status|get|dismiss|enter|return]', positionalArgs: ['action?'], @@ -149,6 +172,11 @@ export const orientationCliReader: CliReader = (positionals, flags) => ({ orientation: parseDeviceRotation(positionals[0]), }); +export const foldCliReader: CliReader = (positionals, flags) => ({ + ...commonInputFromFlags(flags), + pose: parseFoldPose(positionals[0]), +}); + export const keyboardCliReader: CliReader = (positionals, flags) => ({ ...commonInputFromFlags(flags), ...readKeyboardInput(positionals), @@ -174,6 +202,10 @@ export const orientationDaemonWriter: DaemonWriter = direct(ORIENTATION_COMMAND_ requiredDaemonString(input.orientation, 'orientation requires orientation'), ]); +export const foldDaemonWriter: DaemonWriter = direct(FOLD_COMMAND_NAME, (input) => [ + requiredDaemonString(input.pose, 'fold requires pose'), +]); + export const keyboardDaemonWriter: DaemonWriter = direct(KEYBOARD_COMMAND_NAME, (input) => optionalString(input.action), ); @@ -232,6 +264,21 @@ const orientationCommandFacet = defineCommandFacet({ cliOutputFormatter: systemCliOutputFormatters.orientation, }); +const foldCommandFacet = defineCommandFacet({ + name: FOLD_COMMAND_NAME, + text: { + summary: 'Fold or unfold a foldable iPhone simulator', + cliDetail: + 'iPhone Duo simulators only. Presses the pose control in Xcode Device Hub and confirms the hinge angle through CoreDevice; refs and coordinates do not survive a pose change.', + }, + metadata: foldCommandMetadata, + run: (client, input) => client.command.fold(input), + cliSchema: foldCliSchema, + cliReader: foldCliReader, + daemonWriter: foldDaemonWriter, + cliOutputFormatter: systemCliOutputFormatters.fold, +}); + const appSwitcherCommandFacet = defineParameterlessCommandFacet({ name: APP_SWITCHER_COMMAND_NAME, description: appSwitcherCommandDescription, @@ -301,6 +348,7 @@ export const systemCommandFamily = defineCommandFamilyFromFacets({ backCommandFacet, homeCommandFacet, orientationCommandFacet, + foldCommandFacet, appSwitcherCommandFacet, actionButtonCommandFacet, keyboardCommandFacet, diff --git a/src/commands/system/output.ts b/src/commands/system/output.ts index 47af54a829..92804e4479 100644 --- a/src/commands/system/output.ts +++ b/src/commands/system/output.ts @@ -46,6 +46,7 @@ export const systemCliOutputFormatters = withSettleCapableNotes({ back: messageOutput, home: messageOutput, orientation: messageOutput, + fold: messageOutput, 'app-switcher': messageOutput, 'action-button': messageOutput, keyboard: resultOutput(keyboardCliOutput), diff --git a/src/daemon/__tests__/fold-runtime.test.ts b/src/daemon/__tests__/fold-runtime.test.ts new file mode 100644 index 0000000000..a6ca846e63 --- /dev/null +++ b/src/daemon/__tests__/fold-runtime.test.ts @@ -0,0 +1,157 @@ +import { expect, test, vi } from 'vitest'; + +import { + foldRuntimeOperationFacts, + type SetFoldPoseResult, +} from '@agent-device/contracts/fold-runtime'; +import { + localRuntimeOwner, + narrowDeviceBinding, + type DeviceBinding, + type RuntimeFacts, + type RuntimeOperationFact, +} from '@agent-device/contracts/platform-runtime'; +import { + foldRuntimeUse, + type PlatformRuntimeOperations, +} from '@agent-device/contracts/platform-runtime-operations'; +import { deviceShape } from '@agent-device/kernel/device'; +import { makeSession } from '../../__tests__/test-utils/session-factories.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; +import type { GenericPlatformExecutionParams } from '../request-generic-dispatch.ts'; +import { readRequestedFoldPose, resolveBoundFoldRuntime } from '../fold-runtime.ts'; +import { expectRefusesUnavailableExactOwnerFact } from './runtime-binding-conformance.ts'; + +// File-scoped id: this owner binding's `local-family` kind reaches the real on-disk device-claim +// admission, so a shared id risks a cross-file claim collision under parallel execution. +const testDevice = { + id: 'fold-runtime-device', + name: 'iPhone Duo', + platform: 'apple', + appleOs: 'ios', + kind: 'simulator', + target: 'mobile', + booted: true, +} as const; +const available = Object.freeze({ available: true } as const); +const unavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf' as const, +}); + +function foldExecutionParams(positionals: string[]): GenericPlatformExecutionParams { + const session = makeSession('fold-runtime', { device: testDevice }); + return { + session, + sessionName: session.name, + logPath: '/tmp/daemon.log', + command: 'fold', + request: { command: 'fold', positionals, token: 't', session: session.name }, + positionals, + out: undefined, + dispatchContext: {}, + }; +} + +function runtimeHarness( + fact: RuntimeOperationFact = available, + setFoldPose = vi.fn<() => Promise>(async () => ({ + pose: 'open', + hingeAngleDegrees: 180, + })), +) { + const facts: RuntimeFacts = { + device: { ...deviceShape(testDevice), providerMode: 'local' }, + operations: { setFoldPose: fact } as RuntimeFacts['operations'], + }; + const binding = { + device: testDevice, + owner: localRuntimeOwner('apple'), + facts, + operations: { setFoldPose }, + [Symbol.asyncDispose]: async () => {}, + } satisfies DeviceBinding; + const inspectFacts: InspectDeviceRuntimeFacts = vi.fn(async () => facts); + const bindDevice = vi.fn(async (_device, use) => + narrowDeviceBinding(binding, use), + ) as unknown as BindDeviceRuntime; + return { setFoldPose, inspectFacts, bindDevice }; +} + +test('parses the requested pose with the CLI aliases', () => { + expect(readRequestedFoldPose(['open'])).toBe('open'); + expect(readRequestedFoldPose(['book'])).toBe('half-open'); + expect(() => readRequestedFoldPose(['sideways'])).toThrow(); +}); + +test('resolves one admitted binding and reports the pose the owner read back', async () => { + const setFoldPose = vi.fn(async () => ({ + pose: 'open' as const, + hingeAngleDegrees: 180, + screen: { display: 'LCD-1', widthPt: 669, heightPt: 951 }, + })); + const harness = runtimeHarness( + foldRuntimeOperationFacts({ fold: available }).setFoldPose, + setFoldPose, + ); + + const resolved = await resolveBoundFoldRuntime({ + device: testDevice, + positionals: ['unfolded'], + inspectFacts: harness.inspectFacts, + bindDevice: harness.bindDevice, + }); + + expect(resolved.ok).toBe(true); + if (!resolved.ok) return; + expect(harness.bindDevice).toHaveBeenCalledWith(testDevice, foldRuntimeUse); + expect(await resolved.execute(foldExecutionParams(['unfolded']))).toEqual({ + action: 'fold', + pose: 'open', + hingeAngleDegrees: 180, + screen: { display: 'LCD-1', widthPt: 669, heightPt: 951 }, + message: + 'Folded to open (hinge 180°, LCD-1 lit at 669x951pt); refs from before the pose change are stale', + }); + expect(setFoldPose).toHaveBeenCalledWith({ pose: 'open' }); +}); + +test('reports a pose without a panel reading when the owner could not name the lit panel', async () => { + const harness = runtimeHarness(); + const resolved = await resolveBoundFoldRuntime({ + device: testDevice, + positionals: ['open'], + inspectFacts: harness.inspectFacts, + bindDevice: harness.bindDevice, + }); + expect(resolved.ok).toBe(true); + if (!resolved.ok) return; + expect(await resolved.execute(foldExecutionParams(['open']))).toEqual({ + action: 'fold', + pose: 'open', + hingeAngleDegrees: 180, + message: 'Folded to open (hinge 180°); refs from before the pose change are stale', + }); +}); + +test('rejects an invalid pose before inspection or binding', async () => { + const harness = runtimeHarness(); + await expect( + resolveBoundFoldRuntime({ + device: testDevice, + positionals: ['sideways'], + inspectFacts: harness.inspectFacts, + bindDevice: harness.bindDevice, + }), + ).rejects.toMatchObject({ code: 'INVALID_ARGS', message: expect.stringContaining('sideways') }); + expect(harness.inspectFacts).not.toHaveBeenCalled(); + expect(harness.bindDevice).not.toHaveBeenCalled(); +}); + +test('rejects an unavailable exact-owner fact before binding', async () => { + await expectRefusesUnavailableExactOwnerFact({ + command: 'fold', + device: testDevice, + unavailable, + }); +}); diff --git a/src/daemon/fold-runtime.ts b/src/daemon/fold-runtime.ts new file mode 100644 index 0000000000..8a214a4a66 --- /dev/null +++ b/src/daemon/fold-runtime.ts @@ -0,0 +1,62 @@ +import { parseFoldPose, type FoldPose } from '@agent-device/contracts/device'; +import type { SetFoldPoseInput } from '@agent-device/contracts/fold-runtime'; +import { foldRuntimeUse } from '@agent-device/contracts/platform-runtime-operations'; +import type { BoundDeviceRuntime } from '@agent-device/contracts/platform-runtime'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { successText } from '@agent-device/kernel/success-text'; +import type { ResolvedGenericExecution } from './request-generic-dispatch.ts'; +import { resolveBoundGenericRuntime, type RuntimeAdmissionBindings } from './runtime-admission.ts'; + +/** `fold `, parsed with the same aliases the CLI reader accepts. */ +export function readRequestedFoldPose(positionals: readonly string[]): FoldPose { + return parseFoldPose(positionals[0]); +} + +/** + * The one place `fold` reaches a device (ADR 0019). Admission inspects the exact owner's + * `setFoldPose` fact and binds once, before the dispatcher runs, so an owner that cannot pose a + * hinge is refused rather than discovered mid-execution. + */ +export async function resolveBoundFoldRuntime( + params: { + device: DeviceInfo; + positionals: readonly string[]; + } & RuntimeAdmissionBindings, +): Promise { + const pose = readRequestedFoldPose(params.positionals); + return await resolveBoundGenericRuntime( + { + command: 'fold', + device: params.device, + use: foldRuntimeUse, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }, + (runtime) => executeSetFoldPose(runtime, pose), + ); +} + +/** + * The ONE place a bound `setFoldPose` executes. The owner reports the pose it read back from the + * device, so the response carries that reading rather than the request: an owner that could not + * verify the pose throws instead of answering. + */ +async function executeSetFoldPose( + runtime: BoundDeviceRuntime, + requestedPose: FoldPose, +): Promise> { + const input: SetFoldPoseInput = { pose: requestedPose }; + const result = await runtime.operations.setFoldPose(input); + const screen = result.screen; + return { + action: 'fold', + pose: result.pose, + hingeAngleDegrees: result.hingeAngleDegrees, + ...(screen ? { screen } : {}), + ...successText( + screen + ? `Folded to ${result.pose} (hinge ${result.hingeAngleDegrees}°, ${screen.display} lit at ${screen.widthPt}x${screen.heightPt}pt); refs from before the pose change are stale` + : `Folded to ${result.pose} (hinge ${result.hingeAngleDegrees}°); refs from before the pose change are stale`, + ), + }; +} diff --git a/src/daemon/generic-runtime-execution.ts b/src/daemon/generic-runtime-execution.ts index 9be9a3929e..6c4f748332 100644 --- a/src/daemon/generic-runtime-execution.ts +++ b/src/daemon/generic-runtime-execution.ts @@ -10,6 +10,7 @@ import { resolveBoundViewportRuntime } from './viewport-runtime.ts'; import { resolveBoundBackRuntime } from './back-runtime.ts'; import { isSystemButtonCommand, resolveBoundSystemButtonRuntime } from './system-button-runtime.ts'; import { resolveBoundOrientationRuntime } from './orientation-runtime.ts'; +import { resolveBoundFoldRuntime } from './fold-runtime.ts'; import { resolveBoundTvRemoteRuntime } from './tv-remote-runtime.ts'; import { errorResponse } from '@agent-device/kernel/contracts'; @@ -77,6 +78,13 @@ export async function resolveGenericRuntimeExecution( inspectFacts: params.inspectFacts, bindDevice: params.bindDevice, }); + case 'fold': + return await resolveBoundFoldRuntime({ + device: params.session.device, + positionals: params.req.positionals ?? [], + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); case 'tv-remote': return await resolveBoundTvRemoteRuntime({ device: params.session.device, diff --git a/src/daemon/handlers/__tests__/install-source.test.ts b/src/daemon/handlers/__tests__/install-source.test.ts index 0e065b121f..1ccb5275b0 100644 --- a/src/daemon/handlers/__tests__/install-source.test.ts +++ b/src/daemon/handlers/__tests__/install-source.test.ts @@ -380,6 +380,7 @@ function sourceRuntimeFacts( ...keyboardRuntimeOperationFacts({ unsupported: unavailable }), ...clipboardRuntimeOperationFacts({ unsupported: unavailable }), ...systemButtonRuntimeOperationFacts({ unsupported: unavailable }), + setFoldPose: unavailable, triggerAppEvent: unavailable, setSetting: unavailable, readAlert: unavailable, diff --git a/src/daemon/handlers/__tests__/session-state.test.ts b/src/daemon/handlers/__tests__/session-state.test.ts index f823ecb06a..31c8dde543 100644 --- a/src/daemon/handlers/__tests__/session-state.test.ts +++ b/src/daemon/handlers/__tests__/session-state.test.ts @@ -60,6 +60,7 @@ test('boot rejects --headless outside Android directly', async () => { keyboard: { available: false, reason: 'owner-capability-missing' }, clipboard: { available: false, reason: 'owner-capability-missing' }, systemButton: { available: false, reason: 'owner-capability-missing' }, + fold: { available: false, reason: 'owner-capability-missing' }, triggerAppEvent: { available: false, reason: 'owner-capability-missing' }, setSetting: { available: false, reason: 'owner-capability-missing' }, readAlert: { available: false, reason: 'owner-capability-missing' }, @@ -165,6 +166,7 @@ test('appstate rejects web before Android app-state backend dispatch', async () keyboard: { available: false, reason: 'unsupported-platform-leaf' }, clipboard: { available: false, reason: 'unsupported-platform-leaf' }, systemButton: { available: false, reason: 'unsupported-platform-leaf' }, + fold: { available: false, reason: 'unsupported-platform-leaf' }, triggerAppEvent: { available: false, reason: 'unsupported-platform-leaf' }, setSetting: { available: false, reason: 'unsupported-platform-leaf' }, readAlert: { available: false, reason: 'unsupported-platform-leaf' }, diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 2d4c0834b6..094232fb2f 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -5,7 +5,7 @@ import { booleanSchema, looseObjectSchema, stringSchema } from '../commands/comm import { BACK_MODES } from '@agent-device/contracts/back-mode'; import { NATIVE_PATH_DISPOSITION_VALUES } from '@agent-device/contracts/recording-native-path'; import { RECORDER_OBSERVATION_VALUES } from '@agent-device/contracts/recording-stop-observation'; -import { DEVICE_ROTATIONS } from '@agent-device/contracts/device'; +import { DEVICE_ROTATIONS, FOLD_POSES } from '@agent-device/contracts/device'; import { SESSION_SURFACES } from '@agent-device/contracts/session'; import { TV_REMOTE_BUTTONS } from '@agent-device/contracts/tv-remote'; import { DEVICE_TARGETS, PUBLIC_PLATFORMS } from '@agent-device/kernel/device'; @@ -517,6 +517,23 @@ const BASE_COMMAND_OUTPUT_SCHEMAS = { 'action', 'message', ]), + fold: objectSchema( + { + action: constSchema('fold'), + pose: enumSchema(FOLD_POSES), + hingeAngleDegrees: numberSchema('Hinge angle CoreDevice read back after the pose settled.'), + screen: objectSchema( + { + display: stringSchema('CoreDevice name of the panel the device now lights.'), + widthPt: numberSchema(), + heightPt: numberSchema(), + }, + ['display', 'widthPt', 'heightPt'], + ), + message: stringSchema(), + }, + ['action', 'pose', 'hingeAngleDegrees', 'message'], + ), 'action-button': objectSchema({ action: constSchema('action-button'), message: stringSchema() }, [ 'action', 'message', diff --git a/src/platform-runtime-gateway.test.ts b/src/platform-runtime-gateway.test.ts index d2a644f626..5e6ba396fb 100644 --- a/src/platform-runtime-gateway.test.ts +++ b/src/platform-runtime-gateway.test.ts @@ -68,6 +68,7 @@ describe('composed platform runtime gateway', () => { keyboard: unavailable, clipboard: unavailable, systemButton: unavailable, + fold: unavailable, triggerAppEvent: unavailable, setSetting: unavailable, readAlert: unavailable, @@ -165,6 +166,7 @@ describe('composed platform runtime gateway', () => { keyboard: unavailable, clipboard: unavailable, systemButton: unavailable, + fold: unavailable, triggerAppEvent: unavailable, setSetting: unavailable, readAlert: unavailable, diff --git a/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts b/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts index 308aba4c94..06a4785f7e 100644 --- a/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts +++ b/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts @@ -270,6 +270,8 @@ function providerScenarioRuntimeFacts( // hardware exists on the iPhone/iPad leaf only, so the fixture reads the same kernel rule the // Apple owner's fact reads instead of restating it (#2699). actionButton: hasAppleActionButton(device) ? fakeProviderAvailable : fakeProviderUnavailable, + // A provider session has no Device Hub on the host, so no fixture leg poses a hinge. + setFoldPose: fakeProviderUnavailable, // Provider-owned iOS keyboard actions ride the same runner transport the shared interactor // does (#1297): a fixture scenario that can drive the interactor at all can drive these. ...keyboardRuntimeOperationFacts({ diff --git a/test/integration/provider-scenarios/provider-ios-runner-transport.test.ts b/test/integration/provider-scenarios/provider-ios-runner-transport.test.ts index ed29366d80..a9094e6f01 100644 --- a/test/integration/provider-scenarios/provider-ios-runner-transport.test.ts +++ b/test/integration/provider-scenarios/provider-ios-runner-transport.test.ts @@ -18,7 +18,7 @@ import type { } from '@agent-device/platform-apple/runner'; import { withAppleRunnerProvider } from '@agent-device/platform-apple/runner'; import { providerRuntimeOwner } from '@agent-device/contracts/platform-runtime'; -import { assertRpcOk } from './assertions.ts'; +import { assertRpcError, assertRpcOk } from './assertions.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createProviderScenarioLifecycleModule } from './provider-device-runtime.fixtures.ts'; @@ -132,6 +132,25 @@ test('provider transport carries the Action Button press without an app activati }); }); +// A hinge pose is posed through Xcode Device Hub on the host, which a provider-owned device has no +// access to, so the provider fixture states the refusal cell and admission refuses before anything +// reaches the transport: no runner call, no local Device Hub press. +test('provider transport refuses a fold before any runner traffic', async () => { + await withProviderScenarioResource(createInteractorSeamWorld, async ({ daemon, calls }) => { + const lease = await allocateLease(daemon); + const request = { flags: leaseFlags(lease.leaseId), meta: leaseMeta(lease.leaseId) }; + assertRpcOk(await daemon.callCommand('open', ['com.example.app'], request.flags, request)); + + calls.runner.length = 0; + assertRpcError( + await daemon.callCommand('fold', ['open'], request.flags, request), + 'UNSUPPORTED_OPERATION', + /fold/, + ); + assert.deepEqual(calls.runner, [], 'a refused fold must not reach the provider transport'); + }); +}); + // Daemon routes that issue runner commands OUTSIDE interactor methods (keyboard, // native alert, point read, iOS sequences) must reach the provider transport via // the request-boundary `appleRunnerProvider` scope instead of escaping to the diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 47fa882e8a..3e6d226cd6 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -272,6 +272,7 @@ await client.command.tvRemote({ await client.command.appSwitcher(); await client.command.actionButton(); +await client.command.fold({ pose: 'open' }); ``` Vega OS client support is currently VVD-only and covers device discovery, app open/close, `back`, `home`, and `tvRemote`. Physical Fire TV, capture, selector, install, logging, and performance methods report unsupported for Vega targets. @@ -286,6 +287,7 @@ Supported command methods: - `orientation` - `appSwitcher` - `actionButton` +- `fold` - `keyboard` - `clipboard` - `tvRemote` diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index f30353e430..2d83b7d419 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -54,6 +54,9 @@ agent-device orientation portrait agent-device orientation landscape-left agent-device app-switcher agent-device action-button +agent-device fold closed +agent-device fold half-open +agent-device fold open ``` - `boot` ensures the selected target is ready without launching an app. @@ -88,6 +91,9 @@ agent-device action-button - `action-button` asks the device whether it has the button before pressing it. A target whose model has none — an iPhone SE beside an iPhone 15, or most iPad simulators — fails with `UNSUPPORTED_OPERATION` rather than reporting a press that never happened. - `action-button` does not activate or relaunch the session's app, and it takes no `--settle`: pressing a hardware button is not a navigation, so the app stays where it was. - `action-button` reports that the press was dispatched, not what the system did with it. Simulators run no Shortcuts and no App Intents, so what a press triggers can only be verified on a physical iPhone; on a Simulator the command proves the press was accepted and that the session app was not brought forward. +- `fold ` puts a foldable iPhone simulator (iPhone Duo) into a hinge pose. No official host API sets a pose (ADR 0025), so the command presses the pose control in the Xcode Device Hub window through macOS accessibility, then reads the hinge angle back with `devicectl device motion hinge-angle` and reports the pose only when that reading agrees: `closed` is 0°, `open` is 180°, and `half-open` is any angle between them (Device Hub's Book preset, 130° on iOS 27.1), reported once the hinge stops moving or when the read budget ends while it still reads half-open. The response names the panel the device now lights and its point size, because a pose change moves the app to a different coordinate space: re-snapshot afterwards, and never carry refs or coordinates across a `fold`. +- `fold` is simulator-only and needs Xcode 27.1 or newer with Device Hub. The host needs Accessibility permission (`settings permission grant accessibility --platform macos`), and Device Hub must list the simulator; the command reopens Device Hub's window when it shows none, selects the device through the sidebar row keyed by its UDID, and restores the sidebar afterwards. A single-panel simulator such as an iPhone 17 fails with `UNSUPPORTED_OPERATION`; Android, web, Linux, HarmonyOS, Vega, physical devices, and the tvOS, macOS, and visionOS leaves refuse it. +- `fold` costs one bounded hinge stream per read, and devicectl's smallest stream is five seconds: `closed` and `open` take about ten seconds, `half-open` about sixteen, because the hinge animates and the command waits for it to stop. A refusal never names the pose that was asked for: only a hinge whose last reading is some other pose fails, with `COMMAND_FAILED` and `reason: fold-pose-unverified`, naming the angle CoreDevice still reports. - `action-button` is not a cheap command to loop. On an iPhone 17 Pro Simulator the press itself spent about five seconds inside XCUITest, while `home` and `app-switcher` on the same session took under two seconds each. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom scheme URLs require an active app in the session. - Commands that need one concrete device refuse to guess: if no `--device`/`--udid`/`--serial` is given and several candidates are equally preferred (for example two booted emulators), the command fails with `AMBIGUOUS_MATCH` and lists them, rather than picking one and returning a successful answer about a device you did not select. Preferences still apply first — virtual over physical, booted over offline — so one booted emulator beside offline ones resolves normally, as does any command running inside an existing session. `devices` lists everything as before. @@ -325,7 +331,7 @@ agent-device snapshot -i --platform apple --target desktop - Prefer selector or `@ref`-driven interactions on macOS. Window position can shift between runs, so raw x/y point commands are less stable than snapshot-derived targets. - Use `click --button secondary` for context menus on macOS, then run `snapshot -i` again. - On `frontmost-app`, `desktop`, and `menubar` surfaces, `press` and `click` post synthetic mouse events through the macOS helper: `--hold-ms` is how long the button stays down (at least 40 ms, 60 ms by default, because AppKit drops a release posted in the same tick as its press), `--count` is that many independent clicks, and `--double-tap` posts each click as a double-click pair. A long schedule such as `--hold-ms 10000 --count 4` is given the time it needs, and a helper interrupted mid-hold releases the button before it exits. `--jitter-px` is not applied on these surfaces. -- Mobile-only helpers remain unsupported on macOS: `boot`, `shutdown`, `home`, `orientation`, `app-switcher`, `action-button`, `install`, `reinstall`, `install-from-source`, and `push`. +- Mobile-only helpers remain unsupported on macOS: `boot`, `shutdown`, `home`, `orientation`, `app-switcher`, `action-button`, `fold`, `install`, `reinstall`, `install-from-source`, and `push`. Recommended loops: From c9e6cdef618a64fe7d4cdc1a72ff1ba436118d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 20 Sep 2026 22:41:53 +0200 Subject: [PATCH 2/2] chore(gates): register fold in the exports snapshot, coverage tables, and conformance tables The contracts export snapshot names the new `fold-runtime` subpath, the six-platform coverage declaration and its Android evidence row classify the command, the runtime-binding conformance table resolves it, and the provider output guard drives it. Co-Authored-By: Claude Fable 5.1 --- .../layering/contracts-exports.snapshot.json | 1 + .../command-descriptor-timeout-policy.test.ts | 3 ++ .../__tests__/runtime-binding-conformance.ts | 5 +++ .../command-coverage/declarations.ts | 32 +++++++++++++++++++ test/integration/command-coverage/evidence.ts | 6 ++++ .../apple-platform-output-guard.test.ts | 1 + 6 files changed, 48 insertions(+) diff --git a/scripts/layering/contracts-exports.snapshot.json b/scripts/layering/contracts-exports.snapshot.json index 915a85dfc6..f1e855b964 100644 --- a/scripts/layering/contracts-exports.snapshot.json +++ b/scripts/layering/contracts-exports.snapshot.json @@ -45,6 +45,7 @@ "@agent-device/contracts/durable-resource-envelope", "@agent-device/contracts/element-text-runtime", "@agent-device/contracts/focus-runtime", + "@agent-device/contracts/fold-runtime", "@agent-device/contracts/gesture-admission", "@agent-device/contracts/gesture-input", "@agent-device/contracts/gesture-normalization", diff --git a/src/__tests__/command-descriptor-timeout-policy.test.ts b/src/__tests__/command-descriptor-timeout-policy.test.ts index 017390dbe3..bafd275314 100644 --- a/src/__tests__/command-descriptor-timeout-policy.test.ts +++ b/src/__tests__/command-descriptor-timeout-policy.test.ts @@ -143,6 +143,9 @@ test('request envelopes deviating from the default are bounded, reviewed sets', reinstall: 180_000, install_source: 180_000, longpress: 210_000, + // fold: one macOS helper press (30s) plus up to four bounded CoreDevice hinge reads (20s + // each on a wedged host) can pass the default envelope; the policy covers that worst case. + fold: 150_000, // #1774: base allocation budget (300s) + client/daemon race margin (30s). lease_allocate: 330_000, test: 'unbounded', diff --git a/src/daemon/__tests__/runtime-binding-conformance.ts b/src/daemon/__tests__/runtime-binding-conformance.ts index cbb5a1b506..87d12bc1ee 100644 --- a/src/daemon/__tests__/runtime-binding-conformance.ts +++ b/src/daemon/__tests__/runtime-binding-conformance.ts @@ -16,6 +16,7 @@ import { makeSession } from '../../__tests__/test-utils/session-factories.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { resolveBoundBackRuntime } from '../back-runtime.ts'; import { resolveBoundFocusRuntime } from '../focus-runtime.ts'; +import { resolveBoundFoldRuntime } from '../fold-runtime.ts'; import { resolveBoundGestureRuntime } from '../gesture-runtime.ts'; import { resolveBoundOrientationRuntime } from '../orientation-runtime.ts'; import type { ResolvedGenericExecution } from '../request-generic-dispatch.ts'; @@ -100,6 +101,10 @@ export const conformedRuntimeBindings = { resolve: async (device, bindings) => refusable(await resolveBoundBackRuntime({ device, ...bindings })), }, + fold: { + resolve: async (device, bindings) => + refusable(await resolveBoundFoldRuntime({ device, positionals: ['open'], ...bindings })), + }, focus: { resolve: async (device, bindings) => refusable(await resolveBoundFocusRuntime({ device, positionals: ['40', '90'], ...bindings })), diff --git a/test/integration/command-coverage/declarations.ts b/test/integration/command-coverage/declarations.ts index ffd0945c3a..17168c0c5a 100644 --- a/test/integration/command-coverage/declarations.ts +++ b/test/integration/command-coverage/declarations.ts @@ -16,6 +16,7 @@ import { import { ANDROID_ACTION_BUTTON_RUNTIME_CONTRACT_EVIDENCE, ANDROID_APPLICATION_LIFECYCLE_CONTRACT_EVIDENCE, + ANDROID_FOLD_RUNTIME_CONTRACT_EVIDENCE, ANDROID_HOVER_RUNTIME_CONTRACT_EVIDENCE, ANDROID_TV_REMOTE_RUNTIME_CONTRACT_EVIDENCE, ANDROID_VIEWPORT_RUNTIME_CONTRACT_EVIDENCE, @@ -1329,6 +1330,37 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'the exact-owner runtime fact refuses an Action Button press on the Linux desktop', ), }, + [C.fold]: { + androidEmulator: androidEmulator.contract( + ANDROID_FOLD_RUNTIME_CONTRACT_EVIDENCE, + 'the Android runtime fact refuses a foldable hinge pose on every kind', + ), + iosSimulator: iosSimulator.contract( + 'packages/platform-apple/src/runtime.test.ts', + 'classifies the fold fact for the %s leaf', + 'the iOS simulator leaf advertises the hinge pose and binds it; a single-panel simulator is refused by the operation itself', + ), + macos: macos.contract( + 'packages/platform-apple/src/runtime.test.ts', + 'classifies the fold fact for the %s leaf', + 'the exact-owner runtime fact refuses fold on the macOS host leaf, which is not a simulator', + ), + tvos: tvos.contract( + 'packages/platform-apple/src/runtime.test.ts', + 'classifies the fold fact for the %s leaf', + 'the exact-owner runtime fact refuses fold on the tvOS leaf, which has no hinge', + ), + web: web.contract( + 'packages/platform-web/src/runtime.test.ts', + 'clipboard, the app switcher, app events, settings and alerts carry no web bucket', + 'the exact-owner runtime fact refuses a hinge pose on the web target', + ), + linux: linux.contract( + LINUX_RUNTIME_EVIDENCE.path, + LINUX_RUNTIME_EVIDENCE.test, + 'the exact-owner runtime fact refuses a hinge pose on the Linux desktop', + ), + }, [C.installFromSource]: { androidEmulator: androidEmulator.contract( ANDROID_INSTALL_SOURCE_CONTRACT_EVIDENCE, diff --git a/test/integration/command-coverage/evidence.ts b/test/integration/command-coverage/evidence.ts index 742c64d30d..42aed9865b 100644 --- a/test/integration/command-coverage/evidence.ts +++ b/test/integration/command-coverage/evidence.ts @@ -44,6 +44,12 @@ export const ANDROID_ACTION_BUTTON_RUNTIME_CONTRACT_EVIDENCE: AndroidContractEvi [C.actionButton], 'Android refuses the action-button fact on every kind', ); +export const ANDROID_FOLD_RUNTIME_CONTRACT_EVIDENCE: AndroidContractEvidence = + defineAndroidContractEvidence( + 'packages/platform-android/src/runtime.test.ts', + [C.fold], + 'Android refuses the fold fact on every kind', + ); export const ANDROID_VIEWPORT_RUNTIME_CONTRACT_EVIDENCE: AndroidContractEvidence = defineAndroidContractEvidence( 'src/daemon/__tests__/viewport-runtime.test.ts', diff --git a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts index 80449de6d2..4131ca25f8 100644 --- a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts +++ b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts @@ -127,6 +127,7 @@ const DRIVEN_COMMANDS: Record = { [PUBLIC_COMMANDS.tvRemote]: () => one(['select']), [PUBLIC_COMMANDS.appSwitcher]: () => one(), [PUBLIC_COMMANDS.actionButton]: () => one(), + [PUBLIC_COMMANDS.fold]: () => one(['open']), // -- orchestration (drive to an error response; still scanned) -- [PUBLIC_COMMANDS.artifacts]: () => one(),