From 77a0817ee4290f8630374bcbe40fb49be1215043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 6 Sep 2026 11:08:57 +0200 Subject: [PATCH 1/3] test(android-e2e): record rotation state and logcat rotation decisions on a failed step The Android smoke has failed on the post-alert canary since 2026-09-03, and the failed-step screenshot from run 34021894996 shows why the reads miss: the device is in landscape at that point, with the canary below the fold, although `orientation portrait` had taken effect (the fixture confirmed it and every tap before the alert landed at x=540). Nothing we keep says what rotated it. A failed step now also writes failed-step-N-device.txt with the two rotation settings, the display's rotation lines, and WindowManager's rotation decisions from logcat, read through adb so they stand even when the CLI path failed. --- .../android-emulator-e2e/live-harness.ts | 53 +++++++++++++++++++ test/integration/live-device-e2e/runtime.ts | 21 +++++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/test/integration/android-emulator-e2e/live-harness.ts b/test/integration/android-emulator-e2e/live-harness.ts index ed04b1c841..8b4421fb84 100644 --- a/test/integration/android-emulator-e2e/live-harness.ts +++ b/test/integration/android-emulator-e2e/live-harness.ts @@ -1,5 +1,7 @@ +import { execFile } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; +import { promisify } from 'node:util'; import { createLiveDeviceContext, @@ -37,9 +39,60 @@ export function createContext(): LiveContext { }; } +const execFileAsync = promisify(execFile); + +/** + * What the OS says about rotation when a step fails: the two settings `orientation` writes, the + * display's current rotation, and every WindowManager rotation decision logcat still holds (with + * the reason it gives). Read through adb, not agent-device, so it stands even when the CLI path + * is what failed. + */ +async function readAndroidRotationEvidence(context: LiveContext): Promise { + const probes: readonly [string, string[]][] = [ + ['accelerometer_rotation', ['shell', 'settings', 'get', 'system', 'accelerometer_rotation']], + ['user_rotation', ['shell', 'settings', 'get', 'system', 'user_rotation']], + ['display rotation', ['shell', 'dumpsys', 'display']], + ['logcat rotation decisions', ['logcat', '-d', '-v', 'time']], + ]; + const sections: string[] = []; + for (const [title, args] of probes) { + try { + const { stdout } = await execFileAsync('adb', ['-s', context.serial, ...args], { + maxBuffer: 64 * 1024 * 1024, + timeout: 20_000, + }); + sections.push(`## ${title}\n${selectRotationLines(title, stdout)}`); + } catch (error) { + sections.push( + `## ${title}\n(failed: ${error instanceof Error ? error.message : String(error)})`, + ); + } + } + return `${sections.join('\n\n')}\n`; +} + +function selectRotationLines(title: string, output: string): string { + if (title === 'display rotation') { + return output + .split('\n') + .filter((line) => /rotation|orientation/i.test(line)) + .slice(0, 12) + .join('\n'); + } + if (title === 'logcat rotation decisions') { + return output + .split('\n') + .filter((line) => /rotation|orientation/i.test(line) && !/AccessibilityNodeInfo/.test(line)) + .slice(-60) + .join('\n'); + } + return output.trim(); +} + const harness = createLiveDeviceHarness({ behaviorsForScenario: liveBehaviorsForScenario, commandsForScenario: liveCommandsForScenario, + deviceEvidence: readAndroidRotationEvidence, commonFlags: (context, args) => [ ...args, '--platform', diff --git a/test/integration/live-device-e2e/runtime.ts b/test/integration/live-device-e2e/runtime.ts index ff1976929c..731621cf67 100644 --- a/test/integration/live-device-e2e/runtime.ts +++ b/test/integration/live-device-e2e/runtime.ts @@ -49,6 +49,12 @@ type HarnessOptions = { env: NodeJS.ProcessEnv, options?: { timeoutMs?: number }, ) => Promise; + /** + * Platform-owned device facts for a failed step (rotation state, system logs), read outside + * agent-device so they describe the device even when the CLI path is what failed. Best-effort: + * a throw or undefined records nothing. + */ + deviceEvidence?: (context: Context) => Promise; writeCoverageReport: (context: Context) => void; }; @@ -176,6 +182,7 @@ export function createLiveDeviceHarness< `artifacts: ${context.artifactDir}`, `screenshot: ${evidence.screenshotPath ?? '(capture failed)'}`, `snapshot: ${evidence.snapshotPath ?? '(capture failed)'}`, + `device: ${evidence.devicePath ?? '(not collected)'}`, ].join('\n'); fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message); assert.fail(message); @@ -191,12 +198,22 @@ export function createLiveDeviceHarness< */ async function captureFailedStepEvidence( context: Context, - ): Promise<{ screenshotPath?: string; snapshotPath?: string }> { + ): Promise<{ screenshotPath?: string; snapshotPath?: string; devicePath?: string }> { const stem = path.join(context.artifactDir, `failed-step-${context.stepHistory.length}`); const screenshotPath = `${stem}.png`; const snapshotPath = `${stem}-snapshot.json`; + const devicePath = `${stem}-device.txt`; const runCli = options.runCli ?? runBuiltCliJson; - const evidence: { screenshotPath?: string; snapshotPath?: string } = {}; + const evidence: { screenshotPath?: string; snapshotPath?: string; devicePath?: string } = {}; + try { + const facts = await options.deviceEvidence?.(context); + if (facts !== undefined) { + fs.writeFileSync(devicePath, facts); + evidence.devicePath = devicePath; + } + } catch { + // evidence only + } try { const screenshot = await runCli( options.commonFlags(context, ['screenshot', screenshotPath]), From 825faf094ca67fbf73b652ea52d13f5127b135a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 6 Sep 2026 11:09:23 +0200 Subject: [PATCH 2/3] test(android-e2e): keep the rotation evidence to WindowManager decisions and display rotation fields --- .../android-emulator-e2e/live-harness.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/integration/android-emulator-e2e/live-harness.ts b/test/integration/android-emulator-e2e/live-harness.ts index 8b4421fb84..b84f4643d3 100644 --- a/test/integration/android-emulator-e2e/live-harness.ts +++ b/test/integration/android-emulator-e2e/live-harness.ts @@ -75,14 +75,22 @@ function selectRotationLines(title: string, output: string): string { if (title === 'display rotation') { return output .split('\n') - .filter((line) => /rotation|orientation/i.test(line)) - .slice(0, 12) + .filter((line) => + /mCurrentOrientation|mRotation=|installOrientation|\brotation \d/.test(line), + ) + .map((line) => line.trim().slice(0, 200)) + .slice(0, 8) .join('\n'); } if (title === 'logcat rotation decisions') { return output .split('\n') - .filter((line) => /rotation|orientation/i.test(line) && !/AccessibilityNodeInfo/.test(line)) + .filter( + (line) => + /(WindowManager|DisplayRotation|WindowOrientationListener|RotationResolver|DisplayContent|SensorService)/.test( + line, + ) && /rotat|orient/i.test(line), + ) .slice(-60) .join('\n'); } From 1595e3327d5f3fd848e6af8258598a81d9736ad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 6 Sep 2026 11:45:47 +0200 Subject: [PATCH 3/3] test(e2e): own failed-step evidence in one collector, bound the device probes, test it Review follow-up on the rotation evidence. The collectors move out of the harness closure into failed-step-evidence.ts (fallow complexity), where the platform hook runs alongside the screenshot and snapshot and is bounded as a group (15s) so it can never delay them; a hook that throws, answers nothing, or never answers records nothing for the device file and leaves the CLI evidence in place. The Android probes get a 5s per-command bound, and logcat lines are capped in count and length. Deterministic tests cover the file contents, the hook failure and timeout cases, and the harness naming every evidence file, device file included, in failed-step.txt. --- .../android-emulator-e2e/live-harness.ts | 15 +- ...ve-device-e2e-failed-step-evidence.test.ts | 133 ++++++++++++++++++ .../live-device-e2e/failed-step-evidence.ts | 102 ++++++++++++++ test/integration/live-device-e2e/runtime.ts | 48 ++----- 4 files changed, 253 insertions(+), 45 deletions(-) create mode 100644 test/integration/live-device-e2e-failed-step-evidence.test.ts create mode 100644 test/integration/live-device-e2e/failed-step-evidence.ts diff --git a/test/integration/android-emulator-e2e/live-harness.ts b/test/integration/android-emulator-e2e/live-harness.ts index b84f4643d3..93ad8cad96 100644 --- a/test/integration/android-emulator-e2e/live-harness.ts +++ b/test/integration/android-emulator-e2e/live-harness.ts @@ -41,11 +41,15 @@ export function createContext(): LiveContext { const execFileAsync = promisify(execFile); +const ROTATION_PROBE_TIMEOUT_MS = 5_000; +const ROTATION_LOG_LINES = 60; +const ROTATION_LOG_LINE_LENGTH = 240; + /** * What the OS says about rotation when a step fails: the two settings `orientation` writes, the - * display's current rotation, and every WindowManager rotation decision logcat still holds (with + * display's current rotation, and the WindowManager rotation decisions logcat still holds (with * the reason it gives). Read through adb, not agent-device, so it stands even when the CLI path - * is what failed. + * is what failed; the shared collector bounds the whole read so it never delays the screenshot. */ async function readAndroidRotationEvidence(context: LiveContext): Promise { const probes: readonly [string, string[]][] = [ @@ -59,7 +63,7 @@ async function readAndroidRotationEvidence(context: LiveContext): Promise /mCurrentOrientation|mRotation=|installOrientation|\brotation \d/.test(line), ) - .map((line) => line.trim().slice(0, 200)) + .map((line) => line.trim().slice(0, ROTATION_LOG_LINE_LENGTH)) .slice(0, 8) .join('\n'); } @@ -91,7 +95,8 @@ function selectRotationLines(title: string, output: string): string { line, ) && /rotat|orient/i.test(line), ) - .slice(-60) + .slice(-ROTATION_LOG_LINES) + .map((line) => line.slice(0, ROTATION_LOG_LINE_LENGTH)) .join('\n'); } return output.trim(); diff --git a/test/integration/live-device-e2e-failed-step-evidence.test.ts b/test/integration/live-device-e2e-failed-step-evidence.test.ts new file mode 100644 index 0000000000..957def2b24 --- /dev/null +++ b/test/integration/live-device-e2e-failed-step-evidence.test.ts @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +import type { CliJsonResult } from './cli-json.ts'; +import { collectFailedStepEvidence } from './live-device-e2e/failed-step-evidence.ts'; +import { createLiveDeviceContext, createLiveDeviceHarness } from './live-device-e2e/runtime.ts'; + +function fakeCli(artifacts: { screenshot?: boolean; snapshot?: boolean } = {}) { + const calls: string[][] = []; + const runCli = async (args: string[]): Promise => { + calls.push(args); + if (args[0] === 'screenshot') { + if (artifacts.screenshot === false) return { status: 1, stdout: '', stderr: 'no screen' }; + fs.writeFileSync(args[1]!, 'png'); + return { status: 0, stdout: '', stderr: '', json: { success: true } }; + } + if (args[0] === 'snapshot') { + if (artifacts.snapshot === false) return { status: 1, stdout: '', stderr: 'no tree' }; + return { status: 0, stdout: '', stderr: '', json: { success: true, data: { nodes: [] } } }; + } + return { status: 1, stdout: '', stderr: 'step failed', json: { success: false } }; + }; + return { calls, runCli }; +} + +function tempStem(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'failed-step-evidence-')); + return path.join(dir, 'failed-step-3'); +} + +test('device facts land in their own file next to the screenshot and snapshot', async () => { + const stem = tempStem(); + const cli = fakeCli(); + + const evidence = await collectFailedStepEvidence({ + stem, + runCli: cli.runCli, + deviceEvidence: async () => '## user_rotation\n1\n', + }); + + assert.deepEqual(evidence, { + screenshotPath: `${stem}.png`, + snapshotPath: `${stem}-snapshot.json`, + devicePath: `${stem}-device.txt`, + }); + assert.equal(fs.readFileSync(`${stem}-device.txt`, 'utf8'), '## user_rotation\n1\n'); + assert.deepEqual(JSON.parse(fs.readFileSync(`${stem}-snapshot.json`, 'utf8')), { + success: true, + data: { nodes: [] }, + }); + assert.deepEqual( + cli.calls.map((args) => args[0]), + ['screenshot', 'snapshot'], + ); +}); + +test('a device hook that throws or returns nothing still leaves the CLI evidence in place', async () => { + for (const deviceEvidence of [ + async () => { + throw new Error('adb unavailable'); + }, + async () => undefined, + ]) { + const stem = tempStem(); + const evidence = await collectFailedStepEvidence({ + stem, + runCli: fakeCli().runCli, + deviceEvidence, + }); + + assert.deepEqual(evidence, { + screenshotPath: `${stem}.png`, + snapshotPath: `${stem}-snapshot.json`, + }); + assert.equal(fs.existsSync(`${stem}-device.txt`), false); + } +}); + +test('a device hook that never answers is bounded and never delays the CLI evidence', async () => { + const stem = tempStem(); + const startedAt = Date.now(); + + const evidence = await collectFailedStepEvidence({ + stem, + runCli: fakeCli().runCli, + deviceEvidence: () => new Promise(() => undefined), + deviceEvidenceTimeoutMs: 50, + }); + + assert.deepEqual(evidence, { + screenshotPath: `${stem}.png`, + snapshotPath: `${stem}-snapshot.json`, + }); + assert.ok(Date.now() - startedAt < 1_000); +}); + +test('a failed step names its evidence files, including the device file, in failed-step.txt', async () => { + const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'failed-step-harness-')); + const cli = fakeCli(); + const harness = createLiveDeviceHarness< + ReturnType>, + string + >({ + behaviorsForScenario: () => [], + commandsForScenario: () => [], + commonFlags: (_context, args) => [...args, '--json'], + runCli: cli.runCli, + deviceEvidence: async () => 'accelerometer_rotation=1\n', + writeCoverageReport: () => undefined, + }); + const context = createLiveDeviceContext({ artifactRoot, session: 'evidence' }); + + await assert.rejects( + harness.runStep(context, 'read the canary', ['get', 'text', 'id="canary"']), + (error: Error) => { + assert.match(error.message, /step: read the canary/); + assert.match(error.message, /device: .*failed-step-1-device\.txt/); + assert.match(error.message, /screenshot: .*failed-step-1\.png/); + assert.match(error.message, /snapshot: .*failed-step-1-snapshot\.json/); + return true; + }, + ); + + const report = fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8'); + assert.match(report, /device: .*failed-step-1-device\.txt/); + assert.equal( + fs.readFileSync(path.join(context.artifactDir, 'failed-step-1-device.txt'), 'utf8'), + 'accelerometer_rotation=1\n', + ); +}); diff --git a/test/integration/live-device-e2e/failed-step-evidence.ts b/test/integration/live-device-e2e/failed-step-evidence.ts new file mode 100644 index 0000000000..985194b535 --- /dev/null +++ b/test/integration/live-device-e2e/failed-step-evidence.ts @@ -0,0 +1,102 @@ +import fs from 'node:fs'; + +import type { CliJsonResult } from '../cli-json.ts'; + +export type FailedStepEvidence = { + screenshotPath?: string; + snapshotPath?: string; + devicePath?: string; +}; + +export type FailedStepEvidenceInput = { + /** Artifact path prefix, e.g. `/failed-step-7`. */ + stem: string; + /** The CLI bound to the failed step's device and session. */ + runCli: (args: string[]) => Promise; + /** Platform-owned device facts, read outside the CLI. */ + deviceEvidence?: () => Promise; + /** Upper bound for the device facts as a group; the CLI evidence never waits on them. */ + deviceEvidenceTimeoutMs?: number; +}; + +const DEVICE_EVIDENCE_TIMEOUT_MS = 15_000; + +/** + * What the device showed when a step failed: the pixels and the accessibility tree the next + * capture would have read, plus whatever the platform can say about the device outside + * agent-device. Every collector is best-effort and independent: a throw, a non-zero exit, or a + * timed-out hook records nothing for that item and nothing else. + */ +export async function collectFailedStepEvidence( + input: FailedStepEvidenceInput, +): Promise { + const [cli, devicePath] = await Promise.all([ + collectCliEvidence(input), + collectDeviceEvidence(input), + ]); + return { ...cli, ...(devicePath ? { devicePath } : {}) }; +} + +async function collectCliEvidence( + input: FailedStepEvidenceInput, +): Promise> { + const screenshotPath = await captureScreenshot(input); + const snapshotPath = await captureSnapshot(input); + return { + ...(screenshotPath ? { screenshotPath } : {}), + ...(snapshotPath ? { snapshotPath } : {}), + }; +} + +async function captureScreenshot(input: FailedStepEvidenceInput): Promise { + const screenshotPath = `${input.stem}.png`; + try { + const result = await input.runCli(['screenshot', screenshotPath]); + return result.status === 0 ? screenshotPath : undefined; + } catch { + return undefined; + } +} + +async function captureSnapshot(input: FailedStepEvidenceInput): Promise { + const snapshotPath = `${input.stem}-snapshot.json`; + try { + const result = await input.runCli(['snapshot']); + if (result.status !== 0 || result.json === undefined) return undefined; + fs.writeFileSync(snapshotPath, JSON.stringify(result.json, null, 2)); + return snapshotPath; + } catch { + return undefined; + } +} + +async function collectDeviceEvidence(input: FailedStepEvidenceInput): Promise { + if (!input.deviceEvidence) return undefined; + const devicePath = `${input.stem}-device.txt`; + try { + const facts = await withinTimeout( + input.deviceEvidence(), + input.deviceEvidenceTimeoutMs ?? DEVICE_EVIDENCE_TIMEOUT_MS, + ); + if (facts === undefined) return undefined; + fs.writeFileSync(devicePath, facts); + return devicePath; + } catch { + return undefined; + } +} + +async function withinTimeout(pending: Promise, timeoutMs: number): Promise { + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`device evidence exceeded ${timeoutMs}ms`)), + timeoutMs, + ); + }); + try { + return await Promise.race([pending, expired]); + } finally { + clearTimeout(timer); + } +} diff --git a/test/integration/live-device-e2e/runtime.ts b/test/integration/live-device-e2e/runtime.ts index 731621cf67..d0d2acf6a9 100644 --- a/test/integration/live-device-e2e/runtime.ts +++ b/test/integration/live-device-e2e/runtime.ts @@ -3,6 +3,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { type CliJsonResult, formatResultDebug, runBuiltCliJson } from '../cli-json.ts'; +import { collectFailedStepEvidence, type FailedStepEvidence } from './failed-step-evidence.ts'; export type StepRecord = { accepted: boolean; @@ -192,47 +193,14 @@ export function createLiveDeviceHarness< } } - /** - * What the device showed when a step failed: the pixels and the accessibility tree the - * next capture would have read. Best-effort, never throws; a failed capture yields undefined. - */ - async function captureFailedStepEvidence( - context: Context, - ): Promise<{ screenshotPath?: string; snapshotPath?: string; devicePath?: string }> { - const stem = path.join(context.artifactDir, `failed-step-${context.stepHistory.length}`); - const screenshotPath = `${stem}.png`; - const snapshotPath = `${stem}-snapshot.json`; - const devicePath = `${stem}-device.txt`; + function captureFailedStepEvidence(context: Context): Promise { const runCli = options.runCli ?? runBuiltCliJson; - const evidence: { screenshotPath?: string; snapshotPath?: string; devicePath?: string } = {}; - try { - const facts = await options.deviceEvidence?.(context); - if (facts !== undefined) { - fs.writeFileSync(devicePath, facts); - evidence.devicePath = devicePath; - } - } catch { - // evidence only - } - try { - const screenshot = await runCli( - options.commonFlags(context, ['screenshot', screenshotPath]), - context.env, - ); - if (screenshot.status === 0) evidence.screenshotPath = screenshotPath; - } catch { - // evidence only - } - try { - const snapshot = await runCli(options.commonFlags(context, ['snapshot']), context.env); - if (snapshot.status === 0 && snapshot.json !== undefined) { - fs.writeFileSync(snapshotPath, JSON.stringify(snapshot.json, null, 2)); - evidence.snapshotPath = snapshotPath; - } - } catch { - // evidence only - } - return evidence; + const deviceEvidence = options.deviceEvidence; + return collectFailedStepEvidence({ + stem: path.join(context.artifactDir, `failed-step-${context.stepHistory.length}`), + runCli: (args) => runCli(options.commonFlags(context, args), context.env), + ...(deviceEvidence ? { deviceEvidence: () => deviceEvidence(context) } : {}), + }); } function updateSessionState(context: Context, command: string | undefined, status: number): void {