Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions src/__tests__/cli-diff.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,46 @@ describe('cli diff commands', () => {
}
});

test.each([false, true])(
'diff screenshot honors threshold 1 for saved images (json=%s)',
async (json) => {
const dir = mkdtempForTestSync('cli-diff-threshold-');
const baseline = path.join(dir, 'baseline.png');
const current = path.join(dir, 'current.png');
const diffOut = path.join(dir, 'diff.png');
fs.writeFileSync(baseline, solidPngBuffer(2, 2, { r: 0, g: 0, b: 0 }));
fs.writeFileSync(current, solidPngBuffer(2, 2, { r: 255, g: 255, b: 255 }));
fs.writeFileSync(diffOut, 'stale diff');

const result = await runCliCapture([
'diff',
'screenshot',
'--baseline',
baseline,
current,
'--threshold',
'1',
'--out',
diffOut,
...(json ? ['--json'] : []),
]);
assert.equal(result.code, null);
assert.equal(result.calls.length, 0);
assert.equal(result.stderr, '');
if (json) {
const payload = JSON.parse(result.stdout);
assert.equal(payload.success, true);
assert.equal(payload.data.match, true);
assert.equal(payload.data.differentPixels, 0);
assert.equal(payload.data.diffPath, undefined);
} else {
assert.match(result.stdout, /Screenshots match\./);
assert.doesNotMatch(result.stdout, /Diff image:/);
}
assert.equal(fs.existsSync(diffOut), false);
},
);

test('diff screenshot rejects overlay refs with supplied current image', async () => {
const dir = mkdtempForTestSync('cli-diff-test-');
const baseline = path.join(dir, 'baseline.png');
Expand Down
2 changes: 1 addition & 1 deletion src/commands/capture/diff.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ export const diffCommandFacet = defineCommandFacet({
text: {
summary: 'Diff snapshot or screenshot',
cliDetail:
'Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines.',
'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; image dimensions must still match. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines.',
},
metadata: diffCommandMetadata,
run: (client, input) => client.capture.diff(input),
Expand Down
43 changes: 43 additions & 0 deletions src/screenshot-diff/__tests__/screenshot-diff.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,49 @@ test('threshold controls sensitivity: small differences ignored at default thres
assert.equal(strict.differentPixels, 25);
});

test.each([
[
{ r: 0, g: 0, b: 0 },
{ r: 255, g: 255, b: 255 },
],
[
{ r: 255, g: 0, b: 0 },
{ r: 0, g: 255, b: 255 },
],
[
{ r: 0, g: 255, b: 0 },
{ r: 255, g: 0, b: 255 },
],
[
{ r: 0, g: 0, b: 255 },
{ r: 255, g: 255, b: 0 },
],
])('threshold 1 includes the maximum RGB distance from %j to %j', async (before, after) => {
const dir = tmpDir();
const baseline = path.join(dir, 'baseline.png');
const current = path.join(dir, 'current.png');
const outputPath = path.join(dir, 'diff.png');
writeSolidPng(baseline, 2, 2, before);
writeSolidPng(current, 2, 2, after);

const belowMaximum = await compareScreenshots(baseline, current, {
threshold: 1 - Number.EPSILON / 2,
outputPath,
});
assert.equal(belowMaximum.match, false);
assert.equal(belowMaximum.differentPixels, 4);
assert.equal(fs.existsSync(outputPath), true);

const maximum = await compareScreenshots(baseline, current, { threshold: 1, outputPath });
assert.equal(maximum.match, true);
assert.equal(maximum.differentPixels, 0);
assert.equal(maximum.mismatchPercentage, 0);
assert.equal(maximum.totalPixels, 4);
assert.equal(maximum.regions, undefined);
assert.equal(maximum.diffPath, undefined);
assert.equal(fs.existsSync(outputPath), false);
});

test('throws INVALID_ARGS when baseline file does not exist', async () => {
const dir = tmpDir();
const current = path.join(dir, 'current.png');
Expand Down
3 changes: 2 additions & 1 deletion src/screenshot-diff/screenshot-diff.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ export type ScreenshotDiffOptions = {
// white (255,255,255): √(255² + 255² + 255²) = 255√3 ≈ 441.67.
// We use this as the denominator so threshold 0–1 maps linearly to the full
// color distance range: 0 = exact match only, 1 = everything matches.
const COLOR_DISTANCE_SCALE = 255 * Math.sqrt(3);
// Match the per-pixel square-root rounding so the maximum stays inclusive.
const COLOR_DISTANCE_SCALE = Math.sqrt(3 * 255 ** 2);

export async function compareScreenshots(
baselinePath: string,
Expand Down
1 change: 1 addition & 0 deletions website/docs/docs/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -952,6 +952,7 @@ agent-device record stop # Stop active recording
- `screenshot --scale <factor> --overlay-refs` writes a smaller image and draws refs for that final image size; avoid very small scales when text, icons, or labels need to remain readable.
- `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session. Its text output reports ranked changed regions with screen-space rectangles, changed-pixel counts, and each region's share of the diff; JSON also includes normalized rectangles. The earlier best-effort `ocr` and `nonTextDeltas` analyzers are retired; their optional result fields remain for source compatibility but are no longer emitted, so use the baseline/current images and diff artifact with vision for qualitative interpretation. It writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines.
- `diff screenshot --overlay-refs` additionally writes a separate current-screen overlay guide for live captures without using that annotated image for the pixel comparison. If current-screen refs intersect changed regions, the output lists the best ref matches under those regions. Saved-image comparisons do not have live accessibility refs, so `--overlay-refs` is unavailable when a `current.png` path is provided.
- `diff screenshot --threshold <0-1>` sets the per-pixel RGB tolerance (default `0.1`): `0` requires exact colors and `1` ignores all color differences. Image dimensions must still match at every threshold.
- In `--json` mode, each overlay ref also includes a screenshot-space `center` point for coordinate fallback like `press <x> <y>`.
- Burned-in touch overlays are exported only on macOS hosts, because the overlay pipeline depends on Swift + AVFoundation helpers.
- On Linux or other non-macOS hosts, `record stop` still succeeds and returns the raw video plus telemetry sidecar, and includes `overlayWarning` when burn-in overlays were skipped.
Expand Down
Loading