diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fd138dec7..fc8e46e1a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +- Added (diff): `diff screenshot` accepts a JPEG baseline or current image. Both inputs had to be PNG, + so a capture exported by another tool had to be converted first and a HarmonyOS capture — which the + platform serves as JPEG under whatever name the command was given — could never be compared. Each + input is now decoded from its own bytes, so the container is sniffed and a `.png` name holding JPEG + decodes as JPEG. `png-transcode.ts` became `screenshot-image.ts`, the one owner of that sniffing for + both the decode and the provider transcode path, and the PNG worker gained a `decode-image` job that + answers pixels instead of PNG bytes. The `--out` diff image stays PNG, as do the crop, overlay, and + resize passes that rewrite a screenshot in place and could not survive a lossy container. + - Added (limrun): `record start` and `record stop` on Limrun iOS and Android direct sessions. The runtime declared recording unavailable although the Limrun SDK exposes a server-side recorder. Start asks the instance to record (`--quality medium` maps to Limrun quality 5, `high` to 8); diff --git a/packages/capture-kit/src/png-worker-client.test.ts b/packages/capture-kit/src/png-worker-client.test.ts index 6a09d8efe5..b24bedb577 100644 --- a/packages/capture-kit/src/png-worker-client.test.ts +++ b/packages/capture-kit/src/png-worker-client.test.ts @@ -6,6 +6,7 @@ import { computePngRgbDifferenceAsync, computeScreenshotDiffPixelsAsync, decodePngAsync, + decodeScreenshotImageAsync, encodePngAsync, terminatePngWorker, transcodeScreenshotToPngAsync, @@ -110,7 +111,7 @@ test('decodePngAsync rejects invalid PNG data with the canonical decode AppError test('transcodeScreenshotToPngAsync matches the synchronous transcoder byte for byte', async () => { const { encode } = await import('jpeg-js'); - const { transcodeScreenshotToPng } = await import('./png-transcode.ts'); + const { transcodeScreenshotToPng } = await import('./screenshot-image.ts'); const rgba = Buffer.alloc(9 * 7 * 4, 0x66); const jpeg = encode({ width: 9, height: 7, data: rgba }, 90).data; @@ -138,3 +139,50 @@ test('transcodeScreenshotToPngAsync rejects a corrupt JPEG with the canonical de return true; }); }); + +test('decodeScreenshotImageAsync answers the synchronous decoder for both containers', async () => { + const { encode } = await import('jpeg-js'); + const { decodeScreenshotImage } = await import('./screenshot-image.ts'); + const png = PNG.sync.write(buildPatternPng(11, 8, 4)); + const jpeg = encode({ width: 11, height: 8, data: buildPatternPng(11, 8, 4).data }, 90).data; + + for (const bytes of [png, jpeg]) { + const fromWorker = await decodeScreenshotImageAsync(bytes, 'fixture'); + const fromSync = decodeScreenshotImage(bytes, 'fixture'); + + assert.deepEqual([fromWorker.width, fromWorker.height], [fromSync.width, fromSync.height]); + assert.deepEqual(fromWorker.data, fromSync.data); + } +}); + +test('decodeScreenshotImageAsync decodes a JPEG by its bytes, not its label', async () => { + const { encode } = await import('jpeg-js'); + const jpeg = encode({ width: 9, height: 7, data: buildPatternPng(9, 7, 5).data }, 90).data; + + const decoded = await decodeScreenshotImageAsync(jpeg, 'fixture.png'); + + assert.deepEqual([decoded.width, decoded.height], [9, 7]); + assert.equal(decoded.data.length, 9 * 7 * 4); +}); + +test('decodeScreenshotImageAsync rejects a container it cannot read with the canonical AppError', async () => { + await assert.rejects( + () => decodeScreenshotImageAsync(Buffer.from('GIF89a'), 'fixture'), + (error) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'COMMAND_FAILED'); + assert.equal((error as AppError).message, 'fixture is neither PNG nor JPEG'); + assert.equal((error as AppError).details?.label, 'fixture'); + return true; + }, + ); + + const corrupt = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(16, 0)]); + await assert.rejects( + () => decodeScreenshotImageAsync(corrupt, 'fixture'), + (error) => { + assert.match((error as AppError).message, /Failed to decode fixture as JPEG/); + return true; + }, + ); +}); diff --git a/packages/capture-kit/src/png-worker-client.ts b/packages/capture-kit/src/png-worker-client.ts index 1111259af7..994e1cedf0 100644 --- a/packages/capture-kit/src/png-worker-client.ts +++ b/packages/capture-kit/src/png-worker-client.ts @@ -201,6 +201,31 @@ export async function decodePngAsync(buffer: Buffer, label: string): Promise { + const result = await runPngJob({ kind: 'decode-image', image: bytes, label }, async () => { + // Read on demand so the JPEG decoder stays out of the import closure of every entry that only + // needs the worker's other jobs. + const { decodeScreenshotImage } = await import('./screenshot-image.ts'); + const image = decodeScreenshotImage(bytes, label); + return { kind: 'decode-image', width: image.width, height: image.height, data: image.data }; + }); + return toDecodedPng(result); +} + +/** + * Rebuilds the decoded-image handle on this side of the worker boundary. `PNG` is the shape every + * pixel reader here consumes: size plus RGBA rows. + */ +function toDecodedPng(result: { width: number; height: number; data: Uint8Array }): PNG { const png = new PNG({ width: result.width, height: result.height }); png.data = toBuffer(result.data); return png; @@ -267,7 +292,7 @@ export async function transcodeScreenshotToPngAsync(bytes: Buffer, label: string const result = await runPngJob({ kind: 'jpeg-to-png', image: bytes, label }, async () => { // Read on demand so the JPEG decoder stays out of the import closure of every entry that only // needs the worker's other jobs. - const { transcodeScreenshotToPng } = await import('./png-transcode.ts'); + const { transcodeScreenshotToPng } = await import('./screenshot-image.ts'); return { kind: 'jpeg-to-png', png: transcodeScreenshotToPng(bytes, label) }; }); return toBuffer(result.png); diff --git a/packages/capture-kit/src/png-worker-contract.ts b/packages/capture-kit/src/png-worker-contract.ts index 0511b11541..a24767da8f 100644 --- a/packages/capture-kit/src/png-worker-contract.ts +++ b/packages/capture-kit/src/png-worker-contract.ts @@ -16,6 +16,8 @@ import type { PngRgbDifferenceResult } from './png-rgb-difference.ts'; export type PngWorkerJob = | { kind: 'decode'; png: Uint8Array; label: string } + // A screenshot in whatever container it arrived in; answers decoded pixels, not PNG bytes. + | { kind: 'decode-image'; image: Uint8Array; label: string } | { kind: 'encode'; width: number; height: number; data: Uint8Array } | { kind: 'crop'; png: Uint8Array; label: string; box: Rect } | { kind: 'rgb-difference'; firstPng: Uint8Array; secondPng: Uint8Array; label: string } @@ -25,6 +27,7 @@ export type PngWorkerJob = export type PngWorkerJobResult = | { kind: 'decode'; width: number; height: number; data: Uint8Array } + | { kind: 'decode-image'; width: number; height: number; data: Uint8Array } | { kind: 'encode'; png: Uint8Array } // A crop answers `null` when the box already covers the image, so the caller keeps the file. | { kind: 'crop'; png: Uint8Array | null } diff --git a/packages/capture-kit/src/png-worker.test.ts b/packages/capture-kit/src/png-worker.test.ts index 9bc7f3b901..be597d4bef 100644 --- a/packages/capture-kit/src/png-worker.test.ts +++ b/packages/capture-kit/src/png-worker.test.ts @@ -28,6 +28,18 @@ test('resultTransferList transfers a cropped encoding and skips an untouched fil assert.deepEqual(resultTransferList({ kind: 'crop', png: null }), []); }); +test('resultTransferList transfers decoded pixels for both screenshot decode jobs', () => { + const owned = Buffer.alloc(16); // Buffer.alloc never uses the shared pool + const pooled = new Uint8Array(new ArrayBuffer(32), 4, 8); // offset view, pooled-Buffer shape + const decoded = { width: 2, height: 2 }; + + assert.deepEqual(resultTransferList({ kind: 'decode', ...decoded, data: owned }), [owned.buffer]); + assert.deepEqual(resultTransferList({ kind: 'decode-image', ...decoded, data: owned }), [ + owned.buffer, + ]); + assert.deepEqual(resultTransferList({ kind: 'decode-image', ...decoded, data: pooled }), []); +}); + test('resultTransferList transfers only the fully-owned views of a mixed result', () => { const ownedDiffData = Buffer.alloc(16); // Buffer.alloc never uses the shared pool const pooledMask = new Uint8Array(new ArrayBuffer(32), 4, 8); // offset view, pooled-Buffer shape diff --git a/packages/capture-kit/src/png-worker.ts b/packages/capture-kit/src/png-worker.ts index 68a66bf06d..9508d73f9a 100644 --- a/packages/capture-kit/src/png-worker.ts +++ b/packages/capture-kit/src/png-worker.ts @@ -17,13 +17,19 @@ import { * `png-worker-client.ts`; published as the `internal/png-worker` build entry. */ -// The daemon prewarms this worker at startup, so the JPEG decoder loads only for a transcode job. +// The daemon prewarms this worker at startup, so the JPEG decoder loads only for a job that can +// carry JPEG bytes. async function runJob(request: PngWorkerRequest): Promise { switch (request.kind) { case 'decode': { const png = decodePng(toBuffer(request.png), request.label); return { kind: 'decode', width: png.width, height: png.height, data: png.data }; } + case 'decode-image': { + const { decodeScreenshotImage } = await import('./screenshot-image.ts'); + const image = decodeScreenshotImage(toBuffer(request.image), request.label); + return { kind: 'decode-image', width: image.width, height: image.height, data: image.data }; + } case 'encode': { const png = new PNG({ width: request.width, height: request.height }); png.data = toBuffer(request.data); @@ -44,7 +50,7 @@ async function runJob(request: PngWorkerRequest): Promise { return { kind: 'diff-pixels', ...computeScreenshotDiffPixels(request) }; } case 'jpeg-to-png': { - const { transcodeScreenshotToPng } = await import('./png-transcode.ts'); + const { transcodeScreenshotToPng } = await import('./screenshot-image.ts'); return { kind: 'jpeg-to-png', png: transcodeScreenshotToPng(toBuffer(request.image), request.label), @@ -80,6 +86,7 @@ export function resultTransferList(result: PngWorkerJobResult): ArrayBuffer[] { function resultBufferViews(result: PngWorkerJobResult): Uint8Array[] { switch (result.kind) { case 'decode': + case 'decode-image': return [result.data]; case 'encode': return [result.png]; diff --git a/packages/capture-kit/src/png-transcode.test.ts b/packages/capture-kit/src/screenshot-image.test.ts similarity index 61% rename from packages/capture-kit/src/png-transcode.test.ts rename to packages/capture-kit/src/screenshot-image.test.ts index 91872ce16d..172f56b363 100644 --- a/packages/capture-kit/src/png-transcode.test.ts +++ b/packages/capture-kit/src/screenshot-image.test.ts @@ -1,7 +1,11 @@ import { expect, test } from 'vitest'; import { encode as encodeJpeg } from 'jpeg-js'; import { PNG } from './png.ts'; -import { detectScreenshotImageFormat, transcodeScreenshotToPng } from './png-transcode.ts'; +import { + decodeScreenshotImage, + detectScreenshotImageFormat, + transcodeScreenshotToPng, +} from './screenshot-image.ts'; function solidRgba(width: number, height: number, rgba: readonly [number, number, number, number]) { const data = Buffer.alloc(width * height * 4); @@ -74,3 +78,43 @@ test('a truncated JPEG body is refused with the same typed decode error', () => message: 'Failed to decode Limrun iOS screenshot as JPEG', }); }); + +test('decoding a PNG answers its pixels without re-encoding the container', () => { + const png = new PNG({ width: 3, height: 2 }); + png.data = solidRgba(3, 2, [12, 240, 60, 255]); + + const decoded = decodeScreenshotImage(PNG.sync.write(png), 'test screenshot'); + + expect([decoded.width, decoded.height]).toEqual([3, 2]); + expect([...decoded.data.subarray(0, 4)]).toEqual([12, 240, 60, 255]); +}); + +test('decoding a JPEG answers RGBA rows, with opaque alpha where the container has none', () => { + const jpeg = encodeJpeg({ width: 5, height: 3, data: solidRgba(5, 3, [12, 240, 60, 255]) }, 100); + + const decoded = decodeScreenshotImage(jpeg.data, 'test screenshot'); + + expect([decoded.width, decoded.height]).toEqual([5, 3]); + expect(decoded.data.length).toBe(5 * 3 * 4); + const [r = -1, g = -1, b = -1, a = -1] = decoded.data.subarray(0, 4); + // JPEG is lossy; a flat field survives within a few levels per channel. + expect(Math.abs(r - 12)).toBeLessThanOrEqual(4); + expect(Math.abs(g - 240)).toBeLessThanOrEqual(4); + expect(Math.abs(b - 60)).toBeLessThanOrEqual(4); + expect(a).toBe(255); +}); + +test('decoding refuses bytes in neither container and a JPEG body that cannot be decoded', () => { + expect(thrownBy(() => decodeScreenshotImage(Buffer.from('GIF89a'), 'fixture'))).toMatchObject({ + code: 'COMMAND_FAILED', + message: 'fixture is neither PNG nor JPEG', + details: { label: 'fixture', leadingBytes: '47494638' }, + }); + + const corrupt = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 0x41)]); + expect(thrownBy(() => decodeScreenshotImage(corrupt, 'fixture'))).toMatchObject({ + code: 'COMMAND_FAILED', + message: 'Failed to decode fixture as JPEG', + details: { label: 'fixture', reason: expect.any(String) }, + }); +}); diff --git a/packages/capture-kit/src/png-transcode.ts b/packages/capture-kit/src/screenshot-image.ts similarity index 56% rename from packages/capture-kit/src/png-transcode.ts rename to packages/capture-kit/src/screenshot-image.ts index 121360b95a..687ac566cb 100644 --- a/packages/capture-kit/src/png-transcode.ts +++ b/packages/capture-kit/src/screenshot-image.ts @@ -1,6 +1,6 @@ import { AppError } from '@agent-device/kernel/errors'; import { decode as decodeJpeg } from 'jpeg-js'; -import { hasPngSignature, PNG } from './png.ts'; +import { decodePng, hasPngSignature, PNG } from './png.ts'; const JPEG_SIGNATURE = Buffer.from([0xff, 0xd8, 0xff]); @@ -13,10 +13,26 @@ export function detectScreenshotImageFormat(bytes: Buffer): ScreenshotImageForma return undefined; } +/** + * Decodes a screenshot in whatever container it arrived in into the RGBA rows every pixel reader in + * this repository consumes. The container is sniffed, never trusted from a file name or a caller's + * guess, so a JPEG stored under a `.png` name still decodes. + * + * Synchronous and CPU-bound: daemon request paths reach it through the PNG worker + * (`decodeScreenshotImageAsync` in `png-worker-client.ts`), like every other codec job. + */ +export function decodeScreenshotImage(bytes: Buffer, label: string): PNG { + const format = detectScreenshotImageFormat(bytes); + if (format === 'png') return decodePng(bytes, label); + if (format === 'jpeg') return decodeJpegImage(bytes, label); + throw unsupportedContainer(label, bytes); +} + /** * Returns PNG bytes for a screenshot a provider handed back in whatever container it prefers. PNG * passes through untouched; JPEG is decoded and re-encoded losslessly from the decoded pixels, so - * every PNG-only reader downstream (size, crop, overlay, diff) sees the format the path promises. + * every PNG-only reader downstream (size, crop, overlay) sees the format the path promises. Those + * readers rewrite the file in place, which is why a JPEG cannot be handed to them undecoded. * * Synchronous and CPU-bound: daemon request paths reach it through the PNG worker * (`transcodeScreenshotToPngAsync` in `png-worker-client.ts`), like every other codec job. @@ -24,20 +40,17 @@ export function detectScreenshotImageFormat(bytes: Buffer): ScreenshotImageForma export function transcodeScreenshotToPng(bytes: Buffer, label: string): Buffer { const format = detectScreenshotImageFormat(bytes); if (format === 'png') return bytes; - if (format === 'jpeg') return jpegToPng(bytes, label); - throw new AppError('COMMAND_FAILED', `${label} is neither PNG nor JPEG`, { - label, - leadingBytes: bytes.subarray(0, 4).toString('hex'), - }); + if (format === 'jpeg') return PNG.sync.write(decodeJpegImage(bytes, label)); + throw unsupportedContainer(label, bytes); } /** A JPEG signature does not prove a decodable body; a failure keeps the label and the decoder's reason. */ -function jpegToPng(bytes: Buffer, label: string): Buffer { +function decodeJpegImage(bytes: Buffer, label: string): PNG { try { const decoded = decodeJpeg(bytes, { useTArray: true, formatAsRGBA: true }); const png = new PNG({ width: decoded.width, height: decoded.height }); png.data = Buffer.from(decoded.data.buffer, decoded.data.byteOffset, decoded.data.byteLength); - return PNG.sync.write(png); + return png; } catch (error) { throw new AppError('COMMAND_FAILED', `Failed to decode ${label} as JPEG`, { label, @@ -45,3 +58,10 @@ function jpegToPng(bytes: Buffer, label: string): Buffer { }); } } + +function unsupportedContainer(label: string, bytes: Buffer): AppError { + return new AppError('COMMAND_FAILED', `${label} is neither PNG nor JPEG`, { + label, + leadingBytes: bytes.subarray(0, 4).toString('hex'), + }); +} diff --git a/src/commands/capture/diff.ts b/src/commands/capture/diff.ts index 4e50a78596..20c9194e79 100644 --- a/src/commands/capture/diff.ts +++ b/src/commands/capture/diff.ts @@ -59,7 +59,7 @@ export const diffCommandFacet = defineCommandFacet({ text: { summary: 'Diff snapshot or screenshot', cliDetail: - 'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; image dimensions must still match. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines.', + 'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; image dimensions must still match. Both screenshot inputs are decoded from their bytes, so a baseline or current image may be PNG or JPEG, and the diff image is always PNG. JPEG is lossy, so keep --threshold above 0 whenever either input is JPEG. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines.', }, metadata: diffCommandMetadata, run: (client, input) => client.capture.diff(input), diff --git a/src/commands/schema/cli-help-command-usage.test.ts b/src/commands/schema/cli-help-command-usage.test.ts index ae965b9697..1d25e03127 100644 --- a/src/commands/schema/cli-help-command-usage.test.ts +++ b/src/commands/schema/cli-help-command-usage.test.ts @@ -39,6 +39,14 @@ test('usageForCommand documents screenshot diff normalization', async () => { assert.match(help, /screenshot --normalize-status-bar/); }); +test('usageForCommand documents the screenshot diff input containers', async () => { + const help = await usageForCommand('diff'); + if (help === null) throw new Error('Expected diff help text'); + assert.match(help, /a baseline or current image may be PNG or JPEG/); + assert.match(help, /the diff image is always PNG/); + assert.match(help, /keep --threshold above 0 whenever either input is JPEG/); +}); + test('usageForCommand resolves longpress help', async () => { const help = await usageForCommand('longpress'); assert.equal(help === null, false); diff --git a/src/screenshot-diff/__tests__/screenshot-diff.test.ts b/src/screenshot-diff/__tests__/screenshot-diff.test.ts index db52ce175a..ad3a447f6f 100644 --- a/src/screenshot-diff/__tests__/screenshot-diff.test.ts +++ b/src/screenshot-diff/__tests__/screenshot-diff.test.ts @@ -1,25 +1,10 @@ -import { test, vi } from 'vitest'; +import { test } from 'vitest'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; -vi.mock('@agent-device/capture-kit/png-worker-client', async () => { - const [{ PNG }, { decodePng }, { computeScreenshotDiffPixels }] = await Promise.all([ - import('@agent-device/capture-kit/png'), - import('@agent-device/capture-kit/png'), - import('@agent-device/capture-kit/screenshot-diff-pixels'), - ]); - return { - decodePngAsync: async (buffer: Buffer, label: string) => decodePng(buffer, label), - encodePngAsync: async (png: InstanceType) => PNG.sync.write(png), - computeScreenshotDiffPixelsAsync: async ( - job: Parameters[0], - ) => computeScreenshotDiffPixels(job), - }; -}); - -import { PNG } from '@agent-device/capture-kit/png'; +import { hasPngSignature, PNG } from '@agent-device/capture-kit/png'; import { compareScreenshots } from '../screenshot-diff.ts'; function tmpDir(): string { @@ -43,6 +28,26 @@ function writeSolidPng( fs.writeFileSync(filePath, PNG.sync.write(png)); } +/** Encode a solid-color JPEG, so a test can hand the comparison a lossy container. */ +async function writeSolidJpeg( + filePath: string, + width: number, + height: number, + color: { r: number; g: number; b: number }, +): Promise { + const { encode } = await import('jpeg-js'); + const data = Buffer.alloc(width * height * 4); + for (let i = 0; i < data.length; i += 4) { + data[i] = color.r; + data[i + 1] = color.g; + data[i + 2] = color.b; + data[i + 3] = 255; + } + const jpeg = Buffer.from(encode({ width, height, data }, 90).data); + fs.writeFileSync(filePath, jpeg); + return jpeg; +} + function paintRect( png: PNG, rect: { x: number; y: number; width: number; height: number }, @@ -416,7 +421,7 @@ test('throws INVALID_ARGS when current file does not exist', async () => { ); }); -test('throws COMMAND_FAILED for invalid PNG data', async () => { +test('throws COMMAND_FAILED for bytes in neither supported container', async () => { const dir = tmpDir(); const baseline = path.join(dir, 'baseline.png'); const current = path.join(dir, 'current.png'); @@ -428,7 +433,78 @@ test('throws COMMAND_FAILED for invalid PNG data', async () => { () => compareScreenshots(baseline, current), (err: any) => { assert.equal(err.code, 'COMMAND_FAILED'); - assert.match(err.message, /Failed to decode baseline screenshot/); + assert.match(err.message, /baseline screenshot is neither PNG nor JPEG/); + return true; + }, + ); +}); + +test('a JPEG stored under a .png name compares as a match', async () => { + const dir = tmpDir(); + const baseline = path.join(dir, 'baseline.png'); + const current = path.join(dir, 'current.png'); + + // The `.png` names hold JPEG bytes on purpose: the container is sniffed from the bytes. + const jpeg = await writeSolidJpeg(baseline, 8, 6, { r: 20, g: 200, b: 40 }); + fs.writeFileSync(current, jpeg); + + // The same stored bytes decode to the same pixels, so this pins an exact comparison. + const result = await compareScreenshots(baseline, current, { threshold: 0 }); + + assert.equal(result.match, true); + assert.equal(result.differentPixels, 0); + assert.equal(result.totalPixels, 48); +}); + +test('a JPEG comparison writes the diff artifact as PNG', async () => { + const dir = tmpDir(); + const baseline = path.join(dir, 'baseline.jpg'); + const current = path.join(dir, 'current.jpg'); + const diffPath = path.join(dir, 'diff.png'); + + await writeSolidJpeg(baseline, 8, 6, { r: 20, g: 200, b: 40 }); + await writeSolidJpeg(current, 8, 6, { r: 200, g: 20, b: 40 }); + + const result = await compareScreenshots(baseline, current, { outputPath: diffPath }); + + assert.equal(result.match, false); + assert.equal(result.diffPath, diffPath); + assert.equal(hasPngSignature(fs.readFileSync(diffPath)), true); +}); + +test('a JPEG input is measured at its decoded dimensions', async () => { + const dir = tmpDir(); + const baseline = path.join(dir, 'baseline.jpg'); + const current = path.join(dir, 'current.png'); + + await writeSolidJpeg(baseline, 8, 6, { r: 20, g: 200, b: 40 }); + writeSolidPng(current, 5, 5, { r: 20, g: 200, b: 40 }); + + const result = await compareScreenshots(baseline, current); + + assert.equal(result.match, false); + assert.deepEqual(result.dimensionMismatch, { + expected: { width: 8, height: 6 }, + actual: { width: 5, height: 5 }, + }); +}); + +test('JPEG bytes that do not decode are refused with the JPEG decode error', async () => { + const dir = tmpDir(); + const baseline = path.join(dir, 'baseline.jpg'); + const current = path.join(dir, 'current.png'); + + fs.writeFileSync( + baseline, + Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 0x41)]), + ); + writeSolidPng(current, 5, 5, { r: 0, g: 0, b: 0 }); + + await assert.rejects( + () => compareScreenshots(baseline, current), + (err: any) => { + assert.equal(err.code, 'COMMAND_FAILED'); + assert.match(err.message, /Failed to decode baseline screenshot as JPEG/); return true; }, ); diff --git a/src/screenshot-diff/screenshot-diff.ts b/src/screenshot-diff/screenshot-diff.ts index 1c2d034197..657d104591 100644 --- a/src/screenshot-diff/screenshot-diff.ts +++ b/src/screenshot-diff/screenshot-diff.ts @@ -5,7 +5,7 @@ import type { Rect } from '@agent-device/kernel/snapshot'; import { PNG } from '@agent-device/capture-kit/png'; import { computeScreenshotDiffPixelsAsync, - decodePngAsync, + decodeScreenshotImageAsync, encodePngAsync, } from '@agent-device/capture-kit/png-worker-client'; import { annotateDiffRegions } from './screenshot-diff-region-overlay.ts'; @@ -75,6 +75,11 @@ export type ScreenshotDiffOptions = { // Match the per-pixel square-root rounding so the maximum stays inclusive. const COLOR_DISTANCE_SCALE = Math.sqrt(3 * 255 ** 2); +/** + * Compares two screenshots pixel by pixel. Each input may be PNG or JPEG: the container is sniffed + * from the bytes rather than the file name, so a baseline exported by any tool that writes JPEG + * still compares. The diff image itself is always PNG. + */ export async function compareScreenshots( baselinePath: string, currentPath: string, @@ -91,8 +96,8 @@ export async function compareScreenshots( ]); const [baseline, current] = await Promise.all([ - decodePngAsync(baselineBuffer, 'baseline screenshot'), - decodePngAsync(currentBuffer, 'current screenshot'), + decodeScreenshotImageAsync(baselineBuffer, 'baseline screenshot'), + decodeScreenshotImageAsync(currentBuffer, 'current screenshot'), ]); validateMaxPixels(baseline.width, baseline.height, 'baseline screenshot', options.maxPixels); validateMaxPixels(current.width, current.height, 'current screenshot', options.maxPixels); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index c44dc890a3..ff6c28e4b0 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1008,6 +1008,7 @@ agent-device screenshot --fullscreen # Force full-screen capture on macOS app agent-device open --platform macos --surface desktop && agent-device screenshot desktop.png agent-device diff screenshot --baseline baseline.png --out diff.png agent-device diff screenshot --baseline baseline.png current.png --out diff.png +agent-device diff screenshot --baseline baseline.jpg --out diff.png # JPEG inputs are accepted too agent-device diff screenshot --baseline baseline.png --out diff.png --overlay-refs agent-device record start # Start app-scoped recording after open agent-device record start session.mp4 # Start app-scoped recording to explicit path @@ -1025,9 +1026,9 @@ agent-device record stop # Stop active recording - `screenshot --crop-on ` captures a fresh full snapshot of the same screen and crops the saved PNG to the frame the selector resolves to. The crop is re-encoded, so byte-comparing it against an older crop of the same frame is unreliable; a crop whose pixels are all opaque is written as truecolor RGB, while one containing transparency keeps RGBA. The selector must resolve to exactly one framed node; the result carries a `warnings` entry when the frame is clipped to the image. Currently accepted on iOS simulators and Android emulators — every other target is refused before any device work, and the flag cannot be combined with `--overlay-refs` or `--fullscreen` because both move the captured frame away from the snapshot viewport the crop is measured against. - `screenshot --normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for deterministic screenshot baselines; ordinary screenshots leave the simulator's current chrome visible. - `screenshot --scale --overlay-refs` writes a smaller image and draws refs for that final image size; avoid very small scales when text, icons, or labels need to remain readable. -- `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session. Its text output reports ranked changed regions with screen-space rectangles, changed-pixel counts, and each region's share of the diff; JSON also includes normalized rectangles. The earlier best-effort `ocr` and `nonTextDeltas` analyzers are retired; their optional result fields remain for source compatibility but are no longer emitted, so use the baseline/current images and diff artifact with vision for qualitative interpretation. It writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines. +- `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session. Each input is decoded from its own bytes, so `--baseline` and a saved current image may be PNG or JPEG whatever their extension says. Most `agent-device screenshot` artifacts are PNG; a HarmonyOS capture is the JPEG its device serves, stored under the requested name. Its text output reports ranked changed regions with screen-space rectangles, changed-pixel counts, and each region's share of the diff; JSON also includes normalized rectangles. The earlier best-effort `ocr` and `nonTextDeltas` analyzers are retired; their optional result fields remain for source compatibility but are no longer emitted, so use the baseline/current images and diff artifact with vision for qualitative interpretation. It writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines. - `diff screenshot --overlay-refs` additionally writes a separate current-screen overlay guide for live captures without using that annotated image for the pixel comparison. If current-screen refs intersect changed regions, the output lists the best ref matches under those regions. Saved-image comparisons do not have live accessibility refs, so `--overlay-refs` is unavailable when a `current.png` path is provided. -- `diff screenshot --threshold <0-1>` sets the per-pixel RGB tolerance (default `0.1`): `0` requires exact colors and `1` ignores all color differences. Image dimensions must still match at every threshold. +- `diff screenshot --threshold <0-1>` sets the per-pixel RGB tolerance (default `0.1`): `0` requires exact colors and `1` ignores all color differences. Image dimensions must still match at every threshold. JPEG is lossy and shifts pixels around hard edges, so keep the threshold above `0` when comparing a JPEG against anything it was not encoded from. - In `--json` mode, each overlay ref also includes a screenshot-space `center` point for coordinate fallback like `press `. - Burned-in touch overlays are exported only on macOS hosts, because the overlay pipeline depends on Swift + AVFoundation helpers. - On Linux or other non-macOS hosts, `record stop` still succeeds and returns the raw video plus telemetry sidecar, and includes `overlayWarning` when burn-in overlays were skipped.