feat(diff): accept JPEG inputs for screenshot comparison

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## Unreleased
 
+- Added (diff): `diff screenshot` accepts a JPEG baseline or current image. Both inputs had to be PNG,
+  so a capture exported by another tool had to be converted first and a HarmonyOS capture — which the
+  platform serves as JPEG under whatever name the command was given — could never be compared. Each
+  input is now decoded from its own bytes, so the container is sniffed and a `.png` name holding JPEG
+  decodes as JPEG. `png-transcode.ts` became `screenshot-image.ts`, the one owner of that sniffing for
+  both the decode and the provider transcode path, and the PNG worker gained a `decode-image` job that
+  answers pixels instead of PNG bytes. The `--out` diff image stays PNG, as do the crop, overlay, and
+  resize passes that rewrite a screenshot in place and could not survive a lossy container.
+
 - Added (limrun): `record start` and `record stop` on Limrun iOS and Android direct sessions. The
   runtime declared recording unavailable although the Limrun SDK exposes a server-side recorder.
   Start asks the instance to record (`--quality medium` maps to Limrun quality 5, `high` to 8);

packages/capture-kit/src/png-worker-client.test.ts

@@ -6,6 +6,7 @@ import {
   computePngRgbDifferenceAsync,
   computeScreenshotDiffPixelsAsync,
   decodePngAsync,
+  decodeScreenshotImageAsync,
   encodePngAsync,
   terminatePngWorker,
   transcodeScreenshotToPngAsync,
@@ -110,7 +111,7 @@ test('decodePngAsync rejects invalid PNG data with the canonical decode AppError
 
 test('transcodeScreenshotToPngAsync matches the synchronous transcoder byte for byte', async () => {
   const { encode } = await import('jpeg-js');
-  const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
+  const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
   const rgba = Buffer.alloc(9 * 7 * 4, 0x66);
   const jpeg = encode({ width: 9, height: 7, data: rgba }, 90).data;
 
@@ -138,3 +139,50 @@ test('transcodeScreenshotToPngAsync rejects a corrupt JPEG with the canonical de
     return true;
   });
 });
+
+test('decodeScreenshotImageAsync answers the synchronous decoder for both containers', async () => {
+  const { encode } = await import('jpeg-js');
+  const { decodeScreenshotImage } = await import('./screenshot-image.ts');
+  const png = PNG.sync.write(buildPatternPng(11, 8, 4));
+  const jpeg = encode({ width: 11, height: 8, data: buildPatternPng(11, 8, 4).data }, 90).data;
+
+  for (const bytes of [png, jpeg]) {
+    const fromWorker = await decodeScreenshotImageAsync(bytes, 'fixture');
+    const fromSync = decodeScreenshotImage(bytes, 'fixture');
+
+    assert.deepEqual([fromWorker.width, fromWorker.height], [fromSync.width, fromSync.height]);
+    assert.deepEqual(fromWorker.data, fromSync.data);
+  }
+});
+
+test('decodeScreenshotImageAsync decodes a JPEG by its bytes, not its label', async () => {
+  const { encode } = await import('jpeg-js');
+  const jpeg = encode({ width: 9, height: 7, data: buildPatternPng(9, 7, 5).data }, 90).data;
+
+  const decoded = await decodeScreenshotImageAsync(jpeg, 'fixture.png');
+
+  assert.deepEqual([decoded.width, decoded.height], [9, 7]);
+  assert.equal(decoded.data.length, 9 * 7 * 4);
+});
+
+test('decodeScreenshotImageAsync rejects a container it cannot read with the canonical AppError', async () => {
+  await assert.rejects(
+    () => decodeScreenshotImageAsync(Buffer.from('GIF89a'), 'fixture'),
+    (error) => {
+      assert.equal(error instanceof AppError, true);
+      assert.equal((error as AppError).code, 'COMMAND_FAILED');
+      assert.equal((error as AppError).message, 'fixture is neither PNG nor JPEG');
+      assert.equal((error as AppError).details?.label, 'fixture');
+      return true;
+    },
+  );
+
+  const corrupt = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(16, 0)]);
+  await assert.rejects(
+    () => decodeScreenshotImageAsync(corrupt, 'fixture'),
+    (error) => {
+      assert.match((error as AppError).message, /Failed to decode fixture as JPEG/);
+      return true;
+    },
+  );
+});

packages/capture-kit/src/png-worker-client.ts

@@ -201,6 +201,31 @@ export async function decodePngAsync(buffer: Buffer, label: string): Promise<PNG
     const png = decodePng(buffer, label);
     return { kind: 'decode', width: png.width, height: png.height, data: png.data };
   });
+  return toDecodedPng(result);
+}
+
+/**
+ * Decodes a screenshot in whatever container it arrived in into RGBA pixels, sniffing the container
+ * instead of trusting a file name. Use it where a command reads an image someone else produced; a
+ * command that rewrites a screenshot in place stays on `decodePngAsync`, which keeps the file's
+ * container honest. Decode failures carry the canonical `AppError`.
+ */
+export async function decodeScreenshotImageAsync(bytes: Buffer, label: string): Promise<PNG> {
+  const result = await runPngJob({ kind: 'decode-image', image: bytes, label }, async () => {
+    // Read on demand so the JPEG decoder stays out of the import closure of every entry that only
+    // needs the worker's other jobs.
+    const { decodeScreenshotImage } = await import('./screenshot-image.ts');
+    const image = decodeScreenshotImage(bytes, label);
+    return { kind: 'decode-image', width: image.width, height: image.height, data: image.data };
+  });
+  return toDecodedPng(result);
+}
+
+/**
+ * Rebuilds the decoded-image handle on this side of the worker boundary. `PNG` is the shape every
+ * pixel reader here consumes: size plus RGBA rows.
+ */
+function toDecodedPng(result: { width: number; height: number; data: Uint8Array }): PNG {
   const png = new PNG({ width: result.width, height: result.height });
   png.data = toBuffer(result.data);
   return png;
@@ -267,7 +292,7 @@ export async function transcodeScreenshotToPngAsync(bytes: Buffer, label: string
   const result = await runPngJob({ kind: 'jpeg-to-png', image: bytes, label }, async () => {
     // Read on demand so the JPEG decoder stays out of the import closure of every entry that only
     // needs the worker's other jobs.
-    const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
+    const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
     return { kind: 'jpeg-to-png', png: transcodeScreenshotToPng(bytes, label) };
   });
   return toBuffer(result.png);

packages/capture-kit/src/png-worker-contract.ts

@@ -16,6 +16,8 @@ import type { PngRgbDifferenceResult } from './png-rgb-difference.ts';
 
 export type PngWorkerJob =
   | { kind: 'decode'; png: Uint8Array; label: string }
+  // A screenshot in whatever container it arrived in; answers decoded pixels, not PNG bytes.
+  | { kind: 'decode-image'; image: Uint8Array; label: string }
   | { kind: 'encode'; width: number; height: number; data: Uint8Array }
   | { kind: 'crop'; png: Uint8Array; label: string; box: Rect }
   | { kind: 'rgb-difference'; firstPng: Uint8Array; secondPng: Uint8Array; label: string }
@@ -25,6 +27,7 @@ export type PngWorkerJob =
 
 export type PngWorkerJobResult =
   | { kind: 'decode'; width: number; height: number; data: Uint8Array }
+  | { kind: 'decode-image'; width: number; height: number; data: Uint8Array }
   | { kind: 'encode'; png: Uint8Array }
   // A crop answers `null` when the box already covers the image, so the caller keeps the file.
   | { kind: 'crop'; png: Uint8Array | null }

packages/capture-kit/src/png-worker.test.ts

@@ -28,6 +28,18 @@ test('resultTransferList transfers a cropped encoding and skips an untouched fil
   assert.deepEqual(resultTransferList({ kind: 'crop', png: null }), []);
 });
 
+test('resultTransferList transfers decoded pixels for both screenshot decode jobs', () => {
+  const owned = Buffer.alloc(16); // Buffer.alloc never uses the shared pool
+  const pooled = new Uint8Array(new ArrayBuffer(32), 4, 8); // offset view, pooled-Buffer shape
+  const decoded = { width: 2, height: 2 };
+
+  assert.deepEqual(resultTransferList({ kind: 'decode', ...decoded, data: owned }), [owned.buffer]);
+  assert.deepEqual(resultTransferList({ kind: 'decode-image', ...decoded, data: owned }), [
+    owned.buffer,
+  ]);
+  assert.deepEqual(resultTransferList({ kind: 'decode-image', ...decoded, data: pooled }), []);
+});
+
 test('resultTransferList transfers only the fully-owned views of a mixed result', () => {
   const ownedDiffData = Buffer.alloc(16); // Buffer.alloc never uses the shared pool
   const pooledMask = new Uint8Array(new ArrayBuffer(32), 4, 8); // offset view, pooled-Buffer shape

packages/capture-kit/src/png-worker.ts

@@ -17,13 +17,19 @@ import {
  * `png-worker-client.ts`; published as the `internal/png-worker` build entry.
  */
 
-// The daemon prewarms this worker at startup, so the JPEG decoder loads only for a transcode job.
+// The daemon prewarms this worker at startup, so the JPEG decoder loads only for a job that can
+// carry JPEG bytes.
 async function runJob(request: PngWorkerRequest): Promise<PngWorkerJobResult> {
   switch (request.kind) {
     case 'decode': {
       const png = decodePng(toBuffer(request.png), request.label);
       return { kind: 'decode', width: png.width, height: png.height, data: png.data };
     }
+    case 'decode-image': {
+      const { decodeScreenshotImage } = await import('./screenshot-image.ts');
+      const image = decodeScreenshotImage(toBuffer(request.image), request.label);
+      return { kind: 'decode-image', width: image.width, height: image.height, data: image.data };
+    }
     case 'encode': {
       const png = new PNG({ width: request.width, height: request.height });
       png.data = toBuffer(request.data);
@@ -44,7 +50,7 @@ async function runJob(request: PngWorkerRequest): Promise<PngWorkerJobResult> {
       return { kind: 'diff-pixels', ...computeScreenshotDiffPixels(request) };
     }
     case 'jpeg-to-png': {
-      const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
+      const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
       return {
         kind: 'jpeg-to-png',
         png: transcodeScreenshotToPng(toBuffer(request.image), request.label),
@@ -80,6 +86,7 @@ export function resultTransferList(result: PngWorkerJobResult): ArrayBuffer[] {
 function resultBufferViews(result: PngWorkerJobResult): Uint8Array[] {
   switch (result.kind) {
     case 'decode':
+    case 'decode-image':
       return [result.data];
     case 'encode':
       return [result.png];

packages/capture-kit/src/png-transcode.test.ts → packages/capture-kit/src/screenshot-image.test.ts

@@ -1,7 +1,11 @@
 import { expect, test } from 'vitest';
 import { encode as encodeJpeg } from 'jpeg-js';
 import { PNG } from './png.ts';
-import { detectScreenshotImageFormat, transcodeScreenshotToPng } from './png-transcode.ts';
+import {
+  decodeScreenshotImage,
+  detectScreenshotImageFormat,
+  transcodeScreenshotToPng,
+} from './screenshot-image.ts';
 
 function solidRgba(width: number, height: number, rgba: readonly [number, number, number, number]) {
   const data = Buffer.alloc(width * height * 4);
@@ -74,3 +78,43 @@ test('a truncated JPEG body is refused with the same typed decode error', () =>
     message: 'Failed to decode Limrun iOS screenshot as JPEG',
   });
 });
+
+test('decoding a PNG answers its pixels without re-encoding the container', () => {
+  const png = new PNG({ width: 3, height: 2 });
+  png.data = solidRgba(3, 2, [12, 240, 60, 255]);
+
+  const decoded = decodeScreenshotImage(PNG.sync.write(png), 'test screenshot');
+
+  expect([decoded.width, decoded.height]).toEqual([3, 2]);
+  expect([...decoded.data.subarray(0, 4)]).toEqual([12, 240, 60, 255]);
+});
+
+test('decoding a JPEG answers RGBA rows, with opaque alpha where the container has none', () => {
+  const jpeg = encodeJpeg({ width: 5, height: 3, data: solidRgba(5, 3, [12, 240, 60, 255]) }, 100);
+
+  const decoded = decodeScreenshotImage(jpeg.data, 'test screenshot');
+
+  expect([decoded.width, decoded.height]).toEqual([5, 3]);
+  expect(decoded.data.length).toBe(5 * 3 * 4);
+  const [r = -1, g = -1, b = -1, a = -1] = decoded.data.subarray(0, 4);
+  // JPEG is lossy; a flat field survives within a few levels per channel.
+  expect(Math.abs(r - 12)).toBeLessThanOrEqual(4);
+  expect(Math.abs(g - 240)).toBeLessThanOrEqual(4);
+  expect(Math.abs(b - 60)).toBeLessThanOrEqual(4);
+  expect(a).toBe(255);
+});
+
+test('decoding refuses bytes in neither container and a JPEG body that cannot be decoded', () => {
+  expect(thrownBy(() => decodeScreenshotImage(Buffer.from('GIF89a'), 'fixture'))).toMatchObject({
+    code: 'COMMAND_FAILED',
+    message: 'fixture is neither PNG nor JPEG',
+    details: { label: 'fixture', leadingBytes: '47494638' },
+  });
+
+  const corrupt = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 0x41)]);
+  expect(thrownBy(() => decodeScreenshotImage(corrupt, 'fixture'))).toMatchObject({
+    code: 'COMMAND_FAILED',
+    message: 'Failed to decode fixture as JPEG',
+    details: { label: 'fixture', reason: expect.any(String) },
+  });
+});

packages/capture-kit/src/png-transcode.ts → packages/capture-kit/src/screenshot-image.ts

@@ -1,6 +1,6 @@
 import { AppError } from '@agent-device/kernel/errors';
 import { decode as decodeJpeg } from 'jpeg-js';
-import { hasPngSignature, PNG } from './png.ts';
+import { decodePng, hasPngSignature, PNG } from './png.ts';
 
 const JPEG_SIGNATURE = Buffer.from([0xff, 0xd8, 0xff]);
 
@@ -13,35 +13,55 @@ export function detectScreenshotImageFormat(bytes: Buffer): ScreenshotImageForma
   return undefined;
 }
 
+/**
+ * Decodes a screenshot in whatever container it arrived in into the RGBA rows every pixel reader in
+ * this repository consumes. The container is sniffed, never trusted from a file name or a caller's
+ * guess, so a JPEG stored under a `.png` name still decodes.
+ *
+ * Synchronous and CPU-bound: daemon request paths reach it through the PNG worker
+ * (`decodeScreenshotImageAsync` in `png-worker-client.ts`), like every other codec job.
+ */
+export function decodeScreenshotImage(bytes: Buffer, label: string): PNG {
+  const format = detectScreenshotImageFormat(bytes);
+  if (format === 'png') return decodePng(bytes, label);
+  if (format === 'jpeg') return decodeJpegImage(bytes, label);
+  throw unsupportedContainer(label, bytes);
+}
+
 /**
  * Returns PNG bytes for a screenshot a provider handed back in whatever container it prefers. PNG
  * passes through untouched; JPEG is decoded and re-encoded losslessly from the decoded pixels, so
- * every PNG-only reader downstream (size, crop, overlay, diff) sees the format the path promises.
+ * every PNG-only reader downstream (size, crop, overlay) sees the format the path promises. Those
+ * readers rewrite the file in place, which is why a JPEG cannot be handed to them undecoded.
  *
  * Synchronous and CPU-bound: daemon request paths reach it through the PNG worker
  * (`transcodeScreenshotToPngAsync` in `png-worker-client.ts`), like every other codec job.
  */
 export function transcodeScreenshotToPng(bytes: Buffer, label: string): Buffer {
   const format = detectScreenshotImageFormat(bytes);
   if (format === 'png') return bytes;
-  if (format === 'jpeg') return jpegToPng(bytes, label);
-  throw new AppError('COMMAND_FAILED', `${label} is neither PNG nor JPEG`, {
-    label,
-    leadingBytes: bytes.subarray(0, 4).toString('hex'),
-  });
+  if (format === 'jpeg') return PNG.sync.write(decodeJpegImage(bytes, label));
+  throw unsupportedContainer(label, bytes);
 }
 
 /** A JPEG signature does not prove a decodable body; a failure keeps the label and the decoder's reason. */
-function jpegToPng(bytes: Buffer, label: string): Buffer {
+function decodeJpegImage(bytes: Buffer, label: string): PNG {
   try {
     const decoded = decodeJpeg(bytes, { useTArray: true, formatAsRGBA: true });
     const png = new PNG({ width: decoded.width, height: decoded.height });
     png.data = Buffer.from(decoded.data.buffer, decoded.data.byteOffset, decoded.data.byteLength);
-    return PNG.sync.write(png);
+    return png;
   } catch (error) {
     throw new AppError('COMMAND_FAILED', `Failed to decode ${label} as JPEG`, {
       label,
       reason: error instanceof Error ? error.message : String(error),
     });
   }
 }
+
+function unsupportedContainer(label: string, bytes: Buffer): AppError {
+  return new AppError('COMMAND_FAILED', `${label} is neither PNG nor JPEG`, {
+    label,
+    leadingBytes: bytes.subarray(0, 4).toString('hex'),
+  });
+}

src/commands/capture/diff.ts

@@ -59,7 +59,7 @@ export const diffCommandFacet = defineCommandFacet({
   text: {
     summary: 'Diff snapshot or screenshot',
     cliDetail:
-      'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; image dimensions must still match. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines.',
+      'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; image dimensions must still match. Both screenshot inputs are decoded from their bytes, so a baseline or current image may be PNG or JPEG, and the diff image is always PNG. JPEG is lossy, so keep --threshold above 0 whenever either input is JPEG. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines.',
   },
   metadata: diffCommandMetadata,
   run: (client, input) => client.capture.diff(input),

src/commands/schema/cli-help-command-usage.test.ts

@@ -39,6 +39,14 @@ test('usageForCommand documents screenshot diff normalization', async () => {
   assert.match(help, /screenshot --normalize-status-bar/);
 });
 
+test('usageForCommand documents the screenshot diff input containers', async () => {
+  const help = await usageForCommand('diff');
+  if (help === null) throw new Error('Expected diff help text');
+  assert.match(help, /a baseline or current image may be PNG or JPEG/);
+  assert.match(help, /the diff image is always PNG/);
+  assert.match(help, /keep --threshold above 0 whenever either input is JPEG/);
+});
+
 test('usageForCommand resolves longpress help', async () => {
   const help = await usageForCommand('longpress');
   assert.equal(help === null, false);