diff --git a/packages/capture-kit/src/png-codec.fixtures.ts b/packages/capture-kit/src/png-codec.fixtures.ts new file mode 100644 index 000000000..5a2e97f82 --- /dev/null +++ b/packages/capture-kit/src/png-codec.fixtures.ts @@ -0,0 +1,140 @@ +import zlib from 'node:zlib'; +import { predictByte } from './png-predictor.ts'; +import { PNG } from './png.ts'; + +/** + * Test-only PNG writer. `pngjs` picks one filter for a whole image and cannot express the + * per-row mixtures, palettes, transparency, and header variations the crop paths have to + * classify, so fixtures assemble the chunks directly. Every fixture is read back by `pngjs` + * in the test that uses it, which is what proves this writer is honest. + */ + +const SIGNATURE = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const FILTER_CYCLE: readonly number[] = [0, 1, 2, 3, 4]; +const IHDR_COLOR_TYPE_BYTE = 25; +const IHDR_BIT_DEPTH_BYTE = 24; + +export type PngFixture = Readonly<{ + pixels: Uint8Array; + width: number; + height: number; + channels: number; + colorType: number; + bitDepth?: number; + interlace?: number; + compressionMethod?: number; + filterMethod?: number; + filterFor?: (row: number) => number; + palette?: Uint8Array; + transparency?: Uint8Array; + ancillary?: Readonly<{ type: string; data: Uint8Array }>; +}>; + +export function encodeFixturePng(fixture: PngFixture): Buffer { + const { height, width } = fixture; + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = fixture.bitDepth ?? 8; + header[9] = fixture.colorType; + header[10] = fixture.compressionMethod ?? 0; + header[11] = fixture.filterMethod ?? 0; + header[12] = fixture.interlace ?? 0; + const chunks: Uint8Array[] = [SIGNATURE, pngChunk('IHDR', header)]; + if (fixture.ancillary) chunks.push(pngChunk(fixture.ancillary.type, fixture.ancillary.data)); + if (fixture.palette) chunks.push(pngChunk('PLTE', fixture.palette)); + if (fixture.transparency) chunks.push(pngChunk('tRNS', fixture.transparency)); + chunks.push(pngChunk('IDAT', zlib.deflateSync(filterScanlines(fixture), { level: 6 }))); + chunks.push(pngChunk('IEND', new Uint8Array(0))); + return Buffer.concat(chunks); +} + +/** A pixel grid where neighbouring pixels differ, so a wrong row or column cannot hide. */ +export function rampPixels( + width: number, + height: number, + channels: number, + alpha: (x: number, y: number) => number = () => 255, +): Uint8Array { + const pixels = new Uint8Array(width * height * channels); + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const offset = (y * width + x) * channels; + pixels[offset] = (x * 7 + y * 3) & 0xff; + if (channels >= 3) { + pixels[offset + 1] = (x * 11 + y * 5) & 0xff; + pixels[offset + 2] = (x * 13 + y * 17) & 0xff; + } + if (channels === 2) pixels[offset + 1] = alpha(x, y) & 0xff; + if (channels === 4) pixels[offset + 3] = alpha(x, y) & 0xff; + } + } + return pixels; +} + +/** Reads any encoded PNG back as RGBA through `pngjs`, with the layout bytes it declares. */ +export function readPngForTest(buffer: Buffer): { + width: number; + height: number; + colorType: number; + bitDepth: number; + rgba: Uint8Array; +} { + const png = PNG.sync.read(buffer); + return { + width: png.width, + height: png.height, + colorType: buffer.readUInt8(IHDR_COLOR_TYPE_BYTE), + bitDepth: buffer.readUInt8(IHDR_BIT_DEPTH_BYTE), + rgba: png.data, + }; +} + +function filterScanlines(fixture: PngFixture): Uint8Array { + const { channels, height, width } = fixture; + const stride = width * channels; + const scanlines = new Uint8Array(height * (stride + 1)); + for (let row = 0; row < height; row += 1) { + const data = row * (stride + 1) + 1; + const filter = fixture.filterFor?.(row) ?? FILTER_CYCLE[row % FILTER_CYCLE.length]!; + scanlines[row * (stride + 1)] = filter; + const from = row * stride; + for (let offset = 0; offset < stride; offset += 1) { + scanlines[data + offset] = + (fixture.pixels[from + offset]! - + predictByte( + filter, + offset >= channels ? fixture.pixels[from + offset - channels]! : 0, + row > 0 ? fixture.pixels[from - stride + offset]! : 0, + row > 0 && offset >= channels ? fixture.pixels[from - stride + offset - channels]! : 0, + )) & + 0xff; + } + } + return scanlines; +} + +function pngChunk(type: string, data: Uint8Array): Buffer { + const chunk = Buffer.alloc(12 + data.length); + chunk.writeUInt32BE(data.length, 0); + chunk.write(type, 4, 'ascii'); + chunk.set(data, 8); + chunk.writeUInt32BE(zlib.crc32(chunk.subarray(4, 8 + data.length)) >>> 0, 8 + data.length); + return chunk; +} + +/** Flips a byte of the named chunk's checksum, so a reader that verifies checksums must notice. */ +export function corruptChunkChecksum(buffer: Buffer, type: string): Buffer { + const corrupted = Buffer.from(buffer); + let offset = 8; + while (offset + 12 <= corrupted.length) { + const length = corrupted.readUInt32BE(offset); + if (corrupted.toString('ascii', offset + 4, offset + 8) === type) { + const checksum = offset + 8 + length; + corrupted[checksum] = (corrupted[checksum] ?? 0) ^ 0xff; + return corrupted; + } + offset += 12 + length; + } + throw new Error(`fixture PNG carries no ${type} chunk to corrupt`); +} diff --git a/packages/capture-kit/src/png-crop-bytes.test.ts b/packages/capture-kit/src/png-crop-bytes.test.ts new file mode 100644 index 000000000..a12dee1c4 --- /dev/null +++ b/packages/capture-kit/src/png-crop-bytes.test.ts @@ -0,0 +1,304 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fc from 'fast-check'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import { cropPngBytes } from './png-crop-bytes.ts'; +import { + corruptChunkChecksum, + encodeFixturePng, + rampPixels, + readPngForTest, + type PngFixture, +} from './png-codec.fixtures.ts'; + +const OPAQUE = 255; +const LABEL = 'screenshot'; +const PROPERTY_RUNS = 100; + +test('a mixed-filter RGBA capture crops to the box pixels and drops its alpha channel', () => { + const image = claimable(9, 6); + const box: Rect = { x: 2, y: 1, width: 4, height: 3 }; + + const cropped = cropPngBytes(encodeFixturePng(image), box, LABEL)!; + + assert.equal(readPngForTest(cropped).colorType, 2, 'an opaque crop keeps no alpha channel'); + assertCroppedPixels(cropped, image, box); +}); + +test('an RGB capture crops without gaining an alpha channel', () => { + const image: PngFixture = { + pixels: rampPixels(7, 5, 3), + width: 7, + height: 5, + channels: 3, + colorType: 2, + }; + const box: Rect = { x: 1, y: 2, width: 3, height: 2 }; + + const cropped = cropPngBytes(encodeFixturePng(image), box, LABEL)!; + + assert.equal(readPngForTest(cropped).colorType, 2); + assertCroppedPixels(cropped, image, box); +}); + +test('a one-pixel crop of the last row and column reads the right pixel', () => { + const image = claimable(6, 4); + const box: Rect = { x: 5, y: 3, width: 1, height: 1 }; + + assertCroppedPixels(cropPngBytes(encodeFixturePng(image), box, LABEL)!, image, box); +}); + +test('a box holding a translucent pixel keeps the alpha channel', () => { + const image = translucentAt(8, 5, { x: 3, y: 2 }); + const box: Rect = { x: 2, y: 1, width: 4, height: 3 }; + + const cropped = cropPngBytes(encodeFixturePng(image), box, LABEL)!; + + const decoded = readPngForTest(cropped); + assert.equal(decoded.colorType, 6); + assert.equal(decoded.rgba[(1 * box.width + 1) * 4 + 3], 128); + assertCroppedPixels(cropped, image, box); +}); + +test('translucency outside the box still lets the crop drop alpha', () => { + const image = translucentAt(8, 5, { x: 7, y: 4 }); + const box: Rect = { x: 0, y: 0, width: 4, height: 4 }; + + const cropped = cropPngBytes(encodeFixturePng(image), box, LABEL)!; + + assert.equal(readPngForTest(cropped).colorType, 2); + assertCroppedPixels(cropped, image, box); +}); + +test('a palette capture crops through the general reader with its colors intact', () => { + const width = 6; + const height = 5; + const palette = new Uint8Array(3 * 4); + for (let color = 0; color < 4; color += 1) { + palette[color * 3] = (color * 60) & 0xff; + palette[color * 3 + 1] = (color * 90) & 0xff; + palette[color * 3 + 2] = (color * 120) & 0xff; + } + const indices = new Uint8Array(width * height); + for (let pixel = 0; pixel < indices.length; pixel += 1) indices[pixel] = pixel % 4; + const box: Rect = { x: 1, y: 1, width: 3, height: 2 }; + + const cropped = cropPngBytes( + encodeFixturePng({ + pixels: indices, + width, + height, + channels: 1, + colorType: 3, + palette, + filterFor: () => 0, + }), + box, + LABEL, + )!; + + const decoded = readPngForTest(cropped); + assert.equal(decoded.colorType, 6, 'the general reader keeps its RGBA output'); + for (let row = 0; row < box.height; row += 1) { + for (let column = 0; column < box.width; column += 1) { + const color = indices[(row + box.y) * width + column + box.x]!; + const to = (row * box.width + column) * 4; + assert.deepEqual( + [decoded.rgba[to], decoded.rgba[to + 1], decoded.rgba[to + 2]], + [palette[color * 3], palette[color * 3 + 1], palette[color * 3 + 2]], + `pixel ${column},${row}`, + ); + } + } +}); + +test('a grayscale capture crops through the general reader', () => { + const width = 5; + const height = 4; + const gray = new Uint8Array(width * height); + for (let pixel = 0; pixel < gray.length; pixel += 1) gray[pixel] = (pixel * 21) & 0xff; + const box: Rect = { x: 2, y: 1, width: 2, height: 2 }; + + const cropped = cropPngBytes( + encodeFixturePng({ pixels: gray, width, height, channels: 1, colorType: 0 }), + box, + LABEL, + )!; + + const decoded = readPngForTest(cropped); + for (let row = 0; row < box.height; row += 1) { + for (let column = 0; column < box.width; column += 1) { + const wanted = gray[(row + box.y) * width + column + box.x]!; + const to = (row * box.width + column) * 4; + assert.deepEqual( + [decoded.rgba[to], decoded.rgba[to + 1], decoded.rgba[to + 2]], + [wanted, wanted, wanted], + `pixel ${column},${row}`, + ); + } + } +}); + +test('a box covering the whole image returns no replacement bytes', () => { + assert.equal(cropPngBytes(encodeFixturePng(claimable(6, 4)), fullBox(6, 4), LABEL), null); +}); + +test('a box covering the whole image still refuses a file that cannot be decoded', () => { + const buffer = encodeFixturePng(claimable(12, 9)); + + assert.throws( + () => cropPngBytes(Buffer.from(buffer.subarray(0, buffer.length - 8)), fullBox(12, 9), LABEL), + /Failed to decode screenshot as PNG/, + ); +}); + +test('an unreadable scanline filter below the box reports the decode failure', () => { + const buffer = encodeFixturePng({ + ...claimable(6, 8), + filterFor: (row) => (row === 7 ? 9 : 0), + }); + + assert.throws( + () => cropPngBytes(buffer, { x: 0, y: 0, width: 3, height: 3 }, LABEL), + /Failed to decode screenshot as PNG/, + ); +}); + +test('bytes trailing the end of the image report the canonical decode failure', () => { + const withTrailer = Buffer.concat([ + encodeFixturePng(claimable(12, 9)), + Buffer.from([0, 0, 1, 2]), + ]); + + assert.throws( + () => cropPngBytes(withTrailer, { x: 1, y: 1, width: 4, height: 4 }, LABEL), + /Failed to decode screenshot as PNG/, + ); +}); + +test('a corrupted image header is refused instead of cropping by its claimed dimensions', () => { + const corrupted = corruptChunkChecksum(encodeFixturePng(claimable(12, 9)), 'IHDR'); + + assert.throws( + () => cropPngBytes(corrupted, { x: 0, y: 0, width: 4, height: 4 }, LABEL), + /Failed to decode screenshot as PNG/, + ); +}); + +test('a box beyond the image reports the box instead of clamping the crop', () => { + assert.throws( + () => + cropPngBytes(encodeFixturePng(claimable(6, 4)), { x: 4, y: 0, width: 3, height: 2 }, LABEL), + /Screenshot crop box 3x2 at \(4, 0\) exceeds the 6x4 image/, + ); +}); + +test('bytes that are not a PNG report the canonical decode failure', () => { + assert.throws( + () => cropPngBytes(Buffer.from('not a png at all'), { x: 0, y: 0, width: 2, height: 2 }, LABEL), + /Failed to decode screenshot as PNG/, + ); +}); + +test('a file truncated inside its image data reports the decode failure', () => { + const buffer = encodeFixturePng(claimable(12, 9)); + + assert.throws( + () => + cropPngBytes( + Buffer.from(buffer.subarray(0, buffer.length - 8)), + { ...fullBox(12, 9), width: 4, height: 4 }, + LABEL, + ), + /Failed to decode screenshot as PNG/, + ); +}); + +test('a corrupted image-data checksum is refused instead of cropping wrong pixels', () => { + const corrupted = corruptChunkChecksum(encodeFixturePng(claimable(12, 9)), 'IDAT'); + + assert.throws( + () => cropPngBytes(corrupted, { x: 0, y: 0, width: 4, height: 4 }, LABEL), + /Failed to decode screenshot as PNG/, + ); +}); + +test('every crop of a random capture matches the pixels the file declares', () => { + fc.assert( + fc.property( + fc.integer({ min: 1, max: 24 }), + fc.integer({ min: 1, max: 24 }), + fc.nat(), + fc.array(fc.integer({ min: 0, max: 4 }), { minLength: 1 }), + (width, height, boxSeed, filterPlan) => { + const image: PngFixture = { + ...claimable(width, height), + filterFor: (row) => filterPlan[row % filterPlan.length]!, + }; + const x = boxSeed % width; + const y = (boxSeed >>> 4) % height; + const box: Rect = { + x, + y, + width: 1 + (boxSeed % (width - x)), + height: 1 + ((boxSeed >>> 8) % (height - y)), + }; + + const cropped = cropPngBytes(encodeFixturePng(image), box, LABEL); + + if (box.x === 0 && box.y === 0 && box.width === width && box.height === height) { + assert.equal(cropped, null); + return; + } + assert.notEqual(cropped, null); + assertCroppedPixels(cropped!, image, box); + }, + ), + { numRuns: PROPERTY_RUNS }, + ); +}); + +function claimable(width: number, height: number): PngFixture { + return { + pixels: rampPixels(width, height, 4, () => OPAQUE), + width, + height, + channels: 4, + colorType: 6, + }; +} + +function translucentAt( + width: number, + height: number, + translucent: Readonly<{ x: number; y: number }>, +): PngFixture { + return { + ...claimable(width, height), + pixels: rampPixels(width, height, 4, (x, y) => + x === translucent.x && y === translucent.y ? 128 : OPAQUE, + ), + }; +} + +function fullBox(width: number, height: number): Rect { + return { x: 0, y: 0, width, height }; +} + +function assertCroppedPixels(cropped: Buffer, image: PngFixture, box: Rect): void { + const decoded = readPngForTest(cropped); + assert.deepEqual([decoded.width, decoded.height], [box.width, box.height]); + for (let row = 0; row < box.height; row += 1) { + for (let column = 0; column < box.width; column += 1) { + const from = ((row + box.y) * image.width + column + box.x) * image.channels; + const to = (row * box.width + column) * 4; + for (let channel = 0; channel < 3; channel += 1) { + assert.equal( + decoded.rgba[to + channel]!, + image.pixels[from + channel]!, + `pixel ${column},${row} channel ${channel}`, + ); + } + } + } +} diff --git a/packages/capture-kit/src/png-crop-bytes.ts b/packages/capture-kit/src/png-crop-bytes.ts new file mode 100644 index 000000000..261b62c10 --- /dev/null +++ b/packages/capture-kit/src/png-crop-bytes.ts @@ -0,0 +1,56 @@ +import { AppError } from '@agent-device/kernel/errors'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import { decodePng, PNG } from './png.ts'; +import { encodePngPixels } from './png-encode.ts'; +import { decodePngRegion, readPngRegionHeader } from './png-region-decode.ts'; + +/** + * Crops PNG bytes to `box` (positive integer pixels), synchronously and without handing the + * decoded image to anybody else. Returns `null` when the box already covers the image, which + * tells the caller its file is already the answer. + * + * The region reader serves the 8-bit truecolor layout device captures arrive in and writes the + * result without an alpha channel when the cropped pixels carry none. Every other layout, and + * any file the region reader declines to interpret, goes through the general PNG reader, which + * owns the canonical decode error and keeps the previous RGBA output. + */ +export function cropPngBytes(source: Buffer, box: Rect, label: string): Buffer | null { + const header = readPngRegionHeader(source); + if (header !== null && !isFullImageBox(box, header.width, header.height)) { + assertCropBoxFits(box, header.width, header.height); + const region = decodePngRegion(source, header, box); + if (region !== null) { + return encodePngPixels(region.pixels, region.width, region.height, region.channels); + } + } + return cropDecodedPng(source, box, label); +} + +function cropDecodedPng(bytes: Buffer, box: Rect, label: string): Buffer | null { + const decoded = decodePng(bytes, label); + assertCropBoxFits(box, decoded.width, decoded.height); + if (isFullImageBox(box, decoded.width, decoded.height)) return null; + return PNG.sync.write(copyPngBox(decoded, box)); +} + +function copyPngBox(source: PNG, box: Rect): PNG { + const output = new PNG({ width: box.width, height: box.height }); + for (let row = 0; row < box.height; row += 1) { + const sourceStart = ((row + box.y) * source.width + box.x) * 4; + source.data.copy(output.data, row * output.width * 4, sourceStart, sourceStart + box.width * 4); + } + return output; +} + +function assertCropBoxFits(box: Rect, width: number, height: number): void { + if (box.x + box.width > width || box.y + box.height > height) { + throw new AppError( + 'INVALID_ARGS', + `Screenshot crop box ${box.width}x${box.height} at (${box.x}, ${box.y}) exceeds the ${width}x${height} image`, + ); + } +} + +function isFullImageBox(box: Rect, width: number, height: number): boolean { + return box.x === 0 && box.y === 0 && box.width === width && box.height === height; +} diff --git a/packages/capture-kit/src/png-crop.test.ts b/packages/capture-kit/src/png-crop.test.ts index ce7cd1742..7498b8e89 100644 --- a/packages/capture-kit/src/png-crop.test.ts +++ b/packages/capture-kit/src/png-crop.test.ts @@ -2,9 +2,11 @@ import { afterAll, test } from 'vitest'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; +import type { Rect } from '@agent-device/kernel/snapshot'; import { PNG } from './png.ts'; import { cropPngFile } from './png-crop.ts'; import { terminatePngWorker } from './png-worker-client.ts'; +import { encodeFixturePng, rampPixels, readPngForTest } from './png-codec.fixtures.ts'; import { mkdtempForTestSync } from './tmp-dir.fixtures.ts'; afterAll(async () => { @@ -64,6 +66,63 @@ test('non-integer or non-positive boxes refuse', async () => { } }); +test('a crop of an opaque capture is written as truecolor without an alpha channel', async () => { + const filePath = writeFixturePng(opaqueFixture(9, 6)); + + await cropPngFile(filePath, { x: 3, y: 2, width: 4, height: 3 }); + + const written = readPngForTest(fs.readFileSync(filePath)); + assert.equal(written.colorType, 2); + assert.deepEqual([written.width, written.height], [4, 3]); + assertCropMatchesSource(written, opaqueFixture(9, 6), { x: 3, y: 2, width: 4, height: 3 }); +}); + +test('a file that is not a PNG keeps the canonical decode failure', async () => { + const filePath = path.join(mkdtempForTestSync('agent-device-png-junk-'), 'image.png'); + fs.writeFileSync(filePath, Buffer.from('not a png at all')); + + await assert.rejects( + () => cropPngFile(filePath, { x: 0, y: 0, width: 2, height: 2 }), + /Failed to decode screenshot as PNG/, + ); +}); + +function opaqueFixture(width: number, height: number) { + return { + pixels: rampPixels(width, height, 4, () => 255), + width, + height, + channels: 4, + colorType: 6, + }; +} + +function writeFixturePng(fixture: Parameters[0]): string { + const filePath = path.join(mkdtempForTestSync('agent-device-png-fixture-'), 'image.png'); + fs.writeFileSync(filePath, encodeFixturePng(fixture)); + return filePath; +} + +function assertCropMatchesSource( + written: ReturnType, + fixture: Readonly<{ pixels: Uint8Array; width: number; channels: number }>, + box: Rect, +): void { + for (let row = 0; row < box.height; row += 1) { + for (let column = 0; column < box.width; column += 1) { + const from = ((row + box.y) * fixture.width + column + box.x) * fixture.channels; + const to = (row * box.width + column) * 4; + for (let channel = 0; channel < 3; channel += 1) { + assert.equal( + written.rgba[to + channel]!, + fixture.pixels[from + channel]!, + `pixel ${column},${row} channel ${channel}`, + ); + } + } + } +} + // A 6x4 grid whose pixel (x, y) carries (x*10, y*10) so a wrong source offset // is caught by the value, not just the size. function writeCheckedPng(): string { diff --git a/packages/capture-kit/src/png-crop.ts b/packages/capture-kit/src/png-crop.ts index 5ccedda5a..cf8e4dde2 100644 --- a/packages/capture-kit/src/png-crop.ts +++ b/packages/capture-kit/src/png-crop.ts @@ -1,13 +1,13 @@ import { promises as fs } from 'node:fs'; import { AppError } from '@agent-device/kernel/errors'; import type { Rect } from '@agent-device/kernel/snapshot'; -import { PNG } from './png.ts'; -import { decodePngAsync, encodePngAsync } from './png-worker-client.ts'; +import { cropPngBytesAsync } from './png-worker-client.ts'; /** * Crops `filePath` in place to `box` (positive integer pixels). `box` is the caller's * already-intersected region, so one outside the image is a caller bug — refused, not clamped. - * Decode and encode run on the PNG worker thread; a full-image box is a no-op. + * One PNG worker job turns the captured bytes into the cropped bytes, so the decoded image never + * leaves the worker thread; a box that already covers the image leaves the file untouched. */ export async function cropPngFile(filePath: string, box: Rect): Promise { if (!isCropBox(box)) { @@ -16,19 +16,10 @@ export async function cropPngFile(filePath: string, box: Rect): Promise { 'Screenshot crop box must be positive integer pixel offsets', ); } - - const source = await decodePngAsync(await fs.readFile(filePath), 'screenshot'); - if (box.x + box.width > source.width || box.y + box.height > source.height) { - throw new AppError( - 'INVALID_ARGS', - `Screenshot crop box ${box.width}x${box.height} at (${box.x}, ${box.y}) exceeds the ${source.width}x${source.height} image`, - ); - } - if (box.x === 0 && box.y === 0 && box.width === source.width && box.height === source.height) { - return; + const cropped = await cropPngBytesAsync(await fs.readFile(filePath), box, 'screenshot'); + if (cropped !== null) { + await fs.writeFile(filePath, cropped); } - - await fs.writeFile(filePath, await encodePngAsync(cropPngBox(source, box))); } function isCropBox(box: Rect): boolean { @@ -43,12 +34,3 @@ function isCropBox(box: Rect): boolean { box.height > 0 ); } - -function cropPngBox(source: PNG, box: Rect): PNG { - const output = new PNG({ width: box.width, height: box.height }); - for (let row = 0; row < box.height; row += 1) { - const sourceStart = ((row + box.y) * source.width + box.x) * 4; - source.data.copy(output.data, row * output.width * 4, sourceStart, sourceStart + box.width * 4); - } - return output; -} diff --git a/packages/capture-kit/src/png-encode.test.ts b/packages/capture-kit/src/png-encode.test.ts new file mode 100644 index 000000000..393c9f5ae --- /dev/null +++ b/packages/capture-kit/src/png-encode.test.ts @@ -0,0 +1,88 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import zlib from 'node:zlib'; +import { encodePngPixels } from './png-encode.ts'; +import { decodePngRegion, readPngRegionHeader } from './png-region-decode.ts'; +import { corruptChunkChecksum, rampPixels, readPngForTest } from './png-codec.fixtures.ts'; + +const WIDTH = 5; +const HEIGHT = 3; +const FULL_BOX = { x: 0, y: 0, width: WIDTH, height: HEIGHT }; + +test('RGB pixels round-trip through a decoder as truecolor', () => { + const pixels = rampPixels(WIDTH, HEIGHT, 3); + + const encoded = encodePngPixels(pixels, WIDTH, HEIGHT, 3); + + const decoded = readPngForTest(encoded); + assert.equal(decoded.colorType, 2); + assert.equal(decoded.bitDepth, 8); + assert.deepEqual([decoded.width, decoded.height], [WIDTH, HEIGHT]); + assertSameChannelOrder(decoded.rgba, pixels, 3, 4); +}); + +test('RGBA pixels round-trip through a decoder keeping their alpha', () => { + const pixels = rampPixels(WIDTH, HEIGHT, 4, (x) => (x % 2 === 0 ? 128 : 255)); + + const encoded = encodePngPixels(pixels, WIDTH, HEIGHT, 4); + + const decoded = readPngForTest(encoded); + assert.equal(decoded.colorType, 6); + assertSameChannelOrder(decoded.rgba, pixels, 4, 4); +}); + +test('every written scanline declares the None filter', () => { + const encoded = encodePngPixels(rampPixels(WIDTH, HEIGHT, 3), WIDTH, HEIGHT, 3); + + assert.deepEqual(scanlineFilters(encoded), [0, 0, 0]); +}); + +test('the encoding is readable by the region reader, so its checksums are real', () => { + const pixels = rampPixels(WIDTH, HEIGHT, 3); + const encoded = encodePngPixels(pixels, WIDTH, HEIGHT, 3); + + const header = readPngRegionHeader(encoded); + assert.notEqual(header, null); + const region = decodePngRegion(encoded, header!, FULL_BOX); + assert.notEqual(region, null); + assertSameChannelOrder(region!.pixels, pixels, 3, 3); + + const corrupted = corruptChunkChecksum(encoded, 'IDAT'); + assert.equal(decodePngRegion(corrupted, readPngRegionHeader(corrupted)!, FULL_BOX), null); +}); + +function scanlineFilters(buffer: Buffer): number[] { + const stride = WIDTH * 3; + const scanlines = zlib.inflateSync(idatOf(buffer)); + return Array.from({ length: HEIGHT }, (_unused, row) => scanlines[row * (stride + 1)]!); +} + +function idatOf(buffer: Buffer): Buffer { + let offset = 8; + while (offset + 12 <= buffer.length) { + const length = buffer.readUInt32BE(offset); + if (buffer.toString('ascii', offset + 4, offset + 8) === 'IDAT') { + return Buffer.from(buffer.subarray(offset + 8, offset + 8 + length)); + } + offset += 12 + length; + } + throw new Error('encoded PNG carries no image data'); +} + +/** `pngjs` always hands back RGBA, so the comparison walks the channels that were written. */ +function assertSameChannelOrder( + readBack: Uint8Array, + written: Uint8Array, + channels: number, + readBackChannels: number, +): void { + for (let pixel = 0; pixel < WIDTH * HEIGHT; pixel += 1) { + for (let channel = 0; channel < channels; channel += 1) { + assert.equal( + readBack[pixel * readBackChannels + channel]!, + written[pixel * channels + channel]!, + `pixel ${pixel} channel ${channel}`, + ); + } + } +} diff --git a/packages/capture-kit/src/png-encode.ts b/packages/capture-kit/src/png-encode.ts new file mode 100644 index 000000000..e597d2734 --- /dev/null +++ b/packages/capture-kit/src/png-encode.ts @@ -0,0 +1,67 @@ +import zlib from 'node:zlib'; + +/** + * Writes decoded pixels as a PNG file: truecolor, 8-bit, `None` on every scanline, deflate level 6. + * + * The per-row filter search the general PNG writer runs is not worth its cost here. Measured on + * real simulator and emulator captures, scoring the five filters cost 1.4x to 2.6x the encode time + * and came out *larger* than `None` on UI captures, where the sum-of-absolute-differences heuristic + * prefers Sub or Up on text rows that deflate smaller as raw bytes. Vertical redundancy is not lost + * either: deflate matches whole rows, which is why repeated scanlines stay small unfiltered. + * Deflate level 9 bought 1 to 2 kB for double the time, and level 3 saved 6ms and cost a third more + * bytes, so both are declined. + */ + +const SIGNATURE = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const IHDR_CHUNK_TYPE = Buffer.from('IHDR', 'ascii'); +const IDAT_CHUNK_TYPE = Buffer.from('IDAT', 'ascii'); +const IEND_CHUNK_TYPE = Buffer.from('IEND', 'ascii'); +const COLOR_TYPE_RGB = 2; +const COLOR_TYPE_RGBA = 6; +const BIT_DEPTH_8 = 8; +const PNG_IDAT_DEFLATE_LEVEL = 6; +const EMPTY: Uint8Array = new Uint8Array(0); + +export function encodePngPixels( + pixels: Uint8Array, + width: number, + height: number, + channels: 3 | 4, +): Buffer { + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = BIT_DEPTH_8; + header[9] = channels === 3 ? COLOR_TYPE_RGB : COLOR_TYPE_RGBA; + return Buffer.concat([ + SIGNATURE, + writePngChunk(IHDR_CHUNK_TYPE, header), + writePngChunk(IDAT_CHUNK_TYPE, deflateScanlines(pixels, width, height, channels)), + writePngChunk(IEND_CHUNK_TYPE, EMPTY), + ]); +} + +/** One filtered (filter `None`) scanline per image row, compressed as a single IDAT. */ +function deflateScanlines( + pixels: Uint8Array, + width: number, + height: number, + channels: number, +): Uint8Array { + const stride = width * channels; + const scanlines = new Uint8Array(height * (stride + 1)); + for (let row = 0; row < height; row += 1) { + const offset = row * (stride + 1) + 1; + scanlines.set(pixels.subarray(row * stride, (row + 1) * stride), offset); + } + return zlib.deflateSync(scanlines, { level: PNG_IDAT_DEFLATE_LEVEL }); +} + +function writePngChunk(type: Buffer, data: Uint8Array): Buffer { + const chunk = Buffer.allocUnsafe(12 + data.length); + chunk.writeUInt32BE(data.length, 0); + chunk.set(type, 4); + chunk.set(data, 8); + chunk.writeUInt32BE(zlib.crc32(data, zlib.crc32(type)) >>> 0, 8 + data.length); + return chunk; +} diff --git a/packages/capture-kit/src/png-format.test.ts b/packages/capture-kit/src/png-format.test.ts new file mode 100644 index 000000000..5b350433d --- /dev/null +++ b/packages/capture-kit/src/png-format.test.ts @@ -0,0 +1,99 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { pngTruecolorChannels, readPngChunks, readPngHeader } from './png-format.ts'; +import { corruptChunkChecksum, encodeFixturePng, rampPixels } from './png-codec.fixtures.ts'; + +const TRUECOLOR_RGB = 2; +const TRUECOLOR_RGBA = 6; + +test('the header reports the layout the IHDR declares', () => { + const buffer = encodeFixturePng(rgbaFixture(7, 3)); + + assert.deepEqual(readPngHeader(buffer), { + width: 7, + height: 3, + bitDepth: 8, + colorType: TRUECOLOR_RGBA, + compressionMethod: 0, + filterMethod: 0, + interlace: 0, + }); +}); + +test('bytes that are too short or mis-signed have no header', () => { + assert.equal(readPngHeader(Buffer.alloc(0)), null); + assert.equal(readPngHeader(Buffer.alloc(28)), null); + assert.equal(readPngHeader(Buffer.from('png-looking bytes that are long enough')), null); +}); + +test('a first chunk that is not IHDR has no header', () => { + const buffer = Buffer.from(encodeFixturePng(rgbaFixture(4, 4))); + buffer.write('tEXt', 12, 'ascii'); + + assert.equal(readPngHeader(buffer), null); +}); + +test('only the truecolor layouts have a channel count here', () => { + assert.equal(pngTruecolorChannels(TRUECOLOR_RGB), 3); + assert.equal(pngTruecolorChannels(TRUECOLOR_RGBA), 4); + assert.equal(pngTruecolorChannels(0), null); + assert.equal(pngTruecolorChannels(3), null); + assert.equal(pngTruecolorChannels(4), null); +}); + +test('the chunk walk starts at IHDR and runs through IEND', () => { + const buffer = encodeFixturePng({ + ...rgbaFixture(4, 3), + ancillary: { type: 'tEXt', data: Buffer.from('label\0crop', 'ascii') }, + }); + + assert.deepEqual( + readPngChunks(buffer)!.map((chunk) => chunk.type), + ['IHDR', 'tEXt', 'IDAT', 'IEND'], + ); +}); + +test('a chunk sequence that is truncated, corrupted, or never ends is refused', () => { + const buffer = encodeFixturePng(rgbaFixture(4, 3)); + + assert.equal(readPngChunks(buffer.subarray(0, buffer.length - 6)), null); + assert.equal(readPngChunks(corruptChunkChecksum(buffer, 'IEND')), null); + assert.equal(readPngChunks(Buffer.from(SIGNATURE_BYTES)), null); +}); + +test('bytes that do not start with the PNG signature have no chunks', () => { + assert.equal(readPngChunks(Buffer.from('JFIF-looking bytes that are long enough to walk')), null); +}); + +test('a chunk that claims to be critical but is not one of the four is refused', () => { + const buffer = encodeFixturePng({ + ...rgbaFixture(4, 3), + ancillary: { type: 'CUty', data: new Uint8Array([1, 2, 3]) }, + }); + + assert.equal(readPngChunks(buffer), null); +}); + +test('bytes trailing IEND are refused, because no conforming reader reaches them', () => { + const buffer = encodeFixturePng(rgbaFixture(4, 3)); + + assert.equal(readPngChunks(Buffer.concat([buffer, Buffer.from([0, 0, 1, 2, 255])])), null); +}); + +test('a corrupted IHDR checksum is refused, so its dimensions are never trusted', () => { + const buffer = corruptChunkChecksum(encodeFixturePng(rgbaFixture(4, 3)), 'IHDR'); + + assert.equal(readPngChunks(buffer), null); +}); + +const SIGNATURE_BYTES = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; + +function rgbaFixture(width: number, height: number) { + return { + pixels: rampPixels(width, height, 4), + width, + height, + channels: 4, + colorType: TRUECOLOR_RGBA, + }; +} diff --git a/packages/capture-kit/src/png-format.ts b/packages/capture-kit/src/png-format.ts new file mode 100644 index 000000000..0e7a77bed --- /dev/null +++ b/packages/capture-kit/src/png-format.ts @@ -0,0 +1,107 @@ +import zlib from 'node:zlib'; + +/** + * The layout a PNG file declares in its IHDR chunk and the chunks that follow it, read without + * decoding pixels. A declared layout whose checksum does not match is declined like any other + * unrecognised layout, so nobody acts on dimensions the file itself does not vouch for. + * + * `null` means the bytes do not describe something this package understands, which is a decline + * rather than a diagnosis: callers route to the general PNG reader, which owns the canonical + * decode error. + */ + +const SIGNATURE: readonly number[] = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; +const IHDR_DATA_OFFSET = 16; +const HEADER_BYTES = IHDR_DATA_OFFSET + 13; +const IHDR_TYPE_OFFSET = 12; +const IHDR_CHECKSUM_OFFSET = HEADER_BYTES; +const FIRST_CHUNK_OFFSET = 8; +const CHUNK_ENVELOPE_BYTES = 12; +const MAX_CHUNK_BYTES = 0x7fffffff; +const COLOR_TYPE_RGB = 2; +const COLOR_TYPE_RGBA = 6; +const CRITICAL_CHUNKS: readonly string[] = ['IHDR', 'PLTE', 'IDAT', 'IEND']; +const CHUNK_TYPE_PATTERN = /^[A-Za-z]{4}$/; + +export type PngHeader = Readonly<{ + width: number; + height: number; + bitDepth: number; + colorType: number; + compressionMethod: number; + filterMethod: number; + interlace: number; +}>; + +export type PngChunk = Readonly<{ type: string; data: Uint8Array }>; + +// A chunk name is four ASCII letters, and one starting with an uppercase letter is critical. The +// format defines exactly four critical names, so any other claim to be critical is a file no +// conforming reader accepts. +function isUntrustworthyChunkType(type: string): boolean { + if (!CHUNK_TYPE_PATTERN.test(type)) return true; + return type[0]! <= 'Z' && !CRITICAL_CHUNKS.includes(type); +} + +function hasSignature(bytes: Uint8Array): boolean { + return SIGNATURE.every((byte, index) => bytes[index] === byte); +} + +export function readPngHeader(bytes: Uint8Array): PngHeader | null { + if (bytes.length < IHDR_CHECKSUM_OFFSET + 4 || !hasSignature(bytes)) return null; + const view = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if ( + view.readUInt32BE(8) !== 13 || + view.toString('ascii', IHDR_TYPE_OFFSET, IHDR_DATA_OFFSET) !== 'IHDR' + ) + return null; + const checksummed = view.subarray(IHDR_TYPE_OFFSET, IHDR_CHECKSUM_OFFSET); + if (view.readUInt32BE(IHDR_CHECKSUM_OFFSET) !== zlib.crc32(checksummed) >>> 0) return null; + return { + width: view.readUInt32BE(IHDR_DATA_OFFSET), + height: view.readUInt32BE(IHDR_DATA_OFFSET + 4), + bitDepth: bytes[IHDR_DATA_OFFSET + 8]!, + colorType: bytes[IHDR_DATA_OFFSET + 9]!, + compressionMethod: bytes[IHDR_DATA_OFFSET + 10]!, + filterMethod: bytes[IHDR_DATA_OFFSET + 11]!, + interlace: bytes[IHDR_DATA_OFFSET + 12]!, + }; +} + +/** Bytes per pixel of a truecolor layout, or `null` for a layout the region reader declines. */ +export function pngTruecolorChannels(colorType: number): 3 | 4 | null { + if (colorType === COLOR_TYPE_RGB) return 3; + if (colorType === COLOR_TYPE_RGBA) return 4; + return null; +} + +/** + * The chunk sequence from `IHDR` to `IEND`, each with its checksum verified. + * + * `null` reports a sequence that cannot be trusted: a length running past the end of the buffer, + * a checksum that does not match, an unrecognised critical chunk, a file that never reaches `IEND`, + * or bytes trailing it. Callers own any further chunk they decline to interpret. + */ +export function readPngChunks(bytes: Uint8Array): PngChunk[] | null { + if (!hasSignature(bytes)) return null; + const reader = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const chunks: PngChunk[] = []; + let offset = FIRST_CHUNK_OFFSET; + while (offset + CHUNK_ENVELOPE_BYTES <= bytes.length) { + const length = reader.readUInt32BE(offset); + if (length > MAX_CHUNK_BYTES || offset + CHUNK_ENVELOPE_BYTES + length > bytes.length) { + return null; + } + const checksummed = reader.subarray(offset + 4, offset + 8 + length); + if (reader.readUInt32BE(offset + 8 + length) !== zlib.crc32(checksummed) >>> 0) return null; + const type = reader.toString('ascii', offset + 4, offset + 8); + if (isUntrustworthyChunkType(type)) return null; + chunks.push({ type, data: reader.subarray(offset + 8, offset + 8 + length) }); + // `IEND` is the end of the stream, so bytes after it are content no conforming reader sees. + if (type === 'IEND') { + return offset + CHUNK_ENVELOPE_BYTES + length === bytes.length ? chunks : null; + } + offset += CHUNK_ENVELOPE_BYTES + length; + } + return null; +} diff --git a/packages/capture-kit/src/png-predictor.test.ts b/packages/capture-kit/src/png-predictor.test.ts new file mode 100644 index 000000000..4aec571ec --- /dev/null +++ b/packages/capture-kit/src/png-predictor.test.ts @@ -0,0 +1,29 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { addByte, paethPredictor, predictByte } from './png-predictor.ts'; + +test('each filter predicts from the neighbours the format names', () => { + assert.equal(predictByte(0, 40, 60, 20), 0); + assert.equal(predictByte(1, 40, 60, 20), 40); + assert.equal(predictByte(2, 40, 60, 20), 60); + assert.equal(predictByte(3, 40, 60, 20), 50); + assert.equal(predictByte(4, 40, 60, 20), 60); +}); + +test('a missing neighbour reads as zero', () => { + assert.equal(predictByte(1, 0, 0, 0), 0); + assert.equal(predictByte(3, 0, 80, 0), 40); + assert.equal(predictByte(4, 0, 80, 0), 80); +}); + +test('Paeth takes the nearest of the three neighbours', () => { + assert.equal(paethPredictor(10, 9, 8), 10); + assert.equal(paethPredictor(100, 10, 90), 10); + assert.equal(paethPredictor(10, 200, 100), 100); + assert.equal(paethPredictor(0, 80, 0), 80); +}); + +test('reconstruction wraps at one byte, as the format requires', () => { + assert.equal(addByte(250, 10), 4); + assert.equal(addByte(4, -10), 250); +}); diff --git a/packages/capture-kit/src/png-predictor.ts b/packages/capture-kit/src/png-predictor.ts new file mode 100644 index 000000000..caeb1a106 --- /dev/null +++ b/packages/capture-kit/src/png-predictor.ts @@ -0,0 +1,51 @@ +/** + * The row predictors PNG defines, shared by whoever reads or writes a filtered scanline. + * + * Both sides must agree byte for byte: a writer and a reader that each carry their own copy of + * these rules drift apart silently, and the damage shows up as wrong pixels in a file that opens + * fine. + */ + +export const PREDICT_NONE = 0; +export const PREDICT_SUB = 1; +export const PREDICT_UP = 2; +const PREDICT_AVERAGE = 3; +export const PREDICT_PAETH = 4; + +/** The scanline filter types a reader must be able to undo. */ +export const PNG_ROW_FILTERS: readonly number[] = [ + PREDICT_NONE, + PREDICT_SUB, + PREDICT_UP, + PREDICT_AVERAGE, + PREDICT_PAETH, +]; + +/** Reconstructs one byte from its neighbours, where a missing neighbour reads as zero. */ +export function predictByte(filter: number, left: number, up: number, upperLeft: number): number { + switch (filter) { + case PREDICT_SUB: + return left; + case PREDICT_UP: + return up; + case PREDICT_AVERAGE: + return (left + up) >> 1; + case PREDICT_PAETH: + return paethPredictor(left, up, upperLeft); + default: + return 0; + } +} + +export function paethPredictor(left: number, up: number, upperLeft: number): number { + const estimated = left + up - upperLeft; + const leftDistance = Math.abs(estimated - left); + const upDistance = Math.abs(estimated - up); + const upperLeftDistance = Math.abs(estimated - upperLeft); + if (leftDistance <= upDistance && leftDistance <= upperLeftDistance) return left; + return upDistance <= upperLeftDistance ? up : upperLeft; +} + +export function addByte(value: number, predictor: number): number { + return (value + predictor) & 0xff; +} diff --git a/packages/capture-kit/src/png-region-decode.test.ts b/packages/capture-kit/src/png-region-decode.test.ts new file mode 100644 index 000000000..f84a74dc4 --- /dev/null +++ b/packages/capture-kit/src/png-region-decode.test.ts @@ -0,0 +1,187 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import { + decodePngRegion, + readPngRegionHeader, + type PngDecodedRegion, +} from './png-region-decode.ts'; +import { + corruptChunkChecksum, + encodeFixturePng, + rampPixels, + type PngFixture, +} from './png-codec.fixtures.ts'; + +const FULL_ALPHA = 255; +const BOX: Rect = { x: 1, y: 1, width: 3, height: 2 }; + +test('the region reader returns the box pixels for each scanline filter', () => { + for (let filter = 0; filter <= 4; filter += 1) { + const pixels = rampPixels(5, 4, 3); + const region = readRegion( + encodeFixturePng({ + pixels, + width: 5, + height: 4, + channels: 3, + colorType: 2, + filterFor: () => filter, + }), + BOX, + ); + + assert.deepEqual([region.width, region.height, region.channels], [3, 2, 3]); + assertSamePixels(region, pixels, 5, 3, BOX, `filter ${filter}`); + } +}); + +test('the first row reconstructs even though it has no row above it', () => { + for (let filter = 0; filter <= 4; filter += 1) { + const pixels = rampPixels(4, 3, 4, () => FULL_ALPHA); + const firstRow: Rect = { x: 0, y: 0, width: 4, height: 1 }; + const region = readRegion( + encodeFixturePng({ + pixels, + width: 4, + height: 3, + channels: 4, + colorType: 6, + filterFor: () => filter, + }), + firstRow, + ); + + assertSamePixels(region, pixels, 4, 4, firstRow, `filter ${filter} of row 0`); + } +}); + +test('an opaque region of an RGBA capture is returned without an alpha channel', () => { + const buffer = encodeFixturePng({ + pixels: rampPixels(4, 3, 4, (x, y) => (x === 3 && y === 2 ? 10 : FULL_ALPHA)), + width: 4, + height: 3, + channels: 4, + colorType: 6, + }); + + assert.equal(readRegion(buffer, { x: 0, y: 0, width: 2, height: 2 }).channels, 3); + assert.equal(readRegion(buffer, { x: 2, y: 1, width: 2, height: 2 }).channels, 4); +}); + +test('an unknown scanline filter is declined rather than guessed at', () => { + assertStreamDecline(encodeFixturePng({ ...claimable(4, 3), filterFor: () => 9 })); +}); + +test('an unknown filter on a row below the region is declined too', () => { + assertStreamDecline( + encodeFixturePng({ ...claimable(4, 4), filterFor: (row) => (row === 3 ? 9 : 0) }), + ); +}); + +test('a file truncated inside its image data is declined', () => { + const buffer = Buffer.from(encodeFixturePng(claimable(6, 5))); + + assertStreamDecline(buffer.subarray(0, buffer.length - 12)); +}); + +test('a chunk whose checksum does not match is declined', () => { + const buffer = corruptChunkChecksum(encodeFixturePng(claimable(4, 3)), 'IDAT'); + + assertStreamDecline(buffer); +}); + +test('a transparency chunk is left to the general reader', () => { + assertStreamDecline( + encodeFixturePng({ ...claimable(4, 3), transparency: new Uint8Array([0, 0, 0, 128]) }), + ); +}); + +test('an unrelated ancillary chunk does not stop the region reader', () => { + const image = claimable(5, 4); + const buffer = encodeFixturePng({ + ...image, + ancillary: { type: 'tEXt', data: new TextEncoder().encode('comment\0kept') }, + }); + + assertSamePixels(readRegion(buffer, BOX), image.pixels, 5, 4, BOX, 'with a tEXt chunk'); +}); + +test('a layout outside 8-bit non-interlaced truecolor is declined', () => { + assertLayoutDecline({ ...claimable(4, 3), interlace: 1 }); + assertLayoutDecline({ ...claimable(4, 3), bitDepth: 16 }); + assertLayoutDecline({ ...claimable(4, 3), compressionMethod: 1 }); + assertLayoutDecline({ ...claimable(4, 3), filterMethod: 1 }); + assertLayoutDecline({ ...claimable(4, 3), colorType: 0, channels: 1 }); + assertLayoutDecline({ ...claimable(4, 3), colorType: 4, channels: 2 }); +}); + +test('bytes that do not open with a readable IHDR are declined', () => { + assert.equal(readPngRegionHeader(Buffer.alloc(0)), null); + assert.equal(readPngRegionHeader(Buffer.from('not a png, and too short to be one')), null); + assert.equal(readPngRegionHeader(withFirstChunkType(claimable(4, 3), 'tEXt')), null); + assert.equal( + readPngRegionHeader(corruptChunkChecksum(encodeFixturePng(claimable(4, 3)), 'IHDR')), + null, + ); +}); + +function claimable(width: number, height: number): PngFixture { + return { + pixels: rampPixels(width, height, 4, () => FULL_ALPHA), + width, + height, + channels: 4, + colorType: 6, + }; +} + +function withFirstChunkType(fixture: PngFixture, type: string): Buffer { + const buffer = Buffer.from(encodeFixturePng(fixture)); + buffer.write(type, 12, 'ascii'); + return buffer; +} + +function readRegion(buffer: Buffer, box: Rect) { + const header = readPngRegionHeader(buffer); + assert.notEqual(header, null, 'the layout should be claimable'); + const region = decodePngRegion(buffer, header!, box); + assert.notEqual(region, null, 'the image stream should be readable'); + return region!; +} + +function assertStreamDecline(buffer: Buffer, box: Rect = BOX): void { + const header = readPngRegionHeader(buffer); + assert.notEqual(header, null, 'the layout should be claimable'); + assert.equal(decodePngRegion(buffer, header!, box), null, 'the image stream should be declined'); +} + +function assertLayoutDecline(fixture: PngFixture): void { + assert.equal(readPngRegionHeader(encodeFixturePng(fixture)), null); +} + +/** + * The reader keeps alpha only when the region needs it, so the comparison walks the channel + * count it returned. Channel order is the same in both layouts, so index `c` matches itself. + */ +function assertSamePixels( + region: PngDecodedRegion, + expected: Uint8Array, + sourceWidth: number, + sourceChannels: number, + box: Rect, + message: string, +): void { + const { channels } = region; + for (let row = 0; row < box.height; row += 1) { + for (let column = 0; column < box.width; column += 1) { + for (let channel = 0; channel < channels; channel += 1) { + assert.equal( + region.pixels[(row * box.width + column) * channels + channel]!, + expected[((row + box.y) * sourceWidth + column + box.x) * sourceChannels + channel]!, + `${message}: pixel ${column},${row} channel ${channel}`, + ); + } + } + } +} diff --git a/packages/capture-kit/src/png-region-decode.ts b/packages/capture-kit/src/png-region-decode.ts new file mode 100644 index 000000000..2bcac371a --- /dev/null +++ b/packages/capture-kit/src/png-region-decode.ts @@ -0,0 +1,251 @@ +import zlib from 'node:zlib'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import { + type PngHeader, + pngTruecolorChannels, + readPngChunks, + readPngHeader, +} from './png-format.ts'; +import { + addByte, + PNG_ROW_FILTERS, + predictByte, + PREDICT_NONE, + PREDICT_PAETH, + PREDICT_SUB, + PREDICT_UP, +} from './png-predictor.ts'; + +/** + * Reads one rectangular region out of a PNG file without materializing the image's pixels. + * + * The reader claims 8-bit non-interlaced truecolor (RGB and RGBA) — the layout every supported + * device screenshot backend emits. A deflate stream cannot be cut short: this inflates the + * complete image into a buffer as large as the whole image's filtered rows, and reconstructs every + * row above the region, because a row filter can depend on the row above it. What stops at the + * region is the pixel work — reconstruction ends at the region's last row, and the pixels allocated + * and copied are the region's alone. Anything it does not claim returns `null`, which sends the + * caller to the general PNG reader; that reader owns the canonical decode error, so a decline here + * never becomes a worse diagnosis there. + * + * `box` is expected to lie inside the header's image; the caller checks that first, because an + * oversized box is a caller error rather than a layout this reader declines. + */ + +const COMPRESSION_DEFLATE = 0; +const FILTER_METHOD_ADAPTIVE = 0; +const BIT_DEPTH_8 = 8; +const NOT_INTERLACED = 0; +const OPAQUE_ALPHA = 255; + +export type PngDecodedRegion = Readonly<{ + width: number; + height: number; + channels: 3 | 4; + pixels: Uint8Array; +}>; + +/** The IHDR of a PNG this reader can produce region pixels for, or `null` when it cannot. */ +export function readPngRegionHeader(bytes: Uint8Array): PngHeader | null { + const header = readPngHeader(bytes); + if ( + header === null || + header.bitDepth !== BIT_DEPTH_8 || + header.compressionMethod !== COMPRESSION_DEFLATE || + header.filterMethod !== FILTER_METHOD_ADAPTIVE || + header.interlace !== NOT_INTERLACED || + pngTruecolorChannels(header.colorType) === null + ) { + return null; + } + return header; +} + +export function decodePngRegion( + source: Uint8Array, + header: PngHeader, + box: Rect, +): PngDecodedRegion | null { + const channels = pngTruecolorChannels(header.colorType); + if (channels === null) return null; + const imageStream = readImageStream(source); + if (imageStream === null) return null; + + const stride = header.width * channels; + const scanlineBytes = header.height * (stride + 1); + const scanlines = inflateScanlines(imageStream, scanlineBytes); + if (scanlines === null || scanlines.length < scanlineBytes) return null; + + for (let row = 0; row < box.y + box.height; row += 1) { + if (!unfilterScanline(scanlines, row, stride, channels)) return null; + } + if (!hasReadableFiltersBelow(scanlines, box.y + box.height, header.height, stride)) return null; + + // An RGB source has no alpha to carry. An RGBA source keeps it only when a pixel in the + // region actually needs it; otherwise the output pays for a channel nobody reads. + const outChannels: 3 | 4 = channels === 4 && !isRegionOpaque(scanlines, box, stride) ? 4 : 3; + return { + width: box.width, + height: box.height, + channels: outChannels, + pixels: copyRegionRows(scanlines, box, stride, channels, outChannels), + }; +} + +/** + * Reconstructs one scanline in place. `false` reports a filter type this reader does not + * implement, which declines the whole region rather than guessing at pixels. + */ +function unfilterScanline( + scanlines: Uint8Array, + row: number, + stride: number, + channels: number, +): boolean { + const data = row * (stride + 1) + 1; + const filter = scanlines[data - 1]!; + if (!PNG_ROW_FILTERS.includes(filter)) return false; + if (filter === PREDICT_NONE) return true; + // Row zero has no row above it, so every neighbour that reads upward is zero: `Up` adds + // nothing, and `Paeth` collapses to `Sub` because the left byte wins the tie. + if (row === 0) return unfilterFirstRow(scanlines, data, stride, channels, filter); + return unfilterRow(scanlines, data, stride, channels, filter); +} + +function unfilterFirstRow( + scanlines: Uint8Array, + data: number, + stride: number, + channels: number, + filter: number, +): boolean { + const end = data + stride; + if (filter === PREDICT_SUB || filter === PREDICT_PAETH) { + for (let at = data + channels; at < end; at += 1) { + scanlines[at] = addByte(scanlines[at]!, scanlines[at - channels]!); + } + return true; + } + for (let at = data; at < end; at += 1) { + const left = at >= data + channels ? scanlines[at - channels]! : 0; + scanlines[at] = addByte(scanlines[at]!, predictByte(filter, left, 0, 0)); + } + return true; +} + +function unfilterRow( + scanlines: Uint8Array, + data: number, + stride: number, + channels: number, + filter: number, +): boolean { + const end = data + stride; + const above = data - (stride + 1); + if (filter === PREDICT_SUB) { + for (let at = data + channels; at < end; at += 1) { + scanlines[at] = addByte(scanlines[at]!, scanlines[at - channels]!); + } + return true; + } + if (filter === PREDICT_UP) { + for (let at = data; at < end; at += 1) { + scanlines[at] = addByte(scanlines[at]!, scanlines[above + (at - data)]!); + } + return true; + } + for (let at = data; at < end; at += 1) { + const offset = at - data; + const left = at >= data + channels ? scanlines[at - channels]! : 0; + scanlines[at] = addByte( + scanlines[at]!, + predictByte( + filter, + left, + scanlines[above + offset]!, + at >= data + channels ? scanlines[above + offset - channels]! : 0, + ), + ); + } + return true; +} + +/** + * Rows below the region are never reconstructed, so their filter bytes are read instead of applied. + * A filter this reader does not implement disqualifies a file wherever it sits, and the general + * reader is the one that says so. + */ +function hasReadableFiltersBelow( + scanlines: Uint8Array, + firstRow: number, + imageHeight: number, + stride: number, +): boolean { + for (let row = firstRow; row < imageHeight; row += 1) { + if (!PNG_ROW_FILTERS.includes(scanlines[row * (stride + 1)]!)) return false; + } + return true; +} + +/** True when every pixel of the region is fully opaque, so the output can drop its alpha. */ +function isRegionOpaque(scanlines: Uint8Array, box: Rect, stride: number): boolean { + let alphaAnd = OPAQUE_ALPHA; + for (let row = box.y; row < box.y + box.height && alphaAnd === OPAQUE_ALPHA; row += 1) { + // Row layout is [filter byte][pixel bytes], and this runs only for a 4-channel source, + // so the first alpha sits one byte past the filter byte plus three into the first pixel. + const firstAlpha = row * (stride + 1) + 4; + for (let column = 0; column < box.width; column += 1) { + alphaAnd &= scanlines[firstAlpha + (box.x + column) * 4]!; + } + } + return alphaAnd === OPAQUE_ALPHA; +} + +function copyRegionRows( + scanlines: Uint8Array, + box: Rect, + stride: number, + channels: number, + outChannels: number, +): Uint8Array { + const outStride = box.width * outChannels; + const pixels = new Uint8Array(box.height * outStride); + for (let row = 0; row < box.height; row += 1) { + const data = (row + box.y) * (stride + 1) + 1 + box.x * channels; + const offset = row * outStride; + if (channels === outChannels) { + pixels.set(scanlines.subarray(data, data + box.width * channels), offset); + } else { + for (let column = 0; column < box.width; column += 1) { + const from = data + column * channels; + const to = offset + column * outChannels; + pixels[to] = scanlines[from]!; + pixels[to + 1] = scanlines[from + 1]!; + pixels[to + 2] = scanlines[from + 2]!; + } + } + } + return pixels; +} + +function inflateScanlines(imageStream: Uint8Array, scanlineBytes: number): Uint8Array | null { + try { + return zlib.inflateSync(imageStream, { maxOutputLength: scanlineBytes }); + } catch { + return null; + } +} + +/** The concatenated image data, or `null` when a chunk this reader will not interpret appears. */ +function readImageStream(bytes: Uint8Array): Uint8Array | null { + const chunks = readPngChunks(bytes); + if (chunks === null) return null; + const imageStreams: Uint8Array[] = []; + for (const chunk of chunks) { + // A palette or a single transparent color needs interpretation before its pixels mean + // anything, which is the general reader's job. + if (chunk.type === 'PLTE' || chunk.type === 'tRNS') return null; + if (chunk.type === 'IDAT') imageStreams.push(chunk.data); + } + return imageStreams.length === 0 ? null : Buffer.concat(imageStreams); +} diff --git a/packages/capture-kit/src/png-worker-client.ts b/packages/capture-kit/src/png-worker-client.ts index 9208ba1d3..fba83ff10 100644 --- a/packages/capture-kit/src/png-worker-client.ts +++ b/packages/capture-kit/src/png-worker-client.ts @@ -1,6 +1,7 @@ import { Worker } from 'node:worker_threads'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { AppError, toAppErrorCode } from '@agent-device/kernel/errors'; +import type { Rect } from '@agent-device/kernel/snapshot'; import { resolveInternalEntryModulePath } from './internal-entry.ts'; import { decodePng, PNG } from './png.ts'; import { @@ -19,7 +20,7 @@ import { } from './png-worker-contract.ts'; /** - * Async wrappers that offload CPU-heavy PNG decode/encode and screenshot + * Async wrappers that offload CPU-heavy PNG decode/encode/crop and screenshot * pixel diffing to a worker thread so daemon request handlers do not block * the shared event loop. When the worker entry cannot be resolved or fails * to start, every call transparently falls back to the in-process @@ -170,12 +171,12 @@ function runWorkerJob( /** Runs a job on the worker, falling back to `runSync` when it is unavailable. */ async function runPngJob( job: PngWorkerJobFor, - runSync: () => PngWorkerJobResultFor, + runSync: () => PngWorkerJobResultFor | Promise>, ): Promise> { try { return await runWorkerJob(job); } catch (error) { - if (error instanceof PngWorkerUnavailableError) return runSync(); + if (error instanceof PngWorkerUnavailableError) return await runSync(); throw error; } } @@ -213,6 +214,24 @@ export async function encodePngAsync(png: PNG): Promise { return toBuffer(result.png); } +/** + * Crops encoded PNG bytes to `box`, returning the new encoding, or `null` when `box` already + * covers the image so the caller keeps the bytes it has. + */ +export async function cropPngBytesAsync( + source: Buffer, + box: Rect, + label: string, +): Promise { + const result = await runPngJob({ kind: 'crop', png: source, label, box }, async () => { + // Read on demand so the region reader's modules stay out of the import closure of every + // entry that only needs the worker's other jobs. + const { cropPngBytes } = await import('./png-crop-bytes.ts'); + return { kind: 'crop', png: cropPngBytes(source, box, label) }; + }); + return result.png === null ? null : toBuffer(result.png); +} + export async function computePngRgbDifferenceAsync( firstPng: Buffer, secondPng: Buffer, diff --git a/packages/capture-kit/src/png-worker-contract.ts b/packages/capture-kit/src/png-worker-contract.ts index 3d0a76363..67dfd0b8c 100644 --- a/packages/capture-kit/src/png-worker-contract.ts +++ b/packages/capture-kit/src/png-worker-contract.ts @@ -1,4 +1,5 @@ import type { NormalizedError } from '@agent-device/kernel/errors'; +import type { Rect } from '@agent-device/kernel/snapshot'; import type { ScreenshotDiffPixelsJob, ScreenshotDiffPixelsResult, @@ -8,7 +9,7 @@ import type { PngRgbDifferenceResult } from './png-rgb-difference.ts'; /** * Message contract between the daemon-side PNG worker client * (`png-worker-client.ts`) and the worker thread entry (`png-worker.ts`). - * One message = one decode, encode, or diff job. Binary payloads cross the + * One message = one decode, encode, crop, or diff job. Binary payloads cross the * thread boundary via structured clone (or transfer), so `Buffer` fields * arrive as plain `Uint8Array` views on the receiving side. */ @@ -16,12 +17,15 @@ import type { PngRgbDifferenceResult } from './png-rgb-difference.ts'; export type PngWorkerJob = | { kind: 'decode'; png: Uint8Array; label: string } | { kind: 'encode'; width: number; height: number; data: Uint8Array } + | { kind: 'crop'; png: Uint8Array; label: string; box: Rect } | { kind: 'rgb-difference'; firstPng: Uint8Array; secondPng: Uint8Array; label: string } | ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsJob); export type PngWorkerJobResult = | { kind: 'decode'; width: number; height: number; data: Uint8Array } | { kind: 'encode'; png: Uint8Array } + // A crop answers `null` when the box already covers the image, so the caller keeps the file. + | { kind: 'crop'; png: Uint8Array | null } | ({ kind: 'rgb-difference' } & PngRgbDifferenceResult) | ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsResult); diff --git a/packages/capture-kit/src/png-worker.test.ts b/packages/capture-kit/src/png-worker.test.ts index e4a2d9b14..9bc7f3b90 100644 --- a/packages/capture-kit/src/png-worker.test.ts +++ b/packages/capture-kit/src/png-worker.test.ts @@ -21,6 +21,13 @@ test('resultTransferList skips views that do not own their whole ArrayBuffer', ( assert.deepEqual(resultTransferList({ kind: 'encode', png: shortView }), []); }); +test('resultTransferList transfers a cropped encoding and skips an untouched file', () => { + const cropped = Buffer.alloc(32); // Buffer.alloc never uses the shared pool + + assert.deepEqual(resultTransferList({ kind: 'crop', png: cropped }), [cropped.buffer]); + assert.deepEqual(resultTransferList({ kind: 'crop', png: null }), []); +}); + test('resultTransferList transfers only the fully-owned views of a mixed result', () => { const ownedDiffData = Buffer.alloc(16); // Buffer.alloc never uses the shared pool const pooledMask = new Uint8Array(new ArrayBuffer(32), 4, 8); // offset view, pooled-Buffer shape diff --git a/packages/capture-kit/src/png-worker.ts b/packages/capture-kit/src/png-worker.ts index 3da1a5dbe..3d980393f 100644 --- a/packages/capture-kit/src/png-worker.ts +++ b/packages/capture-kit/src/png-worker.ts @@ -1,5 +1,6 @@ import { parentPort } from 'node:worker_threads'; import { normalizeError } from '@agent-device/kernel/errors'; +import { cropPngBytes } from './png-crop-bytes.ts'; import { decodePng, PNG } from './png.ts'; import { computeScreenshotDiffPixels } from './screenshot-diff-pixels.ts'; import { computePngRgbDifference } from './png-rgb-difference.ts'; @@ -27,6 +28,12 @@ function runJob(request: PngWorkerRequest): PngWorkerJobResult { png.data = toBuffer(request.data); return { kind: 'encode', png: PNG.sync.write(png) }; } + case 'crop': { + return { + kind: 'crop', + png: cropPngBytes(toBuffer(request.png), request.box, request.label), + }; + } case 'rgb-difference': { const first = decodePng(toBuffer(request.firstPng), request.label); const second = decodePng(toBuffer(request.secondPng), request.label); @@ -68,6 +75,8 @@ function resultBufferViews(result: PngWorkerJobResult): Uint8Array[] { return [result.data]; case 'encode': return [result.png]; + case 'crop': + return result.png === null ? [] : [result.png]; case 'rgb-difference': return []; case 'diff-pixels': diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 932c34255..aa27d2521 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -947,7 +947,7 @@ agent-device record stop # Stop active recording - Set `AGENT_DEVICE_SCREENSHOT_SCALE=0.3` (or `screenshotScale` in config) as a token-conscious screenshot default for agent workflows. An explicit `--scale` overrides it. - Keep the scale default unset, or use `--scale 1`, when full-resolution screenshots are required for reusable pixel-diff baselines. - `screenshot --overlay-refs` captures a fresh full snapshot and burns visible `@eN` refs plus their target rectangles into the saved PNG. -- `screenshot --crop-on ` captures a fresh full snapshot of the same screen and crops the saved PNG to the frame the selector resolves to. The selector must resolve to exactly one framed node; the result carries a `warnings` entry when the frame is clipped to the image. Currently accepted on iOS simulators and Android emulators — every other target is refused before any device work, and the flag cannot be combined with `--overlay-refs` or `--fullscreen` because both move the captured frame away from the snapshot viewport the crop is measured against. +- `screenshot --crop-on ` captures a fresh full snapshot of the same screen and crops the saved PNG to the frame the selector resolves to. The crop is re-encoded, so byte-comparing it against an older crop of the same frame is unreliable; a crop whose pixels are all opaque is written as truecolor RGB, while one containing transparency keeps RGBA. The selector must resolve to exactly one framed node; the result carries a `warnings` entry when the frame is clipped to the image. Currently accepted on iOS simulators and Android emulators — every other target is refused before any device work, and the flag cannot be combined with `--overlay-refs` or `--fullscreen` because both move the captured frame away from the snapshot viewport the crop is measured against. - `screenshot --normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for deterministic screenshot baselines; ordinary screenshots leave the simulator's current chrome visible. - `screenshot --scale --overlay-refs` writes a smaller image and draws refs for that final image size; avoid very small scales when text, icons, or labels need to remain readable. - `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session. Its text output reports ranked changed regions with screen-space rectangles, changed-pixel counts, and each region's share of the diff; JSON also includes normalized rectangles. The earlier best-effort `ocr` and `nonTextDeltas` analyzers are retired; their optional result fields remain for source compatibility but are no longer emitted, so use the baseline/current images and diff artifact with vision for qualitative interpretation. It writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines.