From 1c5d93f5bae91232459f7d42dc7d7733bc77b465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 11 Sep 2026 19:05:29 +0200 Subject: [PATCH 1/2] perf(scripts): add a device-free PNG crop benchmark `pnpm bench:png-crop` runs the whole-image pipeline and the shipped region crop over the same bytes in one process, so the comparison holds the capture content, the deflate stream, and the machine fixed. The corpus is generated, which keeps a run at seconds with no device; real captures join the same table via `--file`, and each corpus entry prints its compressed size so an unrealistic corpus is visible. The README records what the measurements said, including the case the encoder policy loses: `None` on every scanline is faster everywhere but writes about 1.7x more bytes than a filtered encoding on smooth low-frequency content. --- package.json | 1 + scripts/png-crop-benchmark/README.md | 51 ++++++ scripts/png-crop-benchmark/args.test.ts | 41 +++++ scripts/png-crop-benchmark/args.ts | 36 ++++ scripts/png-crop-benchmark/corpus.test.ts | 68 ++++++++ scripts/png-crop-benchmark/corpus.ts | 155 ++++++++++++++++++ scripts/png-crop-benchmark/pipelines.ts | 74 +++++++++ scripts/png-crop-benchmark/report.ts | 49 ++++++ scripts/png-crop-benchmark/run.ts | 51 ++++++ scripts/png-crop-benchmark/statistics.test.ts | 20 +++ scripts/png-crop-benchmark/statistics.ts | 34 ++++ 11 files changed, 580 insertions(+) create mode 100644 scripts/png-crop-benchmark/README.md create mode 100644 scripts/png-crop-benchmark/args.test.ts create mode 100644 scripts/png-crop-benchmark/args.ts create mode 100644 scripts/png-crop-benchmark/corpus.test.ts create mode 100644 scripts/png-crop-benchmark/corpus.ts create mode 100644 scripts/png-crop-benchmark/pipelines.ts create mode 100644 scripts/png-crop-benchmark/report.ts create mode 100644 scripts/png-crop-benchmark/run.ts create mode 100644 scripts/png-crop-benchmark/statistics.test.ts create mode 100644 scripts/png-crop-benchmark/statistics.ts diff --git a/package.json b/package.json index 4041a0a28..64284a4a3 100644 --- a/package.json +++ b/package.json @@ -124,6 +124,7 @@ "bench:ios-snapshot": "node --experimental-strip-types scripts/ios-snapshot-benchmark/run.ts", "bench:ios-snapshot:deep-button": "node --experimental-strip-types scripts/ios-snapshot-benchmark/deep-button.ts", "bench:ios-snapshot:evidence": "node --experimental-strip-types scripts/ios-snapshot-benchmark/evidence.ts", + "bench:png-crop": "node --experimental-strip-types scripts/png-crop-benchmark/run.ts", "bench:ios-ax-bridge:targeted": "node --experimental-strip-types scripts/ios-ax-bridge-spike/targeted-run.ts", "mutation:run": "node --experimental-strip-types scripts/mutation/run.ts", "mutation:check": "node --experimental-strip-types scripts/mutation/run.ts --no-run", diff --git a/scripts/png-crop-benchmark/README.md b/scripts/png-crop-benchmark/README.md new file mode 100644 index 000000000..26b6f6580 --- /dev/null +++ b/scripts/png-crop-benchmark/README.md @@ -0,0 +1,51 @@ +# PNG crop benchmark + +```sh +pnpm bench:png-crop -- --rounds 5 +pnpm bench:png-crop -- --rounds 5 --file /path/to/real-capture.png +``` + +Compares the two ways this repository can crop a screenshot. Both run over the same bytes as PNG +worker jobs in one process, so what differs is the algorithm and not the thread it happens to land +on, or a file write on one side only: + +- `whole-image` — the previous crop: one job decodes the whole capture to RGBA, the box rows are + copied out of that bitmap, and a second job encodes the box. +- `region` — the shipped crop: one job reads the box's rows and encodes them, as RGB rather than + RGBA whenever the cropped pixels are all opaque. + +The corpus is generated, so a full run costs seconds and needs no device. Generated captures are +written to `.tmp/png-crop-benchmark/` and each one's compressed size is printed under the table: +a corpus that stops resembling a real capture becomes visible there instead of flattering the +result. Pass real captures with `--file` (repeatable) to put their numbers in the same table; the +real captures decide the verdict, since generated content cannot match a device's deflate stream. + +## What is actually saved + +Neither pipeline reads less of the file: a deflate stream has to be inflated to its end, so both +inflate the whole compressed image, and the region path inflates it into a buffer sized for every +filtered row in the capture. What the region path saves is the pixel work — reconstructing only down +to the box's last row and producing only the box's pixels, instead of a full RGBA bitmap for the +whole capture — plus one worker round trip, and the RGBA re-encode of the answer. + +## What the measurements have said + +Measured at that matched boundary over 7 rounds, on captures taken from an iOS Simulator and an +Android Emulator: the iOS captures go 2.5x to 5.6x faster and their crops come out 1.04x to 2.16x +smaller, mostly because an opaque crop is written as RGB instead of RGBA. A flat UI capture gains +the most, because the previous pipeline still expands the whole capture to an RGBA bitmap whatever +the filters look like, while the region path skips the pixels above the box. + +The noisiest capture in that set — a full-screen Android `screencap`, 1.4 MB compressed — is a wash +on time (1.0x to 1.6x) and its crop comes out up to 1.13x *larger*. Inflating and reconstructing +that much entropy dominates both pipelines, and the region writer's `None` filter cannot beat the +general writer's filter search on content that noisy. Check your own captures with `--file` before +reading a win or a loss into any number here. + +The encoder keeps the `None` filter on every scanline. Scoring the five PNG filters per row is +1.4x to 2.6x slower and produces a *larger* file on UI captures, where the smallest-sum-of-absolute- +differences heuristic prefers Sub or Up on text rows that deflate smaller unfiltered. On synthetic +low-frequency content — the `photo` captures here, which are smooth 8px blocks — `None` is still +faster but writes about 1.7x more bytes than a filtered encoding would. Real photo-filled captures +still come out smaller with `None` than with the general writer's own filter search, so the corpus +here overstates that case; check your own captures with `--file` before treating it as a limit. diff --git a/scripts/png-crop-benchmark/args.test.ts b/scripts/png-crop-benchmark/args.test.ts new file mode 100644 index 000000000..d239a4a98 --- /dev/null +++ b/scripts/png-crop-benchmark/args.test.ts @@ -0,0 +1,41 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { parseBenchmarkArgs } from './args.ts'; + +test('an empty command line keeps the default round count', () => { + assert.deepEqual(parseBenchmarkArgs([]), { + rounds: 5, + jsonPath: undefined, + captureFiles: [], + }); +}); + +test('rounds, json path, and every capture file are read from their flags', () => { + assert.deepEqual( + parseBenchmarkArgs([ + '--rounds', + '9', + '--json', + 'out/bench.json', + '--file', + 'a.png', + '--file', + 'b.png', + ]), + { rounds: 9, jsonPath: 'out/bench.json', captureFiles: ['a.png', 'b.png'] }, + ); +}); + +test('a round count that is not a positive number falls back to the default', () => { + assert.equal(parseBenchmarkArgs(['--rounds', '0']).rounds, 5); + assert.equal(parseBenchmarkArgs(['--rounds', 'many']).rounds, 5); + assert.equal(parseBenchmarkArgs(['--rounds', '2.4']).rounds, 2); +}); + +test('a flag with no value after it is ignored', () => { + assert.deepEqual(parseBenchmarkArgs(['--json']), { + rounds: 5, + jsonPath: undefined, + captureFiles: [], + }); +}); diff --git a/scripts/png-crop-benchmark/args.ts b/scripts/png-crop-benchmark/args.ts new file mode 100644 index 000000000..b0f3c3363 --- /dev/null +++ b/scripts/png-crop-benchmark/args.ts @@ -0,0 +1,36 @@ +/** The command line `pnpm bench:png-crop` accepts. */ + +export type BenchmarkOptions = Readonly<{ + rounds: number; + jsonPath: string | undefined; + captureFiles: readonly string[]; +}>; + +const DEFAULT_ROUNDS = 5; + +export function parseBenchmarkArgs(argv: readonly string[]): BenchmarkOptions { + return { + rounds: readNumber(argv, '--rounds') ?? DEFAULT_ROUNDS, + jsonPath: readString(argv, '--json'), + captureFiles: readAll(argv, '--file'), + }; +} + +function readNumber(argv: readonly string[], flag: string): number | undefined { + const value = readString(argv, flag); + const parsed = value === undefined ? Number.NaN : Number(value); + return Number.isFinite(parsed) && parsed > 0 ? Math.round(parsed) : undefined; +} + +function readString(argv: readonly string[], flag: string): string | undefined { + const index = argv.indexOf(flag); + return index >= 0 ? argv[index + 1] : undefined; +} + +function readAll(argv: readonly string[], flag: string): string[] { + const values: string[] = []; + argv.forEach((entry, index) => { + if (entry === flag && argv[index + 1] !== undefined) values.push(argv[index + 1]!); + }); + return values; +} diff --git a/scripts/png-crop-benchmark/corpus.test.ts b/scripts/png-crop-benchmark/corpus.test.ts new file mode 100644 index 000000000..20c5528ad --- /dev/null +++ b/scripts/png-crop-benchmark/corpus.test.ts @@ -0,0 +1,68 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { PNG } from '@agent-device/capture-kit/png'; +import { buildCorpus, cropBoxOf, CROP_SCENARIOS } from './corpus.ts'; + +const SCENARIO_BY_NAME = new Map(CROP_SCENARIOS.map((scenario) => [scenario.name, scenario])); +const CAPTURE = { + name: 'phone', + label: 'phone', + width: 1200, + height: 2400, + bytes: Buffer.alloc(0), +}; +const SMALL = [{ name: 'tiny', label: 'tiny', width: 24, height: 40 }]; + +test('a card crop keeps the framed fraction of the capture', () => { + const box = cropBoxOf(CAPTURE, SCENARIO_BY_NAME.get('card')!); + + assert.deepEqual(box, { x: 96, y: 600, width: 960, height: 480 }); +}); + +test('a full-bleed header is clamped to the image, never one pixel past it', () => { + const box = cropBoxOf(CAPTURE, SCENARIO_BY_NAME.get('header')!); + + assert.equal(box.x, 0); + assert.equal(box.width, CAPTURE.width); + assert.equal(box.y + box.height, 288); +}); + +test('the same generated capture comes out byte for byte every run', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'png-crop-corpus-')); + try { + const first = buildCorpus(path.join(directory, 'first'), SMALL); + const second = buildCorpus(path.join(directory, 'second'), SMALL); + + assert.equal(first.length, 2); + assert.deepEqual( + first.map((capture) => capture.bytes), + second.map((capture) => capture.bytes), + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test('interface and photo captures differ, and both decode at their declared size', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'png-crop-corpus-')); + try { + const [interfaceCapture, photoCapture] = buildCorpus(directory, SMALL); + + const decoded = [interfaceCapture, photoCapture].map((capture) => + PNG.sync.read(capture!.bytes), + ); + assert.deepEqual( + decoded.map((png) => [png.width, png.height]), + [ + [24, 40], + [24, 40], + ], + ); + assert.notDeepEqual(decoded[0]!.data, decoded[1]!.data); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/scripts/png-crop-benchmark/corpus.ts b/scripts/png-crop-benchmark/corpus.ts new file mode 100644 index 000000000..a6dd43ba2 --- /dev/null +++ b/scripts/png-crop-benchmark/corpus.ts @@ -0,0 +1,155 @@ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { PNG } from '@agent-device/capture-kit/png'; + +/** + * The captures and crop boxes this benchmark measures. They are generated rather than recorded, + * so a run costs seconds and needs no device; each one reports its own compressed size, which is + * how a corpus that stopped resembling a real capture becomes visible instead of flattering. + * Pass real captures with `--file` to put their numbers in the same table. + */ + +export type CaptureResolution = Readonly<{ + name: string; + label: string; + width: number; + height: number; +}>; + +const RESOLUTIONS: readonly CaptureResolution[] = [ + { name: 'ios-phone', label: 'iPhone-class 1206x2622', width: 1206, height: 2622 }, + { name: 'android-phone', label: 'Android-class 1080x2400', width: 1080, height: 2400 }, + { name: 'ios-pad', label: 'iPad-class 2048x2732', width: 2048, height: 2732 }, +]; + +export type Capture = Readonly<{ + name: string; + label: string; + width: number; + height: number; + bytes: Buffer; +}>; + +export type CropScenario = Readonly<{ + name: string; + /** Fractions of the capture, so one scenario reads well at every resolution. */ + fraction: Readonly<{ x: number; y: number; width: number; height: number }>; +}>; + +/** The box shapes `--crop-on` resolves: a card, a full-bleed header, and a small control. */ +export const CROP_SCENARIOS: readonly CropScenario[] = [ + { name: 'card', fraction: { x: 0.08, y: 0.25, width: 0.8, height: 0.2 } }, + { name: 'header', fraction: { x: 0, y: 0, width: 1, height: 0.12 } }, + { name: 'control', fraction: { x: 0.3, y: 0.6, width: 0.3, height: 0.04 } }, +]; + +export function buildCorpus( + outDir: string, + resolutions: readonly CaptureResolution[] = RESOLUTIONS, +): Capture[] { + mkdirSync(outDir, { recursive: true }); + return resolutions.flatMap(({ name, label, width, height }) => + (['interface', 'photo'] as const).map((kind) => { + const bytes = PNG.sync.write(encodeCapture(width, height, kind)); + writeFileSync(path.join(outDir, `${name}-${kind}.png`), bytes); + return { name: `${name}-${kind}`, label: `${label} ${kind}`, width, height, bytes }; + }), + ); +} + +export function readCaptureFile(filePath: string, index: number): Capture { + const bytes = readFileSync(filePath); + const png = PNG.sync.read(bytes); + return { + name: `capture-${index}`, + label: path.basename(filePath), + width: png.width, + height: png.height, + bytes, + }; +} + +type Content = 'interface' | 'photo'; + +/** + * A capture a device could have produced. `interface` is flat panels with high-frequency text + * where the copy is drawn, which is what a settings or list screen looks like. `photo` is + * low-frequency detail in 8px blocks plus a gradient, which is what a photo, artwork, or a blurred + * background does to the deflate stream. + */ +function encodeCapture(width: number, height: number, content: Content): PNG { + const png = new PNG({ width, height }); + const tones = content === 'photo' ? photoToneField(width, height) : null; + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const offset = (y * width + x) * 4; + const [red, green, blue] = tones ? photoPixel(tones, x, y) : interfacePixel(x, y); + png.data[offset] = red; + png.data[offset + 1] = green; + png.data[offset + 2] = blue; + png.data[offset + 3] = 255; + } + } + return png; +} + +/** Flat panels with high-frequency text where the copy is drawn. */ +function interfacePixel(x: number, y: number): readonly [number, number, number] { + const ink = x % 320 < 2 || y % 96 < 2 || (y % 24 > 6 && y % 24 < 18 && x % 7 < 3); + if (ink) return [24, 26, 30]; + const band = Math.floor(y / 96); + return [(band * 11) % 240, (band * 17) % 244, (band * 23) % 248]; +} + +/** The 8px-block detail and soft gradient of a photo, scaled by the pixel position. */ +function photoPixel( + tones: readonly (readonly number[])[], + x: number, + y: number, +): readonly [number, number, number] { + const tone = tones[y >> 3]?.[x >> 3] ?? 0; + return [ + clampByte(tone + y / 6), + clampByte(tone * 0.8 + x / 9), + clampByte(tone * 0.6 + (x + y) / 24), + ]; +} + +/** Smooth 8px-block luminance in [40, 215], so a photo corpus compresses like a real photo. */ +function photoToneField(width: number, height: number): number[][] { + let state = 0x9e3779b9; + const next = () => { + state = (state * 1664525 + 1013904223) >>> 0; + return state / 0x100000000; + }; + return Array.from({ length: Math.ceil(height / 8) }, (_row, blockRow) => + Array.from({ length: Math.ceil(width / 8) }, (_column, blockColumn) => { + const drift = Math.sin(blockRow / 26) * 45 + Math.cos(blockColumn / 19) * 45; + return clampByte(128 + drift + (next() - 0.5) * 70); + }), + ); +} + +function clampByte(value: number): number { + return Math.max(0, Math.min(255, Math.round(value))); +} + +export function cropBoxOf( + capture: Capture, + scenario: CropScenario, +): { + x: number; + y: number; + width: number; + height: number; +} { + const { fraction } = scenario; + const x = Math.round(capture.width * fraction.x); + const y = Math.round(capture.height * fraction.y); + return { + x, + y, + width: Math.max(1, Math.min(capture.width - x, Math.round(capture.width * fraction.width))), + height: Math.max(1, Math.min(capture.height - y, Math.round(capture.height * fraction.height))), + }; +} diff --git a/scripts/png-crop-benchmark/pipelines.ts b/scripts/png-crop-benchmark/pipelines.ts new file mode 100644 index 000000000..58ec2dc4e --- /dev/null +++ b/scripts/png-crop-benchmark/pipelines.ts @@ -0,0 +1,74 @@ +import type { Rect } from '@agent-device/kernel/snapshot'; +import { PNG } from '@agent-device/capture-kit/png'; +import { + cropPngBytesAsync, + decodePngAsync, + encodePngAsync, +} from '@agent-device/capture-kit/png-worker-client'; +import type { Capture, CropScenario } from './corpus.ts'; +import { cropBoxOf } from './corpus.ts'; +import { measureAsync } from './statistics.ts'; + +/** + * The two ways this repository can crop a capture, both run as PNG worker jobs over the same + * bytes, so what differs between them is the algorithm rather than the thread it lands on: + * + * - `wholeImage`: the previous crop. One job decodes the whole capture to RGBA, the box rows are + * copied out of that bitmap, and a second job encodes the box. + * - `region`: the shipped crop. One job reads the box's rows and encodes them. + * + * Neither writes a file. Publishing the artifact is the same work on both sides, so it would only + * dilute the ratio; the byte length of the encoded answer is reported instead. + */ + +export type PipelineSample = Readonly<{ medianMs: number; bestMs: number; outBytes: number }>; + +export type ScenarioSample = Readonly<{ + capture: string; + label: string; + scenario: string; + captureBytes: number; + box: Rect; + wholeImage: PipelineSample; + region: PipelineSample; +}>; + +export async function sampleScenario( + capture: Capture, + scenario: CropScenario, + rounds: number, +): Promise { + const box = cropBoxOf(capture, scenario); + const wholeImageBytes = await wholeImageCrop(capture.bytes, box); + const regionBytes = await regionCrop(capture.bytes, box); + const wholeImage = await measureAsync(rounds, async () => { + await wholeImageCrop(capture.bytes, box); + }); + const region = await measureAsync(rounds, async () => { + await regionCrop(capture.bytes, box); + }); + return { + capture: capture.name, + label: capture.label, + scenario: scenario.name, + captureBytes: capture.bytes.length, + box, + wholeImage: { ...wholeImage, outBytes: wholeImageBytes }, + region: { ...region, outBytes: regionBytes }, + }; +} + +async function wholeImageCrop(source: Buffer, box: Rect): Promise { + const decoded = await decodePngAsync(source, 'capture'); + const cropped = new PNG({ width: box.width, height: box.height }); + for (let row = 0; row < box.height; row += 1) { + const from = ((row + box.y) * decoded.width + box.x) * 4; + decoded.data.copy(cropped.data, row * cropped.width * 4, from, from + box.width * 4); + } + return (await encodePngAsync(cropped)).length; +} + +async function regionCrop(source: Buffer, box: Rect): Promise { + const cropped = await cropPngBytesAsync(source, box, 'capture'); + return cropped === null ? source.length : cropped.length; +} diff --git a/scripts/png-crop-benchmark/report.ts b/scripts/png-crop-benchmark/report.ts new file mode 100644 index 000000000..1c82d6f47 --- /dev/null +++ b/scripts/png-crop-benchmark/report.ts @@ -0,0 +1,49 @@ +import type { ScenarioSample } from './pipelines.ts'; +import { speedup } from './statistics.ts'; + +/** The benchmark's report: one row per capture and crop box, plus the corpus honesty check. */ + +const COLUMN_WIDTHS: readonly number[] = [30, 11, 13, 11, 10, 22]; + +export function renderReport(samples: readonly ScenarioSample[]): string { + const rows = [ + renderRow(['capture / crop', 'box', 'whole-image', 'region', 'speed-up', 'artifact']), + renderRow(['-', '-', '-', '-', '-', '-']), + ...samples.map((sample) => + renderRow([ + `${sample.capture} ${sample.scenario}`, + `${sample.box.width}x${sample.box.height}`, + `${sample.wholeImage.medianMs.toFixed(1)}ms`, + `${sample.region.medianMs.toFixed(1)}ms`, + `${speedup(sample.wholeImage.medianMs, sample.region.medianMs).toFixed(2)}x`, + `${kilobytes(sample.wholeImage.outBytes)} -> ${kilobytes(sample.region.outBytes)}`, + ]), + ), + ]; + return [...rows, '', footnotes(samples)].join('\n'); +} + +function footnotes(samples: readonly ScenarioSample[]): string { + const timings = 'median of the measured rounds; artifact = encoded crop, whole-image -> region'; + const corpus = [...new Map(samples.map((sample) => [sample.capture, sample])).values()].map( + (sample) => ` ${sample.label}: ${(sample.captureBytes / 1024).toFixed(0)} kB compressed`, + ); + return [ + timings, + '', + 'Corpus compressed sizes, so an unrealistic corpus is visible. A real 1206x2622 simulator', + 'capture of a UI screen is ~245 kB, and the same device showing a photo screen is ~3 MB.', + ...corpus, + ].join('\n'); +} + +function renderRow(cells: readonly string[]): string { + return cells + .map((cell, index) => cell.padEnd(COLUMN_WIDTHS[index] ?? cell.length)) + .join(' ') + .trimEnd(); +} + +function kilobytes(bytes: number): string { + return `${(bytes / 1024).toFixed(0)}kB`; +} diff --git a/scripts/png-crop-benchmark/run.ts b/scripts/png-crop-benchmark/run.ts new file mode 100644 index 000000000..c3e000852 --- /dev/null +++ b/scripts/png-crop-benchmark/run.ts @@ -0,0 +1,51 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { terminatePngWorker } from '@agent-device/capture-kit/png-worker-client'; +import { parseBenchmarkArgs } from './args.ts'; +import { buildCorpus, CROP_SCENARIOS, readCaptureFile, type Capture } from './corpus.ts'; +import { sampleScenario, type ScenarioSample } from './pipelines.ts'; +import { renderReport } from './report.ts'; + +/** + * `pnpm bench:png-crop` — a cheap, device-free comparison of the two crop pipelines over generated + * captures. Pass `--file ` (repeatable) to add real captures to the same table. + */ + +const CORPUS_DIR = path.join('.tmp', 'png-crop-benchmark'); + +async function main(): Promise { + const options = parseBenchmarkArgs(process.argv.slice(2)); + const captures: Capture[] = [ + ...buildCorpus(CORPUS_DIR), + ...options.captureFiles.map((filePath, index) => readCaptureFile(filePath, index + 1)), + ]; + process.stderr.write( + `[png-crop-bench] ${captures.length} captures x ${CROP_SCENARIOS.length} crop boxes, ${options.rounds} rounds each\n`, + ); + try { + const samples = await sampleAll(captures, options.rounds); + process.stdout.write(`${renderReport(samples)}\n`); + writeJsonReport(options.jsonPath, options.rounds, samples); + } finally { + await terminatePngWorker(); + } +} + +async function sampleAll(captures: readonly Capture[], rounds: number): Promise { + const samples: ScenarioSample[] = []; + for (const capture of captures) { + for (const scenario of CROP_SCENARIOS) { + samples.push(await sampleScenario(capture, scenario, rounds)); + } + } + return samples; +} + +function writeJsonReport(jsonPath: string | undefined, rounds: number, samples: unknown): void { + if (jsonPath === undefined) return; + mkdirSync(path.dirname(path.resolve(jsonPath)), { recursive: true }); + writeFileSync(jsonPath, `${JSON.stringify({ rounds, samples }, null, 2)}\n`); + process.stderr.write(`[png-crop-bench] json: ${jsonPath}\n`); +} + +await main(); diff --git a/scripts/png-crop-benchmark/statistics.test.ts b/scripts/png-crop-benchmark/statistics.test.ts new file mode 100644 index 000000000..9f837af48 --- /dev/null +++ b/scripts/png-crop-benchmark/statistics.test.ts @@ -0,0 +1,20 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { speedup, summarize } from './statistics.ts'; + +test('summarize reports the middle of an odd sample set', () => { + assert.deepEqual(summarize([9, 1, 5]), { medianMs: 5, bestMs: 1, worstMs: 9 }); +}); + +test('summarize averages the two middle samples of an even set', () => { + assert.equal(summarize([4, 8, 2, 6]).medianMs, 5); +}); + +test('summarize survives an empty sample set', () => { + assert.deepEqual(summarize([]), { medianMs: 0, bestMs: 0, worstMs: 0 }); +}); + +test('speedup expresses the old pipeline in multiples of the new one', () => { + assert.equal(speedup(120, 40), 3); + assert.equal(speedup(120, 0), 0); +}); diff --git a/scripts/png-crop-benchmark/statistics.ts b/scripts/png-crop-benchmark/statistics.ts new file mode 100644 index 000000000..6a8ff7422 --- /dev/null +++ b/scripts/png-crop-benchmark/statistics.ts @@ -0,0 +1,34 @@ +import { performance } from 'node:perf_hooks'; + +/** A timing summary cheap enough to compute on every scenario. */ + +export type Timing = Readonly<{ medianMs: number; bestMs: number; worstMs: number }>; + +export async function measureAsync(rounds: number, run: () => Promise): Promise { + const samples: number[] = []; + for (let round = 0; round < rounds; round += 1) { + const started = performance.now(); + await run(); + samples.push(performance.now() - started); + } + return summarize(samples); +} + +export function summarize(samples: readonly number[]): Timing { + if (samples.length === 0) return { medianMs: 0, bestMs: 0, worstMs: 0 }; + const sorted = [...samples].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + const median = + sorted.length % 2 === 1 + ? sorted[middle]! + : ((sorted.at(middle - 1) ?? 0) + (sorted[middle] ?? 0)) / 2; + return { + medianMs: median, + bestMs: sorted[0] ?? 0, + worstMs: sorted.at(-1) ?? 0, + }; +} + +export function speedup(wholeImageMs: number, regionMs: number): number { + return regionMs > 0 ? wholeImageMs / regionMs : 0; +} From 03baaa59c3621ce05d10d073002a30df2ff2f39e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 11 Sep 2026 19:05:29 +0200 Subject: [PATCH 2/2] chore(gates): run the PNG crop benchmark's model tests in unit-core Registers scripts/png-crop-benchmark/*.test.ts so the timing summary that the report is built from stays covered without a device lane. --- vitest.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/vitest.config.ts b/vitest.config.ts index 133fd9370..f8013dfae 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -158,6 +158,7 @@ export default defineConfig({ // the fixed mcp subcommand, or registry-format launchers run the bare CLI. 'scripts/__tests__/mcp-metadata.test.ts', 'scripts/ios-snapshot-benchmark/*.test.ts', + 'scripts/png-crop-benchmark/*.test.ts', 'scripts/ios-ax-bridge-spike/*.test.ts', // Parses CI configuration only, so this action guard needs no device or subprocess lane. 'test/ci/upload-agent-device-artifacts.test.ts',