From e5312bd753534b06ca7dbdd8a5b1d2ae1955db91 Mon Sep 17 00:00:00 2001 From: Nigro Simone Date: Sun, 13 Sep 2026 10:29:01 +0200 Subject: [PATCH 1/2] ci: benchmark every pull request against its base branch --- .github/workflows/benchmark.yml | 87 ++++++++++++++ benchmark/README.md | 23 ++++ benchmark/compare.js | 173 ++++++++++++++++++++++++++++ benchmark/worker.js | 194 ++++++++++++++++++++++++++++++++ package.json | 3 +- 5 files changed, 479 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmark/README.md create mode 100644 benchmark/compare.js create mode 100644 benchmark/worker.js diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..8b75f1a5e --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,87 @@ +name: Benchmark + +on: + pull_request: + branches: [master] + workflow_dispatch: + inputs: + base: + description: 'Ref to compare this branch against' + default: master + +permissions: + contents: read + +jobs: + benchmark: + timeout-minutes: 30 + runs-on: ubuntu-latest + permissions: + contents: read + # the comment on the pull request. Read-only on a pull request from a fork, where the + # job summary is the only output + pull-requests: write + env: + # the PostgreSQL that comes with the runner image, over its unix socket: no container + # and no TCP between the client and the server, so the numbers are the client's. + # The role is named after the OS user, which is what peer authentication wants + PGHOST: /var/run/postgresql + PGDATABASE: benchmark + steps: + - name: Start PostgreSQL + run: | + sudo systemctl start postgresql.service + sudo -u postgres createuser "$USER" + sudo -u postgres createdb -O "$USER" benchmark + - uses: actions/checkout@v4 + with: + path: head + persist-credentials: false + - uses: actions/checkout@v4 + with: + path: base + ref: ${{ github.event.pull_request.base.sha || inputs.base }} + persist-credentials: false + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: 26 + cache: yarn + cache-dependency-path: | + head/yarn.lock + base/yarn.lock + - name: Build both arms + run: | + for arm in base head; do + (cd $arm && yarn install --frozen-lockfile && yarn build) + done + - name: Run benchmark + working-directory: head + run: node benchmark/compare.js --base ../base --head . --rounds 4 --duration 3 --output ../benchmark_summary.md + - name: Publish the summary + run: cat benchmark_summary.md >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + with: + name: benchmark-summary + path: benchmark_summary.md + - name: Comment on the pull request + if: github.event_name == 'pull_request' + continue-on-error: true + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs') + const marker = '' + const body = fs.readFileSync('benchmark_summary.md', 'utf8') + const issue_number = context.payload.pull_request.number + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number, + }) + const existing = comments.find((c) => c.user?.type === 'Bot' && c.body?.includes(marker)) + if (existing) { + await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body }) + } else { + await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number, body }) + } diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 000000000..48dfceda3 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,23 @@ +# Benchmark + +A/B of two checkouts of this repo against the same database: the `pg`, `pg-pool`, `pg-cursor` and +`pg-query-stream` of `--head` over the same packages of `--base`, scenario by scenario. The CI runs +it on every pull request with the base branch in `--base` and the PR in `--head`, over a unix +socket, and posts the table as a comment on the PR. + +Both checkouts must be installed and built (`yarn install && yarn build`). The database comes from +the usual `PG*` environment variables. + +```bash +yarn benchmark --base ../node-postgres-master --head . --rounds 4 --duration 3 +``` + +Both arms stay up in their own process and the load alternates between them one scenario at a +time, swapping which goes first on every round, so the two measurements behind a ratio are seconds +apart and a drift of the machine lands on both. Only the ratio is comparable across runs: the +absolute queries per second depend on the machine. + +The scenarios cover the row parser on every family of type, the parameter encoding on writes, +prepared and unnamed statements, array and binary result modes, transactions, the pool, cursors, +streams and pipeline mode. Each scenario is one iteration of a function in `worker.js`, which +returns how many queries it ran. diff --git a/benchmark/compare.js b/benchmark/compare.js new file mode 100644 index 000000000..94f28e6ab --- /dev/null +++ b/benchmark/compare.js @@ -0,0 +1,173 @@ +'use strict' + +// A/B of two checkouts of this repo against the same database: the pg of `--head` over the pg of +// `--base`, scenario by scenario. Both arms run on the same machine in the same run, so what the +// machine does moves them together and the ratio holds still; that is the only number worth +// reading on a hosted runner, where the absolute queries per second are never the same twice. +// +// Each arm runs twice, in two processes, so every round also measures base against base and head +// against head: the same code on both sides, so whatever those ratios do is the noise of this +// run on this machine, and a head/base ratio is marked only when it moved further than that. +// All four stay up but only one measures at a time, the others wait: the load alternates between +// them one scenario at a time, in an order that changes every round, so the measurements behind +// a ratio are seconds apart and a drift of the machine lands on all of them. The reported +// speedup is the median of the per-round ratios. + +const fs = require('fs') +const os = require('os') +const path = require('path') +const { execFileSync, fork } = require('child_process') + +// below this nothing is marked whatever the noise said: a same-code band can come out very +// narrow by luck on a handful of rounds, and a change this small is not worth a look anyway +const FLOOR = 0.02 + +const parseArgs = (argv) => { + const args = {} + for (let i = 0; i < argv.length; i++) { + if (argv[i].startsWith('--')) args[argv[i].slice(2)] = argv[++i] + } + return args +} + +const median = (values) => { + const sorted = [...values].sort((a, b) => a - b) + const mid = Math.floor(sorted.length / 2) + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2 +} + +const label = (dir) => { + try { + return execFileSync('git', ['-C', dir, 'rev-parse', '--short', 'HEAD'], { encoding: 'utf8' }).trim() + } catch { + return path.basename(path.resolve(dir)) + } +} + +// one worker per arm, driven by messages: each send resolves with the worker's reply +const startArm = (dir, index) => { + const child = fork(path.join(__dirname, 'worker.js'), [], { + execArgv: ['--expose-gc'], + env: { ...process.env, PG_BENCH_MODULE: path.resolve(dir), PG_BENCH_ARM: String(index) }, + }) + const waiting = [] + child.on('message', (msg) => { + const { resolve, reject } = waiting.shift() + msg.ok ? resolve(msg) : reject(new Error(msg.error)) + }) + child.on('exit', (code) => { + for (const { reject } of waiting.splice(0)) reject(new Error(`worker for ${dir} exited with ${code}`)) + }) + const ready = new Promise((resolve, reject) => waiting.push({ resolve, reject })) + return { + ready, + send: (msg) => + new Promise((resolve, reject) => { + waiting.push({ resolve, reject }) + child.send(msg) + }), + } +} + +const percent = (ratio) => `${ratio >= 1 ? '+' : ''}${((ratio - 1) * 100).toFixed(1)}%` + +const markdown = (labels, rows, rounds, duration) => { + const lines = ['', '', `## Benchmark: \`${labels.head}\` against \`${labels.base}\``, ''] + lines.push('| Scenario | Base q/s | Head q/s | Head / base | Rounds | Noise |') + lines.push('| --- | ---: | ---: | ---: | ---: | ---: |') + for (const row of rows) { + const mark = row.notable ? (row.speedup < 1 ? ' :eyes:' : ' :trophy:') : '' + const change = row.notable ? `**${percent(row.speedup)}**` : percent(row.speedup) + lines.push( + `| ${row.name}${mark} | ${row.base.toFixed(0)} | ${row.head.toFixed(0)} | ` + + `${row.speedup.toFixed(3)}x (${change}) | ${row.min.toFixed(2)} to ${row.max.toFixed(2)} | ` + + `±${(row.noise * 100).toFixed(1)}% |` + ) + } + lines.push('') + lines.push( + `${rounds} rounds of ${duration}s per scenario, each round measuring base, head and a second process of ` + + `each one after the other, in an order that changes every round. "Head / base" is the median of the ` + + `per-round ratios and "Rounds" their range. "Noise" is how far base/base and head/head, the same code on ` + + `both sides, got from 1 in this same run: that is what the machine did, so a row is marked only when the ` + + `median moved further than that, at least ${Math.round(FLOOR * 100)}%, and every round moved the same ` + + `way: :eyes: slower, :trophy: faster. Only the ratio is comparable across runs, the absolute q/s depend ` + + `on the runner.` + ) + lines.push('') + lines.push(`Node ${process.version}, ${os.cpus()[0]?.model || 'unknown cpu'}, ${os.cpus().length} cores.`) + return lines.join('\n') + '\n' +} + +const main = async () => { + const args = parseArgs(process.argv.slice(2)) + if (!args.base || !args.head) { + throw new Error( + 'usage: node benchmark/compare.js --base --head [--rounds 4] [--duration 3] [--output file]' + ) + } + const rounds = Number(args.rounds || 4) + const duration = Number(args.duration || 3) + const warmup = Number(args.warmup || 1) + const labels = { base: label(args.base), head: label(args.head) } + + // two processes per arm: base2 and head2 are the same code as base and head, measured + // alongside them so the run can tell how much two identical arms differ on this machine + const arms = { + base: startArm(args.base, 0), + head: startArm(args.head, 1), + base2: startArm(args.base, 2), + head2: startArm(args.head, 3), + } + const names = Object.keys(arms) + const { scenarios } = await arms.base.ready + for (const name of names) await arms[name].ready + await arms.base.send({ type: 'setup' }) + + const ratio = (qps, over, under) => qps[over].map((value, i) => value / qps[under][i]) + const rows = [] + for (const scenario of scenarios) { + process.stderr.write(`${scenario}\n`) + const qps = Object.fromEntries(names.map((name) => [name, []])) + for (let i = -1; i < rounds; i++) { + // the first round is a warmup on cold code and is thrown away + const ms = (i < 0 ? warmup : duration) * 1000 + // rotates one place per round and flips on odd rounds, so each arm sees every position + const rotated = names.map((_, k) => names[(k + i + 1) % names.length]) + const order = i % 2 ? rotated.reverse() : rotated + for (const arm of order) { + const reply = await arms[arm].send({ type: 'run', scenario, ms }) + if (i >= 0) qps[arm].push(reply.qps) + } + } + const ratios = ratio(qps, 'head', 'base') + const same = [...ratio(qps, 'base2', 'base'), ...ratio(qps, 'head2', 'head')] + const speedup = median(ratios) + const min = Math.min(...ratios) + const max = Math.max(...ratios) + const noise = Math.max(...same.map((value) => Math.abs(value - 1))) + process.stderr.write( + ` ${speedup.toFixed(3)}x (${min.toFixed(2)} to ${max.toFixed(2)}), noise ±${(noise * 100).toFixed(1)}%\n` + ) + rows.push({ + name: scenario, + base: median(qps.base), + head: median(qps.head), + speedup, + min, + max, + noise, + notable: Math.abs(speedup - 1) >= Math.max(noise, FLOOR) && (min > 1 || max < 1), + }) + } + await Promise.all(names.map((name) => arms[name].send({ type: 'end' }))) + + const summary = markdown(labels, rows, rounds, duration) + process.stdout.write(summary) + if (args.output) fs.writeFileSync(args.output, summary) +} + +main().catch((err) => { + process.stderr.write(`${err.stack || err}\n`) + process.exit(1) +}) diff --git a/benchmark/worker.js b/benchmark/worker.js new file mode 100644 index 000000000..42b823813 --- /dev/null +++ b/benchmark/worker.js @@ -0,0 +1,194 @@ +'use strict' + +// One arm: loads the packages at PG_BENCH_MODULE and stays up, so compare.js can alternate the +// two arms scenario by scenario. Each message names a scenario and a duration, the reply is the +// queries per second it reached. The arms never share a heap or a JIT. + +const path = require('path') + +const root = path.resolve(process.env.PG_BENCH_MODULE) +const pg = require(path.join(root, 'packages/pg')) +const Cursor = require(path.join(root, 'packages/pg-cursor')) +const QueryStream = require(path.join(root, 'packages/pg-query-stream')) + +const BATCH = 10 +const ROWS = 500 +// the scratch table holds one range of ids per arm, so the arms' deletes never meet +const ARMS = 4 +const SCRATCH_PER_ARM = 200000 +const scratchFrom = Number(process.env.PG_BENCH_ARM || 0) * SCRATCH_PER_ARM + 1 + +// one column per family of pg-types parser, so a row parse pays for every conversion +const COLUMNS = `id int, small smallint, big bigint, real_value real, double_value double precision, + numeric_value numeric(12, 4), string_value text, varchar_value varchar(20), null_value text, + bool_value boolean, ts timestamptz, day date, json_value json, jsonb_value jsonb, uuid_value uuid, + int_array int[], text_array text[], bytea_value bytea` +const COUNT = 18 +// the columns with nothing to convert beyond the row itself, for the protocol-only scenarios +const SIMPLE = 'id, small, string_value, null_value, bool_value' +const ROW = `i, i, i * 100000, i / 3.0, i / 7.0, i / 11.0, 'wat', 'varchar', NULL, i % 2 = 0, + now(), current_date, '{"a": 1}', '{"b": [1, 2]}', gen_random_uuid(), ARRAY[i, i + 1], ARRAY['x', 'y'], '\\xdeadbeef'` + +// reads every field of every row, so the result is really consumed, and checks the shape +const consume = (rows, count = COUNT) => { + for (const row of rows) { + let seen = 0 + for (const key in row) if (row[key] !== undefined) seen++ + if (seen !== count) throw new Error(`expected ${count} fields, got ${seen}`) + } + return 1 +} + +const select = (limit, columns = '*') => ({ + text: `SELECT ${columns} FROM benchmark_rows ORDER BY id LIMIT ${limit}`, + name: `benchmark_${limit}_${columns === '*' ? 'all' : 'simple'}`, +}) + +// what the parameterized writes send: one value per family of parameter encoding +const values = () => [ + 1, + 2, + '3000000000', + 1.5, + 2.25, + '12.3456', + 'wat', + 'varchar', + null, + true, + new Date(), + '2024-01-02', + { a: 1 }, + { b: [1, 2] }, + '00000000-0000-4000-8000-000000000000', + [1, 2, 3], + ['x', 'y'], + Buffer.from('deadbeef', 'hex'), +] +const placeholders = values() + .map((_, i) => `$${i + 1}`) + .join(', ') + +// each scenario runs one iteration and returns how many queries it ran +const scenarios = ({ client, pipelined, pool }) => { + let deleted = 0 + let updated = 0 + return { + 'select 1 row, all types, prepared': async () => consume((await client.query(select(1))).rows), + 'select 100 rows, all types, prepared': async () => consume((await client.query(select(100))).rows), + 'select 500 rows, all types, prepared': async () => consume((await client.query(select(ROWS))).rows), + 'select 500 rows, simple types, prepared': async () => consume((await client.query(select(ROWS, SIMPLE))).rows, 5), + 'select 500 rows, simple types, array mode': async () => + consume((await client.query({ ...select(ROWS, SIMPLE), rowMode: 'array' })).rows, 5), + 'select 500 rows, simple types, binary': async () => + consume((await client.query({ ...select(ROWS, SIMPLE), binary: true })).rows, 5), + // unnamed, so parse and bind are paid on every query + 'select by id, parameterized': async () => + consume((await client.query('SELECT * FROM benchmark_rows WHERE id = $1', [42])).rows), + 'insert, all types, returning': async () => + consume( + (await client.query(`INSERT INTO benchmark_scratch VALUES (${placeholders}) RETURNING *`, values())).rows + ), + 'update, parameterized': async () => { + const id = scratchFrom + SCRATCH_PER_ARM - 1 - (updated++ % 1000) + await client.query('UPDATE benchmark_scratch SET small = $1, string_value = $2, ts = $3 WHERE id = $4', [ + 7, + 'updated', + new Date(), + id, + ]) + return 1 + }, + 'delete, parameterized': async () => { + await client.query('DELETE FROM benchmark_scratch WHERE id = $1', [scratchFrom + deleted++]) + return 1 + }, + // begin and commit go through the simple query protocol, the insert through the extended one + 'transaction, 3 queries': async () => { + await client.query('BEGIN') + await client.query(`INSERT INTO benchmark_scratch (id, small, string_value) VALUES ($1, $2, $3)`, [0, 1, 'tx']) + await client.query('COMMIT') + return 3 + }, + 'pool query, select 1 row': async () => consume((await pool.query(select(1))).rows), + 'cursor, 500 rows in 5 reads': async () => { + const cursor = client.query(new Cursor(select(ROWS).text)) + let rows + do { + rows = await cursor.read(100) + consume(rows) + } while (rows.length > 0) + await cursor.close() + return 1 + }, + 'query stream, 500 rows': async () => { + const rows = [] + for await (const row of client.query(new QueryStream(select(ROWS).text))) rows.push(row) + consume(rows) + return 1 + }, + [`select 100 rows, all types, pipelined x${BATCH}`]: async () => { + const batch = await Promise.all(Array.from({ length: BATCH }, () => pipelined.query(select(100)))) + for (const res of batch) consume(res.rows) + return BATCH + }, + } +} + +const measure = async (fn, ms) => { + // what the previous measurement left behind is not this one's to collect. The collections + // inside the window stay: a change that allocates more pays for it here, as it does in production + global.gc() + let count = 0 + const start = process.hrtime.bigint() + let elapsed = 0 + while (elapsed < ms) { + count += await fn() + elapsed = Number(process.hrtime.bigint() - start) / 1e6 + } + return (count * 1000) / elapsed +} + +const setup = async (client) => { + await client.query('DROP TABLE IF EXISTS benchmark_rows, benchmark_scratch') + await client.query(`CREATE TABLE benchmark_rows (${COLUMNS}, PRIMARY KEY (id))`) + await client.query(`INSERT INTO benchmark_rows SELECT ${ROW} FROM generate_series(1, ${ROWS}) i`) + await client.query('CREATE UNLOGGED TABLE benchmark_scratch (LIKE benchmark_rows)') + await client.query('CREATE INDEX ON benchmark_scratch (id)') + await client.query( + `INSERT INTO benchmark_scratch (id, small, string_value) SELECT i, i % 1000, 'x' FROM generate_series(1, ${ + ARMS * SCRATCH_PER_ARM + }) i` + ) +} + +const main = async () => { + const client = new pg.Client() + const pipelined = new pg.Client({ pipeline: true }) + const pool = new pg.Pool({ max: 2 }) + await client.connect() + await pipelined.connect() + const all = scenarios({ client, pipelined, pool }) + + process.on('message', async (msg) => { + try { + if (msg.type === 'setup') { + await setup(client) + process.send({ ok: true }) + } else if (msg.type === 'run') { + process.send({ ok: true, qps: await measure(all[msg.scenario], msg.ms) }) + } else if (msg.type === 'end') { + await Promise.all([client.end(), pipelined.end(), pool.end()]) + process.send({ ok: true }, () => process.exit(0)) + } + } catch (err) { + process.send({ ok: false, error: String(err.stack || err) }) + } + }) + process.send({ ok: true, scenarios: Object.keys(all) }) +} + +main().catch((err) => { + process.stderr.write(`${err.stack || err}\n`) + process.exit(1) +}) diff --git a/package.json b/package.json index e30454007..02581d784 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "docs:start": "cd docs && yarn dev", "pretest": "yarn build", "prepublish": "yarn build", - "lint": "eslint --cache 'packages/**/*.{js,ts,tsx}'" + "lint": "eslint --cache 'packages/**/*.{js,ts,tsx}' 'benchmark/*.js'", + "benchmark": "node benchmark/compare.js" }, "devDependencies": { "@eslint/eslintrc": "^3.3.5", From e00ad24a85237ed540b3755f8f976c7cef92ad89 Mon Sep 17 00:00:00 2001 From: Nigro Simone Date: Sun, 13 Sep 2026 11:04:33 +0200 Subject: [PATCH 2/2] ci: re-run after a pg-native crash in the many connections test