diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 551388fcc..c0c4dab9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,3 +93,54 @@ jobs: cache: yarn - run: yarn install --frozen-lockfile - run: yarn test + + # A few fuzz rounds on a seed nobody chose, every push and every pull request. One run proves + # nothing; a few hundred runs walk over ground no hand written test reaches, and the seed + # changes every time so the coverage accumulates across pushes. A divergence turns this job + # red and prints the seed that reproduces it, with the case shrunk to the lines worth pasting + # into a test: replay it locally with `node fuzz/.js --seed N --rounds 1`. + # + # What the first runs found, each fixed in its own pull request: #3776 (a copyData chunk + # changing under whoever kept it), #3777 (binary results losing every byte utf8 cannot + # carry), #3778 (pg-native reporting NaN and '' for a missing row count or command), #3780 + # (native errors without detail and hint), #3781 (a named empty statement failing from its + # second run). And one it cannot fix here: a Buffer parameter reaches libpq as a C string, + # #980, so the native arm draws none. + fuzz: + timeout-minutes: 15 + needs: lint + services: + postgres: + image: ghcr.io/railwayapp-templates/postgres-ssl:18 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: 'md5' + POSTGRES_DB: ci_db_test + PGDATA: /var/lib/postgresql/data + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + runs-on: ubuntu-latest + env: + PGUSER: postgres + PGPASSWORD: postgres + PGHOST: localhost + PGDATABASE: ci_db_test + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: 26 + cache: yarn + - run: yarn install --frozen-lockfile + - run: yarn build + - name: Fuzz the protocol parser against the bytes it was given, cut at random points + run: node fuzz/wire.js --rounds 500 + - name: Fuzz pg against itself in every query mode + run: node fuzz/modes.js --rounds 100 + - name: Fuzz pg against pg-native + run: node fuzz/native.js --rounds 100 diff --git a/fuzz/README.md b/fuzz/README.md new file mode 100644 index 000000000..2c8e5b142 --- /dev/null +++ b/fuzz/README.md @@ -0,0 +1,26 @@ +# Fuzzing + +Three differential fuzzers. Each draws random cases from a seed, runs them two ways that must +agree, and on a divergence prints the seed that reproduces it and the case shrunk to the lines +worth pasting into a test. The CI runs a few hundred rounds of each on a seed nobody chose, on +every push and pull request. + +| Tool | Arms | Needs a server | +| --- | --- | --- | +| `node fuzz/wire.js` | the pg-protocol parser on random backend messages, in one buffer and cut at random points, against what was written | no | +| `node fuzz/modes.js` | `client.query` against the extended protocol forced, a named statement run twice, rowMode array, the binary result format, pipeline mode, pg-cursor and pg-query-stream | yes, `PG*` variables | +| `node fuzz/native.js` | pg against pg-native, as a plain query, a named statement and rowMode array | yes, and pg-native built | + +```bash +node fuzz/modes.js --rounds 200 # longer +node fuzz/modes.js --seed 12345 --rounds 1 # replay what a past run printed +node fuzz/modes.js --keep-going # do not stop at the first divergence +node fuzz/modes.js --no-shrink # print the round as drawn +``` + +The queries come from `queries.js`: selects over `generate_series` with a column per type family, +sometimes as a parameter instead of a literal, writes on a temp table, and statements that fail +on purpose, some of them only after rows were sent. What the binary arm can compare is limited to +the types pg-types has a binary parser for, the `binary` flag of each type says which. The native +arm draws no Buffer parameter: pg-native hands it to libpq as a C string, cut at its first zero +byte (#980), and the fix for that is in node-libpq. diff --git a/fuzz/lib.js b/fuzz/lib.js new file mode 100644 index 000000000..477ef161a --- /dev/null +++ b/fuzz/lib.js @@ -0,0 +1,131 @@ +'use strict' + +// What the three fuzzers share: a seeded generator, so a failure prints the seed that reproduces +// it; the loop over rounds; and the shrinking, which drops the parts of a failing case one at a +// time for as long as the failure survives, so what gets printed is the few lines worth pasting +// into a test rather than the whole round. + +/** @param {number} seed @returns {() => number} the same sequence for the same seed */ +const mulberry32 = (seed) => { + let a = seed >>> 0 + return () => { + a = (a + 0x6d2b79f5) >>> 0 + let t = Math.imul(a ^ (a >>> 15), 1 | a) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +const int = (rng, min, max) => min + Math.floor(rng() * (max - min + 1)) +const pick = (rng, items) => items[Math.floor(rng() * items.length)] +const chance = (rng, p) => rng() < p + +// strings a server can send: no NUL, since the protocol delimits them with one, and a spread +// from empty to long with multibyte and awkward characters in between +const ALPHABET = ['a', 'Z', '0', ' ', '_', "'", '"', '\\', '{', '}', ',', 'é', '€', '😀', '\n', '\t', 'ÿ'] +const string = (rng, max = 12) => { + const length = chance(rng, 0.1) ? 0 : chance(rng, 0.05) ? int(rng, 100, max * 40) : int(rng, 1, max) + let out = '' + for (let i = 0; i < length; i++) out += chance(rng, 0.7) ? pick(rng, ALPHABET.slice(0, 3)) : pick(rng, ALPHABET) + return out +} + +const bytes = (rng, max = 16) => { + const length = chance(rng, 0.1) ? 0 : int(rng, 1, max) + const out = Buffer.alloc(length) + for (let i = 0; i < length; i++) out[i] = chance(rng, 0.2) ? 0 : int(rng, 0, 255) + return out +} + +// one canonical text for a value, so two results compare as strings: a Buffer by its bytes, a +// Date by its instant, and the rest by JSON with the keys in insertion order +const canon = (value) => { + const replacer = (_, v) => { + if (Buffer.isBuffer(v)) return `<${v.toString('hex')}>` + if (v && v.type === 'Buffer' && Array.isArray(v.data)) return `<${Buffer.from(v.data).toString('hex')}>` + if (typeof v === 'bigint') return `${v}n` + if (v === undefined) return '' + if (typeof v === 'number' && !Number.isFinite(v)) return `<${v}>` + return v + } + return JSON.stringify(value instanceof Date ? value.toISOString() : value, replacer) +} + +const parseArgs = (argv) => { + const flag = (name, fallback) => { + const at = argv.indexOf(`--${name}`) + return at === -1 ? fallback : Number(argv[at + 1]) + } + return { + rounds: flag('rounds', 100), + seed: flag('seed', (Date.now() ^ (process.pid << 16)) >>> 0), + keepGoing: argv.includes('--keep-going'), + noShrink: argv.includes('--no-shrink'), + } +} + +/** + * Drops items from a failing case one at a time, keeping every drop the failure survives. + * + * @template T + * @param {T} plan + * @param {(plan: T) => T[]} variants the smaller plans to try, each with one thing removed + * @param {(plan: T) => Promise} fails + */ +const shrink = async (plan, variants, fails) => { + let current = plan + let progress = true + while (progress) { + progress = false + for (const smaller of variants(current)) { + if (await fails(smaller)) { + current = smaller + progress = true + break + } + } + } + return current +} + +/** + * The loop every fuzzer runs: draw a case per round, run it, and on a divergence print the seed, + * shrink the case and print it as source. + * + * @param {object} fuzzer + * @param {string} fuzzer.name + * @param {(rng: () => number) => any} fuzzer.draw + * @param {(plan: any) => Promise} fuzzer.run a description of the divergence, or null + * @param {(plan: any) => any[]} fuzzer.variants + * @param {(plan: any) => string} fuzzer.source + * @param {() => Promise} [fuzzer.close] + */ +const main = async (fuzzer) => { + const args = parseArgs(process.argv.slice(2)) + console.log(`${fuzzer.name}: ${args.rounds} rounds from seed ${args.seed}`) + let found = 0 + for (let round = 0; round < args.rounds; round++) { + const seed = (args.seed + round) >>> 0 + const plan = fuzzer.draw(mulberry32(seed)) + const divergence = await fuzzer.run(plan) + if (!divergence) { + if (round % 25 === 24) console.log(` ${round + 1} rounds, no divergence`) + continue + } + found++ + console.log(`\n=== divergence in round ${round}, seed ${seed} (replay: --seed ${seed} --rounds 1)`) + console.log(divergence) + if (!args.noShrink) { + console.log('\nshrinking...') + const small = await shrink(plan, fuzzer.variants, async (p) => Boolean(await fuzzer.run(p))) + console.log(`\n${fuzzer.source(small)}`) + console.log(await fuzzer.run(small)) + } + if (!args.keepGoing) break + } + if (fuzzer.close) await fuzzer.close() + console.log(`\n${found} divergence${found === 1 ? '' : 's'}`) + process.exit(found ? 1 : 0) +} + +module.exports = { mulberry32, int, pick, chance, string, bytes, canon, shrink, main } diff --git a/fuzz/modes.js b/fuzz/modes.js new file mode 100644 index 000000000..ce20fe50e --- /dev/null +++ b/fuzz/modes.js @@ -0,0 +1,144 @@ +'use strict' + +// pg against itself: the same statements, on one connection per mode, and every mode must +// answer what a plain query on a plain client answered. +// +// The reference arm is `client.query(text, values)`. The others are the paths a program can +// take to the same rows: the extended protocol forced on a query that would have used the +// simple one, a named prepared statement run twice, rowMode array, the binary result format, +// pipeline mode with the whole round in flight at once, pg-cursor reading in batches and +// pg-query-stream. Each keeps its own connection and runs the whole round in order, so the +// temp table and the transaction state are the same on every arm. +// +// node fuzz/modes.js fifty rounds, PG* variables say where the server is +// node fuzz/modes.js --seed 12345 replay what a past run did +// node fuzz/modes.js --keep-going do not stop at the first divergence + +const { createHash } = require('crypto') +const pg = require('../packages/pg') +const Cursor = require('../packages/pg-cursor') +const QueryStream = require('../packages/pg-query-stream') +const { int, mulberry32, main } = require('./lib') +const { draw, render, summarize, QUIET, rowsOnly, source, variants, SETUP } = require('./queries') + +// the outcome of a query, whichever way it went +const outcome = (promise) => + promise.then( + (result) => ({ result }), + (error) => ({ error }) + ) +const settle = (promise) => outcome(promise).then(({ result, error }) => summarize(result, error)) +const settleRows = (promise) => outcome(promise).then(({ result, error }) => rowsOnly(result, error)) + +const name = (text) => `fuzz_${createHash('sha1').update(text).digest('hex')}` + +// each arm runs one query of the round on its own connection and answers the canonical text. +// `rng` is the round's, so a batch size is drawn the same on a replay +const ARMS = { + extended: (client, q) => settle(client.query({ text: q.text, values: q.values, queryMode: 'extended' })), + // twice, so the second run goes through the statement cache; only a select, since running a + // write twice would leave this arm's table different from the others + prepared: async (client, q) => { + const first = await settle(client.query({ text: q.text, values: q.values, name: name(q.text) })) + // and not after a failure either, which would have aborted an open transaction + if (q.kind !== 'select' || first.startsWith('{"error"')) return first + const second = await settle(client.query({ text: q.text, values: q.values, name: name(q.text) })) + return first === second ? first : `first run: ${first}\n second run: ${second}` + }, + array: async (client, q, rng, reference) => { + const got = await settle(client.query({ text: q.text, values: q.values, rowMode: 'array' })) + // the reference rows as arrays, which is the only thing this mode changes + const { result, error } = reference + const expected = summarize(result && { ...result, rows: result.rows.map(Object.values) }, error) + return got === expected ? summarize(result, error) : got + }, + binary: (client, q) => + q.binary + ? settle(client.query({ text: q.text, values: q.values, binary: true })) + : settle(client.query(q.text, q.values)), + cursor: async (client, q, rng) => { + if (q.kind !== 'select') return settleRows(client.query(q.text, q.values)) + const cursor = client.query(new Cursor(q.text, q.values)) + const rows = [] + try { + for (;;) { + const batch = await cursor.read(int(rng, 1, 40)) + if (batch.length === 0) break + rows.push(...batch) + } + await cursor.close() + return rowsOnly({ rows }) + } catch (error) { + return rowsOnly(null, error) + } + }, + stream: async (client, q, rng) => { + if (q.kind !== 'select') return settleRows(client.query(q.text, q.values)) + const rows = [] + try { + const stream = client.query( + new QueryStream(q.text, q.values, { batchSize: int(rng, 1, 40), highWaterMark: int(rng, 1, 40) }) + ) + for await (const row of stream) rows.push(row) + return rowsOnly({ rows }) + } catch (error) { + return rowsOnly(null, error) + } + }, +} + +const clients = {} +const connect = async () => { + clients.reference = new pg.Client() + clients.pipeline = new pg.Client({ pipeline: true }) + for (const arm of Object.keys(ARMS)) clients[arm] = new pg.Client() + for (const client of Object.values(clients)) { + await client.connect() + await client.query(QUIET) + } +} + +// every round starts from the same state on every arm: no transaction open, an empty table +const reset = async (client) => { + await client.query('ROLLBACK').catch(() => {}) + await client.query('DROP TABLE IF EXISTS fuzz_rows') + await client.query(SETUP) +} + +const run = async (plan) => { + if (!clients.reference) await connect() + for (const client of Object.values(clients)) await reset(client) + const rng = mulberry32(plan.queries.length) + const queries = plan.queries.map(render) + const references = [] + for (const q of queries) references.push(await outcome(clients.reference.query(q.text, q.values))) + const full = references.map(({ result, error }) => summarize(result, error)) + + // the whole round in flight at once on the pipelined connection + const pipelined = await Promise.all(queries.map((q) => settle(clients.pipeline.query(q.text, q.values)))) + for (let i = 0; i < queries.length; i++) { + if (pipelined[i] !== full[i]) + return `pipeline, query ${i} + reference: ${full[i]} + pipeline: ${pipelined[i]}` + } + + for (const [arm, runArm] of Object.entries(ARMS)) { + for (let i = 0; i < queries.length; i++) { + const got = await runArm(clients[arm], queries[i], rng, references[i]) + const { result, error } = references[i] + const expected = arm === 'cursor' || arm === 'stream' ? rowsOnly(result, error) : full[i] + if (got !== expected) + return `${arm}, query ${i} + reference: ${expected} + ${arm}: ${got}` + } + } + return null +} + +const close = async () => { + for (const client of Object.values(clients)) await client.end() +} + +main({ name: 'modes', draw, run, variants, source, close }) diff --git a/fuzz/native.js b/fuzz/native.js new file mode 100644 index 000000000..cbb1649d5 --- /dev/null +++ b/fuzz/native.js @@ -0,0 +1,80 @@ +'use strict' + +// pg against pg-native: the same statements on the pure javascript client and on the libpq one, +// which promise the same API, and every result and every error compared. +// +// Three arms on each side: a plain query, a named prepared statement and rowMode array. Each +// keeps its own connection and runs the whole round in order, so the temp table and the +// transaction state are the same everywhere. Needs pg-native built, which is what the CI has. +// +// node fuzz/native.js fifty rounds, PG* variables say where the server is +// node fuzz/native.js --seed 12345 replay what a past run did +// node fuzz/native.js --keep-going do not stop at the first divergence + +const { createHash } = require('crypto') +const pg = require('../packages/pg') +const { main } = require('./lib') +const { draw, render, summarize, QUIET, source, variants, SETUP } = require('./queries') + +if (!pg.native) { + console.error('pg-native is not available, install libpq and rebuild') + process.exit(1) +} + +const settle = (promise) => + promise.then( + (result) => summarize(result), + (error) => summarize(null, error) + ) + +const name = (text) => `fuzz_${createHash('sha1').update(text).digest('hex')}` + +// how one query is run on a client, for each of the three ways the two clients share +const ARMS = { + query: (client, q) => settle(client.query(q.text, q.values)), + prepared: (client, q) => settle(client.query({ text: q.text, values: q.values, name: name(q.text) })), + array: (client, q) => settle(client.query({ text: q.text, values: q.values, rowMode: 'array' })), +} + +const clients = {} +const connect = async () => { + for (const arm of Object.keys(ARMS)) { + clients[arm] = { js: new pg.Client(), native: new pg.native.Client() } + await clients[arm].js.connect() + await clients[arm].native.connect() + await clients[arm].js.query(QUIET) + await clients[arm].native.query(QUIET) + } +} + +// every round starts from the same state on every connection: no transaction open, an empty table +const reset = async (client) => { + await client.query('ROLLBACK').catch(() => {}) + await client.query('DROP TABLE IF EXISTS fuzz_rows') + await client.query(SETUP) +} + +const run = async (plan) => { + if (!clients.query) await connect() + const queries = plan.queries.map(render) + for (const [arm, runArm] of Object.entries(ARMS)) { + const { js, native } = clients[arm] + await reset(js) + await reset(native) + for (let i = 0; i < queries.length; i++) { + const expected = await runArm(js, queries[i]) + const got = await runArm(native, queries[i]) + if (got !== expected) return `${arm}, query ${i}\n pg: ${expected}\n pg-native: ${got}` + } + } + return null +} + +const close = async () => { + for (const { js, native } of Object.values(clients)) { + await js.end() + await native.end() + } +} + +main({ name: 'native', draw: (rng) => draw(rng, { buffers: false }), run, variants, source, close }) diff --git a/fuzz/queries.js b/fuzz/queries.js new file mode 100644 index 000000000..52ed46247 --- /dev/null +++ b/fuzz/queries.js @@ -0,0 +1,341 @@ +'use strict' + +// Random queries for the two fuzzers that need a server: what to send, and one canonical text +// for what came back, so two arms compare as strings. +// +// A round is a short sequence of statements on one connection: selects over generate_series +// with a column per type family, sometimes as a parameter instead of a literal, and writes on a +// temp table so the command tags and row counts of INSERT, UPDATE and DELETE are covered too. +// Some statements fail on purpose, a few of them only after rows were already sent. + +const { int, pick, chance, string, bytes, canon } = require('./lib') + +const quote = (s) => `'${s.replace(/'/g, "''")}'` +// how pg writes an array parameter, so a literal and a parameter carry the same value +const arrayLiteral = (items) => + `{${items + .map((v) => + v === null ? 'NULL' : Array.isArray(v) ? arrayLiteral(v) : `"${String(v).replace(/(["\\])/g, '\\$1')}"` + ) + .join(',')}}` + +const ints = (rng, bits) => { + const max = 2 ** (bits - 1) - 1 + return pick(rng, [0, 1, -1, 42, max, -max - 1, int(rng, -1000, 1000), int(rng, -max, max)]) +} +const floats = (rng) => + pick(rng, [0, -0, 1.5, -2.25, 1e-7, 123456.789, 1e300, -1e-300, rng() * 1000, 'NaN', 'Infinity', '-Infinity']) +const numerics = (rng) => + pick(rng, [ + '0', + '-0.00', + '1.5', + '123456789012345678901234567890.123456789', + '-0.000001', + 'NaN', + String(int(rng, -99999, 99999)), + ]) +const texts = (rng) => string(rng, 20) +const dates = (rng) => + pick(rng, ['2024-02-29', '1970-01-01', '0001-01-01', '9999-12-31', '2000-01-01', 'infinity', '-infinity']) +const stamps = (rng) => + pick(rng, [ + '2024-02-29 12:34:56.789', + '1969-12-31 23:59:59.999999', + '2000-01-01 00:00:00', + '1900-06-15 01:02:03.5', + 'infinity', + ]) +const jsons = (rng) => + pick(rng, ['{}', '[]', 'null', '1', '"x"', '{"a":[1,2,{"b":null}],"c":"é"}', '[1.5,true,"\\u00e9"]', '{"k":"a\\"b"}']) +const uuid = () => '123e4567-e89b-12d3-a456-426614174000' +// flat most of the time, sometimes two levels of the same width, which is what an array type takes +const arrayOf = (rng, item) => { + const n = chance(rng, 0.15) ? 0 : int(rng, 1, 5) + const one = () => Array.from({ length: n }, () => (chance(rng, 0.15) ? null : item(rng))) + return chance(rng, 0.15) && n > 0 ? [one(), one()] : one() +} +const hex = (rng) => `\\x${bytes(rng, 8).toString('hex')}` + +// one entry per type family. `literal` is a SQL expression, `param` a JS value pg sends for it, +// `binary` says whether the binary result format gives the same JS value as the text one, which +// is only true for the types pg-types has a binary parser for, or decodes as utf8 anyway +const TYPES = [ + { type: 'int2', binary: true, literal: (rng) => `${ints(rng, 16)}::int2`, param: (rng) => ints(rng, 16) }, + { type: 'int4', binary: true, literal: (rng) => `${ints(rng, 32)}::int4`, param: (rng) => ints(rng, 32) }, + { type: 'int8', binary: true, literal: (rng) => `${ints(rng, 53)}::int8`, param: (rng) => String(ints(rng, 53)) }, + { + type: 'float8', + binary: true, + literal: (rng) => `${quote(String(floats(rng)))}::float8`, + param: (rng) => floats(rng), + }, + { + type: 'float4', + binary: false, + literal: (rng) => `${quote(String(floats(rng)))}::float4`, + param: (rng) => floats(rng), + }, + { + type: 'numeric', + binary: false, + literal: (rng) => `${quote(numerics(rng))}::numeric`, + param: (rng) => numerics(rng), + }, + { type: 'bool', binary: true, literal: (rng) => pick(rng, ['true', 'false']), param: (rng) => chance(rng, 0.5) }, + { type: 'text', binary: true, literal: (rng) => `${quote(texts(rng))}::text`, param: (rng) => texts(rng) }, + { type: 'varchar', binary: true, literal: (rng) => `${quote(texts(rng))}::varchar(30)`, param: (rng) => texts(rng) }, + { + type: 'char', + binary: true, + literal: (rng) => `${quote(texts(rng).slice(0, 4))}::char(6)`, + param: (rng) => texts(rng).slice(0, 4), + }, + { type: 'name', binary: true, literal: (rng) => `${quote(texts(rng))}::name`, param: (rng) => texts(rng) }, + // pg-native gets a Buffer parameter as a C string, cut at its first zero byte (#980), so the + // arm that compares against it draws no Buffer parameter + { + type: 'bytea', + binary: false, + literal: (rng) => `${quote(hex(rng))}::bytea`, + param: (rng) => bytes(rng, 8), + buffer: true, + }, + { type: 'date', binary: false, literal: (rng) => `${quote(dates(rng))}::date`, param: (rng) => dates(rng) }, + { + type: 'timestamp', + binary: false, + literal: (rng) => `${quote(stamps(rng))}::timestamp`, + param: (rng) => stamps(rng), + }, + { + type: 'timestamptz', + binary: false, + literal: (rng) => `${quote(stamps(rng))}::timestamptz`, + param: (rng) => stamps(rng), + }, + { type: 'time', binary: false, literal: () => `'12:34:56.789'::time`, param: () => '12:34:56.789' }, + { + type: 'interval', + binary: false, + literal: (rng) => + `${quote(pick(rng, ['1 day', '-3 hours 2 minutes', '1 year 2 mons 3 days 04:05:06.789', '0']))}::interval`, + }, + { + type: 'json', + binary: false, + literal: (rng) => `${quote(jsons(rng))}::json`, + param: (rng) => JSON.parse(jsons(rng)), + }, + { + type: 'jsonb', + binary: false, + literal: (rng) => `${quote(jsons(rng))}::jsonb`, + param: (rng) => JSON.parse(jsons(rng)), + }, + { type: 'uuid', binary: false, literal: () => `${quote(uuid())}::uuid`, param: () => uuid() }, + { + type: 'oid', + binary: true, + literal: (rng) => `${int(rng, 0, 2147483647)}::oid`, + param: (rng) => int(rng, 0, 2147483647), + }, + { + type: 'int4[]', + binary: false, + literal: (rng) => `${quote(arrayLiteral(arrayOf(rng, (r) => ints(r, 32))))}::int4[]`, + param: (rng) => arrayOf(rng, (r) => ints(r, 32)), + }, + { + type: 'int8[]', + binary: false, + literal: (rng) => `${quote(arrayLiteral(arrayOf(rng, (r) => ints(r, 53))))}::int8[]`, + param: (rng) => arrayOf(rng, (r) => String(ints(r, 53))), + }, + { + type: 'float8[]', + binary: false, + literal: (rng) => `${quote(arrayLiteral(arrayOf(rng, floats)))}::float8[]`, + param: (rng) => arrayOf(rng, floats), + }, + { + type: 'text[]', + binary: true, + literal: (rng) => `${quote(arrayLiteral(arrayOf(rng, texts)))}::text[]`, + param: (rng) => arrayOf(rng, texts), + }, + { + type: 'bool[]', + binary: false, + literal: (rng) => `${quote(arrayLiteral(arrayOf(rng, (r) => chance(r, 0.5))))}::bool[]`, + param: (rng) => arrayOf(rng, (r) => chance(r, 0.5)), + }, + { + type: 'timestamptz[]', + binary: false, + literal: (rng) => `${quote(arrayLiteral(arrayOf(rng, stamps)))}::timestamptz[]`, + }, + { + type: 'numeric[]', + binary: false, + literal: (rng) => `${quote(arrayLiteral(arrayOf(rng, numerics)))}::numeric[]`, + param: (rng) => arrayOf(rng, numerics), + }, + { type: 'jsonb[]', binary: false, literal: (rng) => `${quote(arrayLiteral(arrayOf(rng, jsons)))}::jsonb[]` }, + { type: 'bytea[]', binary: false, literal: (rng) => `${quote(arrayLiteral(arrayOf(rng, hex)))}::bytea[]` }, + { type: 'uuid[]', binary: false, literal: (rng) => `${quote(arrayLiteral(arrayOf(rng, uuid)))}::uuid[]` }, + { type: 'point', binary: false, literal: (rng) => `point(${int(rng, -10, 10)}, ${rng() * 10})` }, + { + type: 'inet', + binary: false, + literal: (rng) => `${quote(pick(rng, ['127.0.0.1', '10.0.0.0/8', '::1', 'fe80::1/64']))}::inet`, + }, + { + type: 'int4range', + binary: false, + literal: (rng) => `${quote(pick(rng, ['[1,10)', 'empty', '(,)', '[5,5]']))}::int4range`, + }, + { type: 'record', binary: false, literal: (rng) => `ROW(${ints(rng, 32)}, ${quote(texts(rng))}, NULL)` }, + { type: 'unknown', binary: true, literal: (rng) => quote(texts(rng)) }, + { type: 'null', binary: true, literal: () => 'NULL', param: () => null }, + // something that changes with the row, so a result is not one value repeated + { type: 'row', binary: true, literal: () => 'g' }, + { type: 'rowtext', binary: true, literal: () => `'r' || g::text` }, +] + +// the ways a statement can fail: a text nothing can run, an expression that fails before any +// row, and one that fails only once some rows were already sent +const FAILURES = [ + { text: 'SELEC 1' }, + { column: '1/0' }, + { column: `'abc'::int4` }, + { column: 'nosuchcol' }, + { column: 'CASE WHEN g > 3 THEN 1/0 ELSE 1 END' }, +] + +const drawSelect = (rng, { buffers }) => { + const columns = [] + for (let i = int(rng, 1, 8); i > 0; i--) { + const t = pick(rng, TYPES) + columns.push( + t.param && (buffers || !t.buffer) && chance(rng, 0.35) + ? { sql: `::${t.type}`, param: t.param(rng), binary: t.binary } + : { sql: t.literal(rng), binary: t.binary } + ) + } + const failure = chance(rng, 0.15) ? pick(rng, FAILURES) : null + if (failure && failure.column) columns.splice(int(rng, 0, columns.length), 0, { sql: failure.column, binary: true }) + return { + kind: 'select', + columns, + rows: chance(rng, 0.1) ? 0 : chance(rng, 0.1) ? int(rng, 200, 1500) : int(rng, 1, 30), + where: chance(rng, 0.2) ? ' WHERE g % 2 = 1' : '', + order: chance(rng, 0.2) ? ' ORDER BY g DESC' : '', + text: failure && failure.text, + } +} + +// the reset of a round sends ROLLBACK whether a transaction is open or not, and the warning it +// gets otherwise would go to stderr through libpq; nothing here compares notices +const QUIET = 'SET client_min_messages = error' +// the writes go to a temp table every arm creates on its own connection +const SETUP = 'CREATE TEMP TABLE fuzz_rows (id serial PRIMARY KEY, n int4, t text, j jsonb)' +const drawWrite = (rng) => { + const n = int(rng, 0, 20) + const write = pick(rng, [ + { text: `INSERT INTO fuzz_rows (n, t) SELECT g, 'v' || g FROM generate_series(1, ${n}) g` }, + { + text: `INSERT INTO fuzz_rows (n, t, j) VALUES ($1, $2, $3) RETURNING *`, + values: [ints(rng, 32), texts(rng), { k: n }], + }, + { text: `INSERT INTO fuzz_rows (n) SELECT g FROM generate_series(1, ${n}) g RETURNING id, n` }, + { text: `UPDATE fuzz_rows SET n = n + $1 WHERE n > $2`, values: [1, int(rng, -5, 5)] }, + { text: `UPDATE fuzz_rows SET t = upper(t) WHERE id % 3 = 0 RETURNING id, t` }, + { text: `DELETE FROM fuzz_rows WHERE n < $1`, values: [int(rng, 0, 10)] }, + { text: `DELETE FROM fuzz_rows WHERE id % 2 = 0 RETURNING id` }, + { text: `SELECT count(*)::int AS c, sum(n)::int AS s, array_agg(t ORDER BY id) AS ts FROM fuzz_rows` }, + { text: 'BEGIN' }, + { text: 'COMMIT' }, + { text: 'ROLLBACK' }, + { text: pick(rng, ['', ' ', ';', '-- nothing']) }, + // a duplicate key once a row with id 1 exists + { text: `INSERT INTO fuzz_rows (id) VALUES (1)` }, + ]) + return { kind: 'write', text: write.text, values: write.values || [], binary: false } +} + +// `buffers: false` leaves Buffer parameters out, see the bytea entry above +const draw = (rng, { buffers = true } = {}) => ({ + queries: Array.from({ length: int(rng, 1, 6) }, () => + chance(rng, 0.7) ? drawSelect(rng, { buffers }) : drawWrite(rng) + ), +}) + +// the text and values of a query, numbering the parameters in the order the columns have now, +// which is what lets shrinking drop a column without leaving a gap in the $n +const render = (query) => { + if (query.kind === 'write') return { kind: 'write', text: query.text, values: query.values, binary: false } + const values = [] + const names = query.columns.map((c, i) => { + if ('param' in c) { + values.push(c.param) + return `$${values.length}${c.sql} AS c${i}` + } + return `${c.sql} AS c${i}` + }) + return { + kind: 'select', + text: + query.text || + `SELECT ${names.join(', ')} FROM generate_series(1, ${query.rows}) AS g${query.where}${query.order}`, + values, + binary: !query.text && query.columns.every((c) => c.binary), + } +} + +// one text for a result, or for an error, so arms compare by string equality. Fields keep only +// what every arm reports the same way, the format is the arm's own choice +const summarize = (result, error) => { + if (error) { + return canon({ + error: error.code, + message: error.message, + position: error.position, + severity: error.severity, + detail: error.detail, + hint: error.hint, + }) + } + return canon({ + command: result.command, + rowCount: result.rowCount, + fields: (result.fields || []).map((f) => ({ name: f.name, dataTypeID: f.dataTypeID })), + rows: result.rows, + }) +} + +const rowsOnly = (result, error) => (error ? summarize(result, error) : canon({ rows: result.rows })) + +const source = (plan) => + plan.queries + .map((q) => { + const { text, values } = render(q) + return `await client.query(${JSON.stringify(text)}${values.length ? `, ${canon(values)}` : ''})` + }) + .join('\n') + +// a smaller plan for each query that can be dropped, then for each column of a select +const variants = (plan) => { + const out = [] + for (let i = 0; i < plan.queries.length; i++) out.push({ queries: plan.queries.filter((_, k) => k !== i) }) + plan.queries.forEach((q, i) => { + if (q.kind !== 'select' || q.columns.length < 2) return + for (let k = 0; k < q.columns.length; k++) { + const smaller = { ...q, columns: q.columns.filter((_, j) => j !== k) } + out.push({ queries: plan.queries.map((other, j) => (j === i ? smaller : other)) }) + } + }) + return out +} + +module.exports = { draw, render, summarize, rowsOnly, source, variants, SETUP, QUIET } diff --git a/fuzz/wire.js b/fuzz/wire.js new file mode 100644 index 000000000..23a94339c --- /dev/null +++ b/fuzz/wire.js @@ -0,0 +1,310 @@ +'use strict' + +// The pg-protocol parser against the bytes it was given, and against itself. +// +// A round is a random sequence of backend messages, written here byte by byte from the protocol +// description, so what every message should parse to is known before the parser sees it. The +// bytes are then parsed twice: in one buffer, and cut into chunks at random points, the way a +// socket delivers them. Both parses must give the messages that were written. A chunk boundary +// inside a header, inside a field, or one byte before the end is where a parser keeps state, and +// the hand written tests cover three such cuts. +// +// node fuzz/wire.js a hundred rounds +// node fuzz/wire.js --seed 12345 replay what a past run did +// node fuzz/wire.js --keep-going do not stop at the first divergence + +const { Parser } = require('../packages/pg-protocol/dist/parser') +const { int, pick, chance, string, bytes, canon, main } = require('./lib') + +const int32 = (n) => { + const b = Buffer.alloc(4) + b.writeInt32BE(n) + return b +} +const uint32 = (n) => { + const b = Buffer.alloc(4) + b.writeUInt32BE(n) + return b +} +const int16 = (n) => { + const b = Buffer.alloc(2) + b.writeInt16BE(n) + return b +} +const cstring = (s) => Buffer.concat([Buffer.from(s, 'utf8'), Buffer.from([0])]) +// one message: the type byte, the length of everything after the type byte, the body +const message = (code, ...parts) => { + const body = Buffer.concat(parts) + return Buffer.concat([Buffer.from(code), int32(body.length + 4), body]) +} + +// the error and notice fields the parser reads, by the type byte the protocol gives them +const ERROR_FIELDS = { + S: 'severity', + C: 'code', + M: 'message', + D: 'detail', + H: 'hint', + P: 'position', + p: 'internalPosition', + q: 'internalQuery', + W: 'where', + s: 'schema', + t: 'table', + c: 'column', + d: 'dataType', + n: 'constraint', + F: 'file', + L: 'line', + R: 'routine', +} + +// each kind draws a message and returns its bytes and what parsing them must give. Only the +// properties listed in `expect` are compared, since the parser adds `length` and the classes +// carry more than the protocol does +const KINDS = { + readyForQuery: (rng) => { + const status = pick(rng, ['I', 'T', 'E']) + return { bytes: message('Z', Buffer.from(status)), expect: { name: 'readyForQuery', status } } + }, + commandComplete: (rng) => { + const text = pick(rng, ['SELECT 1', 'INSERT 0 1', 'UPDATE 0', 'BEGIN', 'COPY 12', string(rng, 40)]) + return { bytes: message('C', cstring(text)), expect: { name: 'commandComplete', text } } + }, + dataRow: (rng) => { + const fields = Array.from({ length: int(rng, 0, 20) }, () => (chance(rng, 0.2) ? null : string(rng, 30))) + const parts = [int16(fields.length)] + for (const field of fields) { + if (field === null) parts.push(int32(-1)) + else { + const buf = Buffer.from(field, 'utf8') + parts.push(int32(buf.length), buf) + } + } + return { bytes: message('D', ...parts), expect: { name: 'dataRow', fields } } + }, + rowDescription: (rng) => { + const fields = Array.from({ length: int(rng, 0, 12) }, () => ({ + name: string(rng), + tableID: chance(rng, 0.3) ? int(rng, 0x80000000, 0xffffffff) : int(rng, 0, 100000), + columnID: int(rng, -1, 3000), + dataTypeID: chance(rng, 0.3) ? int(rng, 0x80000000, 0xffffffff) : pick(rng, [23, 25, 16, 1184, 3802, 114]), + dataTypeSize: pick(rng, [-1, 1, 2, 4, 8, 16]), + dataTypeModifier: pick(rng, [-1, 0, 104, 2147483647]), + format: pick(rng, ['text', 'binary']), + })) + const parts = [int16(fields.length)] + for (const f of fields) { + parts.push( + cstring(f.name), + uint32(f.tableID), + int16(f.columnID), + uint32(f.dataTypeID), + int16(f.dataTypeSize), + int32(f.dataTypeModifier), + int16(f.format === 'text' ? 0 : 1) + ) + } + return { bytes: message('T', ...parts), expect: { name: 'rowDescription', fieldCount: fields.length, fields } } + }, + parameterDescription: (rng) => { + const dataTypeIDs = Array.from({ length: int(rng, 0, 8) }, () => int(rng, 0, 0xffffffff)) + return { + bytes: message('t', int16(dataTypeIDs.length), ...dataTypeIDs.map(uint32)), + expect: { name: 'parameterDescription', parameterCount: dataTypeIDs.length, dataTypeIDs }, + } + }, + parameterStatus: (rng) => { + const parameterName = string(rng) + const parameterValue = string(rng) + return { + bytes: message('S', cstring(parameterName), cstring(parameterValue)), + expect: { name: 'parameterStatus', parameterName, parameterValue }, + } + }, + backendKeyData: (rng) => { + const processID = int(rng, 0, 0x7fffffff) + const secretKey = int(rng, -0x80000000, 0x7fffffff) + return { + bytes: message('K', int32(processID), int32(secretKey)), + expect: { name: 'backendKeyData', processID, secretKey }, + } + }, + notification: (rng) => { + const processId = int(rng, 0, 0x7fffffff) + const channel = string(rng) + const payload = string(rng, 60) + return { + bytes: message('A', int32(processId), cstring(channel), cstring(payload)), + expect: { name: 'notification', processId, channel, payload }, + } + }, + errorOrNotice: (rng) => { + const name = pick(rng, ['error', 'notice']) + const fields = {} + const expect = { name } + for (const [type, prop] of Object.entries(ERROR_FIELDS)) { + if (type === 'M' || chance(rng, 0.3)) { + fields[type] = string(rng, 30) + expect[prop] = fields[type] + } + } + const parts = Object.entries(fields).flatMap(([type, value]) => [Buffer.from(type), cstring(value)]) + return { bytes: message(name === 'error' ? 'E' : 'N', ...parts, Buffer.from([0])), expect } + }, + empty: (rng) => { + const [code, name] = pick(rng, [ + ['1', 'parseComplete'], + ['2', 'bindComplete'], + ['3', 'closeComplete'], + ['n', 'noData'], + ['s', 'portalSuspended'], + ['I', 'emptyQuery'], + ['c', 'copyDone'], + ['W', 'replicationStart'], + ]) + return { bytes: message(code), expect: { name } } + }, + authentication: (rng) => { + const code = pick(rng, [0, 3, 5, 10, 11, 12]) + if (code === 0) return { bytes: message('R', int32(0)), expect: { name: 'authenticationOk' } } + if (code === 3) return { bytes: message('R', int32(3)), expect: { name: 'authenticationCleartextPassword' } } + if (code === 5) { + const salt = bytes(rng, 4) + if (salt.length !== 4) return { bytes: message('R', int32(0)), expect: { name: 'authenticationOk' } } + return { bytes: message('R', int32(5), salt), expect: { name: 'authenticationMD5Password', salt } } + } + if (code === 10) { + const mechanisms = Array.from({ length: int(rng, 1, 3) }, () => + pick(rng, ['SCRAM-SHA-256', 'SCRAM-SHA-256-PLUS']) + ) + return { + bytes: message('R', int32(10), ...mechanisms.map(cstring), Buffer.from([0])), + expect: { name: 'authenticationSASL', mechanisms }, + } + } + const data = string(rng, 40) + return { + bytes: message('R', int32(code), Buffer.from(data, 'utf8')), + expect: { name: code === 11 ? 'authenticationSASLContinue' : 'authenticationSASLFinal', data }, + } + }, + copyResponse: (rng) => { + const name = pick(rng, ['copyInResponse', 'copyOutResponse']) + const binary = chance(rng, 0.5) + const columnTypes = Array.from({ length: int(rng, 0, 6) }, () => (binary ? 1 : 0)) + return { + bytes: message( + name === 'copyInResponse' ? 'G' : 'H', + Buffer.from([binary ? 1 : 0]), + int16(columnTypes.length), + ...columnTypes.map(int16) + ), + expect: { name, binary, columnTypes }, + } + }, + copyData: (rng) => { + const chunk = bytes(rng, 200) + return { bytes: message('d', chunk), expect: { name: 'copyData', chunk } } + }, + // a type byte the parser does not know: it must answer an error and go on with the next message + unknown: (rng) => { + const code = pick(rng, ['X', 'Q', 'x', '?']) + return { + bytes: message(code, bytes(rng, 8)), + expect: { name: 'error', message: `received invalid response: ${code.charCodeAt(0).toString(16)}` }, + } + }, +} +const NAMES = Object.keys(KINDS) + +// how the bytes are cut: at random points, and with a bias to the first bytes of a message, where +// the header is, and to cuts one byte apart +const cuts = (rng, total) => { + const points = new Set() + const count = int(rng, 0, Math.min(total, 40)) + for (let i = 0; i < count; i++) { + const at = chance(rng, 0.3) ? int(rng, 1, Math.min(total - 1, 6)) : int(rng, 1, total - 1) + points.add(at) + if (chance(rng, 0.3) && at + 1 < total) points.add(at + 1) + } + return [...points].sort((a, b) => a - b) +} + +const draw = (rng) => { + const messages = Array.from({ length: int(rng, 1, 30) }, () => KINDS[pick(rng, NAMES)](rng)) + const total = Buffer.concat(messages.map((m) => m.bytes)).length + return { messages, cuts: total > 1 ? cuts(rng, total) : [] } +} + +// what the parser gave, with only the properties the protocol promised, so it compares against +// the expectation and between the two parses +const observed = (parsed, expect) => { + const out = {} + for (const key of Object.keys(expect)) out[key] = parsed ? parsed[key] : undefined + return out +} + +// what each message must parse to, in the sequence it is in: a column the last row description +// declared binary comes back as its bytes, the others as text +const expectations = (messages) => { + let binaryColumns = [] + return messages.map(({ expect }) => { + if (expect.name === 'rowDescription') binaryColumns = expect.fields.map((f) => f.format === 'binary') + if (expect.name !== 'dataRow') return expect + const fields = expect.fields.map((field, i) => + field !== null && binaryColumns[i] ? Buffer.from(field, 'utf8') : field + ) + return { ...expect, fields } + }) +} + +const parseAll = (chunks, expects) => { + const parser = new Parser() + const out = [] + for (const chunk of chunks) parser.parse(chunk, (msg) => out.push(msg)) + return out.map((msg, i) => observed(msg, expects[i] || { name: true })) +} + +const run = async (plan) => { + const whole = Buffer.concat(plan.messages.map((m) => m.bytes)) + const expects = expectations(plan.messages) + const chunks = [] + let from = 0 + for (const at of [...plan.cuts, whole.length]) { + // a copy per chunk, as a socket gives one: the parser may keep it + chunks.push(Buffer.from(whole.subarray(from, at))) + from = at + } + const arms = { whole: parseAll([whole], expects), chunked: parseAll(chunks, expects) } + for (const [arm, got] of Object.entries(arms)) { + if (got.length !== expects.length) { + return `${arm}: ${got.length} messages parsed, ${expects.length} written\n got: ${canon(got.map((m) => m.name))}` + } + for (let i = 0; i < expects.length; i++) { + if (canon(got[i]) !== canon(expects[i])) { + return `${arm}: message ${i} (${expects[i].name})\n written: ${canon(expects[i])}\n parsed: ${canon(got[i])}` + } + } + } + return null +} + +const variants = (plan) => { + const out = [] + for (let i = 0; i < plan.messages.length; i++) { + const messages = plan.messages.filter((_, k) => k !== i) + const total = Buffer.concat(messages.map((m) => m.bytes)).length + out.push({ messages, cuts: plan.cuts.filter((at) => at < total) }) + } + for (let i = 0; i < plan.cuts.length; i++) + out.push({ messages: plan.messages, cuts: plan.cuts.filter((_, k) => k !== i) }) + return out +} + +const source = (plan) => { + const lines = plan.messages.map((m) => ` ${canon(m.expect)} // ${m.bytes.toString('hex')}`) + return `messages:\n${lines.join('\n')}\ncut at bytes: ${plan.cuts.join(', ') || 'none'}` +} + +main({ name: 'wire', draw, run, variants, source }) diff --git a/package.json b/package.json index e30454007..23a97faaf 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,10 @@ "docs:start": "cd docs && yarn dev", "pretest": "yarn build", "prepublish": "yarn build", - "lint": "eslint --cache 'packages/**/*.{js,ts,tsx}'" + "lint": "eslint --cache 'packages/**/*.{js,ts,tsx}' 'fuzz/*.js'", + "fuzz:wire": "node fuzz/wire.js", + "fuzz:modes": "node fuzz/modes.js", + "fuzz:native": "node fuzz/native.js" }, "devDependencies": { "@eslint/eslintrc": "^3.3.5", diff --git a/packages/pg-native/lib/build-result.js b/packages/pg-native/lib/build-result.js index 9117a11ef..8c5d02010 100644 --- a/packages/pg-native/lib/build-result.js +++ b/packages/pg-native/lib/build-result.js @@ -14,8 +14,11 @@ class Result { } consumeCommand(pq) { - this.command = pq.cmdStatus().split(' ')[0] - this.rowCount = parseInt(pq.cmdTuples(), 10) + // null when there is none, as pg reports it: BEGIN has no row count, an empty query no command + const status = pq.cmdStatus() + this.command = status ? status.split(' ')[0] : null + const tuples = pq.cmdTuples() + this.rowCount = tuples ? parseInt(tuples, 10) : null } consumeFields(pq) { diff --git a/packages/pg-native/test/empty-query.js b/packages/pg-native/test/empty-query.js index aa3f05a0d..9fd73c64d 100644 --- a/packages/pg-native/test/empty-query.js +++ b/packages/pg-native/test/empty-query.js @@ -13,4 +13,22 @@ describe('empty query', () => { client.end(done) }) }) + + // what pg reports too: a command without a row count and an empty query without a command + // are null, not NaN and an empty string + it('reports no row count and no command as null', (done) => { + const client = new Client() + client.connectSync() + client.query('BEGIN', (err, rows, res) => { + assert(!err) + assert.strictEqual(res.command, 'BEGIN') + assert.strictEqual(res.rowCount, null) + client.query('', (err, rows, res) => { + assert(!err) + assert.strictEqual(res.command, null) + assert.strictEqual(res.rowCount, null) + client.end(done) + }) + }) + }) }) diff --git a/packages/pg-protocol/src/buffer-reader.ts b/packages/pg-protocol/src/buffer-reader.ts index 42a4a23fa..9bfbadb11 100644 --- a/packages/pg-protocol/src/buffer-reader.ts +++ b/packages/pg-protocol/src/buffer-reader.ts @@ -51,7 +51,9 @@ export class BufferReader { } public bytes(length: number): Buffer { - const result = this.buffer.slice(this.offset, this.offset + length) + // a copy, not a view: the parser reuses its buffer for the next chunk, and a view would + // change under a message that was already delivered + const result = Buffer.from(this.buffer.subarray(this.offset, this.offset + length)) this.offset += length return result } diff --git a/packages/pg-protocol/src/inbound-parser.test.ts b/packages/pg-protocol/src/inbound-parser.test.ts index 8687194c3..4369ac23a 100644 --- a/packages/pg-protocol/src/inbound-parser.test.ts +++ b/packages/pg-protocol/src/inbound-parser.test.ts @@ -577,6 +577,35 @@ describe('PgPacketStream', function () { }) }) + // the row description says which columns are binary, and their values are bytes rather than + // text: decoding them as utf8 would lose every byte it cannot carry + it('keeps the bytes of a binary column', async function () { + const description = buffers.rowDescription([ + { name: 'n', dataTypeID: 23, formatCode: 1 }, + { name: 't', dataTypeID: 25, formatCode: 0 }, + ]) + const row = new BufferList() + .addInt16(2) + .addInt32(4) + .add(Buffer.from([0, 0, 0x03, 0xe8])) + .addInt32(2) + .add(Buffer.from('é', 'utf8')) + .join(true, 'D') + const messages = await parseBuffers([description, row]) + assert.deepStrictEqual((messages[1] as any).fields, [Buffer.from([0, 0, 0x03, 0xe8]), 'é']) + }) + + // the parser moves what is left of a chunk to the front of its buffer before reading the next + // one, so a message that kept a view into that buffer would change after being delivered + it('keeps a copyData chunk intact after the parser reuses its buffer', async function () { + const copyData = buffers.copyData(Buffer.alloc(64, 0xaa)) + const commandComplete = buffers.commandComplete('COPY 1') + const fullBuffer = Buffer.concat([copyData, commandComplete]) + const messages = await parseBuffers([fullBuffer.subarray(0, fullBuffer.length - 1), fullBuffer.subarray(-1)]) + assert.strictEqual(messages.length, 2) + assert.deepEqual((messages[0] as any).chunk, Buffer.alloc(64, 0xaa)) + }) + it('cleans up the reader after handling a packet', function () { const parser = new Parser() parser.parse(oneFieldBuf, () => {}) diff --git a/packages/pg-protocol/src/parser.ts b/packages/pg-protocol/src/parser.ts index df48ca4a1..d97ccd3ba 100644 --- a/packages/pg-protocol/src/parser.ts +++ b/packages/pg-protocol/src/parser.ts @@ -83,6 +83,9 @@ export class Parser { private bufferOffset: number = 0 private reader = new BufferReader() private mode: Mode + // which columns of the rows to come are in the binary format, from the last row description: + // a binary value is bytes, and decoding it as text would lose every byte utf8 cannot carry + private binaryColumns: boolean[] | null = null constructor(opts?: StreamOptions) { if (opts?.mode === 'binary') { @@ -188,7 +191,7 @@ export class Parser { message = emptyQuery break case MessageCodes.DataRow: - message = parseDataRowMessage(reader) + message = parseDataRowMessage(reader, this.binaryColumns) break case MessageCodes.CommandComplete: message = parseCommandCompleteMessage(reader) @@ -216,6 +219,7 @@ export class Parser { break case MessageCodes.RowDescriptionMessage: message = parseRowDescriptionMessage(reader) + this.binaryColumns = binaryColumnsOf(message as RowDescriptionMessage) break case MessageCodes.ParameterDescriptionMessage: message = parseParameterDescriptionMessage(reader) @@ -276,6 +280,12 @@ const parseNotificationMessage = (reader: BufferReader) => { return new NotificationResponseMessage(LATEINIT_LENGTH, processId, channel, payload) } +// null when every column is text, which is nearly always, so the row parser has one check to make +const binaryColumnsOf = (message: RowDescriptionMessage): boolean[] | null => { + const formats = message.fields.map((field) => field.format === 'binary') + return formats.includes(true) ? formats : null +} + const parseRowDescriptionMessage = (reader: BufferReader) => { const fieldCount = reader.int16() const message = new RowDescriptionMessage(LATEINIT_LENGTH, fieldCount) @@ -306,13 +316,13 @@ const parseParameterDescriptionMessage = (reader: BufferReader) => { return message } -const parseDataRowMessage = (reader: BufferReader) => { +const parseDataRowMessage = (reader: BufferReader, binaryColumns: boolean[] | null) => { const fieldCount = reader.int16() const fields: any[] = new Array(fieldCount) for (let i = 0; i < fieldCount; i++) { const len = reader.int32() // a -1 for length means the value of the field is null - fields[i] = len === -1 ? null : reader.string(len) + fields[i] = len === -1 ? null : binaryColumns && binaryColumns[i] ? reader.bytes(len) : reader.string(len) } return new DataRowMessage(LATEINIT_LENGTH, fields) } diff --git a/packages/pg/lib/native/query.js b/packages/pg/lib/native/query.js index 8cb561979..22fc87bf6 100644 --- a/packages/pg/lib/native/query.js +++ b/packages/pg/lib/native/query.js @@ -35,6 +35,8 @@ const errorFieldMap = { sqlState: 'code', statementPosition: 'position', messagePrimary: 'message', + messageDetail: 'detail', + messageHint: 'hint', context: 'where', schemaName: 'schema', tableName: 'table', @@ -136,8 +138,9 @@ NativeQuery.prototype.submit = function (client) { const values = (this.values || []).map(utils.prepareValue) // check if the client has already executed this named query - // if so...just execute it again - skip the planning phase - if (client.namedQueries[this.name]) { + // if so...just execute it again - skip the planning phase. By presence, not truth: a + // named statement with an empty text is prepared all the same + if (client.namedQueries[this.name] !== undefined) { if (this.text && client.namedQueries[this.name] !== this.text) { const err = new Error(`Prepared statements must be unique - '${this.name}' was used for a different statement`) return after(err) diff --git a/packages/pg/lib/query.js b/packages/pg/lib/query.js index 6b9214199..7d88b9d36 100644 --- a/packages/pg/lib/query.js +++ b/packages/pg/lib/query.js @@ -183,7 +183,12 @@ class Query extends EventEmitter { } hasBeenParsed(connection) { - return this.name && (connection.parsedStatements[this.name] || connection.submittedNamedStatements[this.name]) + // by presence, not truth: a named statement with an empty text is parsed all the same + return ( + this.name && + (connection.parsedStatements[this.name] !== undefined || + connection.submittedNamedStatements[this.name] !== undefined) + ) } handlePortalSuspended(connection) { diff --git a/packages/pg/test/integration/client/binary-results-tests.js b/packages/pg/test/integration/client/binary-results-tests.js new file mode 100644 index 000000000..db24e9be9 --- /dev/null +++ b/packages/pg/test/integration/client/binary-results-tests.js @@ -0,0 +1,25 @@ +'use strict' +const helper = require('./test-helper') +const assert = require('assert') +const suite = new helper.Suite() + +const Client = helper.Client + +// a binary value is bytes, and any byte utf8 cannot carry used to be lost between the parser +// and the type parsers: 1000 came back as 1007 +suite.test('binary results keep every byte of a value', async function () { + const client = new Client(helper.config) + await client.connect() + try { + const result = await client.query({ + text: 'SELECT $1::int4 AS n, $2::float8 AS f, $3::text AS t', + values: [1000, -2.5, 'é'], + binary: true, + }) + assert.strictEqual(result.rows[0].n, 1000) + assert.strictEqual(result.rows[0].f, -2.5) + assert.strictEqual(result.rows[0].t, 'é') + } finally { + await client.end() + } +}) diff --git a/packages/pg/test/integration/client/empty-query-tests.js b/packages/pg/test/integration/client/empty-query-tests.js index 61d46512e..069422635 100644 --- a/packages/pg/test/integration/client/empty-query-tests.js +++ b/packages/pg/test/integration/client/empty-query-tests.js @@ -19,3 +19,17 @@ suite.test('callback supported', function (done) { client.end(done) }) }) + +// the name was recorded as parsed with its empty text, and then read as not parsed at all, +// so the second run prepared it again and the server refused the duplicate +suite.test('a named empty statement can run more than once', async function () { + const client = helper.client() + try { + for (let i = 0; i < 2; i++) { + const result = await client.query({ text: '', name: 'empty' }) + assert.empty(result.rows) + } + } finally { + await client.end() + } +}) diff --git a/packages/pg/test/native/native-vs-js-error-tests.js b/packages/pg/test/native/native-vs-js-error-tests.js index d61b0c69d..b594e1708 100644 --- a/packages/pg/test/native/native-vs-js-error-tests.js +++ b/packages/pg/test/native/native-vs-js-error-tests.js @@ -6,16 +6,41 @@ const NativeClient = require('../../lib/native') const client = new Client() const nativeClient = new NativeClient() +// every field of an error the native client reports must be on the javascript one under the +// same name, and with the same value +const compare = (err, nativeErr) => { + for (const key in nativeErr) { + assert.equal(err[key], nativeErr[key], `Expected err.${key} to equal nativeErr.${key}`) + } +} + +const bothFail = (text, cb) => { + client.query(text, (err) => { + nativeClient.query(text, (nativeErr) => { + compare(err, nativeErr) + cb() + }) + }) +} + client.connect() nativeClient.connect((err) => { - client.query('SELECT alsdkfj', (err) => { - client.end() - - nativeClient.query('SELECT lkdasjfasd', (nativeErr) => { - for (const key in nativeErr) { - assert.equal(err[key], nativeErr[key], `Expected err.${key} to equal nativeErr.${key}`) - } - nativeClient.end() + assert(!err) + bothFail('SELECT alsdkfj', () => { + // a duplicate key carries a detail, a misspelt column a hint. A real table rather than a + // temp one, whose schema is named after the connection + const setup = + 'DROP TABLE IF EXISTS native_vs_js_dup; CREATE TABLE native_vs_js_dup (id int PRIMARY KEY); INSERT INTO native_vs_js_dup VALUES (1)' + client.query(setup, (err) => { + assert(!err) + bothFail('INSERT INTO native_vs_js_dup VALUES (1)', () => { + bothFail('SELECT cols FROM (SELECT 1 AS col) t', () => { + client.query('DROP TABLE native_vs_js_dup', () => { + client.end() + nativeClient.end() + }) + }) + }) }) }) })