From 5dffdf8f79fe1544d0a2a896fbffea163d8213bc Mon Sep 17 00:00:00 2001 From: u9g Date: Sun, 6 Sep 2026 10:32:22 -0400 Subject: [PATCH] Add the hash datatype ["hash", { alg, type, body }] writes the value serialized as `body`, hashed with `alg`, as `type`; reading yields the digest. crc32 and crc32c are built in, anything else goes through node's crypto and is a Buffer. A CRC written into a signed type takes its two's complement. The compiled writer needs the size of the body before it can serialize it, which no writer could get at until now: WriteCompiler.callTypeSize and SizeOfCompiler.callTypeWrite generate code with the sibling compiler in the current scope and run it against that compiler's context. SizeOfCompiler now remembers fixed-size natives so hashes into a fixed-width type are sized without hashing. --- ProtoDef | 2 +- doc/compiler.md | 17 +++++++- src/compiler.js | 51 ++++++++++++++++++++++ src/datatypes/compiler-utils.js | 26 +++++++++++ src/datatypes/utils.js | 31 +++++++++++++- src/hash.js | 39 +++++++++++++++++ test/misc.js | 76 +++++++++++++++++++++++++++++++++ 7 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 src/hash.js diff --git a/ProtoDef b/ProtoDef index 8e07785..647e754 160000 --- a/ProtoDef +++ b/ProtoDef @@ -1 +1 @@ -Subproject commit 8e07785e94626882fa3333184bb366e3f6625356 +Subproject commit 647e754b8147b40dd05e947801e00ef3f38f704f diff --git a/doc/compiler.md b/doc/compiler.md index a97a078..ac34893 100644 --- a/doc/compiler.md +++ b/doc/compiler.md @@ -228,4 +228,19 @@ compiledProto.setVariable('noArraySizeCheck', true); // Use it as if it were a normal ProtoDef const buffer = compiledProto.createPacketBuffer('mainType', result) const result = compiledProto.parsePacketBuffer('mainType', buffer) -``` \ No newline at end of file +``` +### Sizing inside a writer, writing inside a sizer + +A parametrizable type is compiled by one compiler at a time, so a writer normally has no way to know how large a nested value will be. When it must serialize part of the value before writing (a checksum of it, for example), `WriteCompiler.callTypeSize(value, type)` returns code computing the size of `value` as `type`, and `SizeOfCompiler.callTypeWrite(value, type, offsetExpr)` returns code writing it into `buffer`. Both resolve field references against the current scope and run against the other compiler's context, so they are only available when the types are compiled through `ProtoDefCompiler`. The `hash` datatype is built on them: + +```javascript +Write: { + hash: ['parametrizable', (compiler, { alg, type, body }) => { + let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n` + code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n` + code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n` + code += 'return ' + compiler.callType('hash', type) + return compiler.wrapCode(code) + }] +} +``` diff --git a/src/compiler.js b/src/compiler.js index 9c8f057..4547361 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -12,6 +12,8 @@ class ProtoDefCompiler { this.readCompiler = new ReadCompiler() this.writeCompiler = new WriteCompiler() this.sizeOfCompiler = new SizeOfCompiler() + this.writeCompiler.sizeOfCompiler = this.sizeOfCompiler + this.sizeOfCompiler.writeCompiler = this.writeCompiler } addTypes (types) { @@ -62,6 +64,9 @@ class CompiledProtodef { this.sizeOfCtx = sizeOfCtx this.writeCtx = writeCtx this.readCtx = readCtx + // Code from callTypeSize / callTypeWrite runs against the other context + writeCtx.sizeOfCtx = sizeOfCtx + sizeOfCtx.writeCtx = writeCtx } read (buffer, cursor, type) { @@ -174,6 +179,24 @@ class Compiler { } } + /** + * Generates code with another compiler inside this compiler's scope, so that + * field references resolve to the same variables, and binds it to that + * compiler's context. Natives are reachable through the context as well. + */ + callTypeIn (other, ctxName, generate) { + if (!other) throw new Error(`${ctxName} is only available when compiling with ProtoDefCompiler`) + const scopeStack = other.scopeStack + other.scopeStack = this.scopeStack + try { + const code = generate(other) + if (!isNaN(code)) return code + return `((ctx, native) => ${code})(ctx.${ctxName}, ctx.${ctxName})` + } finally { + other.scopeStack = scopeStack + } + } + addTypesToCompile (types) { for (const [type, json] of Object.entries(types)) { // Replace native type, otherwise first in wins @@ -259,6 +282,7 @@ class Compiler { // Local variable to provide some context to eval() const native = this.native // eslint-disable-line const { PartialReadError } = require('./utils') // eslint-disable-line + const hashDigest = require('./hash').digest // eslint-disable-line return eval(code)() // eslint-disable-line } } @@ -361,11 +385,20 @@ class WriteCompiler extends Compiler { if (args.length > 0) return '(' + code + `)(${value}, buffer, ${offsetExpr}, ` + args.map(name => this.getField(name)).join(', ') + ')' return '(' + code + `)(${value}, buffer, ${offsetExpr})` } + + /** + * Code computing the size of `value` as `type`, for writers that need to + * serialize part of a value before they can write it + */ + callTypeSize (value, type, args = []) { + return this.callTypeIn(this.sizeOfCompiler, 'sizeOfCtx', compiler => compiler.callType(value, type, args)) + } } class SizeOfCompiler extends Compiler { constructor () { super() + this.constants = {} this.addTypes(conditionalDatatypes.SizeOf) this.addTypes(structuresDatatypes.SizeOf) @@ -390,12 +423,22 @@ class SizeOfCompiler extends Compiler { this.primitiveTypes[type] = `native.${type}` if (!isNaN(fn)) { this.native[type] = (value) => { return fn } + this.constants[type] = fn } else { this.native[type] = fn } this.types[type] = 'native' } + /** + * The size of `type` when it doesn't depend on the value, following + * aliases down to a fixed-size native; undefined otherwise + */ + constantSize (type) { + while (typeof type === 'string' && typeof this.types[type] === 'string' && this.types[type] !== 'native') type = this.types[type] + return this.constants[type] + } + compileType (type) { if (type instanceof Array) { if (this.parameterizableTypes[type[0]]) { return this.parameterizableTypes[type[0]](this, type[1]) } @@ -429,6 +472,14 @@ class SizeOfCompiler extends Compiler { if (args.length > 0) return '(' + code + `)(${value}, ` + args.map(name => this.getField(name)).join(', ') + ')' return '(' + code + `)(${value})` } + + /** + * Code writing `value` as `type` into `buffer` at `offsetExpr`, for sizers + * whose result depends on the serialized form of a value + */ + callTypeWrite (value, type, offsetExpr = 'offset', args = []) { + return this.callTypeIn(this.writeCompiler, 'writeCtx', compiler => compiler.callType(value, type, offsetExpr, args)) + } } module.exports = { diff --git a/src/datatypes/compiler-utils.js b/src/datatypes/compiler-utils.js index d60eda5..e528e90 100644 --- a/src/datatypes/compiler-utils.js +++ b/src/datatypes/compiler-utils.js @@ -83,6 +83,9 @@ return { value, size } let code = 'const { value, size } = ' + compiler.callType(mapper.type) + '\n' code += 'return { value: ' + JSON.stringify(sanitizeMappings(mapper.mappings)) + '[value] || value, size }' return compiler.wrapCode(code) + }], + hash: ['parametrizable', (compiler, { type }) => { + return compiler.wrapCode('return ' + compiler.callType(type)) }] }, @@ -163,6 +166,18 @@ return (ctx.${type})(val, buffer, offset) code += 'if (mapped === undefined) throw new Error(value + \' is not in the mappings value\')\n' code += 'return ' + compiler.callType('mapped', mapper.type) return compiler.wrapCode(code) + }], + hash: ['parametrizable', (compiler, { alg, type, body }) => { + let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n` + code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n` + code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n` + code += 'try {\n' + code += ' return ' + compiler.callType('hash', type) + '\n' + code += '} catch (e) {\n' + code += ' if (!(e instanceof RangeError) || typeof hash !== "number") throw e\n' + code += ' return ' + compiler.callType('hash | 0', type) + '\n' + code += '}' + return compiler.wrapCode(code) }] }, @@ -217,6 +232,17 @@ return (ctx.${type})(val) code += 'if (mapped === undefined) throw new Error(value + \' is not in the mappings value\')\n' code += 'return ' + compiler.callType('mapped', mapper.type) return compiler.wrapCode(code) + }], + hash: ['parametrizable', (compiler, { alg, type, body }) => { + const constant = compiler.constantSize(type) + if (constant !== undefined) return String(constant) + const size = compiler.callType('hash', type) + if (!isNaN(size)) return size + let code = `const bodyBuffer = Buffer.alloc(${compiler.callType('value', body)})\n` + code += `;((buffer) => ${compiler.callTypeWrite('value', body, '0')})(bodyBuffer)\n` + code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n` + code += 'return ' + size + return compiler.wrapCode(code) }] } } diff --git a/src/datatypes/utils.js b/src/datatypes/utils.js index 2f1722b..ff380a2 100644 --- a/src/datatypes/utils.js +++ b/src/datatypes/utils.js @@ -1,4 +1,5 @@ -const { getCount, sendCount, calcCount, PartialReadError } = require('../utils') +const { getCount, sendCount, calcCount, getFieldInfo, PartialReadError } = require('../utils') +const { digest } = require('../hash') module.exports = { bool: [readBool, writeBool, 1, require('../../ProtoDef/schemas/utils.json').bool], @@ -9,6 +10,7 @@ module.exports = { bitflags: [readBitflags, writeBitflags, sizeOfBitflags, require('../../ProtoDef/schemas/utils.json').bitflags], cstring: [readCString, writeCString, sizeOfCString, require('../../ProtoDef/schemas/utils.json').cstring], mapper: [readMapper, writeMapper, sizeOfMapper, require('../../ProtoDef/schemas/utils.json').mapper], + hash: [readHash, writeHash, sizeOfHash, require('../../ProtoDef/schemas/utils.json').hash], ...require('./varint') } @@ -280,3 +282,30 @@ function sizeOfBitflags (value, { type, flags, shift, big }, rootNode) { } return this.sizeOf(mappedValue, type, rootNode) } + +function readHash (buffer, offset, { type }, rootNode) { + return this.read(buffer, offset, type, rootNode) +} + +function hashOf (value, { alg, body }, rootNode) { + const bodyBuffer = Buffer.alloc(this.sizeOf(value, body, rootNode)) + this.write(value, bodyBuffer, 0, body, rootNode) + return digest(alg, bodyBuffer) +} + +// A CRC is unsigned; a signed `type` takes its two's complement. +function writeHash (value, buffer, offset, typeArgs, rootNode) { + const hash = hashOf.call(this, value, typeArgs, rootNode) + try { + return this.write(hash, buffer, offset, typeArgs.type, rootNode) + } catch (e) { + if (!(e instanceof RangeError) || typeof hash !== 'number') throw e + return this.write(hash | 0, buffer, offset, typeArgs.type, rootNode) + } +} + +function sizeOfHash (value, typeArgs, rootNode) { + const functions = this.types[getFieldInfo(typeArgs.type).type] + if (functions && typeof functions[2] === 'number') return functions[2] + return this.sizeOf(hashOf.call(this, value, typeArgs, rootNode), typeArgs.type, rootNode) +} diff --git a/src/hash.js b/src/hash.js new file mode 100644 index 0000000..a1023d5 --- /dev/null +++ b/src/hash.js @@ -0,0 +1,39 @@ +const crypto = require('crypto') + +// Reflected table-driven CRC with all-ones init and final xor; `poly` is the +// reversed polynomial. +const tables = {} +function table (poly) { + if (!tables[poly]) { + const t = new Int32Array(256) + for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? poly ^ (c >>> 1) : c >>> 1 + t[n] = c + } + tables[poly] = t + } + return tables[poly] +} + +function crc (poly, buffer) { + const t = table(poly) + let c = -1 + for (let i = 0; i < buffer.length; i++) c = t[(c ^ buffer[i]) & 0xff] ^ (c >>> 8) + return (c ^ -1) >>> 0 +} + +const algorithms = { + crc32: buffer => crc(0xEDB88320, buffer), + crc32c: buffer => crc(0x82F63B78, buffer) +} + +// CRC digests are unsigned integers; every other algorithm is delegated to +// node's crypto and yields a Buffer. +function digest (alg, buffer) { + const algorithm = algorithms[alg] + if (algorithm) return algorithm(buffer) + return crypto.createHash(alg).update(buffer).digest() +} + +module.exports = { digest, algorithms } diff --git a/test/misc.js b/test/misc.js index dab6359..d4a7829 100644 --- a/test/misc.js +++ b/test/misc.js @@ -25,3 +25,79 @@ describe('mapper', () => { }) } }) + +describe('hash', () => { + const { digest } = require('../src/hash') + const types = { + crc32: ['hash', { alg: 'crc32', type: 'u32', body: ['buffer', { count: 9 }] }], + crc32c: ['hash', { alg: 'crc32c', type: 'u32', body: ['buffer', { count: 9 }] }], + signed: ['hash', { alg: 'crc32c', type: 'HashCode', body: ['buffer', { count: 9 }] }], + HashCode: 'i32', + asVarint: ['hash', { alg: 'crc32c', type: 'varint', body: ['buffer', { count: 9 }] }], + sha256: ['hash', { alg: 'sha256', type: ['buffer', { count: 32 }], body: ['buffer', { count: 9 }] }], + // A field of the enclosing container selects the body's type + tagged: ['container', [ + { name: 'kind', type: 'u8' }, + { name: 'hash', type: ['hash', { alg: 'crc32c', type: 'u32', body: ['switch', { compareTo: 'kind', fields: { 0: 'u8', 1: 'u16' } }] }] } + ]], + // A hash over a list of hashes + entry: ['container', [{ name: 'key', type: ['pstring', { countType: 'u8' }] }, { name: 'value', type: 'li32' }]], + list: ['array', { countType: 'u8', type: ['hash', { alg: 'crc32c', type: 'lu32', body: 'entry' }] }], + nested: ['hash', { alg: 'crc32c', type: 'lu32', body: 'list' }] + } + const proto = new ProtoDef() + proto.addTypes(types) + const compiler = new ProtoDefCompiler() + compiler.addTypesToCompile(types) + const compiled = compiler.compileProtoDefSync() + const check = Buffer.from('123456789') + const u32 = n => { const b = Buffer.alloc(4); b.writeUInt32BE(n); return b } + const lu32 = n => { const b = Buffer.alloc(4); b.writeUInt32LE(n); return b } + + it('crc32 and crc32c match their check values', () => { + assert.strictEqual(digest('crc32', check), 0xCBF43926) + assert.strictEqual(digest('crc32c', check), 0xE3069283) + }) + + for (const [label, p] of [['interpreted', proto], ['compiled', compiled]]) { + describe(label, () => { + it('writes the hash of the serialized body', () => { + assert.deepStrictEqual(p.createPacketBuffer('crc32', check), u32(0xCBF43926)) + assert.deepStrictEqual(p.createPacketBuffer('crc32c', check), u32(0xE3069283)) + }) + it('reads the hash, not the value', () => { + assert.strictEqual(p.parsePacketBuffer('crc32c', u32(0xE3069283)).data, 0xE3069283) + }) + it('writes a signed type in two\'s complement', () => { + const buffer = p.createPacketBuffer('signed', check) + assert.deepStrictEqual(buffer, u32(0xE3069283)) + assert.strictEqual(p.parsePacketBuffer('signed', buffer).data, 0xE3069283 | 0) + }) + it('sizes a fixed-size type without hashing', () => { + assert.strictEqual(p.sizeOf(check, 'signed'), 4) + assert.strictEqual(p.sizeOf(check, 'sha256'), 32) + }) + it('sizes a variable-size type from the hash', () => { + const buffer = p.createPacketBuffer('asVarint', check) + assert.strictEqual(p.sizeOf(check, 'asVarint'), buffer.length) + assert.deepStrictEqual(buffer, p.createPacketBuffer('varint', 0xE3069283 | 0)) + }) + it('writes a crypto digest as a buffer', () => { + assert.deepStrictEqual(p.createPacketBuffer('sha256', check), + require('crypto').createHash('sha256').update(check).digest()) + }) + it('resolves body fields against the enclosing container', () => { + assert.deepStrictEqual(p.createPacketBuffer('tagged', { kind: 1, hash: 300 }), + Buffer.concat([Buffer.from([1]), u32(digest('crc32c', Buffer.from([0x01, 0x2C])))])) + assert.deepStrictEqual(p.createPacketBuffer('tagged', { kind: 0, hash: 44 }), + Buffer.concat([Buffer.from([0]), u32(digest('crc32c', Buffer.from([44])))])) + }) + it('nests hashes of hashes', () => { + const value = [{ key: 'a', value: 1 }, { key: 'b', value: 2 }] + const list = Buffer.concat([Buffer.from([2]), ...value.map(entry => lu32(digest('crc32c', p.createPacketBuffer('entry', entry))))]) + assert.deepStrictEqual(p.createPacketBuffer('list', value), list) + assert.deepStrictEqual(p.createPacketBuffer('nested', value), lu32(digest('crc32c', list))) + }) + }) + } +})