Add the hash datatype

doc/compiler.md

@@ -228,4 +228,19 @@ compiledProto.setVariable('noArraySizeCheck', true);
 // Use it as if it were a normal ProtoDef
 const buffer = compiledProto.createPacketBuffer('mainType', result)
 const result = compiledProto.parsePacketBuffer('mainType', buffer)
-```
+```
+### Sizing inside a writer, writing inside a sizer
+
+A parametrizable type is compiled by one compiler at a time, so a writer normally has no way to know how large a nested value will be. When it must serialize part of the value before writing (a checksum of it, for example), `WriteCompiler.callTypeSize(value, type)` returns code computing the size of `value` as `type`, and `SizeOfCompiler.callTypeWrite(value, type, offsetExpr)` returns code writing it into `buffer`. Both resolve field references against the current scope and run against the other compiler's context, so they are only available when the types are compiled through `ProtoDefCompiler`. The `hash` datatype is built on them:
+
+```javascript
+Write: {
+  hash: ['parametrizable', (compiler, { alg, type, body }) => {
+    let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n`
+    code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n`
+    code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n`
+    code += 'return ' + compiler.callType('hash', type)
+    return compiler.wrapCode(code)
+  }]
+}
+```

src/compiler.js

@@ -12,6 +12,8 @@ class ProtoDefCompiler {
     this.readCompiler = new ReadCompiler()
     this.writeCompiler = new WriteCompiler()
     this.sizeOfCompiler = new SizeOfCompiler()
+    this.writeCompiler.sizeOfCompiler = this.sizeOfCompiler
+    this.sizeOfCompiler.writeCompiler = this.writeCompiler
   }
 
   addTypes (types) {
@@ -62,6 +64,9 @@ class CompiledProtodef {
     this.sizeOfCtx = sizeOfCtx
     this.writeCtx = writeCtx
     this.readCtx = readCtx
+    // Code from callTypeSize / callTypeWrite runs against the other context
+    writeCtx.sizeOfCtx = sizeOfCtx
+    sizeOfCtx.writeCtx = writeCtx
   }
 
   read (buffer, cursor, type) {
@@ -174,6 +179,24 @@ class Compiler {
     }
   }
 
+  /**
+   * Generates code with another compiler inside this compiler's scope, so that
+   * field references resolve to the same variables, and binds it to that
+   * compiler's context. Natives are reachable through the context as well.
+   */
+  callTypeIn (other, ctxName, generate) {
+    if (!other) throw new Error(`${ctxName} is only available when compiling with ProtoDefCompiler`)
+    const scopeStack = other.scopeStack
+    other.scopeStack = this.scopeStack
+    try {
+      const code = generate(other)
+      if (!isNaN(code)) return code
+      return `((ctx, native) => ${code})(ctx.${ctxName}, ctx.${ctxName})`
+    } finally {
+      other.scopeStack = scopeStack
+    }
+  }
+
   addTypesToCompile (types) {
     for (const [type, json] of Object.entries(types)) {
       // Replace native type, otherwise first in wins
@@ -259,6 +282,7 @@ class Compiler {
     // Local variable to provide some context to eval()
     const native = this.native // eslint-disable-line
     const { PartialReadError } = require('./utils') // eslint-disable-line
+    const hashDigest = require('./hash').digest // eslint-disable-line
     return eval(code)() // eslint-disable-line
   }
 }
@@ -361,11 +385,20 @@ class WriteCompiler extends Compiler {
     if (args.length > 0) return '(' + code + `)(${value}, buffer, ${offsetExpr}, ` + args.map(name => this.getField(name)).join(', ') + ')'
     return '(' + code + `)(${value}, buffer, ${offsetExpr})`
   }
+
+  /**
+   * Code computing the size of `value` as `type`, for writers that need to
+   * serialize part of a value before they can write it
+   */
+  callTypeSize (value, type, args = []) {
+    return this.callTypeIn(this.sizeOfCompiler, 'sizeOfCtx', compiler => compiler.callType(value, type, args))
+  }
 }
 
 class SizeOfCompiler extends Compiler {
   constructor () {
     super()
+    this.constants = {}
 
     this.addTypes(conditionalDatatypes.SizeOf)
     this.addTypes(structuresDatatypes.SizeOf)
@@ -390,12 +423,22 @@ class SizeOfCompiler extends Compiler {
     this.primitiveTypes[type] = `native.${type}`
     if (!isNaN(fn)) {
       this.native[type] = (value) => { return fn }
+      this.constants[type] = fn
     } else {
       this.native[type] = fn
     }
     this.types[type] = 'native'
   }
 
+  /**
+   * The size of `type` when it doesn't depend on the value, following
+   * aliases down to a fixed-size native; undefined otherwise
+   */
+  constantSize (type) {
+    while (typeof type === 'string' && typeof this.types[type] === 'string' && this.types[type] !== 'native') type = this.types[type]
+    return this.constants[type]
+  }
+
   compileType (type) {
     if (type instanceof Array) {
       if (this.parameterizableTypes[type[0]]) { return this.parameterizableTypes[type[0]](this, type[1]) }
@@ -429,6 +472,14 @@ class SizeOfCompiler extends Compiler {
     if (args.length > 0) return '(' + code + `)(${value}, ` + args.map(name => this.getField(name)).join(', ') + ')'
     return '(' + code + `)(${value})`
   }
+
+  /**
+   * Code writing `value` as `type` into `buffer` at `offsetExpr`, for sizers
+   * whose result depends on the serialized form of a value
+   */
+  callTypeWrite (value, type, offsetExpr = 'offset', args = []) {
+    return this.callTypeIn(this.writeCompiler, 'writeCtx', compiler => compiler.callType(value, type, offsetExpr, args))
+  }
 }
 
 module.exports = {

src/datatypes/compiler-utils.js

@@ -83,6 +83,9 @@ return { value, size }
       let code = 'const { value, size } = ' + compiler.callType(mapper.type) + '\n'
       code += 'return { value: ' + JSON.stringify(sanitizeMappings(mapper.mappings)) + '[value] || value, size }'
       return compiler.wrapCode(code)
+    }],
+    hash: ['parametrizable', (compiler, { type }) => {
+      return compiler.wrapCode('return ' + compiler.callType(type))
     }]
   },
 
@@ -163,6 +166,18 @@ return (ctx.${type})(val, buffer, offset)
       code += 'if (mapped === undefined) throw new Error(value + \' is not in the mappings value\')\n'
       code += 'return ' + compiler.callType('mapped', mapper.type)
       return compiler.wrapCode(code)
+    }],
+    hash: ['parametrizable', (compiler, { alg, type, body }) => {
+      let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n`
+      code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n`
+      code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n`
+      code += 'try {\n'
+      code += '  return ' + compiler.callType('hash', type) + '\n'
+      code += '} catch (e) {\n'
+      code += '  if (!(e instanceof RangeError) || typeof hash !== "number") throw e\n'
+      code += '  return ' + compiler.callType('hash | 0', type) + '\n'
+      code += '}'
+      return compiler.wrapCode(code)
     }]
   },
 
@@ -217,6 +232,17 @@ return (ctx.${type})(val)
       code += 'if (mapped === undefined) throw new Error(value + \' is not in the mappings value\')\n'
       code += 'return ' + compiler.callType('mapped', mapper.type)
       return compiler.wrapCode(code)
+    }],
+    hash: ['parametrizable', (compiler, { alg, type, body }) => {
+      const constant = compiler.constantSize(type)
+      if (constant !== undefined) return String(constant)
+      const size = compiler.callType('hash', type)
+      if (!isNaN(size)) return size
+      let code = `const bodyBuffer = Buffer.alloc(${compiler.callType('value', body)})\n`
+      code += `;((buffer) => ${compiler.callTypeWrite('value', body, '0')})(bodyBuffer)\n`
+      code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n`
+      code += 'return ' + size
+      return compiler.wrapCode(code)
     }]
   }
 }

src/datatypes/utils.js

@@ -1,4 +1,5 @@
-const { getCount, sendCount, calcCount, PartialReadError } = require('../utils')
+const { getCount, sendCount, calcCount, getFieldInfo, PartialReadError } = require('../utils')
+const { digest } = require('../hash')
 
 module.exports = {
   bool: [readBool, writeBool, 1, require('../../ProtoDef/schemas/utils.json').bool],
@@ -9,6 +10,7 @@ module.exports = {
   bitflags: [readBitflags, writeBitflags, sizeOfBitflags, require('../../ProtoDef/schemas/utils.json').bitflags],
   cstring: [readCString, writeCString, sizeOfCString, require('../../ProtoDef/schemas/utils.json').cstring],
   mapper: [readMapper, writeMapper, sizeOfMapper, require('../../ProtoDef/schemas/utils.json').mapper],
+  hash: [readHash, writeHash, sizeOfHash, require('../../ProtoDef/schemas/utils.json').hash],
   ...require('./varint')
 }
 
@@ -280,3 +282,30 @@ function sizeOfBitflags (value, { type, flags, shift, big }, rootNode) {
   }
   return this.sizeOf(mappedValue, type, rootNode)
 }
+
+function readHash (buffer, offset, { type }, rootNode) {
+  return this.read(buffer, offset, type, rootNode)
+}
+
+function hashOf (value, { alg, body }, rootNode) {
+  const bodyBuffer = Buffer.alloc(this.sizeOf(value, body, rootNode))
+  this.write(value, bodyBuffer, 0, body, rootNode)
+  return digest(alg, bodyBuffer)
+}
+
+// A CRC is unsigned; a signed `type` takes its two's complement.
+function writeHash (value, buffer, offset, typeArgs, rootNode) {
+  const hash = hashOf.call(this, value, typeArgs, rootNode)
+  try {
+    return this.write(hash, buffer, offset, typeArgs.type, rootNode)
+  } catch (e) {
+    if (!(e instanceof RangeError) || typeof hash !== 'number') throw e
+    return this.write(hash | 0, buffer, offset, typeArgs.type, rootNode)
+  }
+}
+
+function sizeOfHash (value, typeArgs, rootNode) {
+  const functions = this.types[getFieldInfo(typeArgs.type).type]
+  if (functions && typeof functions[2] === 'number') return functions[2]
+  return this.sizeOf(hashOf.call(this, value, typeArgs, rootNode), typeArgs.type, rootNode)
+}

src/hash.js

@@ -0,0 +1,39 @@
+const crypto = require('crypto')
+
+// Reflected table-driven CRC with all-ones init and final xor; `poly` is the
+// reversed polynomial.
+const tables = {}
+function table (poly) {
+  if (!tables[poly]) {
+    const t = new Int32Array(256)
+    for (let n = 0; n < 256; n++) {
+      let c = n
+      for (let k = 0; k < 8; k++) c = c & 1 ? poly ^ (c >>> 1) : c >>> 1
+      t[n] = c
+    }
+    tables[poly] = t
+  }
+  return tables[poly]
+}
+
+function crc (poly, buffer) {
+  const t = table(poly)
+  let c = -1
+  for (let i = 0; i < buffer.length; i++) c = t[(c ^ buffer[i]) & 0xff] ^ (c >>> 8)
+  return (c ^ -1) >>> 0
+}
+
+const algorithms = {
+  crc32: buffer => crc(0xEDB88320, buffer),
+  crc32c: buffer => crc(0x82F63B78, buffer)
+}
+
+// CRC digests are unsigned integers; every other algorithm is delegated to
+// node's crypto and yields a Buffer.
+function digest (alg, buffer) {
+  const algorithm = algorithms[alg]
+  if (algorithm) return algorithm(buffer)
+  return crypto.createHash(alg).update(buffer).digest()
+}
+
+module.exports = { digest, algorithms }

test/misc.js

@@ -25,3 +25,79 @@ describe('mapper', () => {
     })
   }
 })
+
+describe('hash', () => {
+  const { digest } = require('../src/hash')
+  const types = {
+    crc32: ['hash', { alg: 'crc32', type: 'u32', body: ['buffer', { count: 9 }] }],
+    crc32c: ['hash', { alg: 'crc32c', type: 'u32', body: ['buffer', { count: 9 }] }],
+    signed: ['hash', { alg: 'crc32c', type: 'HashCode', body: ['buffer', { count: 9 }] }],
+    HashCode: 'i32',
+    asVarint: ['hash', { alg: 'crc32c', type: 'varint', body: ['buffer', { count: 9 }] }],
+    sha256: ['hash', { alg: 'sha256', type: ['buffer', { count: 32 }], body: ['buffer', { count: 9 }] }],
+    // A field of the enclosing container selects the body's type
+    tagged: ['container', [
+      { name: 'kind', type: 'u8' },
+      { name: 'hash', type: ['hash', { alg: 'crc32c', type: 'u32', body: ['switch', { compareTo: 'kind', fields: { 0: 'u8', 1: 'u16' } }] }] }
+    ]],
+    // A hash over a list of hashes
+    entry: ['container', [{ name: 'key', type: ['pstring', { countType: 'u8' }] }, { name: 'value', type: 'li32' }]],
+    list: ['array', { countType: 'u8', type: ['hash', { alg: 'crc32c', type: 'lu32', body: 'entry' }] }],
+    nested: ['hash', { alg: 'crc32c', type: 'lu32', body: 'list' }]
+  }
+  const proto = new ProtoDef()
+  proto.addTypes(types)
+  const compiler = new ProtoDefCompiler()
+  compiler.addTypesToCompile(types)
+  const compiled = compiler.compileProtoDefSync()
+  const check = Buffer.from('123456789')
+  const u32 = n => { const b = Buffer.alloc(4); b.writeUInt32BE(n); return b }
+  const lu32 = n => { const b = Buffer.alloc(4); b.writeUInt32LE(n); return b }
+
+  it('crc32 and crc32c match their check values', () => {
+    assert.strictEqual(digest('crc32', check), 0xCBF43926)
+    assert.strictEqual(digest('crc32c', check), 0xE3069283)
+  })
+
+  for (const [label, p] of [['interpreted', proto], ['compiled', compiled]]) {
+    describe(label, () => {
+      it('writes the hash of the serialized body', () => {
+        assert.deepStrictEqual(p.createPacketBuffer('crc32', check), u32(0xCBF43926))
+        assert.deepStrictEqual(p.createPacketBuffer('crc32c', check), u32(0xE3069283))
+      })
+      it('reads the hash, not the value', () => {
+        assert.strictEqual(p.parsePacketBuffer('crc32c', u32(0xE3069283)).data, 0xE3069283)
+      })
+      it('writes a signed type in two\'s complement', () => {
+        const buffer = p.createPacketBuffer('signed', check)
+        assert.deepStrictEqual(buffer, u32(0xE3069283))
+        assert.strictEqual(p.parsePacketBuffer('signed', buffer).data, 0xE3069283 | 0)
+      })
+      it('sizes a fixed-size type without hashing', () => {
+        assert.strictEqual(p.sizeOf(check, 'signed'), 4)
+        assert.strictEqual(p.sizeOf(check, 'sha256'), 32)
+      })
+      it('sizes a variable-size type from the hash', () => {
+        const buffer = p.createPacketBuffer('asVarint', check)
+        assert.strictEqual(p.sizeOf(check, 'asVarint'), buffer.length)
+        assert.deepStrictEqual(buffer, p.createPacketBuffer('varint', 0xE3069283 | 0))
+      })
+      it('writes a crypto digest as a buffer', () => {
+        assert.deepStrictEqual(p.createPacketBuffer('sha256', check),
+          require('crypto').createHash('sha256').update(check).digest())
+      })
+      it('resolves body fields against the enclosing container', () => {
+        assert.deepStrictEqual(p.createPacketBuffer('tagged', { kind: 1, hash: 300 }),
+          Buffer.concat([Buffer.from([1]), u32(digest('crc32c', Buffer.from([0x01, 0x2C])))]))
+        assert.deepStrictEqual(p.createPacketBuffer('tagged', { kind: 0, hash: 44 }),
+          Buffer.concat([Buffer.from([0]), u32(digest('crc32c', Buffer.from([44])))]))
+      })
+      it('nests hashes of hashes', () => {
+        const value = [{ key: 'a', value: 1 }, { key: 'b', value: 2 }]
+        const list = Buffer.concat([Buffer.from([2]), ...value.map(entry => lu32(digest('crc32c', p.createPacketBuffer('entry', entry))))])
+        assert.deepStrictEqual(p.createPacketBuffer('list', value), list)
+        assert.deepStrictEqual(p.createPacketBuffer('nested', value), lu32(digest('crc32c', list)))
+      })
+    })
+  }
+})