From 75f6c07a218451874f569cd08fbd786d36f32c06 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Mon, 24 Aug 2026 14:13:23 -0400 Subject: [PATCH 1/2] ci(libdatadog): add artifact dependency and size reports --- .github/workflows/build.yml | 13 +- crates/libdatadog-wasm/Cargo.toml | 14 + package.json | 1 + packages/libdatadog/package.json | 3 + .../libdatadog/scripts/report-napi-size.js | 183 +++++++++ .../libdatadog/scripts/report-wasm-size.js | 382 ++++++++++++++++++ packages/libdatadog/test/size-report.test.js | 85 ++++ scripts/build-wasm.js | 17 +- scripts/check-dependencies.js | 129 ++++++ test/dependencies.js | 62 +++ 10 files changed, 884 insertions(+), 5 deletions(-) create mode 100644 packages/libdatadog/scripts/report-napi-size.js create mode 100644 packages/libdatadog/scripts/report-wasm-size.js create mode 100644 packages/libdatadog/test/size-report.test.js create mode 100644 scripts/check-dependencies.js create mode 100644 test/dependencies.js diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bb688d0..57a04d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,9 +28,20 @@ jobs: node-version: 24 - name: Add WASM target run: rustup target add wasm32-unknown-unknown + - name: Validate artifact dependencies + run: npm run check:dependencies - run: npm ci --prefix packages/libdatadog - run: cargo install wasm-pack --version 0.14.0 --locked - - run: npm run build:wasm --prefix packages/libdatadog + - run: cargo install cargo-bloat --version 0.12.1 --locked + - run: npm run build --prefix packages/libdatadog + - name: Report N-API binary size + run: npm run report:napi-size --prefix packages/libdatadog + - name: Build symbolized WASM for size attribution + run: npm run size:profile --prefix packages/libdatadog + - name: Report and validate WASM binary size + run: >- + npm run report:wasm-size --prefix packages/libdatadog -- + ../../target/size/libdatadog_wasm_bg.wasm - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: libdatadog-wasm diff --git a/crates/libdatadog-wasm/Cargo.toml b/crates/libdatadog-wasm/Cargo.toml index 91cbc95..71273b5 100644 --- a/crates/libdatadog-wasm/Cargo.toml +++ b/crates/libdatadog-wasm/Cargo.toml @@ -48,3 +48,17 @@ wasm-opt = [ "--enable-multivalue", "--converge", ] + +[package.metadata.wasm-pack.profile.profiling] +# Keep function names for crate attribution after wasm-opt runs in Linux CI. +wasm-opt = [ + "-Oz", + "--enable-mutable-globals", + "--enable-nontrapping-float-to-int", + "--enable-bulk-memory", + "--enable-sign-ext", + "--enable-reference-types", + "--enable-multivalue", + "--converge", + "-g", +] diff --git a/package.json b/package.json index 552476a..0f43068 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "cargo-build-release": "yarn -s cargo-build -- --release", "cargo-build": "cargo build --message-format=json-render-diagnostics", "copy-artifacts": "node ./scripts/copy-artifacts", + "check:dependencies": "node scripts/check-dependencies.js", "lint": "eslint .", "test": "bash scripts/test.sh" }, diff --git a/packages/libdatadog/package.json b/packages/libdatadog/package.json index 8303ac8..2d45cdc 100644 --- a/packages/libdatadog/package.json +++ b/packages/libdatadog/package.json @@ -39,7 +39,10 @@ "build:native": "node scripts/build-native.js", "build:wasm": "npm run build:wasm:binary && npm run inline:wasm", "build:wasm:binary": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm wasm/dist", + "size:profile": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm ../../target/size --profiling", "inline:wasm": "node scripts/inline-wasm.js", + "report:napi-size": "node scripts/report-napi-size.js", + "report:wasm-size": "node scripts/report-wasm-size.js", "package:native": "node scripts/package-native.js", "build": "npm run build:native && npm run build:wasm", "test": "node scripts/run-tests.js && npm run test:types", diff --git a/packages/libdatadog/scripts/report-napi-size.js b/packages/libdatadog/scripts/report-napi-size.js new file mode 100644 index 0000000..63a0066 --- /dev/null +++ b/packages/libdatadog/scripts/report-napi-size.js @@ -0,0 +1,183 @@ +'use strict' + +const fs = require('node:fs') +const path = require('node:path') +const { spawnSync } = require('node:child_process') + +function formatBytes (bytes) { + return bytes.toLocaleString('en-US') +} + +function formatKibibytes (bytes) { + return (bytes / 1024).toFixed(1) +} + +function findArtifact () { + const directory = path.join(__dirname, '..', 'dist', 'native') + const artifacts = fs.readdirSync(directory) + .filter(file => /^libdatadog\..+\.node$/.test(file)) + + if (artifacts.length !== 1) { + throw new Error(`expected one native artifact in ${directory}, found ${artifacts.length}`) + } + + return path.join(directory, artifacts[0]) +} + +function runCommand (command, args) { + const result = spawnSync(command, args, { + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024, + }) + + if (result.error) throw result.error + if (result.status !== 0) { + const detail = result.stderr.trim() || result.stdout.trim() + throw new Error(`${command} failed: ${detail}`) + } + + return result.stdout +} + +function parseDarwinSections (output) { + const sections = [] + + for (const line of output.split('\n')) { + const match = line.match(/^Segment (\S+):\s+(\d+)$/) + if (match) sections.push({ name: match[1], bytes: Number(match[2]) }) + } + + return sections +} + +function parseElfSections (output) { + const sections = [] + + for (const line of output.split('\n')) { + const match = line.trim().match(/^(\S+)\s+(\d+)\s+[0-9a-fA-Fx]+$/) + if (match && match[1] !== 'section') { + sections.push({ name: match[1], bytes: Number(match[2]) }) + } + } + + return sections +} + +function readSections (artifactPath, platform = process.platform) { + const sections = platform === 'darwin' + ? parseDarwinSections(runCommand('size', ['-m', artifactPath])) + : parseElfSections(runCommand('size', ['-A', artifactPath])) + + if (sections.length === 0) throw new Error('could not read native binary sections') + return sections +} + +function readCrateSizes () { + const output = runCommand('cargo', [ + 'bloat', + '--release', + '--crates', + '-n', + '0', + '--message-format', + 'json', + '-p', + 'libdatadog', + '--lib', + ]) + + return JSON.parse(output) +} + +function appendCrateReport (lines, profile) { + const crates = profile.crates + .map(crate => ({ ...crate, name: crate.name.replaceAll('_', '-') })) + const attributedBytes = crates.reduce((total, crate) => total + crate.size, 0) + const overheadBytes = profile['text-section-size'] - attributedBytes + if (overheadBytes > 0) { + crates.push({ name: 'unattributed .text overhead', size: overheadBytes }) + crates.sort((left, right) => right.size - left.size) + } + const visible = crates.filter(crate => crate.size >= 2048) + const otherBytes = crates + .filter(crate => crate.size < 2048) + .reduce((total, crate) => total + crate.size, 0) + + if (otherBytes > 0) { + visible.push({ name: 'other crates (<2 KiB each)', size: otherBytes }) + } + + lines.push( + '', + '### Code by Rust crate', + '', + '| Crate | Bytes | KiB | Share of native code |', + '| --- | ---: | ---: | ---: |', + ) + + for (const crate of visible) { + const share = `${(crate.size / profile['text-section-size'] * 100).toFixed(1)}%` + lines.push( + `| ${crate.name} | ${formatBytes(crate.size)} | ` + + `${formatKibibytes(crate.size)} | ${share} |`, + ) + } + + lines.push( + '', + `.text section: ${formatBytes(profile['text-section-size'])} bytes ` + + `(${formatKibibytes(profile['text-section-size'])} KiB).`, + '', + 'Crate ownership comes from cargo-bloat using a separate symbol-preserving release build. ' + + 'It attributes native code only; data, unwind information, symbols, and file alignment ' + + 'remain in the section and artifact totals above.', + ) +} + +function createReport (artifactPath, profile) { + const artifactBytes = fs.statSync(artifactPath).size + const sections = readSections(artifactPath) + const totalSectionBytes = sections.reduce((total, section) => total + section.bytes, 0) + const lines = [ + '## libdatadog N-API size', + '', + '| Native artifact | Bytes | KiB |', + '| --- | ---: | ---: |', + `| **Shipped .node file** | **${formatBytes(artifactBytes)}** | ` + + `**${formatKibibytes(artifactBytes)}** |`, + '', + '### Native binary sections', + '', + '| Section/segment | Bytes | KiB | Share |', + '| --- | ---: | ---: | ---: |', + ] + + for (const section of sections) { + const share = `${(section.bytes / totalSectionBytes * 100).toFixed(1)}%` + lines.push( + `| ${section.name} | ${formatBytes(section.bytes)} | ` + + `${formatKibibytes(section.bytes)} | ${share} |`, + ) + } + + if (profile) appendCrateReport(lines, profile) + + lines.push('', `Generated from \`${path.relative(process.cwd(), artifactPath)}\`.`) + return lines.join('\n') +} + +if (require.main === module) { + const artifactPath = process.argv[2] ? path.resolve(process.argv[2]) : findArtifact() + const report = createReport(artifactPath, readCrateSizes()) + console.log(report) + + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${report}\n`) + } +} + +module.exports = { + createReport, + parseDarwinSections, + parseElfSections, +} diff --git a/packages/libdatadog/scripts/report-wasm-size.js b/packages/libdatadog/scripts/report-wasm-size.js new file mode 100644 index 0000000..aadf229 --- /dev/null +++ b/packages/libdatadog/scripts/report-wasm-size.js @@ -0,0 +1,382 @@ +'use strict' + +const fs = require('node:fs') +const path = require('node:path') +const { brotliDecompressSync } = require('node:zlib') + +const sectionNames = [ + 'custom', + 'type', + 'import', + 'function', + 'table', + 'memory', + 'global', + 'export', + 'start', + 'element', + 'code', + 'data', + 'data count', + 'tag', +] + +const forbiddenWasmCode = [ + { + dependency: 'regex', + owners: new Set(['aho-corasick', 'regex', 'regex-automata', 'regex-syntax']), + }, + { + dependency: 'zstd', + owners: new Set(['zstd', 'zstd-safe']), + }, + { + dependency: 'zstd-sys', + owners: new Set(['zstd-sys', 'zstd-sys (C)']), + }, +] + +function readUnsignedLeb128 (bytes, start) { + let offset = start + let multiplier = 1 + let value = 0 + + while (offset < bytes.length) { + const byte = bytes[offset++] + value += (byte & 0x7F) * multiplier + if ((byte & 0x80) === 0) return { offset, value } + multiplier *= 128 + } + + throw new Error('WASM contains an unterminated section length') +} + +function validateWasm (wasm) { + const expectedHeader = Buffer.from([ + 0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, + ]) + if (wasm.length < expectedHeader.length || !wasm.subarray(0, 8).equals(expectedHeader)) { + throw new Error('inline payload is not a WebAssembly 1 binary') + } +} + +function readSectionRecords (wasm) { + validateWasm(wasm) + + const sections = [] + let offset = 8 + + while (offset < wasm.length) { + const sectionStart = offset + const id = wasm[offset++] + const length = readUnsignedLeb128(wasm, offset) + const payloadStart = length.offset + const payloadEnd = payloadStart + length.value + if (payloadEnd > wasm.length) throw new Error('WASM section extends past the binary') + sections.push({ + bytes: payloadEnd - sectionStart, + id, + payloadEnd, + payloadStart, + }) + offset = payloadEnd + } + + return sections +} + +function readSections (wasm) { + const sections = [{ name: 'header', bytes: 8 }] + + for (const record of readSectionRecords(wasm)) { + const { bytes, id } = record + const name = sectionNames[id] || `unknown (${id})` + const existing = sections.find(section => section.name === name) + if (existing) existing.bytes += bytes + else sections.push({ name, bytes }) + } + + return sections +} + +function readWasmString (wasm, start) { + const length = readUnsignedLeb128(wasm, start) + const end = length.offset + length.value + if (end > wasm.length) throw new Error('WASM string extends past the binary') + return { + offset: end, + value: wasm.toString('utf8', length.offset, end), + } +} + +function skipLimits (wasm, start) { + const flags = readUnsignedLeb128(wasm, start) + const minimum = readUnsignedLeb128(wasm, flags.offset) + if ((flags.value & 1) === 0) return minimum.offset + return readUnsignedLeb128(wasm, minimum.offset).offset +} + +function readImportedFunctionCount (wasm, sections) { + const section = sections.find(section => section.id === 2) + if (!section) return 0 + + const cursor = readUnsignedLeb128(wasm, section.payloadStart) + let offset = cursor.offset + let functionCount = 0 + + for (let index = 0; index < cursor.value; index++) { + offset = readWasmString(wasm, offset).offset + offset = readWasmString(wasm, offset).offset + const kind = wasm[offset++] + + switch (kind) { + case 0: { + functionCount++ + offset = readUnsignedLeb128(wasm, offset).offset + break + } + case 1: { + offset++ + offset = skipLimits(wasm, offset) + break + } + case 2: { + offset = skipLimits(wasm, offset) + break + } + case 3: { + offset += 2 + break + } + case 4: { + offset++ + offset = readUnsignedLeb128(wasm, offset).offset + break + } + default: { + throw new Error(`unsupported WASM import kind: ${kind}`) + } + } + } + + return functionCount +} + +function readFunctionNames (wasm, sections) { + const names = new Map() + + for (const section of sections) { + if (section.id !== 0) continue + const customName = readWasmString(wasm, section.payloadStart) + if (customName.value !== 'name') continue + + let offset = customName.offset + while (offset < section.payloadEnd) { + const subsectionId = wasm[offset++] + const length = readUnsignedLeb128(wasm, offset) + const subsectionEnd = length.offset + length.value + offset = length.offset + + if (subsectionId === 1) { + const count = readUnsignedLeb128(wasm, offset) + offset = count.offset + for (let index = 0; index < count.value; index++) { + const functionIndex = readUnsignedLeb128(wasm, offset) + const name = readWasmString(wasm, functionIndex.offset) + names.set(functionIndex.value, name.value) + offset = name.offset + } + } + + offset = subsectionEnd + } + } + + return names +} + +function inferCrate (functionName) { + if (/^(COVER|FASTCOVER|FSE|HIST|HUF|POOL|XXH|ZSTD|ZSTDMT)_/.test(functionName)) { + return 'zstd-sys (C)' + } + if (/^__(externref|wbindgen|wbg)/.test(functionName)) return 'wasm-bindgen runtime' + if (/^(__rust|__rg_|dlmalloc::)/.test(functionName)) return 'Rust runtime' + + const match = functionName.match(/(?:^|[< &(,])(?:mut )?([A-Za-z][A-Za-z0-9_]*)::/) + if (!match) return 'bindings / unattributed' + if (['alloc', 'core', 'std'].includes(match[1])) return 'Rust standard library' + return match[1].replaceAll('_', '-') +} + +function insertBySize (entries, entry) { + const index = entries.findIndex(candidate => candidate.bytes < entry.bytes) + if (index === -1) entries.push(entry) + else entries.splice(index, 0, entry) +} + +function readCrateSizes (wasm) { + const sections = readSectionRecords(wasm) + const code = sections.find(section => section.id === 10) + if (!code) throw new Error('symbolized WASM does not contain a code section') + const names = readFunctionNames(wasm, sections) + if (names.size === 0) throw new Error('symbolized WASM does not contain function names') + + const importedFunctions = readImportedFunctionCount(wasm, sections) + const sizes = new Map() + const count = readUnsignedLeb128(wasm, code.payloadStart) + let offset = count.offset + let totalBytes = 0 + + for (let index = 0; index < count.value; index++) { + const bodyStart = offset + const body = readUnsignedLeb128(wasm, bodyStart) + offset = body.offset + body.value + if (offset > code.payloadEnd) throw new Error('WASM function extends past the code section') + + const bytes = offset - bodyStart + const name = names.get(importedFunctions + index) || '' + const crate = inferCrate(name) + sizes.set(crate, (sizes.get(crate) || 0) + bytes) + totalBytes += bytes + } + + const entries = [] + for (const [name, bytes] of sizes) insertBySize(entries, { bytes, name }) + return { entries, totalBytes } +} + +function findForbiddenWasmCode (entries) { + const failures = [] + + for (const entry of entries) { + const forbidden = forbiddenWasmCode.find(candidate => candidate.owners.has(entry.name)) + if (forbidden) failures.push({ ...entry, dependency: forbidden.dependency }) + } + + return failures +} + +function formatBytes (bytes) { + return bytes.toLocaleString('en-US') +} + +function formatKibibytes (bytes) { + return (bytes / 1024).toFixed(1) +} + +function layerRow (name, bytes, emphasis = false) { + const formattedBytes = formatBytes(bytes) + const kibibytes = formatKibibytes(bytes) + if (emphasis) return `| **${name}** | **${formattedBytes}** | **${kibibytes}** |` + return `| ${name} | ${formattedBytes} | ${kibibytes} |` +} + +function appendCrateReport (lines, profilePath) { + const profileWasm = fs.readFileSync(profilePath) + const { entries, totalBytes } = readCrateSizes(profileWasm) + const visibleEntries = entries.filter(entry => entry.bytes >= 2048) + const otherBytes = entries + .filter(entry => entry.bytes < 2048) + .reduce((total, entry) => total + entry.bytes, 0) + if (otherBytes > 0) { + visibleEntries.push({ bytes: otherBytes, name: 'other crates (<2 KiB each)' }) + } + const attributionNote = [ + 'Crate ownership comes from a separate symbol-preserving build with the same size settings.', + 'Debug-name bytes are excluded; generic functions are assigned to their symbol owner.', + ].join(' ') + + lines.push( + '', + '### Code by Rust crate', + '', + '| Crate/function owner | Bytes | KiB | Share |', + '| --- | ---: | ---: | ---: |', + ) + + for (const entry of visibleEntries) { + const share = `${(entry.bytes / totalBytes * 100).toFixed(1)}%` + lines.push( + `| ${entry.name} | ${formatBytes(entry.bytes)} | ` + + `${formatKibibytes(entry.bytes)} | ${share} |`, + ) + } + + lines.push('', attributionNote) +} + +function createReport (gluePath, profilePath) { + const glue = fs.readFileSync(gluePath, 'utf8') + const match = glue.match(/Buffer\.from\('([A-Za-z0-9+/=]+)', 'base64'\)/) + if (!match) throw new Error('could not find the inline base64 WASM payload') + + const base64Bytes = Buffer.byteLength(match[1]) + const compressed = Buffer.from(match[1], 'base64') + const wasm = brotliDecompressSync(compressed) + const glueBytes = Buffer.byteLength(glue) - base64Bytes + const inlineBytes = Buffer.byteLength(glue) + const base64Overhead = base64Bytes - compressed.length + const sections = readSections(wasm) + const lines = [ + '## libdatadog WASM size', + '', + '| Inline artifact layer | Bytes | KiB |', + '| --- | ---: | ---: |', + layerRow('Raw WASM (before Brotli)', wasm.length), + layerRow('Brotli-compressed WASM', compressed.length), + layerRow('Base64 encoding overhead', base64Overhead), + layerRow('JavaScript glue/loader', glueBytes), + layerRow('Final inlined JavaScript', inlineBytes, true), + '', + '### Raw WebAssembly sections', + '', + '| Section | Bytes | KiB | Share |', + '| --- | ---: | ---: | ---: |', + ] + + for (const section of sections) { + const share = `${(section.bytes / wasm.length * 100).toFixed(1)}%` + lines.push( + `| ${section.name} | ${formatBytes(section.bytes)} | ` + + `${formatKibibytes(section.bytes)} | ${share} |`, + ) + } + + if (profilePath) appendCrateReport(lines, profilePath) + + lines.push('', `Generated from \`${path.relative(process.cwd(), gluePath)}\`.`) + return lines.join('\n') +} + +if (require.main === module) { + const gluePath = path.join(__dirname, '..', 'dist', 'wasm', 'libdatadog_wasm.js') + const profilePath = process.argv[2] && path.resolve(process.argv[2]) + const report = createReport(gluePath, profilePath) + console.log(report) + + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${report}\n`) + } + + if (profilePath) { + const profileWasm = fs.readFileSync(profilePath) + const failures = findForbiddenWasmCode(readCrateSizes(profileWasm).entries) + if (failures.length > 0) { + console.error('Forbidden code found in the symbolized WASM binary:') + for (const failure of failures) { + console.error( + `- ${failure.dependency} via ${failure.name}: ${formatBytes(failure.bytes)} bytes`, + ) + } + process.exitCode = 1 + } + } +} + +module.exports = { + createReport, + findForbiddenWasmCode, + inferCrate, + readCrateSizes, + readSections, +} diff --git a/packages/libdatadog/test/size-report.test.js b/packages/libdatadog/test/size-report.test.js new file mode 100644 index 0000000..7917798 --- /dev/null +++ b/packages/libdatadog/test/size-report.test.js @@ -0,0 +1,85 @@ +'use strict' + +const assert = require('node:assert/strict') +const path = require('node:path') +const test = require('node:test') + +const { + createReport: createWasmReport, + findForbiddenWasmCode, + inferCrate, + readSections, +} = require('../scripts/report-wasm-size') +const { + createReport: createNapiReport, + parseDarwinSections, + parseElfSections, +} = require('../scripts/report-napi-size') + +test('reports inline packaging and WASM section sizes', () => { + const gluePath = path.join(__dirname, '..', 'dist', 'wasm', 'libdatadog_wasm.js') + const report = createWasmReport(gluePath) + + assert.match(report, /Raw WASM \(before Brotli\)/) + assert.match(report, /Base64 encoding overhead/) + assert.match(report, /Raw WebAssembly sections/) + assert.match(report, /\| code \|/) + assert.match(report, /\| data \|/) +}) + +test('reports N-API artifact, section, and crate sizes', () => { + const nativeDirectory = path.join(__dirname, '..', 'dist', 'native') + const artifact = require('node:fs').readdirSync(nativeDirectory) + .find(file => /^libdatadog\..+\.node$/.test(file)) + const report = createNapiReport(path.join(nativeDirectory, artifact), { + 'text-section-size': 7000, + 'crates': [ + { name: 'libdatadog', size: 2500 }, + { name: 'small_crate', size: 500 }, + ], + }) + + assert.match(report, /Shipped \.node file/) + assert.match(report, /Native binary sections/) + assert.match(report, /Code by Rust crate/) + assert.match(report, /libdatadog/) + assert.match(report, /unattributed \.text overhead/) + assert.match(report, /other crates \(<2 KiB each\)/) +}) + +test('parses Darwin and ELF section reports', () => { + assert.deepEqual(parseDarwinSections('Segment __TEXT: 123\nSegment __DATA: 45\n'), [ + { name: '__TEXT', bytes: 123 }, + { name: '__DATA', bytes: 45 }, + ]) + assert.deepEqual(parseElfSections('.text 123 0x10\n.data 45 0x20\nTotal 168\n'), [ + { name: '.text', bytes: 123 }, + { name: '.data', bytes: 45 }, + ]) +}) + +test('rejects data that is not a WebAssembly binary', () => { + assert.throws(() => readSections(Buffer.from('not wasm')), /not a WebAssembly 1 binary/) +}) + +test('attributes symbolized functions to their Rust crate', () => { + assert.equal( + inferCrate('libdd_data_pipeline::trace_exporter::send'), + 'libdd-data-pipeline', + ) + assert.equal(inferCrate('::serialize'), 'serde-yaml') + assert.equal(inferCrate('ZSTD_compress'), 'zstd-sys (C)') + assert.equal(inferCrate('core::slice::sort'), 'Rust standard library') +}) + +test('rejects forbidden code linked into WASM', () => { + assert.deepEqual(findForbiddenWasmCode([ + { bytes: 10, name: 'regex-lite' }, + { bytes: 20, name: 'regex-automata' }, + { bytes: 30, name: 'zstd-sys (C)' }, + { bytes: 40, name: 'zrip-encode' }, + ]), [ + { bytes: 20, dependency: 'regex', name: 'regex-automata' }, + { bytes: 30, dependency: 'zstd-sys', name: 'zstd-sys (C)' }, + ]) +}) diff --git a/scripts/build-wasm.js b/scripts/build-wasm.js index e94d0d7..a6717f9 100644 --- a/scripts/build-wasm.js +++ b/scripts/build-wasm.js @@ -58,29 +58,38 @@ if (isMacOS) { * * @param {string} cratePath * @param {string} outputDirectory - * @param {{ skipOptimization?: boolean }} options + * @param {{ profiling?: boolean, skipOptimization?: boolean }} options * @returns {void} */ function buildWasm (cratePath, outputDirectory, options = {}) { - const { skipOptimization = false } = options + const { profiling = false, skipOptimization = false } = options const resolvedOutputDirectory = path.resolve(cratePath, outputDirectory) fs.rmSync(resolvedOutputDirectory, { force: true, recursive: true }) const args = ['build'] + if (profiling) args.push('--profiling') if (skipOptimization) args.push('--no-opt') args.push('--target', 'nodejs', cratePath, '--out-dir', resolvedOutputDirectory) - childProcess.execFileSync('wasm-pack', args, { env }) + childProcess.execFileSync('wasm-pack', args, { + env: { + ...env, + // Cargo's release profile strips the function names needed for size attribution. + ...(profiling && { CARGO_PROFILE_RELEASE_STRIP: 'false' }), + }, + }) // wasm-pack ignores its output by default. These outputs are package inputs, // so remove the nested ignore file and let each npm package's files allowlist // decide whether they are published. fs.rmSync(path.join(resolvedOutputDirectory, '.gitignore'), { force: true }) } -const [cratePath, outputDirectory] = process.argv.slice(2) +const [cratePath, outputDirectory, mode] = process.argv.slice(2) if (cratePath || outputDirectory) { if (!cratePath || !outputDirectory) { throw new Error('Both the WASM crate path and output directory are required') } + if (mode && mode !== '--profiling') throw new Error(`Unknown build mode: ${mode}`) buildWasm(path.resolve(cratePath), path.resolve(outputDirectory), { + profiling: mode === '--profiling', skipOptimization: isMacOS, }) } else { diff --git a/scripts/check-dependencies.js b/scripts/check-dependencies.js new file mode 100644 index 0000000..01222c3 --- /dev/null +++ b/scripts/check-dependencies.js @@ -0,0 +1,129 @@ +'use strict' + +const path = require('node:path') +const { execFileSync } = require('node:child_process') + +const repositoryRoot = path.join(__dirname, '..') +const packageJson = require('../packages/libdatadog/package.json') +const trees = [ + ...packageJson.napi.targets.map(target => ({ package: 'libdatadog', target })), + { package: 'libdatadog-wasm', target: 'wasm32-unknown-unknown' }, +] + +function parseCargoTree (output) { + const paths = [] + const stack = [] + + for (const line of output.split('\n')) { + const match = line.match(/^(\d+)(\S+) v(\S+)/) + if (!match) continue + + const [, depthValue, name, version] = match + const depth = Number(depthValue) + stack.length = depth + stack[depth] = name + paths.push({ depth, name, path: [...stack], version }) + } + + return paths +} + +function findDuplicateVersions (dependencies) { + const versionsByPackage = new Map() + const failures = [] + + for (const { name, version } of dependencies) { + const versions = versionsByPackage.get(name) ?? new Set() + versions.add(version) + versionsByPackage.set(name, versions) + } + + for (const [name, versions] of versionsByPackage) { + if (versions.size > 1) { + failures.push({ name, versions: [...versions] }) + } + } + return failures +} + +function findForbiddenDependencies (dependencies, tree) { + const failures = [] + + for (const dependency of dependencies) { + const isTokio = dependency.name === 'tokio' + const isTokioCompanion = dependency.name.startsWith('tokio-') + if (!isTokio && !isTokioCompanion) continue + + const parent = dependency.path.at(-2) + const allowedNapiBridge = tree.package === 'libdatadog' + && isTokio + && parent === 'napi' + if (!allowedNapiBridge) failures.push(dependency) + } + + return failures +} + +function checkTrees () { + const duplicateFailures = [] + const forbiddenFailures = [] + + for (const tree of trees) { + const output = execFileSync('cargo', [ + 'tree', + '--locked', + '--package', tree.package, + '--target', tree.target, + '--edges', 'normal,build', + '--prefix', 'depth', + '--format', '{p}', + ], { + cwd: repositoryRoot, + encoding: 'utf8', + }) + const dependencies = parseCargoTree(output) + + for (const failure of findDuplicateVersions(dependencies)) { + duplicateFailures.push({ ...failure, ...tree }) + } + for (const failure of findForbiddenDependencies(dependencies, tree)) { + forbiddenFailures.push({ ...failure, ...tree }) + } + } + + if (duplicateFailures.length > 0) { + console.error('Dependencies with multiple versions found:') + for (const failure of duplicateFailures) { + console.error( + `- ${failure.package} (${failure.target}): ` + + `${failure.name} ${failure.versions.join(', ')}`, + ) + } + } else { + console.log(`No dependency version duplicates found in ${trees.length} artifact trees.`) + } + + if (forbiddenFailures.length > 0) { + console.error('Forbidden Tokio dependencies found:') + for (const failure of forbiddenFailures) { + console.error( + `- ${failure.package} (${failure.target}): ` + + `${failure.name} through ${failure.path.join(' -> ')}`, + ) + } + } else { + console.log('Tokio is limited to the NAPI async bridge and absent from WASM.') + } + + if (duplicateFailures.length > 0 || forbiddenFailures.length > 0) { + process.exitCode = 1 + } +} + +if (require.main === module) checkTrees() + +module.exports = { + findDuplicateVersions, + findForbiddenDependencies, + parseCargoTree, +} diff --git a/test/dependencies.js b/test/dependencies.js new file mode 100644 index 0000000..91a7c09 --- /dev/null +++ b/test/dependencies.js @@ -0,0 +1,62 @@ +'use strict' + +const assert = require('node:assert/strict') +const { test } = require('node:test') + +const { + findDuplicateVersions, + findForbiddenDependencies, + parseCargoTree, +} = require('../scripts/check-dependencies') + +test('dependency validation allows Tokio only below the NAPI async bridge', () => { + const dependencies = parseCargoTree([ + '0libdatadog v0.1.0', + '1napi v3.12.1', + '2tokio v1.52.1', + ].join('\n')) + + assert.deepStrictEqual( + findForbiddenDependencies(dependencies, { package: 'libdatadog' }), + [], + ) +}) + +test('dependency validation rejects Tokio outside the NAPI async bridge', () => { + const dependencies = parseCargoTree([ + '0libdatadog v0.1.0', + '1tokio v1.52.1', + '1napi v3.12.1', + '2tokio-util v0.7.18', + ].join('\n')) + const failures = findForbiddenDependencies(dependencies, { package: 'libdatadog' }) + + assert.deepStrictEqual(failures.map(({ name }) => name), ['tokio', 'tokio-util']) +}) + +test('dependency validation rejects all Tokio packages in WASM', () => { + const dependencies = parseCargoTree([ + '0libdatadog-wasm v0.1.0', + '1napi v3.12.1', + '2tokio v1.52.1', + ].join('\n')) + + assert.strictEqual( + findForbiddenDependencies(dependencies, { package: 'libdatadog-wasm' }).length, + 1, + ) +}) + +test('dependency validation still finds multiple versions in one artifact tree', () => { + const dependencies = parseCargoTree([ + '0libdatadog v0.1.0', + '1bytes v1.10.0', + '1dependency v1.0.0', + '2bytes v1.11.0', + ].join('\n')) + + assert.deepStrictEqual(findDuplicateVersions(dependencies), [{ + name: 'bytes', + versions: ['1.10.0', '1.11.0'], + }]) +}) From 1fa777e6851f374e037e898cff858e9849d647a4 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Mon, 24 Aug 2026 20:17:12 -0400 Subject: [PATCH 2/2] ci(libdatadog): fix wasm report artifact path --- .../libdatadog/scripts/report-wasm-size.js | 2 +- packages/libdatadog/test/size-report.test.js | 28 +++++++++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/libdatadog/scripts/report-wasm-size.js b/packages/libdatadog/scripts/report-wasm-size.js index aadf229..16b5294 100644 --- a/packages/libdatadog/scripts/report-wasm-size.js +++ b/packages/libdatadog/scripts/report-wasm-size.js @@ -349,7 +349,7 @@ function createReport (gluePath, profilePath) { } if (require.main === module) { - const gluePath = path.join(__dirname, '..', 'dist', 'wasm', 'libdatadog_wasm.js') + const gluePath = path.join(__dirname, '..', 'wasm', 'dist', 'libdatadog_wasm.js') const profilePath = process.argv[2] && path.resolve(process.argv[2]) const report = createReport(gluePath, profilePath) console.log(report) diff --git a/packages/libdatadog/test/size-report.test.js b/packages/libdatadog/test/size-report.test.js index 7917798..86ba482 100644 --- a/packages/libdatadog/test/size-report.test.js +++ b/packages/libdatadog/test/size-report.test.js @@ -1,6 +1,8 @@ 'use strict' const assert = require('node:assert/strict') +const { spawnSync } = require('node:child_process') +const fs = require('node:fs') const path = require('node:path') const test = require('node:test') @@ -16,8 +18,23 @@ const { parseElfSections, } = require('../scripts/report-napi-size') +const nativeDirectory = path.join(__dirname, '..', 'dist', 'native') +const nativeArtifact = fs.existsSync(nativeDirectory) + ? fs.readdirSync(nativeDirectory) + .find(file => /^libdatadog\..+\.node$/.test(file)) + : undefined + +function napiReportSkipReason () { + if (!nativeArtifact) return 'a native artifact is not installed' + + const size = spawnSync('size', ['--version'], { stdio: 'ignore' }) + if (size.error?.code === 'ENOENT') return 'the size command is not installed' + + return false +} + test('reports inline packaging and WASM section sizes', () => { - const gluePath = path.join(__dirname, '..', 'dist', 'wasm', 'libdatadog_wasm.js') + const gluePath = path.join(__dirname, '..', 'wasm', 'dist', 'libdatadog_wasm.js') const report = createWasmReport(gluePath) assert.match(report, /Raw WASM \(before Brotli\)/) @@ -27,11 +44,10 @@ test('reports inline packaging and WASM section sizes', () => { assert.match(report, /\| data \|/) }) -test('reports N-API artifact, section, and crate sizes', () => { - const nativeDirectory = path.join(__dirname, '..', 'dist', 'native') - const artifact = require('node:fs').readdirSync(nativeDirectory) - .find(file => /^libdatadog\..+\.node$/.test(file)) - const report = createNapiReport(path.join(nativeDirectory, artifact), { +test('reports N-API artifact, section, and crate sizes', { + skip: napiReportSkipReason(), +}, () => { + const report = createNapiReport(path.join(nativeDirectory, nativeArtifact), { 'text-section-size': 7000, 'crates': [ { name: 'libdatadog', size: 2500 },