perf(wasm): publish compressed binaries outside JS glue

.github/workflows/build.yml

@@ -53,7 +53,9 @@ jobs:
           name: libdatadog-wasm-size-${{ matrix.revision }}
           path: |
             packages/libdatadog/wasm/dist/libdatadog_wasm.js
+            packages/libdatadog/wasm/dist/libdatadog_wasm_bg.wasm.br
             packages/libdatadog/wasm/dist/remote-config/remote_config.js
+            packages/libdatadog/wasm/dist/remote-config/remote_config_bg.wasm.br
             target/size/libdatadog-wasm/libdatadog_wasm_bg.wasm
             target/size/remote-config/remote_config_bg.wasm
           if-no-files-found: error

packages/libdatadog/README.md

@@ -29,8 +29,9 @@ the package version.
 `createAgentlessExporter(options, { agent })` accepts an optional borrowed
 Node.js HTTP agent. The caller owns the agent and its lifetime.
 
-The package uses a wasm-bindgen backend with the WebAssembly bytes embedded in
-JavaScript. The canonical inlined output is published as the regular
-`@datadog/libdatadog-wasm` dependency from the `wasm` workspace. The WASM
-package also contains the separate remote configuration artifact. No raw
+The package publishes Brotli-compressed `.wasm.br` files next to the
+wasm-bindgen JavaScript loaders. Each loader reads and decompresses its file
+synchronously before instantiation. The read expression carries a marker that
+allows build tools to inline the compressed bytes. Other bundled distributions
+must copy the referenced asset next to their output JavaScript file. No raw
 `.wasm` asset or native extension is published.

packages/libdatadog/package.json

@@ -53,15 +53,15 @@
     "node": ">=18"
   },
   "scripts": {
-    "benchmark:agentless": "npm run build:wasm:binary && npm run inline:wasm && node benchmark/agentless-pipeline.js",
-    "build:wasm": "npm run build:wasm:binary && npm run build:remote-config:binary && npm run inline:wasm && npm run inline:remote-config",
+    "benchmark:agentless": "npm run build:wasm:binary && npm run compress:wasm && node benchmark/agentless-pipeline.js",
+    "build:wasm": "npm run build:wasm:binary && npm run build:remote-config:binary && npm run compress:wasm && npm run compress:remote-config",
     "build:wasm:binary": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm wasm/dist",
     "build:remote-config:binary": "node ../../scripts/build-wasm.js ../../crates/remote_config wasm/dist/remote-config",
     "size:profile": "npm run size:profile:wasm && npm run size:profile:remote-config",
     "size:profile:wasm": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm ../../target/size/libdatadog-wasm --profiling",
     "size:profile:remote-config": "node ../../scripts/build-wasm.js ../../crates/remote_config ../../target/size/remote-config --profiling",
-    "inline:wasm": "node scripts/inline-wasm.js libdatadog_wasm wasm/dist",
-    "inline:remote-config": "node scripts/inline-wasm.js remote_config wasm/dist/remote-config",
+    "compress:wasm": "node scripts/compress-wasm.js libdatadog_wasm wasm/dist",
+    "compress:remote-config": "node scripts/compress-wasm.js remote_config wasm/dist/remote-config",
     "report:wasm-size": "node scripts/report-wasm-size.js",
     "build": "npm run build:wasm",
     "test": "node scripts/run-tests.js && npm run test:types",

packages/libdatadog/scripts/inline-wasm.js → packages/libdatadog/scripts/compress-wasm.js

@@ -7,35 +7,41 @@ const { constants, brotliCompressSync } = require('node:zlib')
 const [moduleName, relativeOutputDirectory] = process.argv.slice(2)
 
 if (!moduleName || !relativeOutputDirectory) {
-  throw new Error('usage: node scripts/inline-wasm.js <module-name> <output-directory>')
+  throw new Error('usage: node scripts/compress-wasm.js <module-name> <output-directory>')
 }
 
 const outputDirectory = path.join(__dirname, '..', relativeOutputDirectory)
 const gluePath = path.join(outputDirectory, `${moduleName}.js`)
 const wasmPath = path.join(outputDirectory, `${moduleName}_bg.wasm`)
+const compressedWasmPath = `${wasmPath}.br`
 const glue = fs.readFileSync(gluePath, 'utf8')
 const wasm = fs.readFileSync(wasmPath)
-const encodedWasm = brotliCompressSync(wasm, {
+const compressedWasm = brotliCompressSync(wasm, {
   params: {
     [constants.BROTLI_PARAM_QUALITY]: 11,
   },
-}).toString('base64')
+})
 const loader = [
   `const wasmPath = \`\${__dirname}/${moduleName}_bg.wasm\`;`,
   'const wasmBytes = require(\'fs\').readFileSync(wasmPath);',
 ].join('\n')
 
 if (!glue.includes(loader)) {
-  throw new Error('wasm-bindgen loader changed; refusing to publish an external WASM asset')
+  throw new Error('wasm-bindgen loader changed; refusing to publish a compressed WASM asset')
 }
 
 fs.writeFileSync(
   gluePath,
   glue.replace(
     loader,
-    () => `const wasmBytes = require('node:zlib').brotliDecompressSync(Buffer.from('${encodedWasm}', 'base64'));`,
+    () => [
+      'const compressedWasm = /* @datadog/wasm-asset */ '
+      + `require('node:fs').readFileSync(\`\${__dirname}/${moduleName}_bg.wasm.br\`);`,
+      'const wasmBytes = require(\'node:zlib\').brotliDecompressSync(compressedWasm);',
+    ].join('\n'),
   ),
 )
+fs.writeFileSync(compressedWasmPath, compressedWasm)
 fs.rmSync(wasmPath)
 fs.rmSync(path.join(outputDirectory, '.gitignore'), { force: true })
 fs.rmSync(path.join(outputDirectory, 'package.json'), { force: true })

packages/libdatadog/scripts/report-wasm-size.js

@@ -43,14 +43,14 @@ const artifacts = [
     comparisonGluePath: 'packages/libdatadog/wasm/dist/libdatadog_wasm.js',
     gluePath: path.join(__dirname, '..', 'wasm', 'dist', 'libdatadog_wasm.js'),
     name: 'libdatadog',
-    maximumInlineBytes: 240 * 1024,
+    maximumPackagedBytes: 240 * 1024,
     profilePath: 'target/size/libdatadog-wasm/libdatadog_wasm_bg.wasm',
   },
   {
     comparisonGluePath: 'packages/libdatadog/wasm/dist/remote-config/remote_config.js',
     gluePath: path.join(__dirname, '..', 'wasm', 'dist', 'remote-config', 'remote_config.js'),
     name: 'remote config',
-    maximumInlineBytes: 330 * 1024,
+    maximumPackagedBytes: 330 * 1024,
     profilePath: 'target/size/remote-config/remote_config_bg.wasm',
   },
 ]
@@ -75,7 +75,7 @@ function validateWasm (wasm) {
     0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00,
   ])
   if (wasm.length < expectedHeader.length || !wasm.subarray(0, 8).equals(expectedHeader)) {
-    throw new Error('inline payload is not a WebAssembly 1 binary')
+    throw new Error('payload is not a WebAssembly 1 binary')
   }
 }
 
@@ -362,12 +362,18 @@ function appendCrateReport (lines, crateSizes) {
   lines.push('', attributionNote)
 }
 
+/** @param {string} gluePath */
+function getCompressedPath (gluePath) {
+  return `${gluePath.slice(0, -3)}_bg.wasm.br`
+}
+
 /**
  * @param {string} gluePath
  * @param {string | undefined} profilePath
  * @param {string} artifactName
  * @returns {{
  *   artifactName: string,
+ *   compressedPath: string,
  *   crateSizes: { entries: Array<{ bytes: number, name: string }>, totalBytes: number } | undefined,
  *   gluePath: string,
  *   layers: Array<{ bytes: number, name: string }>,
@@ -376,24 +382,34 @@ function appendCrateReport (lines, crateSizes) {
  */
 function readArtifactSizes (gluePath, profilePath, artifactName) {
   const glue = fs.readFileSync(gluePath, 'utf8')
-  const match = glue.match(/Buffer\.from\('([A-Za-z0-9+/=]+)', 'base64'\)/)
-  if (!match) throw new Error('could not find the inline base64 WASM payload')
-
-  const base64Bytes = Buffer.byteLength(match[1])
-  const compressed = Buffer.from(match[1], 'base64')
+  const compressedPath = getCompressedPath(gluePath)
+  const hasCompressedAsset = fs.existsSync(compressedPath)
+  const encodedWasm = hasCompressedAsset
+    ? undefined
+    : glue.match(/Buffer\.from\('([A-Za-z0-9+/=]+)', 'base64'\)/)?.[1]
+  if (!hasCompressedAsset && !encodedWasm) throw new Error('could not find the compressed WASM payload')
+
+  const compressed = hasCompressedAsset ? fs.readFileSync(compressedPath) : Buffer.from(encodedWasm, 'base64')
   const wasm = brotliDecompressSync(compressed)
+  const glueBytes = Buffer.byteLength(glue) - (encodedWasm ? Buffer.byteLength(encodedWasm) : 0)
+  const layers = [
+    { bytes: wasm.length, name: 'Raw WASM (before Brotli)' },
+    { bytes: compressed.length, name: 'Brotli-compressed WASM' },
+  ]
+  if (encodedWasm) {
+    layers.push({ bytes: Buffer.byteLength(encodedWasm) - compressed.length, name: 'Base64 encoding overhead' })
+  }
+  layers.push(
+    { bytes: glueBytes, name: 'JavaScript glue/loader' },
+    { bytes: Buffer.byteLength(glue) + (hasCompressedAsset ? compressed.length : 0), name: 'Final packaged artifacts' },
+  )
 
   return {
     artifactName,
+    compressedPath,
     crateSizes: profilePath ? readCrateSizes(fs.readFileSync(profilePath)) : undefined,
     gluePath,
-    layers: [
-      { bytes: wasm.length, name: 'Raw WASM (before Brotli)' },
-      { bytes: compressed.length, name: 'Brotli-compressed WASM' },
-      { bytes: base64Bytes - compressed.length, name: 'Base64 encoding overhead' },
-      { bytes: Buffer.byteLength(glue) - base64Bytes, name: 'JavaScript glue/loader' },
-      { bytes: Buffer.byteLength(glue), name: 'Final inlined JavaScript' },
-    ],
+    layers,
     sections: readSections(wasm),
   }
 }
@@ -536,19 +552,19 @@ function createComparisonReport (beforeRoot, afterRoot, artifact) {
   const crates = orderChangedFirst(alignCrates(before.crateSizes, after.crateSizes))
   const entries = [...layers, ...sections, ...crates]
   const changedCount = entries.filter(entry => entry.afterBytes !== entry.beforeBytes).length
-  const finalArtifact = layers.find(entry => entry.name === 'Final inlined JavaScript')
+  const finalArtifact = layers.find(entry => entry.name === 'Final packaged artifacts')
   const lines = [
     '<details>',
     `<summary>${artifact.name}: ${changedCount} changed, ${entries.length - changedCount} unchanged</summary>`,
     '',
-    '### Inline artifact layers',
+    '### Packaged artifact layers',
     '',
   ]
   appendComparisonTable(
     lines,
-    'Inline artifact layer',
+    'Packaged artifact layer',
     layers,
-    'Final inlined JavaScript',
+    'Final packaged artifacts',
   )
   lines.push('', '### Raw WebAssembly sections', '')
   appendComparisonTable(lines, 'Section', sections)
@@ -604,14 +620,14 @@ function createComparisonSummary (comparisons) {
  * @param {string} [artifactName]
  */
 function createReport (gluePath, profilePath, artifactName = 'libdatadog') {
-  const { crateSizes, layers, sections } = readArtifactSizes(gluePath, profilePath, artifactName)
+  const { compressedPath, crateSizes, layers, sections } = readArtifactSizes(gluePath, profilePath, artifactName)
   const lines = [
     `## ${artifactName} WASM size`,
     '',
-    '| Inline artifact layer | Bytes | KiB |',
+    '| Packaged artifact layer | Bytes | KiB |',
     '| --- | ---: | ---: |',
   ]
-  for (const layer of layers) lines.push(layerRow(layer.name, layer.bytes, layer.name === 'Final inlined JavaScript'))
+  for (const layer of layers) lines.push(layerRow(layer.name, layer.bytes, layer.name === 'Final packaged artifacts'))
   lines.push('', '### Raw WebAssembly sections', '', '| Section | Bytes | KiB | Share |', '| --- | ---: | ---: | ---: |')
 
   for (const section of sections) {
@@ -624,18 +640,22 @@ function createReport (gluePath, profilePath, artifactName = 'libdatadog') {
 
   if (crateSizes) appendCrateReport(lines, crateSizes)
 
-  lines.push('', `Generated from \`${path.relative(process.cwd(), gluePath)}\`.`)
+  lines.push(
+    '',
+    `Generated from \`${path.relative(process.cwd(), gluePath)}\` and `
+    + `\`${path.relative(process.cwd(), compressedPath)}\`.`,
+  )
   return lines.join('\n')
 }
 
 /**
  * @param {string} artifactName
- * @param {number} inlineBytes
- * @param {number} maximumInlineBytes
+ * @param {number} packagedBytes
+ * @param {number} maximumPackagedBytes
  */
-function getSizeBudgetFailure (artifactName, inlineBytes, maximumInlineBytes) {
-  if (inlineBytes <= maximumInlineBytes) return
-  return `${artifactName}: ${formatBytes(inlineBytes)} bytes exceeds ${formatBytes(maximumInlineBytes)} bytes`
+function getSizeBudgetFailure (artifactName, packagedBytes, maximumPackagedBytes) {
+  if (packagedBytes <= maximumPackagedBytes) return
+  return `${artifactName}: ${formatBytes(packagedBytes)} bytes exceeds ${formatBytes(maximumPackagedBytes)} bytes`
 }
 
 /** @param {string[]} reports */
@@ -665,13 +685,14 @@ function main () {
   const reports = []
 
   for (const [index, artifact] of artifacts.entries()) {
-    const { gluePath, maximumInlineBytes, name } = artifact
+    const { gluePath, maximumPackagedBytes, name } = artifact
     const profilePath = profilePaths[index] && path.resolve(profilePaths[index])
     const report = createReport(gluePath, profilePath, name)
     reports.push(report)
 
-    const inlineBytes = fs.statSync(gluePath).size
-    const budgetFailure = getSizeBudgetFailure(name, inlineBytes, maximumInlineBytes)
+    const compressedPath = getCompressedPath(gluePath)
+    const packagedBytes = fs.statSync(gluePath).size + fs.statSync(compressedPath).size
+    const budgetFailure = getSizeBudgetFailure(name, packagedBytes, maximumPackagedBytes)
     if (budgetFailure) failures.push(budgetFailure)
     if (!profilePath) continue
 

packages/libdatadog/test/bundlers.test.js

@@ -13,36 +13,52 @@ const webpack = require('webpack')
 const packageRoot = path.join(__dirname, '..')
 const webpackAsync = promisify(webpack)
 const entries = new Map([
-  ['package', path.join(packageRoot, 'index.js')],
-  ['WASM', path.join(packageRoot, 'wasm.js')],
-  ['remote config', path.join(packageRoot, 'remote-config.js')],
+  ['package', {
+    asset: path.join(packageRoot, 'wasm', 'dist', 'libdatadog_wasm_bg.wasm.br'),
+    entry: path.join(packageRoot, 'index.js'),
+  }],
+  ['WASM', {
+    asset: path.join(packageRoot, 'wasm', 'dist', 'libdatadog_wasm_bg.wasm.br'),
+    entry: path.join(packageRoot, 'wasm.js'),
+  }],
+  ['remote config', {
+    asset: path.join(packageRoot, 'wasm', 'dist', 'remote-config', 'remote_config_bg.wasm.br'),
+    entry: path.join(packageRoot, 'remote-config.js'),
+  }],
 ])
 
-for (const [name, entry] of entries) {
-  test(`esbuild bundles the ${name} entry point without emitting an asset`, async () => {
-    await assertBundle(entry, bundleWithEsbuild)
+for (const [name, { asset, entry }] of entries) {
+  test(`esbuild bundles the ${name} entry point with its compressed WASM asset`, async () => {
+    await assertBundle(entry, asset, bundleWithEsbuild, name !== 'remote config')
   })
 
-  test(`webpack bundles the ${name} entry point without emitting an asset`, async () => {
-    await assertBundle(entry, bundleWithWebpack)
+  test(`webpack bundles the ${name} entry point with its compressed WASM asset`, async () => {
+    await assertBundle(entry, asset, bundleWithWebpack, true)
   })
 }
 
 /**
  * @param {string} entry
+ * @param {string} asset
  * @param {(entry: string, output: string) => Promise<void>} bundle
+ * @param {boolean} loadBundle
  */
-async function assertBundle (entry, bundle) {
+async function assertBundle (entry, asset, bundle, loadBundle) {
   const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'libdatadog-bundle-'))
   const output = path.join(directory, 'bundle.cjs')
+  const copiedAsset = path.join(directory, path.basename(asset))
 
   try {
     await bundle(entry, output)
+    fs.copyFileSync(asset, copiedAsset)
     const files = fs.readdirSync(directory)
     const contents = fs.readFileSync(output, 'utf8')
 
-    assert.deepStrictEqual(files, ['bundle.cjs'])
-    assert.doesNotMatch(contents, /\.wasm(?:['"`)]|$)/m)
+    assert.equal(files.length, 2)
+    assert(files.includes('bundle.cjs'))
+    assert(files.includes(path.basename(asset)))
+    assert.match(contents, /\.wasm\.br/)
+    if (loadBundle) require(output)
   } finally {
     fs.rmSync(directory, { force: true, recursive: true })
   }

packages/libdatadog/test/package-contents.test.js

@@ -35,11 +35,16 @@ test('published packages contain only the intended artifacts', () => {
   const wasmNames = libdatadogWasm.files.map(file => file.path)
   const standaloneWasm = [...names, ...wasmNames]
     .filter(file => file.endsWith('.wasm'))
+  const compressedWasm = wasmNames.filter(file => file.endsWith('.wasm.br'))
 
   assert.deepStrictEqual(standaloneWasm, [],
-    `packages must not contain standalone WASM files: ${standaloneWasm.join(', ')}`)
+    `packages must not contain raw WASM files: ${standaloneWasm.join(', ')}`)
+  assert.deepStrictEqual(compressedWasm, [
+    'dist/libdatadog_wasm_bg.wasm.br',
+    'dist/remote-config/remote_config_bg.wasm.br',
+  ])
   assert(wasmNames.includes('dist/libdatadog_wasm.js'),
-    'WASM package must contain the inline-WASM JavaScript fallback')
+    'WASM package must contain the JavaScript loader')
   assert(wasmNames.includes('dist/remote-config/remote_config.js'),
     'WASM package must contain the dedicated remote config artifact')
   assert.strictEqual(wasmNames.includes('remote-config.js'), false)