feat(libdatadog): add dedicated remote config WASM

README.md

@@ -21,6 +21,9 @@ Zstandard compression, DDSketch, and the agentless data pipeline use a
 wasm-bindgen backend whose WebAssembly bytes are embedded in JavaScript. No raw
 `.wasm` asset or native extension is published.
 
+Remote configuration is available from `@datadog/libdatadog/remote-config`.
+It uses a dedicated wasm-bindgen artifact that loads only with this entry point.
+
 See the [package README](packages/libdatadog/README.md) for implementation and
 packaging details.
 

eslint.config.js

@@ -90,6 +90,7 @@ module.exports = [
     // generated during its build; root lint cannot resolve them.
     files: [
       'packages/libdatadog/lib/wasm.js',
+      'packages/libdatadog/remote-config.js',
       'packages/libdatadog/test/bundlers.test.js',
       'packages/libdatadog/test/exporter.test.js',
     ],

packages/libdatadog/README.md

@@ -10,10 +10,14 @@ DDSketch, and the agentless data pipeline are maintained in the root
 `crates/libdatadog-wasm` workspace crate. Optional libdatadog functionality is
 published separately as `@datadog/libdatadog-extras`.
 
+Remote configuration is available from `@datadog/libdatadog/remote-config`.
+It uses a dedicated wasm-bindgen artifact that loads only with this entry point.
+
 The package accepts Datadog v0.4 MessagePack payloads and exports them to an
 agentless intake.
 
 The package uses a wasm-bindgen backend with the WebAssembly bytes embedded in
 JavaScript. The canonical inlined output is published as the regular
-`@datadog/libdatadog-wasm` dependency from the `wasm` workspace. No raw `.wasm`
-asset or native extension is published.
+`@datadog/libdatadog-wasm` dependency from the `wasm` workspace. The WASM
+package also contains the separate remote configuration artifact. No raw
+`.wasm` asset or native extension is published.

packages/libdatadog/package.json

@@ -18,12 +18,20 @@
       "import": "./wasm.mjs",
       "require": "./wasm.js",
       "default": "./wasm.js"
+    },
+    "./remote-config": {
+      "types": "./remote-config.d.ts",
+      "import": "./remote-config.js",
+      "require": "./remote-config.js",
+      "default": "./remote-config.js"
     }
   },
   "files": [
     "README.md",
     "index.js",
     "index.mjs",
+    "remote-config.js",
+    "remote-config.d.ts",
     "wasm.js",
     "wasm.mjs",
     "index.d.ts",
@@ -36,10 +44,12 @@
     "node": ">=18"
   },
   "scripts": {
-    "build:wasm": "npm run build:wasm:binary && npm run inline:wasm",
+    "build:wasm": "npm run build:wasm:binary && npm run build:remote-config:binary && npm run inline:wasm && npm run inline:remote-config",
     "build:wasm:binary": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm wasm/dist",
+    "build:remote-config:binary": "node ../../scripts/build-wasm.js ../../crates/remote_config wasm/dist/remote-config",
     "size:profile": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm ../../target/size --profiling",
-    "inline:wasm": "node scripts/inline-wasm.js",
+    "inline:wasm": "node scripts/inline-wasm.js libdatadog_wasm wasm/dist",
+    "inline:remote-config": "node scripts/inline-wasm.js remote_config wasm/dist/remote-config",
     "report:wasm-size": "node scripts/report-wasm-size.js",
     "build": "npm run build:wasm",
     "test": "node scripts/run-tests.js && npm run test:types",

packages/libdatadog/remote-config.d.ts

@@ -0,0 +1 @@
+export * from '@datadog/libdatadog-wasm/remote-config'

packages/libdatadog/remote-config.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('@datadog/libdatadog-wasm/remote-config')

packages/libdatadog/scripts/inline-wasm.js

@@ -4,9 +4,15 @@ const fs = require('node:fs')
 const path = require('node:path')
 const { constants, brotliCompressSync } = require('node:zlib')
 
-const outputDirectory = path.join(__dirname, '..', 'wasm', 'dist')
-const gluePath = path.join(outputDirectory, 'libdatadog_wasm.js')
-const wasmPath = path.join(outputDirectory, 'libdatadog_wasm_bg.wasm')
+const [moduleName, relativeOutputDirectory] = process.argv.slice(2)
+
+if (!moduleName || !relativeOutputDirectory) {
+  throw new Error('usage: node scripts/inline-wasm.js <module-name> <output-directory>')
+}
+
+const outputDirectory = path.join(__dirname, '..', relativeOutputDirectory)
+const gluePath = path.join(outputDirectory, `${moduleName}.js`)
+const wasmPath = path.join(outputDirectory, `${moduleName}_bg.wasm`)
 const glue = fs.readFileSync(gluePath, 'utf8')
 const wasm = fs.readFileSync(wasmPath)
 const encodedWasm = brotliCompressSync(wasm, {
@@ -15,7 +21,7 @@ const encodedWasm = brotliCompressSync(wasm, {
   },
 }).toString('base64')
 const loader = [
-  'const wasmPath = `${__dirname}/libdatadog_wasm_bg.wasm`;',
+  `const wasmPath = \`\${__dirname}/${moduleName}_bg.wasm\`;`,
   'const wasmBytes = require(\'fs\').readFileSync(wasmPath);',
 ].join('\n')
 

packages/libdatadog/test/bundlers.test.js

@@ -5,48 +5,39 @@ const fs = require('node:fs')
 const os = require('node:os')
 const path = require('node:path')
 const { test } = require('node:test')
+const { promisify } = require('node:util')
 
 const esbuild = require('esbuild')
 const webpack = require('webpack')
 
 const packageRoot = path.join(__dirname, '..')
-const entry = path.join(packageRoot, 'index.js')
-
-test('esbuild bundles the package entry point without emitting an asset', async () => {
-  await assertBundle(async (output) => {
-    await esbuild.build({
-      bundle: true,
-      entryPoints: [entry],
-      outfile: output,
-      platform: 'node',
-    })
+const webpackAsync = promisify(webpack)
+const entries = new Map([
+  ['package', path.join(packageRoot, 'index.js')],
+  ['WASM', path.join(packageRoot, 'wasm.js')],
+  ['remote config', path.join(packageRoot, 'remote-config.js')],
+])
+
+for (const [name, entry] of entries) {
+  test(`esbuild bundles the ${name} entry point without emitting an asset`, async () => {
+    await assertBundle(entry, bundleWithEsbuild)
+  })
+
+  test(`webpack bundles the ${name} entry point without emitting an asset`, async () => {
+    await assertBundle(entry, bundleWithWebpack)
   })
-})
-
-test('webpack bundles the package entry point without emitting an asset', async () => {
-  await assertBundle(output => new Promise((resolve, reject) => {
-    webpack({
-      entry,
-      mode: 'production',
-      output: {
-        filename: path.basename(output),
-        path: path.dirname(output),
-      },
-      target: 'node',
-    }, (error, stats) => {
-      if (error) return reject(error)
-      if (stats.hasErrors()) return reject(new Error(stats.toString({ all: false, errors: true })))
-      resolve()
-    })
-  }))
-})
-
-async function assertBundle (bundle) {
+}
+
+/**
+ * @param {string} entry
+ * @param {(entry: string, output: string) => Promise<void>} bundle
+ */
+async function assertBundle (entry, bundle) {
   const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'libdatadog-bundle-'))
   const output = path.join(directory, 'bundle.cjs')
 
   try {
-    await bundle(output)
+    await bundle(entry, output)
     const files = fs.readdirSync(directory)
     const contents = fs.readFileSync(output, 'utf8')
 
@@ -56,3 +47,34 @@ async function assertBundle (bundle) {
     fs.rmSync(directory, { force: true, recursive: true })
   }
 }
+
+/**
+ * @param {string} entry
+ * @param {string} output
+ */
+async function bundleWithEsbuild (entry, output) {
+  await esbuild.build({
+    bundle: true,
+    entryPoints: [entry],
+    outfile: output,
+    platform: 'node',
+  })
+}
+
+/**
+ * @param {string} entry
+ * @param {string} output
+ */
+async function bundleWithWebpack (entry, output) {
+  const stats = await webpackAsync({
+    entry,
+    mode: 'production',
+    output: {
+      filename: path.basename(output),
+      path: path.dirname(output),
+    },
+    target: 'node',
+  })
+
+  if (stats.hasErrors()) throw new Error(stats.toString({ all: false, errors: true }))
+}

packages/libdatadog/test/package-contents.test.js

@@ -40,6 +40,13 @@ test('published packages contain only the intended artifacts', () => {
     `packages must not contain standalone WASM files: ${standaloneWasm.join(', ')}`)
   assert(wasmNames.includes('dist/libdatadog_wasm.js'),
     'WASM package must contain the inline-WASM JavaScript fallback')
+  assert(wasmNames.includes('dist/remote-config/remote_config.js'),
+    'WASM package must contain the dedicated remote config artifact')
+  assert.strictEqual(wasmNames.includes('remote-config.js'), false)
+  assert(wasmNames.includes('remote-config.d.ts'))
+  assert(names.includes('remote-config.js'))
+  assert.strictEqual(names.includes('remote-config.mjs'), false)
+  assert(names.includes('remote-config.d.ts'))
   assert.strictEqual(libdatadogWasm.name, '@datadog/libdatadog-wasm')
   assert.strictEqual(
     metapackageJson.dependencies['@datadog/libdatadog-wasm'],
@@ -92,7 +99,13 @@ test('installed package uses its WASM dependency', () => {
     const requireInstalled = createRequire(path.join(installRoot, 'package.json'))
     const libdatadog = requireInstalled('@datadog/libdatadog')
     const explicitWasm = requireInstalled('@datadog/libdatadog/wasm')
+    const directRemoteConfig = requireInstalled('@datadog/libdatadog-wasm/remote-config')
 
+    assert.strictEqual(libdatadog.RemoteConfigFetcher, undefined)
+    assert.strictEqual(explicitWasm.RemoteConfigFetcher, undefined)
+    const remoteConfig = requireInstalled('@datadog/libdatadog/remote-config')
+    assert.strictEqual(typeof remoteConfig.RemoteConfigFetcher, 'function')
+    assert.strictEqual(remoteConfig.RemoteConfigFetcher, directRemoteConfig.RemoteConfigFetcher)
     assert.strictEqual(libdatadog.backend(), 'wasm')
     assert.strictEqual(explicitWasm.backend(), 'wasm')
     assert(libdatadog.zstd_compress(Buffer.alloc(16), 3) instanceof Uint8Array)
@@ -122,7 +135,22 @@ function assertEsmImports (installRoot, environment) {
       DDSketch as WasmDDSketch,
       zstd_compress as wasmCompress,
     } from '@datadog/libdatadog/wasm'
-
+    import remoteConfig, {
+      RemoteConfigFetcher,
+      setStorage,
+    } from '@datadog/libdatadog/remote-config'
+    import directRemoteConfig, {
+      RemoteConfigFetcher as DirectRemoteConfigFetcher,
+      setStorage as setDirectStorage,
+    } from '@datadog/libdatadog-wasm/remote-config'
+
+    assert.strictEqual(libdatadog.RemoteConfigFetcher, undefined)
+    assert.strictEqual(wasm.RemoteConfigFetcher, undefined)
+    assert.strictEqual(remoteConfig.RemoteConfigFetcher, RemoteConfigFetcher)
+    assert.strictEqual(remoteConfig.setStorage, setStorage)
+    assert.strictEqual(directRemoteConfig.RemoteConfigFetcher, DirectRemoteConfigFetcher)
+    assert.strictEqual(directRemoteConfig.setStorage, setDirectStorage)
+    assert.strictEqual(RemoteConfigFetcher, DirectRemoteConfigFetcher)
     assert.strictEqual(backend(), 'wasm')
     assert.strictEqual(libdatadog.backend, backend)
     assert.strictEqual(libdatadog.createAgentlessExporter, createAgentlessExporter)

packages/libdatadog/test/package.test.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const assert = require('node:assert/strict')
+const { spawnSync } = require('node:child_process')
 const fs = require('node:fs')
 const path = require('node:path')
 const { brotliDecompressSync } = require('node:zlib')
@@ -14,6 +15,16 @@ test('publishes the universal libdatadog package', () => {
 
   assert.strictEqual(packageJson.name, '@datadog/libdatadog')
   assert.strictEqual(packageJson.exports['./wasm'].require, './wasm.js')
+  assert.strictEqual(packageJson.exports['./remote-config'].import, './remote-config.js')
+  assert.strictEqual(packageJson.exports['./remote-config'].require, './remote-config.js')
+
+  const wasmPackageJson = JSON.parse(fs.readFileSync(
+    path.join(packageRoot, 'wasm', 'package.json'),
+  ))
+  assert.strictEqual(
+    wasmPackageJson.exports['./remote-config'].require,
+    './dist/remote-config/remote_config.js',
+  )
 })
 
 test('uses the libdatadog release version', () => {
@@ -38,17 +49,82 @@ test('root entry point uses the WASM backend', () => {
 })
 
 test('embeds a Brotli-compressed WASM fallback below the size budgets', () => {
-  const gluePath = path.join(
+  const { glue, wasm } = readInlineWasm(path.join(
     packageRoot,
     'wasm',
     'dist',
     'libdatadog_wasm.js',
-  )
+  ))
+
+  assert.ok(Buffer.byteLength(glue) < 200 * 1024)
+  assert.ok(wasm.length < 500 * 1024)
+})
+
+test('embeds dedicated remote config WASM below the size budgets', () => {
+  const { glue, wasm } = readInlineWasm(path.join(
+    packageRoot,
+    'wasm',
+    'dist',
+    'remote-config',
+    'remote_config.js',
+  ))
+
+  assert.ok(Buffer.byteLength(glue) < 450 * 1024)
+  assert.ok(wasm.length < 1024 * 1024)
+})
+
+test('requires an artifact name and output directory when inlining WASM', () => {
+  const script = path.join(packageRoot, 'scripts', 'inline-wasm.js')
+
+  for (const scriptArguments of [[], ['fixture']]) {
+    const result = spawnSync(process.execPath, [script, ...scriptArguments])
+
+    assert.notStrictEqual(result.status, 0)
+    assert.match(result.stderr.toString(), /usage: node scripts\/inline-wasm\.js/)
+  }
+})
+
+test('inlines a named WASM artifact into its generated module', () => {
+  const outputDirectory = fs.mkdtempSync(path.join(packageRoot, '.inline-wasm-'))
+  const moduleName = 'fixture'
+  const wasm = Buffer.from('fixture WASM')
+  const loader = [
+    `const wasmPath = \`\${__dirname}/${moduleName}_bg.wasm\`;`,
+    'const wasmBytes = require(\'fs\').readFileSync(wasmPath);',
+  ].join('\n')
+
+  try {
+    fs.writeFileSync(path.join(outputDirectory, `${moduleName}.js`), loader)
+    fs.writeFileSync(path.join(outputDirectory, `${moduleName}_bg.wasm`), wasm)
+    fs.writeFileSync(path.join(outputDirectory, '.gitignore'), '')
+    fs.writeFileSync(path.join(outputDirectory, 'package.json'), '{}')
+
+    const result = spawnSync(process.execPath, [
+      path.join(packageRoot, 'scripts', 'inline-wasm.js'),
+      moduleName,
+      path.relative(packageRoot, outputDirectory),
+    ])
+
+    assert.strictEqual(result.status, 0, result.stderr.toString())
+    assert.deepStrictEqual(readInlineWasm(path.join(outputDirectory, `${moduleName}.js`)).wasm, wasm)
+    assert.strictEqual(fs.existsSync(path.join(outputDirectory, `${moduleName}_bg.wasm`)), false)
+    assert.strictEqual(fs.existsSync(path.join(outputDirectory, '.gitignore')), false)
+    assert.strictEqual(fs.existsSync(path.join(outputDirectory, 'package.json')), false)
+  } finally {
+    fs.rmSync(outputDirectory, { force: true, recursive: true })
+  }
+})
+
+/**
+ * @param {string} gluePath
+ */
+function readInlineWasm (gluePath) {
   const glue = fs.readFileSync(gluePath, 'utf8')
   const encodedWasm = glue.match(/brotliDecompressSync\(Buffer\.from\('([^']+)', 'base64'\)\)/)?.[1]
 
   assert.ok(encodedWasm, 'WASM must be embedded as a Brotli-compressed base64 string')
-  const compressedWasm = Buffer.from(encodedWasm, 'base64')
-  assert.ok(Buffer.byteLength(glue) < 200 * 1024)
-  assert.ok(brotliDecompressSync(compressedWasm).length < 500 * 1024)
-})
+  return {
+    glue,
+    wasm: brotliDecompressSync(Buffer.from(encodedWasm, 'base64')),
+  }
+}