feat!: carve the wasm pipeline into @datadog/libdatadog-wasm-pipeline

.github/actions/build-test-wasm/action.yaml

@@ -8,32 +8,32 @@ runs:
   using: 'composite'
   steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
       - name: Setup Node.js
         uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
       - run: yarn install
         shell: bash
       - name: Install wasm-pack
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
         shell: bash
+      # scripts/wasm-crates.js owns which package each crate is published from,
+      # so the output directory and the artifact name are both derived from it
+      # rather than repeated here.
       - name: Build WASM
+        id: build
         run: |
-          mkdir -p ./prebuilds/${{ inputs.crate }}
-          wasm-pack build --target nodejs ./crates/${{ inputs.crate }} --out-dir ../../prebuilds/${{ inputs.crate }}
+          node scripts/build-wasm.js '${{ inputs.crate }}'
+          echo "prebuilds=$(node scripts/wasm-crates.js --prebuilds-root '${{ inputs.crate }}')" >> "$GITHUB_OUTPUT"
+          echo "artifact=$(node scripts/wasm-crates.js --artifact-name '${{ inputs.crate }}')" >> "$GITHUB_OUTPUT"
         shell: bash
       - name: Test WASM
-        # The pipeline crate's tests are top-level node:test suites that need
-        # --test-force-exit (the wasm exporter keeps the event loop alive after
-        # a flush); the other wasm crates use plain test/wasm/<crate>/ scripts.
-        run: |
-          if [ "${{ inputs.crate }}" = "pipeline" ]; then
-            node --test --test-force-exit test/pipeline.js
-          else
-            node test-wasm.js ${{ inputs.crate }}
-          fi
+        run: bash scripts/test.sh wasm '${{ inputs.crate }}'
         shell: bash
+      # Crates that stay in @datadog/libdatadog are uploaded as `prebuilds-*`,
+      # which action-prebuildify merges into that package's `prebuilds` artifact.
+      # A carved-out crate must not match that pattern or it would be published
+      # by both packages; `wasm-crates.js` is what keeps the two apart.
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: prebuilds-wasm-${{ inputs.crate }}
-          if-no-files-found: ignore
-          path: ./prebuilds/*
+          name: ${{ steps.build.outputs.artifact }}
+          if-no-files-found: error
+          path: ${{ steps.build.outputs.prebuilds }}/*

.github/workflows/build.yml

@@ -17,6 +17,7 @@ jobs:
       - run: yarn install
         working-directory: test/crashtracker
       - run: yarn lint
+      - run: node scripts/check-versions.js
 
   build-test-wasm:
     runs-on: ubuntu-latest
@@ -45,6 +46,26 @@ jobs:
       # Need this, now that libdatadog packages libunwind as a submodule
       prebuild: '(command -v apk >/dev/null && apk add autoconf automake libtool) || (command -v apt-get >/dev/null && apt-get update && apt-get install -y autoconf automake libtool) || true'
 
+  # Asserts that each wasm crate is published by exactly the package that owns
+  # it and that only @datadog/libdatadog ships native binaries, which is
+  # otherwise a silent failure.
+  package-contents:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: prebuilds
+          path: prebuilds
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: wasm-pipeline
+          path: packages/wasm-pipeline/prebuilds
+      - name: Setup Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+      - run: node scripts/check-packages.js
+
   package-size:
     runs-on: ubuntu-latest
     needs: build

.github/workflows/release.yml

@@ -6,6 +6,14 @@ on:
       - v0.x
 
 jobs:
+  versions:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - name: Setup Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+      - run: node scripts/check-versions.js
+
   build-test-wasm:
     runs-on: ubuntu-latest
     strategy:
@@ -31,9 +39,32 @@ jobs:
       rust: true
       only: darwin-arm64,darwin-x64,linux-arm64,linux-x64
 
-  publish:
+  # Asserts that each wasm crate is published by exactly the package that owns
+  # it and that only @datadog/libdatadog ships native binaries, which is
+  # otherwise a silent failure.
+  package-contents:
     runs-on: ubuntu-latest
     needs: build
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: prebuilds
+          path: prebuilds
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: wasm-pipeline
+          path: packages/wasm-pipeline/prebuilds
+      - name: Setup Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+      - run: node scripts/check-packages.js
+
+  publish:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+      - package-contents
+      - versions
     environment: npm
     permissions:
       id-token: write  # Required for OIDC
@@ -66,3 +97,26 @@ jobs:
       - run: |
           git tag v${{ fromJson(steps.pkg.outputs.json).version }}
           git push https://x-access-token:${{ steps.octo-sts.outputs.token }}@github.com/${{ github.repository }}.git v${{ fromJson(steps.pkg.outputs.json).version }}
+
+  # This package has no per-platform build, so it only needs the artifact from
+  # build-test-wasm. It publishes after @datadog/libdatadog so that a failed
+  # native build never leaves half of a lockstep pair on npm.
+  publish-wasm-pipeline:
+    runs-on: ubuntu-latest
+    needs: publish
+    environment: npm
+    permissions:
+      id-token: write  # Required for OIDC
+      contents: read
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: wasm-pipeline
+          path: packages/wasm-pipeline/prebuilds
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: '24'
+          registry-url: 'https://registry.npmjs.org'
+      - run: npm publish
+        working-directory: packages/wasm-pipeline

DEVELOPMENT.md

@@ -40,10 +40,29 @@ The project compiles Rust for both native Node.js addons and WebAssembly. Use [r
 
 ## Building
 
-* `yarn build`: Build the default workspaces in debug mode.
+The repository publishes two packages from the same set of crates:
+`@datadog/libdatadog` (the native bindings plus most WASM modules, from the
+repository root) and `@datadog/libdatadog-wasm-pipeline` (the `pipeline` crate,
+from `packages/wasm-pipeline`). `scripts/wasm-crates.js` is the single source of
+truth for which package publishes which crate.
+
+* `yarn build`: Build the default workspaces in debug mode, then the WASM modules.
 * `yarn build-release`: Build the default workspaces in release mode.
 * `yarn build-all`: Build all workspaces in debug mode. This is useful when working on a workspace that is not a default member yet.
+* `yarn build-wasm`: Build every WASM module. To build just one, run `node scripts/build-wasm.js <crate>`.
+
+Native artifacts land in `build/Release/`. WASM modules land in
+`prebuilds/<crate>/`, except the pipeline module, which belongs to its own
+package and lands in `packages/wasm-pipeline/prebuilds/pipeline/`.
 
 ## Run tests
 
-* `yarn test`: Run the JavaScript test suite
+* `yarn test`: Run the native test suite (needs `yarn build`).
+* `yarn test-wasm`: Run the WASM test suites (needs `yarn build-wasm`).
+* `yarn test-wasm <crate>`: Run only one crate's WASM tests, e.g. `yarn test-wasm pipeline`.
+
+## Versioning
+
+The published packages are versioned in lockstep, so `package.json` and
+`packages/wasm-pipeline/package.json` must always declare the same version.
+`node scripts/check-versions.js` enforces this in CI.

README.md

@@ -2,6 +2,19 @@
 
 Node.js bindings for [libdatadog](https://github.com/DataDog/libdatadog).
 
+## Packages
+
+This repository publishes two packages, versioned in lockstep:
+
+* [`@datadog/libdatadog`](https://www.npmjs.com/package/@datadog/libdatadog) — the
+  native (Node-API) bindings, plus the `library_config` and `datadog-js-zstd`
+  WebAssembly modules.
+* [`@datadog/libdatadog-wasm-pipeline`](https://www.npmjs.com/package/@datadog/libdatadog-wasm-pipeline) — the
+  WebAssembly trace pipeline, carved out so that consumers of it do not also
+  install the per-platform native binaries.
+
+Neither package depends on the other, and both expose the same loader API.
+
 ## Installing
 
 This project is currently meant to be used only by [dd-trace-js](https://github.com/DataDog/dd-trace-js)

eslint.config.js

@@ -44,7 +44,7 @@ module.exports = [
     },
   },
   {
-    files: ['load.js'],
+    files: ['**/load.js'],
     languageOptions: {
       globals: {
         __webpack_require__: 'readonly',
@@ -88,6 +88,6 @@ module.exports = [
     },
   },
   {
-    ignores: ['build/', 'target/', 'prebuilds/'],
+    ignores: ['build/', 'target/', '**/prebuilds/'],
   },
 ]

package.json

@@ -9,12 +9,13 @@
     "build-debug": "mkdir -p target && yarn -s cargo-build > ./target/out.ndjson && yarn -s copy-artifacts",
     "build-release": "mkdir -p target && yarn -s cargo-build-release > ./target/out.ndjson && yarn -s copy-artifacts",
     "build-all": "mkdir -p target && yarn -s cargo-build -- --workspace > ./target/out.ndjson && yarn -s copy-artifacts && yarn -s build-wasm",
-    "build-wasm": "yarn -s install-wasm-pack && node scripts/build-wasm.js library_config && node scripts/build-wasm.js datadog-js-zstd && node scripts/build-wasm.js pipeline",
+    "build-wasm": "yarn -s install-wasm-pack && node scripts/build-wasm.js",
     "cargo-build-release": "yarn -s cargo-build -- --release",
     "cargo-build": "cargo build --message-format=json-render-diagnostics",
     "copy-artifacts": "node ./scripts/copy-artifacts",
     "lint": "eslint .",
-    "test": "bash scripts/test.sh"
+    "test": "bash scripts/test.sh native",
+    "test-wasm": "bash scripts/test.sh wasm"
   },
   "author": "Datadog Inc. <info@datadoghq.com>",
   "license": "Apache-2.0",

packages/wasm-pipeline/.npmignore

@@ -0,0 +1,6 @@
+*
+!prebuilds/**/*
+!index.js
+!load.js
+!LICENSE
+!README.md