From 9862429e5e2f252aa6d64fadc979f568c96a0846 Mon Sep 17 00:00:00 2001 From: Bryan English Date: Fri, 31 Jul 2026 17:12:57 -0400 Subject: [PATCH] feat!: carve the wasm pipeline into its own package The pipeline wasm crate now ships as @datadog/libdatadog-wasm-pipeline so that consumers of the trace pipeline do not also install the per-platform native binaries. Everything else is unchanged: @datadog/libdatadog keeps the native Node-API bindings alongside the library_config and datadog-js-zstd wasm modules, and the loader at the repository root still resolves both. The packages are versioned in lockstep and neither depends on the other. scripts/wasm-crates.js owns the crate to package mapping and derives the wasm-pack output directory, the CI artifact name and the packaging assertions from it. The artifact name matters: action-prebuildify merges every prebuilds-* artifact into the native package's prebuilds artifact, which is how the crates that stay get published and would silently republish a carved-out one, so carved-out crates upload outside that pattern. Because both of those failure modes are silent, check-packages.js asserts the packed contents of every package on each PR and before both publishes, and check-versions.js enforces the lockstep version. build-wasm.js also deletes the .gitignore wasm-pack writes into its output directory; npm honours nested ignore files, so it would otherwise drop the whole module from a locally packed tarball. The test suites are split structurally: test/wasm// is the wasm suite and everything else is native, driven by scripts/test.sh [crate] in place of test-wasm.js. --- .github/actions/build-test-wasm/action.yaml | 30 +-- .github/workflows/build.yml | 21 ++ .github/workflows/release.yml | 56 +++++- DEVELOPMENT.md | 23 ++- README.md | 13 ++ eslint.config.js | 4 +- package.json | 5 +- packages/wasm-pipeline/.npmignore | 6 + packages/wasm-pipeline/LICENSE | 201 +++++++++++++++++++ packages/wasm-pipeline/README.md | 26 +++ packages/wasm-pipeline/index.js | 3 + packages/wasm-pipeline/load.js | 44 ++++ packages/wasm-pipeline/package.json | 19 ++ scripts/build-wasm.js | 28 ++- scripts/check-packages.js | 90 +++++++++ scripts/check-versions.js | 29 +++ scripts/test.sh | 94 +++++---- scripts/wasm-crates.js | 65 ++++++ test-wasm.js | 10 - test/{ => wasm/pipeline}/http_transport.js | 4 +- test/{pipeline.js => wasm/pipeline/index.js} | 15 +- 21 files changed, 702 insertions(+), 84 deletions(-) create mode 100644 packages/wasm-pipeline/.npmignore create mode 100644 packages/wasm-pipeline/LICENSE create mode 100644 packages/wasm-pipeline/README.md create mode 100644 packages/wasm-pipeline/index.js create mode 100644 packages/wasm-pipeline/load.js create mode 100644 packages/wasm-pipeline/package.json create mode 100644 scripts/check-packages.js create mode 100644 scripts/check-versions.js create mode 100644 scripts/wasm-crates.js delete mode 100644 test-wasm.js rename test/{ => wasm/pipeline}/http_transport.js (98%) rename test/{pipeline.js => wasm/pipeline/index.js} (98%) diff --git a/.github/actions/build-test-wasm/action.yaml b/.github/actions/build-test-wasm/action.yaml index 90cafd52..348e2283 100644 --- a/.github/actions/build-test-wasm/action.yaml +++ b/.github/actions/build-test-wasm/action.yaml @@ -8,7 +8,6 @@ runs: using: 'composite' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - name: Setup Node.js uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 - run: yarn install @@ -16,24 +15,25 @@ runs: - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh shell: bash + # scripts/wasm-crates.js owns which package each crate is published from, + # so the output directory and the artifact name are both derived from it + # rather than repeated here. - name: Build WASM + id: build run: | - mkdir -p ./prebuilds/${{ inputs.crate }} - wasm-pack build --target nodejs ./crates/${{ inputs.crate }} --out-dir ../../prebuilds/${{ inputs.crate }} + node scripts/build-wasm.js '${{ inputs.crate }}' + echo "prebuilds=$(node scripts/wasm-crates.js --prebuilds-root '${{ inputs.crate }}')" >> "$GITHUB_OUTPUT" + echo "artifact=$(node scripts/wasm-crates.js --artifact-name '${{ inputs.crate }}')" >> "$GITHUB_OUTPUT" shell: bash - name: Test WASM - # The pipeline crate's tests are top-level node:test suites that need - # --test-force-exit (the wasm exporter keeps the event loop alive after - # a flush); the other wasm crates use plain test/wasm// scripts. - run: | - if [ "${{ inputs.crate }}" = "pipeline" ]; then - node --test --test-force-exit test/pipeline.js - else - node test-wasm.js ${{ inputs.crate }} - fi + run: bash scripts/test.sh wasm '${{ inputs.crate }}' shell: bash + # Crates that stay in @datadog/libdatadog are uploaded as `prebuilds-*`, + # which action-prebuildify merges into that package's `prebuilds` artifact. + # A carved-out crate must not match that pattern or it would be published + # by both packages; `wasm-crates.js` is what keeps the two apart. - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: prebuilds-wasm-${{ inputs.crate }} - if-no-files-found: ignore - path: ./prebuilds/* + name: ${{ steps.build.outputs.artifact }} + if-no-files-found: error + path: ${{ steps.build.outputs.prebuilds }}/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1019981b..e2c4f59c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,6 +17,7 @@ jobs: - run: yarn install working-directory: test/crashtracker - run: yarn lint + - run: node scripts/check-versions.js build-test-wasm: runs-on: ubuntu-latest @@ -45,6 +46,26 @@ jobs: # Need this, now that libdatadog packages libunwind as a submodule prebuild: '(command -v apk >/dev/null && apk add autoconf automake libtool) || (command -v apt-get >/dev/null && apt-get update && apt-get install -y autoconf automake libtool) || true' + # Asserts that each wasm crate is published by exactly the package that owns + # it and that only @datadog/libdatadog ships native binaries, which is + # otherwise a silent failure. + package-contents: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: prebuilds + path: prebuilds + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: wasm-pipeline + path: packages/wasm-pipeline/prebuilds + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - run: node scripts/check-packages.js + package-size: runs-on: ubuntu-latest needs: build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f5ce2754..ed20192c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,6 +6,14 @@ on: - v0.x jobs: + versions: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - run: node scripts/check-versions.js + build-test-wasm: runs-on: ubuntu-latest strategy: @@ -31,9 +39,32 @@ jobs: rust: true only: darwin-arm64,darwin-x64,linux-arm64,linux-x64 - publish: + # Asserts that each wasm crate is published by exactly the package that owns + # it and that only @datadog/libdatadog ships native binaries, which is + # otherwise a silent failure. + package-contents: runs-on: ubuntu-latest needs: build + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: prebuilds + path: prebuilds + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: wasm-pipeline + path: packages/wasm-pipeline/prebuilds + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - run: node scripts/check-packages.js + + publish: + runs-on: ubuntu-latest + needs: + - build + - package-contents + - versions environment: npm permissions: id-token: write # Required for OIDC @@ -66,3 +97,26 @@ jobs: - run: | git tag v${{ fromJson(steps.pkg.outputs.json).version }} git push https://x-access-token:${{ steps.octo-sts.outputs.token }}@github.com/${{ github.repository }}.git v${{ fromJson(steps.pkg.outputs.json).version }} + + # This package has no per-platform build, so it only needs the artifact from + # build-test-wasm. It publishes after @datadog/libdatadog so that a failed + # native build never leaves half of a lockstep pair on npm. + publish-wasm-pipeline: + runs-on: ubuntu-latest + needs: publish + environment: npm + permissions: + id-token: write # Required for OIDC + contents: read + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: wasm-pipeline + path: packages/wasm-pipeline/prebuilds + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + registry-url: 'https://registry.npmjs.org' + - run: npm publish + working-directory: packages/wasm-pipeline diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 4adca209..bc95428b 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -40,10 +40,29 @@ The project compiles Rust for both native Node.js addons and WebAssembly. Use [r ## Building -* `yarn build`: Build the default workspaces in debug mode. +The repository publishes two packages from the same set of crates: +`@datadog/libdatadog` (the native bindings plus most WASM modules, from the +repository root) and `@datadog/libdatadog-wasm-pipeline` (the `pipeline` crate, +from `packages/wasm-pipeline`). `scripts/wasm-crates.js` is the single source of +truth for which package publishes which crate. + +* `yarn build`: Build the default workspaces in debug mode, then the WASM modules. * `yarn build-release`: Build the default workspaces in release mode. * `yarn build-all`: Build all workspaces in debug mode. This is useful when working on a workspace that is not a default member yet. +* `yarn build-wasm`: Build every WASM module. To build just one, run `node scripts/build-wasm.js `. + +Native artifacts land in `build/Release/`. WASM modules land in +`prebuilds//`, except the pipeline module, which belongs to its own +package and lands in `packages/wasm-pipeline/prebuilds/pipeline/`. ## Run tests -* `yarn test`: Run the JavaScript test suite +* `yarn test`: Run the native test suite (needs `yarn build`). +* `yarn test-wasm`: Run the WASM test suites (needs `yarn build-wasm`). +* `yarn test-wasm `: Run only one crate's WASM tests, e.g. `yarn test-wasm pipeline`. + +## Versioning + +The published packages are versioned in lockstep, so `package.json` and +`packages/wasm-pipeline/package.json` must always declare the same version. +`node scripts/check-versions.js` enforces this in CI. diff --git a/README.md b/README.md index 69caebcd..62019903 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,19 @@ Node.js bindings for [libdatadog](https://github.com/DataDog/libdatadog). +## Packages + +This repository publishes two packages, versioned in lockstep: + +* [`@datadog/libdatadog`](https://www.npmjs.com/package/@datadog/libdatadog) — the + native (Node-API) bindings, plus the `library_config` and `datadog-js-zstd` + WebAssembly modules. +* [`@datadog/libdatadog-wasm-pipeline`](https://www.npmjs.com/package/@datadog/libdatadog-wasm-pipeline) — the + WebAssembly trace pipeline, carved out so that consumers of it do not also + install the per-platform native binaries. + +Neither package depends on the other, and both expose the same loader API. + ## Installing This project is currently meant to be used only by [dd-trace-js](https://github.com/DataDog/dd-trace-js) diff --git a/eslint.config.js b/eslint.config.js index e7b67365..b36f1557 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -44,7 +44,7 @@ module.exports = [ }, }, { - files: ['load.js'], + files: ['**/load.js'], languageOptions: { globals: { __webpack_require__: 'readonly', @@ -88,6 +88,6 @@ module.exports = [ }, }, { - ignores: ['build/', 'target/', 'prebuilds/'], + ignores: ['build/', 'target/', '**/prebuilds/'], }, ] diff --git a/package.json b/package.json index d956cc21..2bfecd70 100644 --- a/package.json +++ b/package.json @@ -9,12 +9,13 @@ "build-debug": "mkdir -p target && yarn -s cargo-build > ./target/out.ndjson && yarn -s copy-artifacts", "build-release": "mkdir -p target && yarn -s cargo-build-release > ./target/out.ndjson && yarn -s copy-artifacts", "build-all": "mkdir -p target && yarn -s cargo-build -- --workspace > ./target/out.ndjson && yarn -s copy-artifacts && yarn -s build-wasm", - "build-wasm": "yarn -s install-wasm-pack && node scripts/build-wasm.js library_config && node scripts/build-wasm.js datadog-js-zstd && node scripts/build-wasm.js pipeline", + "build-wasm": "yarn -s install-wasm-pack && node scripts/build-wasm.js", "cargo-build-release": "yarn -s cargo-build -- --release", "cargo-build": "cargo build --message-format=json-render-diagnostics", "copy-artifacts": "node ./scripts/copy-artifacts", "lint": "eslint .", - "test": "bash scripts/test.sh" + "test": "bash scripts/test.sh native", + "test-wasm": "bash scripts/test.sh wasm" }, "author": "Datadog Inc. ", "license": "Apache-2.0", diff --git a/packages/wasm-pipeline/.npmignore b/packages/wasm-pipeline/.npmignore new file mode 100644 index 00000000..605efae7 --- /dev/null +++ b/packages/wasm-pipeline/.npmignore @@ -0,0 +1,6 @@ +* +!prebuilds/**/* +!index.js +!load.js +!LICENSE +!README.md diff --git a/packages/wasm-pipeline/LICENSE b/packages/wasm-pipeline/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/packages/wasm-pipeline/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/wasm-pipeline/README.md b/packages/wasm-pipeline/README.md new file mode 100644 index 00000000..75918698 --- /dev/null +++ b/packages/wasm-pipeline/README.md @@ -0,0 +1,26 @@ +# @datadog/libdatadog-wasm-pipeline + +The WebAssembly trace pipeline binding for [libdatadog](https://github.com/DataDog/libdatadog): +span management, the change buffer protocol, and trace export. + +It is carved out of [`@datadog/libdatadog`](https://www.npmjs.com/package/@datadog/libdatadog) +so that consumers of the trace pipeline do not also pull in the per-platform +native binaries. The two packages are versioned in lockstep, neither depends on +the other, and both expose the same loader API: + +```js +const pipeline = require('@datadog/libdatadog-wasm-pipeline').load('pipeline') +``` + +Everything else libdatadog-nodejs builds — the native Node-API bindings and the +remaining WebAssembly modules — stays in `@datadog/libdatadog`. + +## Installing + +This project is currently meant to be used only by [dd-trace-js](https://github.com/DataDog/dd-trace-js) +and installing it directly is not supported at the moment. + +## Development + +Sources live in the [libdatadog-nodejs](https://github.com/DataDog/libdatadog-nodejs) +repository; see its `DEVELOPMENT.md`. diff --git a/packages/wasm-pipeline/index.js b/packages/wasm-pipeline/index.js new file mode 100644 index 00000000..c694ff81 --- /dev/null +++ b/packages/wasm-pipeline/index.js @@ -0,0 +1,3 @@ +'use strict' + +module.exports = require('./load') diff --git a/packages/wasm-pipeline/load.js b/packages/wasm-pipeline/load.js new file mode 100644 index 00000000..ad543016 --- /dev/null +++ b/packages/wasm-pipeline/load.js @@ -0,0 +1,44 @@ +'use strict' + +// TODO: Extract this file to an external library, shared with the loader at the +// repository root. + +const { existsSync } = require('node:fs') +const path = require('node:path') + +// The indirection through `runtimeRequire` is what keeps a bundler from pulling +// the wasm glue into its output, so this package resolves its module at runtime +// rather than exporting `require('./prebuilds/pipeline/pipeline.js')` directly. +const inWebpack = typeof __webpack_require__ === 'function' +const runtimeRequire = inWebpack ? __non_webpack_require__ : require + +function maybeLoad (name) { + try { + return load(name) + } catch { + // Not found, skip. + } +} + +function load (name) { + const filename = find(name) + + if (filename) { + return runtimeRequire(filename) + } + + throw new Error(`Could not find a ${name} WASM module.`) +} + +// TODO: `find` exists for API parity with the loader at the repository root, +// which needs it to locate the crashtracker receiver binary. Nothing consumes it +// here, so drop it once callers stop treating the two loaders as interchangeable. +function find (name) { + // see https://github.com/rust-lang/cargo/issues/12780 + const filename = `${name.replaceAll('-', '_')}.js` + const file = path.join(__dirname, 'prebuilds', name, filename) + + if (existsSync(file)) return file +} + +module.exports = { find, load, maybeLoad } diff --git a/packages/wasm-pipeline/package.json b/packages/wasm-pipeline/package.json new file mode 100644 index 00000000..fe55002e --- /dev/null +++ b/packages/wasm-pipeline/package.json @@ -0,0 +1,19 @@ +{ + "name": "@datadog/libdatadog-wasm-pipeline", + "version": "0.1.0-pre", + "description": "WebAssembly trace pipeline bindings for libdatadog", + "main": "index.js", + "author": "Datadog Inc. ", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/DataDog/libdatadog-nodejs.git" + }, + "bugs": { + "url": "https://github.com/DataDog/libdatadog-nodejs/issues" + }, + "homepage": "https://github.com/DataDog/libdatadog-nodejs#readme", + "publishConfig": { + "access": "public" + } +} diff --git a/scripts/build-wasm.js b/scripts/build-wasm.js index b5a20a6d..8547979f 100644 --- a/scripts/build-wasm.js +++ b/scripts/build-wasm.js @@ -12,10 +12,14 @@ const os = require('node:os') const childProcess = require('node:child_process') +const fs = require('node:fs') +const path = require('node:path') + +const { prebuildsDir, WASM_CRATES } = require('./wasm-crates') const isMacOS = os.platform() === 'darwin' const noWasmOpt = isMacOS ? '--no-opt' : '' -const library = process.argv[2] +const libraries = process.argv[2] ? [process.argv[2]] : WASM_CRATES const env = { ...process.env, @@ -44,8 +48,20 @@ if (isMacOS) { env.CXX_wasm32_unknown_unknown = `${llvmBinDir}/clang++` } -childProcess.execSync( - `wasm-pack build ${noWasmOpt} --target nodejs ./crates/${library} --out-dir ../../prebuilds/${library}`, { - env, - }, -) +for (const library of libraries) { + // wasm-pack resolves `--out-dir` relative to the crate directory. + const outDir = path.join(__dirname, '..', prebuildsDir(library)) + const cratePath = path.join(__dirname, '..', 'crates', library) + + childProcess.execSync( + `wasm-pack build ${noWasmOpt} --target nodejs ./crates/${library} --out-dir ${path.relative(cratePath, outDir)}`, + { + env, + }, + ) + + // wasm-pack writes a `.gitignore` containing `*` into its output directory. npm + // honours nested ignore files when packing, so leaving it in place would drop + // the whole module from the published tarball. + fs.rmSync(path.join(outDir, '.gitignore'), { force: true }) +} diff --git a/scripts/check-packages.js b/scripts/check-packages.js new file mode 100644 index 00000000..aca27fd2 --- /dev/null +++ b/scripts/check-packages.js @@ -0,0 +1,90 @@ +'use strict' + +// Guards the boundary between the published packages: every wasm crate must be +// packed by exactly the package that owns it, only @datadog/libdatadog ships +// native binaries, and no carved-out crate may leak back into it. +// +// Both failure modes here are silent, which is why this runs in CI: +// +// * action-prebuildify merges every artifact matching `prebuilds-*` into the +// native package's `prebuilds` artifact. That is deliberate for the crates +// that stay, and fatal for a carved-out one — see `wasm-crates.js`. +// * the `.gitignore` wasm-pack writes into its output directory makes npm drop +// the whole module from the tarball. + +const { execFileSync } = require('node:child_process') +const path = require('node:path') + +const { packageDir, WASM_CRATES } = require('./wasm-crates') + +const NATIVE_PACKAGE = '.' + +function packedFiles (dir) { + const stdout = execFileSync('npm', ['pack', '--dry-run', '--json'], { + cwd: path.join(__dirname, '..', dir), + encoding: 'utf8', + }) + + return JSON.parse(stdout)[0].files.map(entry => entry.path) +} + +// Package directory -> the wasm crates it is expected to publish. +const owned = new Map([[NATIVE_PACKAGE, []]]) + +for (const crate of WASM_CRATES) { + const dir = packageDir(crate) + + if (!owned.has(dir)) owned.set(dir, []) + owned.get(dir).push(crate) +} + +let failed = false + +function fail (message, files = []) { + failed = true + console.error(`✖ ${message}`) + for (const file of files) { + console.error(` ${file}`) + } +} + +// Files a package packs for a crate, e.g. `prebuilds/pipeline/pipeline_bg.wasm`. +function packedFor (files, crate) { + return files.filter(f => f.startsWith(`prebuilds/${crate}/`)) +} + +function wasmFor (files, crate) { + return packedFor(files, crate).filter(f => f.endsWith('.wasm')) +} + +for (const [dir, crates] of owned) { + const isNative = dir === NATIVE_PACKAGE + const name = require(path.join('..', dir, 'package.json')).name + const files = packedFiles(dir) + + for (const crate of crates) { + if (wasmFor(files, crate).length === 0) fail(`${name} is missing the ${crate} wasm module`) + } + + for (const crate of WASM_CRATES.filter(c => !crates.includes(c))) { + const leaked = packedFor(files, crate) + if (leaked.length > 0) fail(`${name} must not contain the ${crate} crate:`, leaked) + } + + const native = files.filter(f => f.endsWith('.node')) + + if (isNative && native.length === 0) { + fail(`${name} contains no native .node binaries`) + } else if (!isNative && native.length > 0) { + fail(`${name} must not contain native binaries:`, native) + } + + console.log(` ${name}: ${files.length} packed files, ${native.length} native, ` + + `wasm crates [${crates.filter(c => wasmFor(files, c).length > 0).join(', ')}]`) +} + +if (failed) { + process.exitCode = 1 +} else { + console.log('✔ package boundaries hold') +} diff --git a/scripts/check-versions.js b/scripts/check-versions.js new file mode 100644 index 00000000..5a3fc23e --- /dev/null +++ b/scripts/check-versions.js @@ -0,0 +1,29 @@ +'use strict' + +// The published packages are versioned in lockstep: consumers install +// @datadog/libdatadog and the carved-out wasm packages together, so a mismatch +// would ship an inconsistent set. + +const path = require('node:path') + +const root = require('../package.json') + +const { OWN_PACKAGE } = require('./wasm-crates') + +let failed = false + +for (const dir of new Set(OWN_PACKAGE.values())) { + const pkg = require(path.join('..', dir, 'package.json')) + + if (pkg.version === root.version) { + console.log(`✔ ${pkg.name} matches ${root.name} at ${root.version}`) + } else { + failed = true + console.error(`✖ ${pkg.name}@${pkg.version} does not match ${root.name}@${root.version};` + + ' the packages must be versioned in lockstep') + } +} + +if (failed) { + process.exitCode = 1 +} diff --git a/scripts/test.sh b/scripts/test.sh index 181afd39..e5720054 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,6 +1,32 @@ #!/usr/bin/env bash set -e +# Usage: test.sh [crate] +# +# The two suites are kept apart because the native prebuildify matrix only ever +# has the native prebuilds available: +# +# native Everything loaded through the loader at the repository root. This is +# what `yarn test` runs, including inside that matrix. +# wasm The wasm crates, one directory per crate under test/wasm. Pass a +# crate name to run only that crate's tests, which is what CI does +# after building a single wasm module. + +mode=$1 +crate=$2 + +if [ "$mode" != 'native' ] && [ "$mode" != 'wasm' ]; then + echo "usage: $0 [crate]" >&2 + exit 1 +fi + +# `--test-force-exit` exists on Node >= 20.14/22 but Node 18 rejects it as an +# unknown option. +force_exit=false +if node --test-force-exit --eval '' >/dev/null 2>&1; then + force_exit=true +fi + run_test() { local dir dir=$(dirname "$1") @@ -8,48 +34,48 @@ run_test() { echo "Installing dependencies for $1" yarn --cwd "$dir" install fi - echo "Running $1" + # node:test does not force the process to exit when the event loop is kept # active by async work that has already settled (e.g. the wasm trace # exporter's runtime machinery after a flush). For the long-lived real # consumer that is expected; for the test runner we force a clean exit once - # all tests have finished. Only applies to files that use node:test. - # - # `--test-force-exit` exists on Node >= 20.14/22 but Node 18 rejects it as an - # unknown option. The wasm transport unref's its timeout/backoff timers so the - # process still exits cleanly without the flag; probe for support and degrade - # gracefully on Node 18. - if grep -q "node:test" "$1"; then - if node --test-force-exit --eval '' >/dev/null 2>&1; then - node --test-force-exit "$1" - else - node "$1" + # all tests have finished. Node 18 both lacks the flag and leaves a mock-agent + # socket open in the wasm HTTP client, so node:test cannot exit cleanly there + # at all; those suites are covered by every newer Node in the matrix. + if grep -q 'node:test' "$1"; then + if [ "$force_exit" = 'false' ]; then + echo "Skipping $1 (no --test-force-exit on this Node; covered by newer Node)" + return fi + + echo "Running $1" + node --test-force-exit "$1" else + echo "Running $1" node "$1" fi } -# Run top-level test files -for f in test/*.js; do - # pipeline.js's wasm exporter keeps the event loop alive after a flush, so it - # needs --test-force-exit. Node 18 lacks that flag AND the wasm HTTP client - # leaves a mock-agent socket open, so node:test cannot exit cleanly there. The - # pipeline wasm is fully exercised by the build-test-wasm job and by the - # Node 20/22/24/26 runs here, so skip it on a Node without --test-force-exit. - if [ "$f" = "test/pipeline.js" ] && ! node --test-force-exit --eval '' >/dev/null 2>&1; then - echo "Skipping $f (no --test-force-exit on this Node; covered by build-test-wasm + newer Node)" - continue +if [ "$mode" = 'native' ]; then + # Top-level test files, plus the entry point of every test directory other + # than the wasm ones. + for f in test/*.js; do + run_test "$f" + done + + for d in test/*/; do + [ "$d" = 'test/wasm/' ] && continue + [ -f "${d}index.js" ] && run_test "${d}index.js" + done +else + if [ -n "$crate" ] && [ ! -d "test/wasm/$crate" ]; then + echo "No wasm tests for crate '$crate' (expected test/wasm/$crate)" >&2 + exit 1 fi - run_test "$f" -done - -# Run index.js in test subdirectories (except wasm) -for d in test/*/; do - case "$d" in - *wasm*) ;; - *) - [ -f "${d}index.js" ] && run_test "${d}index.js" - ;; - esac -done + + for d in test/wasm/${crate:-*}/; do + for f in "$d"*.js; do + [ -f "$f" ] && run_test "$f" + done + done +fi diff --git a/scripts/wasm-crates.js b/scripts/wasm-crates.js new file mode 100644 index 00000000..93af66cb --- /dev/null +++ b/scripts/wasm-crates.js @@ -0,0 +1,65 @@ +'use strict' + +// Where each wasm crate is published from. Only the pipeline crate is carved +// out into its own package; the rest ship in @datadog/libdatadog alongside the +// native bindings. +// +// This is the single source of truth for the mapping: `build-wasm.js` writes +// there, `check-packages.js` asserts on it, and the build-test-wasm action +// queries it for the artifact it uploads. + +const path = require('node:path') + +// Every crate built with wasm-pack, in build order. +const WASM_CRATES = ['library_config', 'datadog-js-zstd', 'pipeline'] + +const PIPELINE_PACKAGE = path.join('packages', 'wasm-pipeline') + +// Crates carved out of @datadog/libdatadog, mapped to their package directory. +const OWN_PACKAGE = new Map([ + ['pipeline', PIPELINE_PACKAGE], +]) + +// Package directory (relative to the repository root) publishing a crate. +function packageDir (crate) { + return OWN_PACKAGE.get(crate) ?? '.' +} + +// The `prebuilds` directory a crate's output lands under. Uploading its `*` glob +// keeps the crate directory itself inside the CI artifact, which is what the +// consumers of these artifacts expect. +function prebuildsRoot (crate) { + return path.join(packageDir(crate), 'prebuilds') +} + +// Directory the crate's wasm-pack output belongs in. +function prebuildsDir (crate) { + return path.join(prebuildsRoot(crate), crate) +} + +// CI artifact name. Anything matching `prebuilds-*` is merged into the native +// package's `prebuilds` artifact by action-prebuildify, which is how the crates +// that stay in @datadog/libdatadog get published. The carved-out crates must +// therefore use a name outside that pattern. +function artifactName (crate) { + return OWN_PACKAGE.has(crate) ? `wasm-${crate}` : `prebuilds-wasm-${crate}` +} + +if (require.main === module) { + const [flag, crate] = process.argv.slice(2) + const queries = { + '--package-dir': packageDir, + '--prebuilds-root': prebuildsRoot, + '--prebuilds-dir': prebuildsDir, + '--artifact-name': artifactName, + } + + if (!queries[flag] || !crate) { + console.error(`usage: ${path.basename(__filename)} <${Object.keys(queries).join('|')}> `) + process.exitCode = 1 + } else { + console.log(queries[flag](crate)) + } +} + +module.exports = { artifactName, packageDir, prebuildsDir, prebuildsRoot, OWN_PACKAGE, WASM_CRATES } diff --git a/test-wasm.js b/test-wasm.js deleted file mode 100644 index 47aadf04..00000000 --- a/test-wasm.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict' - -const fs = require('node:fs') - -const crateTestsDir = `./test/wasm/${process.argv[2]}` -const files = fs.readdirSync(crateTestsDir).filter(file => file.endsWith('.js') || !file.includes('.')) - -for (const file of files) { - require(`${crateTestsDir}/${file}`) -} diff --git a/test/http_transport.js b/test/wasm/pipeline/http_transport.js similarity index 98% rename from test/http_transport.js rename to test/wasm/pipeline/http_transport.js index 5a97a58a..40b4a789 100644 --- a/test/http_transport.js +++ b/test/wasm/pipeline/http_transport.js @@ -13,7 +13,7 @@ const os = require('node:os') const path = require('node:path') const fs = require('node:fs') -const transport = require('../crates/capabilities/src/http_transport') +const transport = require('../../../crates/capabilities/src/http_transport') // Distinctive, multi-byte body so the pooled-buffer slicing in httpRequest // (the reason for `new Uint8Array(body)` over `body.buffer`) is exercised: @@ -180,7 +180,7 @@ describe('http_transport IPv6 host', () => { describe('http_transport lazy builtin requires', () => { it('does not require node:http/https/fs at module load', () => { const Module = require('node:module') - const modPath = require.resolve('../crates/capabilities/src/http_transport') + const modPath = require.resolve('../../../crates/capabilities/src/http_transport') const orig = Module.prototype.require const seen = [] Module.prototype.require = function (id) { diff --git a/test/pipeline.js b/test/wasm/pipeline/index.js similarity index 98% rename from test/pipeline.js rename to test/wasm/pipeline/index.js index bc6eabf6..3a7afc69 100644 --- a/test/pipeline.js +++ b/test/wasm/pipeline/index.js @@ -4,15 +4,10 @@ const { describe, it, before, beforeEach } = require('node:test') const assert = require('node:assert') const crypto = require('node:crypto') -const pipeline = require('..').maybeLoad('pipeline') -// The pipeline binding is wasm-only and is absent in the native -// (action-prebuildify) test matrix, where `maybeLoad` returns undefined. Skip -// the suite there instead of crashing on the destructure below; the pipeline -// wasm is built and these tests run for real in the `build-test-wasm` job. -const skip = pipeline === undefined -const { WasmSpanState } = pipeline ?? {} -const OpCode = pipeline ? pipeline.getOpCodes() : {} -const wasmMemory = pipeline ? pipeline.getWasmMemory() : undefined +const pipeline = require('../../../packages/wasm-pipeline').load('pipeline') +const { WasmSpanState } = pipeline +const OpCode = pipeline.getOpCodes() +const wasmMemory = pipeline.getWasmMemory() function getRandomBytes (byteCount) { return new Uint8Array(crypto.randomBytes(byteCount)) @@ -384,7 +379,7 @@ function msgpackOuterArrayLen (buf) { throw new Error('payload is not a msgpack array: 0x' + b.toString(16)) } -describe('pipeline', { skip }, () => { +describe('pipeline', () => { let nativeSpans before(() => {