From 1784c3b9a4d883c350fc0161caac4bde3eeae027 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Fri, 19 Jun 2026 05:10:37 -0700 Subject: [PATCH] perf: restore sidecar reuse fast paths --- crates/secure-exec-client/src/transport.rs | 2 +- examples/native-client/src/index.ts | 4 +- packages/benchmarks/README.md | 80 + packages/benchmarks/bench-utils.ts | 141 +- packages/benchmarks/coldstart.bench.ts | 309 +- .../benchmarks/results/coldstart-final.json | 3907 +++++++++++++++++ .../benchmarks/results/coldstart-final.log | 232 +- ...ldstart-resident-full-matrix-20260619.json | 1276 ++++++ ...oldstart-resident-full-matrix-20260619.log | 75 + packages/benchmarks/run-benchmarks.sh | 10 +- packages/core/src/index.ts | 2 + packages/core/src/kernel-proxy.ts | 17 +- packages/core/src/node-runtime.ts | 265 +- packages/core/src/sidecar-client.ts | 51 +- packages/core/src/test-runtime.ts | 177 +- packages/secure-exec/src/index.ts | 8 +- 16 files changed, 6309 insertions(+), 247 deletions(-) create mode 100644 packages/benchmarks/README.md create mode 100644 packages/benchmarks/results/coldstart-resident-full-matrix-20260619.json create mode 100644 packages/benchmarks/results/coldstart-resident-full-matrix-20260619.log diff --git a/crates/secure-exec-client/src/transport.rs b/crates/secure-exec-client/src/transport.rs index 90a8a184e..d9e9da320 100644 --- a/crates/secure-exec-client/src/transport.rs +++ b/crates/secure-exec-client/src/transport.rs @@ -1,7 +1,7 @@ //! `SidecarTransport`: spawns a native sidecar binary and speaks the existing framed //! BARE protocol over its stdio. //! -//! This mirrors the TypeScript `NativeSidecarProcessClient`. Generated wire payloads are the native +//! This mirrors the TypeScript `Sidecar`. Generated wire payloads are the native //! transport path. //! //! Request-id direction is load-bearing: host-initiated `Request`/`Response` frames use positive ids diff --git a/examples/native-client/src/index.ts b/examples/native-client/src/index.ts index 53648074f..7c1c3e471 100644 --- a/examples/native-client/src/index.ts +++ b/examples/native-client/src/index.ts @@ -1,8 +1,8 @@ -import { NativeSidecarProcessClient } from "@secure-exec/core/sidecar-client"; +import { Sidecar } from "@secure-exec/core/sidecar-client"; const decoder = new TextDecoder(); -const client = NativeSidecarProcessClient.spawn({ +const client = Sidecar.spawn({ cwd: process.cwd(), }); diff --git a/packages/benchmarks/README.md b/packages/benchmarks/README.md new file mode 100644 index 000000000..93c9dff98 --- /dev/null +++ b/packages/benchmarks/README.md @@ -0,0 +1,80 @@ +# Secure Exec Benchmarks + +These benchmarks measure the public `secure-exec` SDK paths used by consumers. + +## Cold Start Matrix + +`coldstart.bench.ts` writes machine-readable JSON to stdout and human-readable progress to stderr. It measures: + +- `owned-sidecar`: `NodeRuntime.create()` owns a fresh sidecar for each runtime. +- `shared-sidecar`: a `Sidecar` is created once per batch and passed to `NodeRuntime.create({ sidecar })`; sidecar setup is measured separately and excluded from cold start. +- `resident-runner`: a shared sidecar plus `runtime.createResidentRunner()`, so repeated tiny snippets reuse one live guest Node process. + +The result JSON includes hardware metadata, aggregate cold/warm latency, and phase timings such as `session_open`, `vm_create`, `runtime_mount_wasm`, `first_exec`, and resident-runner phases. + +## Run + +Build a release sidecar first for meaningful timings: + +```bash +cargo build --release -p secure-exec-sidecar +``` + +Run the full benchmark suite: + +```bash +pnpm --dir packages/benchmarks bench +``` + +This writes timestamped files under `packages/benchmarks/results/`: + +```text +coldstart-YYYYMMDD-HHMMSS.json +coldstart-YYYYMMDD-HHMMSS.log +memory-YYYYMMDD-HHMMSS.json +memory-YYYYMMDD-HHMMSS.log +``` + +Run only the cold-start matrix: + +```bash +SECURE_EXEC_SIDECAR_BIN="$PWD/target/release/secure-exec-sidecar" \ + pnpm --silent --dir packages/benchmarks bench:coldstart \ + > packages/benchmarks/results/coldstart-local.json \ + 2> packages/benchmarks/results/coldstart-local.log +``` + +Quick smoke run: + +```bash +BENCH_BATCH_SIZES=1 \ +BENCH_ITERATIONS=1 \ +BENCH_WARMUP=0 \ +BENCH_SCENARIOS=owned-sidecar,shared-sidecar,resident-runner \ +SECURE_EXEC_SIDECAR_BIN="$PWD/target/release/secure-exec-sidecar" \ + pnpm --silent --dir packages/benchmarks bench:coldstart +``` + +Useful knobs: + +```text +BENCH_BATCH_SIZES=1,10,50,100,200 +BENCH_ITERATIONS=5 +BENCH_WARMUP=1 +BENCH_SCENARIOS=owned-sidecar,shared-sidecar,resident-runner +BENCH_MAX_LIVE_RUNTIMES=8 +BENCH_MAX_RESIDENT_RUNNERS=1 +BENCH_EXEC_TIMEOUT_MS=30000 +SECURE_EXEC_SIDECAR_BIN=/abs/path/to/secure-exec-sidecar +``` + +## Checked-In Results + +Current captured results: + +- `results/coldstart-final.json` +- `results/coldstart-final.log` +- `results/coldstart-resident-full-matrix-20260619.json` +- `results/coldstart-resident-full-matrix-20260619.log` + +`coldstart-final.*` is the latest full run from June 19, 2026. It includes the three SDK scenarios above. It also contains an `isolate-only` reference row from a lower-level one-off V8 snapshot/restore benchmark; that row is captured for comparison but is not part of the normal SDK benchmark command. diff --git a/packages/benchmarks/bench-utils.ts b/packages/benchmarks/bench-utils.ts index 787f02fa9..224dcd567 100644 --- a/packages/benchmarks/bench-utils.ts +++ b/packages/benchmarks/bench-utils.ts @@ -1,31 +1,17 @@ /** * Shared utilities for the Secure Exec cold-start, warm-start, and memory * benchmarks. - * - * The benchmarks drive the public SDK exactly as a consumer would: - * - * import { NodeRuntime } from "secure-exec"; - * const runtime = await NodeRuntime.create(); - * await runtime.exec("export const x = 1;"); - * await runtime.dispose(); - * - * `NodeRuntime.create()` boots an out-of-process sidecar VM: it spawns the - * sidecar binary, opens a session, creates a VM with a bootstrapped root - * filesystem, and mounts the shell + Node runtimes. There is no in-process - * isolate path. The benchmarks therefore measure full VM boot cost, not - * isolate creation. */ +import { readFileSync } from "node:fs"; import os from "node:os"; -import { NodeRuntime } from "secure-exec"; +import { + NodeRuntime, + Sidecar, + type NodeRuntimeBootTiming, + type NodeRuntimeCreateOptions, +} from "secure-exec"; -/** - * The full matrix matches the original harness: batch sizes 1/10/50/100/200, - * 5 recorded iterations, 1 warmup discarded. All four are overridable via env so - * a quick smoke run is possible without editing the file, e.g.: - * - * BENCH_BATCH_SIZES=1,5 BENCH_ITERATIONS=2 BENCH_WARMUP=0 tsx coldstart.bench.ts - */ function numList(envVar: string, fallback: number[]): number[] { const raw = process.env[envVar]; if (!raw) return fallback; @@ -34,7 +20,8 @@ function numList(envVar: string, fallback: number[]): number[] { .map((s) => Number(s.trim())) .filter((n) => Number.isFinite(n) && n > 0); } -function num(envVar: string, fallback: number): number { + +export function num(envVar: string, fallback: number): number { const raw = process.env[envVar]; if (raw === undefined) return fallback; const n = Number(raw); @@ -46,21 +33,47 @@ export const ITERATIONS = num("BENCH_ITERATIONS", 5); export const WARMUP_ITERATIONS = num("BENCH_WARMUP", 1); export const MEMORY_ITERATIONS = num("BENCH_MEMORY_ITERATIONS", 5); -/** - * A trivial guest program: just enough to confirm the runtime is live and the - * first `exec()` round-trips. Keeps the measurement focused on runtime boot, - * not workload. - */ export const TRIVIAL_CODE = "export const x = 1;"; +export const RESIDENT_TRIVIAL_CODE = + "globalThis.__benchValue = (globalThis.__benchValue ?? 0) + 1;"; -/** - * Cap concurrency below available parallelism to leave headroom for the bench - * harness, Node's event loop, and each sidecar's own threads. - */ export const MAX_CONCURRENCY = Math.max(1, os.availableParallelism() - 4); +export const MAX_LIVE_RUNTIMES = Math.max( + 1, + num("BENCH_MAX_LIVE_RUNTIMES", Math.min(8, MAX_CONCURRENCY)), +); +export const MAX_RESIDENT_RUNNERS = Math.max( + 1, + num("BENCH_MAX_RESIDENT_RUNNERS", 1), +); +export const EXEC_TIMEOUT_MS = Math.max( + 1, + num("BENCH_EXEC_TIMEOUT_MS", 30_000), +); -export async function createBenchRuntime(): Promise { - return NodeRuntime.create(); +export type BenchScenario = + | "owned-sidecar" + | "shared-sidecar" + | "resident-runner"; + +export const SCENARIOS: BenchScenario[] = ( + process.env.BENCH_SCENARIOS?.split(",").map((s) => s.trim()) ?? [ + "owned-sidecar", + "shared-sidecar", + "resident-runner", + ] +).filter((s): s is BenchScenario => + ["owned-sidecar", "shared-sidecar", "resident-runner"].includes(s), +); + +export async function createBenchRuntime( + options: Pick = {}, +): Promise { + return NodeRuntime.create(options); +} + +export function createBenchSidecar(): Sidecar { + return Sidecar.spawn(); } export function percentile(sorted: number[], p: number): number { @@ -94,15 +107,36 @@ export function formatBytes(bytes: number): string { return `${round(mb, 2)} MB`; } +function readMemInfo(): Record { + try { + const entries = readFileSync("/proc/meminfo", "utf8") + .trim() + .split("\n") + .map((line) => { + const [key, value] = line.split(":"); + return [key, value.trim()] as const; + }); + return Object.fromEntries(entries); + } catch { + return {}; + } +} + export function getHardware() { const cpus = os.cpus(); + const memInfo = readMemInfo(); return { cpu: cpus[0]?.model ?? "unknown", cores: os.availableParallelism(), ram: `${round(os.totalmem() / 1024 ** 3, 1)} GB`, + memAvailable: memInfo.MemAvailable, + swapTotal: memInfo.SwapTotal, + swapFree: memInfo.SwapFree, + swapCached: memInfo.SwapCached, node: process.version, os: `${os.type()} ${os.release()}`, arch: os.arch(), + loadAverage: os.loadavg().map((n) => round(n, 2)), }; } @@ -118,6 +152,47 @@ export async function sleep(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } +export type PhaseSamples = Record; + +export function createBootTimingRecorder(phases: PhaseSamples) { + return (timing: NodeRuntimeBootTiming) => { + (phases[timing.phase] ??= []).push(timing.durationMs); + }; +} + +export function mergePhaseSamples(target: PhaseSamples, source: PhaseSamples) { + for (const [phase, samples] of Object.entries(source)) { + (target[phase] ??= []).push(...samples); + } +} + +export function summarizePhases(phases: PhaseSamples) { + return Object.fromEntries( + Object.entries(phases).map(([phase, samples]) => [phase, stats(samples)]), + ); +} + +export async function runLimited( + count: number, + concurrency: number, + fn: (index: number) => Promise, +): Promise { + const results: T[] = new Array(count); + let next = 0; + const workers = Array.from( + { length: Math.min(count, Math.max(1, concurrency)) }, + async () => { + for (;;) { + const index = next++; + if (index >= count) return; + results[index] = await fn(index); + } + }, + ); + await Promise.all(workers); + return results; +} + /** Print a table to stderr for human readability. */ export function printTable( headers: string[], diff --git a/packages/benchmarks/coldstart.bench.ts b/packages/benchmarks/coldstart.bench.ts index 461b148b3..81c976340 100644 --- a/packages/benchmarks/coldstart.bench.ts +++ b/packages/benchmarks/coldstart.bench.ts @@ -1,165 +1,290 @@ /** - * Cold-start and warm-start latency benchmark for the Secure Exec runtime. + * Cold-start, sidecar-reuse, and resident-runner latency benchmark for the + * Secure Exec runtime. * - * Measures, against the public `secure-exec` SDK: - * - Cold start: time from `NodeRuntime.create()` through the first `exec()` - * of a trivial program completing. This is the full out-of-process VM boot: - * sidecar spawn, session open, VM create, root-filesystem bootstrap, shell - * and Node runtime mount, plus the first guest execution. - * - Warm start: time for a second `exec()` on the already-booted runtime. The - * VM is reused, so this isolates the per-execution cost. - * - Both sequential and concurrent modes at various batch sizes. - * - * Usage: - * SECURE_EXEC_SIDECAR_BIN=/path/to/secure-exec-sidecar tsx coldstart.bench.ts - * - * Build a release sidecar first for meaningful numbers: - * cargo build --release -p secure-exec-sidecar + * Scenarios: + * - owned-sidecar: `NodeRuntime.create()` owns a fresh sidecar per runtime. + * - shared-sidecar: one `Sidecar` is created outside the measurement and + * passed to `NodeRuntime.create({ sidecar })`. + * - resident-runner: one shared sidecar plus a live guest Node process reused + * through `runtime.createResidentRunner()`. */ import { BATCH_SIZES, + type BenchScenario, createBenchRuntime, + createBenchSidecar, + createBootTimingRecorder, + EXEC_TIMEOUT_MS, getHardware, ITERATIONS, MAX_CONCURRENCY, + MAX_LIVE_RUNTIMES, + MAX_RESIDENT_RUNNERS, + mergePhaseSamples, + type PhaseSamples, printTable, + RESIDENT_TRIVIAL_CODE, + SCENARIOS, stats, + summarizePhases, TRIVIAL_CODE, WARMUP_ITERATIONS, } from "./bench-utils.js"; +type BenchMode = "sequential" | "concurrent"; + +interface Measurement { + coldMs: number; + warmMs: number; + phases: PhaseSamples; +} + interface ColdStartEntry { + scenario: BenchScenario; batchSize: number; - mode: "sequential" | "concurrent"; + mode: BenchMode; iterations: number; + sidecarSetupMs?: ReturnType; coldStart: ReturnType; warmStart: ReturnType; + phases: ReturnType; } -async function measureOne(): Promise<{ coldMs: number; warmMs: number }> { +async function measureRuntime(sidecar?: ReturnType) { + const phases: PhaseSamples = {}; const t0 = performance.now(); - const runtime = await createBenchRuntime(); - await runtime.exec(TRIVIAL_CODE); - const coldMs = performance.now() - t0; + const runtime = await createBenchRuntime({ + ...(sidecar ? { sidecar } : {}), + onBootTiming: createBootTimingRecorder(phases), + }); + (phases.runtime_create_total ??= []).push(performance.now() - t0); + return { runtime, phases }; +} + +async function measureExecPair( + sidecar?: ReturnType, +): Promise { + const { runtime, phases } = await measureRuntime(sidecar); + try { + const firstStart = performance.now(); + await runtime.exec(TRIVIAL_CODE, { timeout: EXEC_TIMEOUT_MS }); + const firstMs = performance.now() - firstStart; + (phases.first_exec ??= []).push(firstMs); - const t1 = performance.now(); - await runtime.exec(TRIVIAL_CODE); - const warmMs = performance.now() - t1; + const warmStart = performance.now(); + await runtime.exec(TRIVIAL_CODE, { timeout: EXEC_TIMEOUT_MS }); + const warmMs = performance.now() - warmStart; + (phases.warm_exec ??= []).push(warmMs); - await runtime.dispose(); - return { coldMs, warmMs }; + return { + coldMs: + (phases.runtime_create_total?.at(-1) ?? 0) + firstMs, + warmMs, + phases, + }; + } finally { + await runtime.dispose(); + } } -async function benchSequential(batchSize: number): Promise { - const coldSamples: number[] = []; - const warmSamples: number[] = []; +async function measureResidentPair( + sidecar: ReturnType, +): Promise { + const { runtime, phases } = await measureRuntime(sidecar); + try { + const runnerStart = performance.now(); + const runner = await runtime.createResidentRunner(); + const runnerCreateMs = performance.now() - runnerStart; + (phases.resident_runner_create ??= []).push(runnerCreateMs); - for (let iter = 0; iter < WARMUP_ITERATIONS + ITERATIONS; iter++) { - const iterCold: number[] = []; - const iterWarm: number[] = []; + try { + const firstStart = performance.now(); + await runner.exec(RESIDENT_TRIVIAL_CODE, { timeout: EXEC_TIMEOUT_MS }); + const firstMs = performance.now() - firstStart; + (phases.resident_first_exec ??= []).push(firstMs); - for (let i = 0; i < batchSize; i++) { - const { coldMs, warmMs } = await measureOne(); - iterCold.push(coldMs); - iterWarm.push(warmMs); - } + const warmStart = performance.now(); + await runner.exec(RESIDENT_TRIVIAL_CODE, { timeout: EXEC_TIMEOUT_MS }); + const warmMs = performance.now() - warmStart; + (phases.resident_warm_exec ??= []).push(warmMs); - // Skip warmup iterations - if (iter >= WARMUP_ITERATIONS) { - coldSamples.push(...iterCold); - warmSamples.push(...iterWarm); + return { + coldMs: + (phases.runtime_create_total?.at(-1) ?? 0) + + runnerCreateMs + + firstMs, + warmMs, + phases, + }; + } finally { + await runner.dispose(); } + } finally { + await runtime.dispose(); } +} - return { - batchSize, - mode: "sequential", - iterations: ITERATIONS, - coldStart: stats(coldSamples), - warmStart: stats(warmSamples), - }; +function appendMeasurements( + target: { + coldSamples: number[]; + warmSamples: number[]; + phaseSamples: PhaseSamples; + }, + measurements: Measurement[], +) { + for (const measurement of measurements) { + target.coldSamples.push(measurement.coldMs); + target.warmSamples.push(measurement.warmMs); + mergePhaseSamples(target.phaseSamples, measurement.phases); + } +} + +async function collectBatch( + scenario: BenchScenario, + batchSize: number, + mode: BenchMode, +): Promise<{ + measurements: Measurement[]; + sidecarSetupMs?: number; +}> { + if (scenario === "owned-sidecar") { + const concurrency = + mode === "sequential" ? 1 : Math.min(batchSize, MAX_LIVE_RUNTIMES); + return { + measurements: await runScenarioBatch(batchSize, concurrency, () => + measureExecPair(), + ), + }; + } + + const sidecarStart = performance.now(); + const sidecar = createBenchSidecar(); + const sidecarSetupMs = performance.now() - sidecarStart; + try { + const concurrency = + scenario === "resident-runner" + ? mode === "sequential" + ? 1 + : Math.min(batchSize, MAX_RESIDENT_RUNNERS) + : mode === "sequential" + ? 1 + : Math.min(batchSize, MAX_LIVE_RUNTIMES); + const measure = + scenario === "resident-runner" + ? () => measureResidentPair(sidecar) + : () => measureExecPair(sidecar); + return { + measurements: await runScenarioBatch(batchSize, concurrency, measure), + sidecarSetupMs, + }; + } finally { + await sidecar.dispose(); + } } -async function benchConcurrent(batchSize: number): Promise { - const effectiveConcurrency = Math.min(batchSize, MAX_CONCURRENCY); +async function runScenarioBatch( + batchSize: number, + concurrency: number, + measure: () => Promise, +) { + const { runLimited } = await import("./bench-utils.js"); + return runLimited(batchSize, concurrency, measure); +} + +async function benchScenario( + scenario: BenchScenario, + batchSize: number, + mode: BenchMode, +): Promise { const coldSamples: number[] = []; const warmSamples: number[] = []; + const sidecarSetupSamples: number[] = []; + const phaseSamples: PhaseSamples = {}; for (let iter = 0; iter < WARMUP_ITERATIONS + ITERATIONS; iter++) { - const iterCold: number[] = []; - const iterWarm: number[] = []; - let remaining = batchSize; - - // Launch in chunks up to MAX_CONCURRENCY so we never oversubscribe the - // host beyond the configured concurrency. - while (remaining > 0) { - const chunk = Math.min(remaining, effectiveConcurrency); - const results = await Promise.all( - Array.from({ length: chunk }, () => measureOne()), + const batch = await collectBatch(scenario, batchSize, mode); + if (iter >= WARMUP_ITERATIONS) { + appendMeasurements( + { coldSamples, warmSamples, phaseSamples }, + batch.measurements, ); - for (const { coldMs, warmMs } of results) { - iterCold.push(coldMs); - iterWarm.push(warmMs); + if (batch.sidecarSetupMs !== undefined) { + sidecarSetupSamples.push(batch.sidecarSetupMs); } - remaining -= chunk; - } - - if (iter >= WARMUP_ITERATIONS) { - coldSamples.push(...iterCold); - warmSamples.push(...iterWarm); } } return { + scenario, batchSize, - mode: "concurrent", + mode, iterations: ITERATIONS, + ...(sidecarSetupSamples.length > 0 + ? { sidecarSetupMs: stats(sidecarSetupSamples) } + : {}), coldStart: stats(coldSamples), warmStart: stats(warmSamples), + phases: summarizePhases(phaseSamples), }; } +function formatPhaseSummary(phases: ColdStartEntry["phases"]): string { + return Object.entries(phases) + .map(([phase, phaseStats]) => `${phase}.p50=${phaseStats.p50}ms`) + .join(" | "); +} + async function main() { const hardware = getHardware(); console.error("=== Cold Start Benchmark ==="); console.error(`CPU: ${hardware.cpu}`); console.error( - `Cores: ${hardware.cores} | Max concurrency: ${MAX_CONCURRENCY}`, + `Cores: ${hardware.cores} | Max concurrency: ${MAX_CONCURRENCY} | Max live runtimes: ${MAX_LIVE_RUNTIMES}`, ); + console.error(`Resident runner live cap: ${MAX_RESIDENT_RUNNERS}`); console.error(`RAM: ${hardware.ram} | Node: ${hardware.node}`); + console.error(`Loadavg: ${hardware.loadAverage.join(", ")}`); + console.error( + `MemAvailable: ${hardware.memAvailable ?? "unknown"} | SwapFree: ${hardware.swapFree ?? "unknown"} | SwapCached: ${hardware.swapCached ?? "unknown"}`, + ); console.error(`Iterations: ${ITERATIONS} (+ ${WARMUP_ITERATIONS} warmup)`); console.error(`Batch sizes: ${BATCH_SIZES.join(", ")}`); + console.error(`Scenarios: ${SCENARIOS.join(", ")}`); console.error( `Sidecar: ${process.env.SECURE_EXEC_SIDECAR_BIN ?? "(resolved from @secure-exec/sidecar)"}\n`, ); const results: ColdStartEntry[] = []; - for (const batchSize of BATCH_SIZES) { - console.error(`\n--- batch=${batchSize}, mode=sequential ---`); - const seq = await benchSequential(batchSize); - results.push(seq); - console.error( - ` cold: mean=${seq.coldStart.mean}ms p50=${seq.coldStart.p50}ms p95=${seq.coldStart.p95}ms`, - ); - console.error( - ` warm: mean=${seq.warmStart.mean}ms p50=${seq.warmStart.p50}ms p95=${seq.warmStart.p95}ms`, - ); - - console.error(`\n--- batch=${batchSize}, mode=concurrent ---`); - const conc = await benchConcurrent(batchSize); - results.push(conc); - console.error( - ` cold: mean=${conc.coldStart.mean}ms p50=${conc.coldStart.p50}ms p95=${conc.coldStart.p95}ms`, - ); - console.error( - ` warm: mean=${conc.warmStart.mean}ms p50=${conc.warmStart.p50}ms p95=${conc.warmStart.p95}ms`, - ); + for (const scenario of SCENARIOS) { + for (const batchSize of BATCH_SIZES) { + for (const mode of ["sequential", "concurrent"] as const) { + console.error( + `\n--- scenario=${scenario}, batch=${batchSize}, mode=${mode} ---`, + ); + const entry = await benchScenario(scenario, batchSize, mode); + results.push(entry); + console.error( + ` cold: mean=${entry.coldStart.mean}ms p50=${entry.coldStart.p50}ms p95=${entry.coldStart.p95}ms`, + ); + console.error( + ` warm: mean=${entry.warmStart.mean}ms p50=${entry.warmStart.p50}ms p95=${entry.warmStart.p95}ms`, + ); + console.error(` phases: ${formatPhaseSummary(entry.phases)}`); + if (entry.sidecarSetupMs) { + console.error( + ` sidecar setup excluded: p50=${entry.sidecarSetupMs.p50}ms p95=${entry.sidecarSetupMs.p95}ms`, + ); + } + } + } } printTable( [ + "scenario", "batch", "mode", "cold mean", @@ -170,6 +295,7 @@ async function main() { "warm p95", ], results.map((r) => [ + r.scenario, r.batchSize, r.mode, `${r.coldStart.mean}ms`, @@ -181,8 +307,7 @@ async function main() { ]), ); - // JSON to stdout for capture. - console.log(JSON.stringify({ hardware, results }, null, 2)); + console.log(JSON.stringify({ hardware, scenarioSetup: {}, results }, null, 2)); } main().catch((err) => { diff --git a/packages/benchmarks/results/coldstart-final.json b/packages/benchmarks/results/coldstart-final.json index e69de29bb..13ddb5f8b 100644 --- a/packages/benchmarks/results/coldstart-final.json +++ b/packages/benchmarks/results/coldstart-final.json @@ -0,0 +1,3907 @@ +{ + "hardware": { + "cpu": "12th Gen Intel(R) Core(TM) i7-12700KF", + "cores": 20, + "ram": "62.6 GB", + "memAvailable": "26424332 kB", + "swapTotal": "1000444 kB", + "swapFree": "0 kB", + "swapCached": "114032 kB", + "node": "v24.13.0", + "os": "Linux 6.1.0-41-amd64", + "arch": "x64", + "loadAverage": [ + 4.8, + 3.73, + 3.23 + ] + }, + "scenarioSetup": { + "owned-sidecar": {}, + "shared-sidecar": {}, + "resident-runner": {}, + "isolate-only": { + "snapshotCreateMs": 123.904, + "snapshotBytes": 3902240, + "bridge": "generated-v8-bridge", + "isolateWarmup": 5 + } + }, + "results": [ + { + "scenario": "owned-sidecar", + "batchSize": 1, + "mode": "sequential", + "iterations": 5, + "coldStart": { + "samples": 5, + "mean": 772.88, + "p50": 771.96, + "p95": 779.42, + "p99": 779.42, + "min": 765.8, + "max": 779.42 + }, + "warmStart": { + "samples": 5, + "mean": 452.68, + "p50": 452.37, + "p95": 457.13, + "p99": 457.13, + "min": 449.32, + "max": 457.13 + }, + "phases": { + "filesystem_snapshot": { + "samples": 5, + "mean": 0.16, + "p50": 0.1, + "p95": 0.29, + "p99": 0.29, + "min": 0.07, + "max": 0.29 + }, + "sidecar_spawn": { + "samples": 5, + "mean": 2.97, + "p50": 2.78, + "p95": 3.76, + "p99": 3.76, + "min": 2.71, + "max": 3.76 + }, + "session_open": { + "samples": 5, + "mean": 2.54, + "p50": 2.34, + "p95": 3.61, + "p99": 3.61, + "min": 2.04, + "max": 3.61 + }, + "vm_create": { + "samples": 5, + "mean": 1.27, + "p50": 1.36, + "p95": 1.38, + "p99": 1.38, + "min": 1.09, + "max": 1.38 + }, + "vm_ready": { + "samples": 5, + "mean": 0.01, + "p50": 0.01, + "p95": 0.02, + "p99": 0.02, + "min": 0.01, + "max": 0.02 + }, + "vm_configure": { + "samples": 5, + "mean": 0.17, + "p50": 0.17, + "p95": 0.26, + "p99": 0.26, + "min": 0.11, + "max": 0.26 + }, + "runtime_mount_wasm": { + "samples": 5, + "mean": 172.92, + "p50": 172.95, + "p95": 177.04, + "p99": 177.04, + "min": 169.07, + "max": 177.04 + }, + "runtime_mount_node": { + "samples": 5, + "mean": 2.83, + "p50": 2.71, + "p95": 3.23, + "p99": 3.23, + "min": 2.61, + "max": 3.23 + }, + "runtime_create_total": { + "samples": 5, + "mean": 175.85, + "p50": 176.3, + "p95": 179.8, + "p99": 179.8, + "min": 171.8, + "max": 179.8 + }, + "first_exec": { + "samples": 5, + "mean": 597.03, + "p50": 596.26, + "p95": 601.45, + "p99": 601.45, + "min": 593.79, + "max": 601.45 + }, + "warm_exec": { + "samples": 5, + "mean": 452.68, + "p50": 452.37, + "p95": 457.13, + "p99": 457.13, + "min": 449.32, + "max": 457.13 + } + } + }, + { + "scenario": "owned-sidecar", + "batchSize": 1, + "mode": "concurrent", + "iterations": 5, + "coldStart": { + "samples": 5, + "mean": 776.22, + "p50": 775.63, + "p95": 782.37, + "p99": 782.37, + "min": 773.07, + "max": 782.37 + }, + "warmStart": { + "samples": 5, + "mean": 449.86, + "p50": 449.59, + "p95": 452.53, + "p99": 452.53, + "min": 447.58, + "max": 452.53 + }, + "phases": { + "filesystem_snapshot": { + "samples": 5, + "mean": 0.19, + "p50": 0.26, + "p95": 0.28, + "p99": 0.28, + "min": 0.05, + "max": 0.28 + }, + "sidecar_spawn": { + "samples": 5, + "mean": 2.84, + "p50": 2.79, + "p95": 3.08, + "p99": 3.08, + "min": 2.65, + "max": 3.08 + }, + "session_open": { + "samples": 5, + "mean": 2.85, + "p50": 2.99, + "p95": 3.63, + "p99": 3.63, + "min": 2.13, + "max": 3.63 + }, + "vm_create": { + "samples": 5, + "mean": 1.2, + "p50": 1.11, + "p95": 1.46, + "p99": 1.46, + "min": 1.05, + "max": 1.46 + }, + "vm_ready": { + "samples": 5, + "mean": 0.02, + "p50": 0.01, + "p95": 0.04, + "p99": 0.04, + "min": 0.01, + "max": 0.04 + }, + "vm_configure": { + "samples": 5, + "mean": 0.11, + "p50": 0.12, + "p95": 0.13, + "p99": 0.13, + "min": 0.08, + "max": 0.13 + }, + "runtime_mount_wasm": { + "samples": 5, + "mean": 175.25, + "p50": 174.2, + "p95": 178.88, + "p99": 178.88, + "min": 173.44, + "max": 178.88 + }, + "runtime_mount_node": { + "samples": 5, + "mean": 2.71, + "p50": 2.67, + "p95": 2.79, + "p99": 2.79, + "min": 2.64, + "max": 2.79 + }, + "runtime_create_total": { + "samples": 5, + "mean": 178.06, + "p50": 176.95, + "p95": 181.63, + "p99": 181.63, + "min": 176.32, + "max": 181.63 + }, + "first_exec": { + "samples": 5, + "mean": 598.17, + "p50": 599.03, + "p95": 600.75, + "p99": 600.75, + "min": 595.48, + "max": 600.75 + }, + "warm_exec": { + "samples": 5, + "mean": 449.86, + "p50": 449.59, + "p95": 452.53, + "p99": 452.53, + "min": 447.58, + "max": 452.53 + } + } + }, + { + "scenario": "owned-sidecar", + "batchSize": 10, + "mode": "sequential", + "iterations": 5, + "coldStart": { + "samples": 50, + "mean": 780.07, + "p50": 778.64, + "p95": 802.66, + "p99": 817.39, + "min": 760.69, + "max": 817.39 + }, + "warmStart": { + "samples": 50, + "mean": 453.92, + "p50": 452.69, + "p95": 463.47, + "p99": 472.79, + "min": 448.9, + "max": 472.79 + }, + "phases": { + "filesystem_snapshot": { + "samples": 50, + "mean": 0.18, + "p50": 0.24, + "p95": 0.29, + "p99": 0.76, + "min": 0.05, + "max": 0.76 + }, + "sidecar_spawn": { + "samples": 50, + "mean": 3.07, + "p50": 2.74, + "p95": 5.08, + "p99": 5.57, + "min": 2.36, + "max": 5.57 + }, + "session_open": { + "samples": 50, + "mean": 2.69, + "p50": 2.44, + "p95": 4.12, + "p99": 5.86, + "min": 1.06, + "max": 5.86 + }, + "vm_create": { + "samples": 50, + "mean": 1.31, + "p50": 1.21, + "p95": 1.93, + "p99": 2.37, + "min": 1.05, + "max": 2.37 + }, + "vm_ready": { + "samples": 50, + "mean": 0.02, + "p50": 0.01, + "p95": 0.08, + "p99": 0.37, + "min": 0.01, + "max": 0.37 + }, + "vm_configure": { + "samples": 50, + "mean": 0.12, + "p50": 0.11, + "p95": 0.21, + "p99": 0.35, + "min": 0.08, + "max": 0.35 + }, + "runtime_mount_wasm": { + "samples": 50, + "mean": 174.85, + "p50": 173.25, + "p95": 187.89, + "p99": 211.45, + "min": 163.97, + "max": 211.45 + }, + "runtime_mount_node": { + "samples": 50, + "mean": 3.07, + "p50": 2.74, + "p95": 3.71, + "p99": 14.98, + "min": 2.53, + "max": 14.98 + }, + "runtime_create_total": { + "samples": 50, + "mean": 178, + "p50": 176.21, + "p95": 190.61, + "p99": 215.85, + "min": 166.69, + "max": 215.85 + }, + "first_exec": { + "samples": 50, + "mean": 602.07, + "p50": 601.18, + "p95": 614.83, + "p99": 618.39, + "min": 593.97, + "max": 618.39 + }, + "warm_exec": { + "samples": 50, + "mean": 453.92, + "p50": 452.69, + "p95": 463.47, + "p99": 472.79, + "min": 448.9, + "max": 472.79 + } + } + }, + { + "scenario": "owned-sidecar", + "batchSize": 10, + "mode": "concurrent", + "iterations": 5, + "coldStart": { + "samples": 50, + "mean": 1000.39, + "p50": 1028.29, + "p95": 1119.05, + "p99": 1137.11, + "min": 798.29, + "max": 1137.11 + }, + "warmStart": { + "samples": 50, + "mean": 477.27, + "p50": 469.19, + "p95": 532.17, + "p99": 550.94, + "min": 451.03, + "max": 550.94 + }, + "phases": { + "filesystem_snapshot": { + "samples": 50, + "mean": 46.54, + "p50": 31.48, + "p95": 120.71, + "p99": 142.03, + "min": 0.03, + "max": 142.03 + }, + "sidecar_spawn": { + "samples": 50, + "mean": 2.64, + "p50": 2.08, + "p95": 4.88, + "p99": 8.98, + "min": 1.62, + "max": 8.98 + }, + "session_open": { + "samples": 50, + "mean": 10.03, + "p50": 7.93, + "p95": 22.99, + "p99": 51.88, + "min": 1.85, + "max": 51.88 + }, + "vm_create": { + "samples": 50, + "mean": 22.65, + "p50": 2.54, + "p95": 91.3, + "p99": 112.29, + "min": 1.06, + "max": 112.29 + }, + "vm_ready": { + "samples": 50, + "mean": 0.01, + "p50": 0.01, + "p95": 0.01, + "p99": 0.02, + "min": 0, + "max": 0.02 + }, + "vm_configure": { + "samples": 50, + "mean": 32.8, + "p50": 22.13, + "p95": 96.03, + "p99": 112.46, + "min": 0.07, + "max": 112.46 + }, + "runtime_mount_wasm": { + "samples": 50, + "mean": 357.03, + "p50": 369.84, + "p95": 470.12, + "p99": 479.73, + "min": 188.48, + "max": 479.73 + }, + "runtime_mount_node": { + "samples": 50, + "mean": 3.5, + "p50": 3.08, + "p95": 5.13, + "p99": 8.36, + "min": 2.54, + "max": 8.36 + }, + "runtime_create_total": { + "samples": 50, + "mean": 360.62, + "p50": 373.11, + "p95": 473.4, + "p99": 482.44, + "min": 191.55, + "max": 482.44 + }, + "first_exec": { + "samples": 50, + "mean": 639.78, + "p50": 642.15, + "p95": 670.64, + "p99": 711.39, + "min": 601.8, + "max": 711.39 + }, + "warm_exec": { + "samples": 50, + "mean": 477.27, + "p50": 469.19, + "p95": 532.17, + "p99": 550.94, + "min": 451.03, + "max": 550.94 + } + } + }, + { + "scenario": "owned-sidecar", + "batchSize": 50, + "mode": "sequential", + "iterations": 5, + "coldStart": { + "samples": 250, + "mean": 777.49, + "p50": 775.3, + "p95": 797.46, + "p99": 810.01, + "min": 758.17, + "max": 842.78 + }, + "warmStart": { + "samples": 250, + "mean": 453.43, + "p50": 452.59, + "p95": 461.71, + "p99": 468.33, + "min": 446.49, + "max": 473.73 + }, + "phases": { + "filesystem_snapshot": { + "samples": 250, + "mean": 0.03, + "p50": 0.03, + "p95": 0.04, + "p99": 0.06, + "min": 0.02, + "max": 0.09 + }, + "sidecar_spawn": { + "samples": 250, + "mean": 3.66, + "p50": 3.13, + "p95": 5.57, + "p99": 7.97, + "min": 2.39, + "max": 11.09 + }, + "session_open": { + "samples": 250, + "mean": 2.63, + "p50": 2.36, + "p95": 3.93, + "p99": 5.17, + "min": 1.78, + "max": 7.13 + }, + "vm_create": { + "samples": 250, + "mean": 1.34, + "p50": 1.22, + "p95": 2.04, + "p99": 2.9, + "min": 1.01, + "max": 4.06 + }, + "vm_ready": { + "samples": 250, + "mean": 0.01, + "p50": 0.01, + "p95": 0.04, + "p99": 0.06, + "min": 0, + "max": 0.07 + }, + "vm_configure": { + "samples": 250, + "mean": 0.12, + "p50": 0.11, + "p95": 0.21, + "p99": 0.27, + "min": 0.07, + "max": 0.72 + }, + "runtime_mount_wasm": { + "samples": 250, + "mean": 173.44, + "p50": 171.68, + "p95": 186.85, + "p99": 202.94, + "min": 159.51, + "max": 214.03 + }, + "runtime_mount_node": { + "samples": 250, + "mean": 2.85, + "p50": 2.72, + "p95": 3.95, + "p99": 4.48, + "min": 2.5, + "max": 5.41 + }, + "runtime_create_total": { + "samples": 250, + "mean": 176.37, + "p50": 174.76, + "p95": 189.74, + "p99": 206.04, + "min": 162.34, + "max": 217.76 + }, + "first_exec": { + "samples": 250, + "mean": 601.12, + "p50": 599.8, + "p95": 612.78, + "p99": 625.02, + "min": 590.75, + "max": 629.36 + }, + "warm_exec": { + "samples": 250, + "mean": 453.43, + "p50": 452.59, + "p95": 461.71, + "p99": 468.33, + "min": 446.49, + "max": 473.73 + } + } + }, + { + "scenario": "owned-sidecar", + "batchSize": 50, + "mode": "concurrent", + "iterations": 5, + "coldStart": { + "samples": 250, + "mean": 1041.81, + "p50": 1044.33, + "p95": 1133.77, + "p99": 1160.99, + "min": 803.82, + "max": 1173.16 + }, + "warmStart": { + "samples": 250, + "mean": 468.05, + "p50": 465.77, + "p95": 491.47, + "p99": 500.52, + "min": 448.81, + "max": 504.51 + }, + "phases": { + "filesystem_snapshot": { + "samples": 250, + "mean": 51.91, + "p50": 45.06, + "p95": 120.47, + "p99": 125.55, + "min": 0.03, + "max": 127.77 + }, + "sidecar_spawn": { + "samples": 250, + "mean": 2.58, + "p50": 2.11, + "p95": 4.93, + "p99": 7.04, + "min": 1.48, + "max": 11.03 + }, + "session_open": { + "samples": 250, + "mean": 13.62, + "p50": 9.75, + "p95": 45.62, + "p99": 98.82, + "min": 2.04, + "max": 105.13 + }, + "vm_create": { + "samples": 250, + "mean": 23.51, + "p50": 21.44, + "p95": 74.05, + "p99": 117.57, + "min": 1.16, + "max": 132.92 + }, + "vm_ready": { + "samples": 250, + "mean": 0.01, + "p50": 0, + "p95": 0.01, + "p99": 0.05, + "min": 0, + "max": 0.24 + }, + "vm_configure": { + "samples": 250, + "mean": 40.77, + "p50": 34.25, + "p95": 109.91, + "p99": 127.38, + "min": 0.07, + "max": 140.87 + }, + "runtime_mount_wasm": { + "samples": 250, + "mean": 390.18, + "p50": 393.55, + "p95": 469.13, + "p99": 489.67, + "min": 195.44, + "max": 494.31 + }, + "runtime_mount_node": { + "samples": 250, + "mean": 3.56, + "p50": 3.23, + "p95": 4.85, + "p99": 5.94, + "min": 2.44, + "max": 14.6 + }, + "runtime_create_total": { + "samples": 250, + "mean": 393.82, + "p50": 396.28, + "p95": 471.79, + "p99": 492.32, + "min": 198.12, + "max": 497.15 + }, + "first_exec": { + "samples": 250, + "mean": 647.99, + "p50": 643.04, + "p95": 687.33, + "p99": 708.65, + "min": 601.99, + "max": 742.52 + }, + "warm_exec": { + "samples": 250, + "mean": 468.05, + "p50": 465.77, + "p95": 491.47, + "p99": 500.52, + "min": 448.81, + "max": 504.51 + } + } + }, + { + "scenario": "owned-sidecar", + "batchSize": 100, + "mode": "sequential", + "iterations": 5, + "coldStart": { + "samples": 500, + "mean": 777.89, + "p50": 776.54, + "p95": 793.79, + "p99": 802.12, + "min": 757.86, + "max": 877.76 + }, + "warmStart": { + "samples": 500, + "mean": 453.66, + "p50": 452.93, + "p95": 460.66, + "p99": 464, + "min": 446.28, + "max": 472.32 + }, + "phases": { + "filesystem_snapshot": { + "samples": 500, + "mean": 0.03, + "p50": 0.02, + "p95": 0.03, + "p99": 0.21, + "min": 0.02, + "max": 0.49 + }, + "sidecar_spawn": { + "samples": 500, + "mean": 4.62, + "p50": 4.22, + "p95": 6.59, + "p99": 7.61, + "min": 3.42, + "max": 14.42 + }, + "session_open": { + "samples": 500, + "mean": 2.51, + "p50": 2.33, + "p95": 3.7, + "p99": 4.36, + "min": 1.9, + "max": 5.92 + }, + "vm_create": { + "samples": 500, + "mean": 1.3, + "p50": 1.2, + "p95": 1.99, + "p99": 2.5, + "min": 1.01, + "max": 5.75 + }, + "vm_ready": { + "samples": 500, + "mean": 0.01, + "p50": 0.01, + "p95": 0.04, + "p99": 0.07, + "min": 0, + "max": 0.09 + }, + "vm_configure": { + "samples": 500, + "mean": 0.13, + "p50": 0.1, + "p95": 0.25, + "p99": 0.73, + "min": 0.07, + "max": 0.95 + }, + "runtime_mount_wasm": { + "samples": 500, + "mean": 174.6, + "p50": 173.29, + "p95": 187.32, + "p99": 194.52, + "min": 161.57, + "max": 247.12 + }, + "runtime_mount_node": { + "samples": 500, + "mean": 2.85, + "p50": 2.73, + "p95": 3.57, + "p99": 4.3, + "min": 2.49, + "max": 10.8 + }, + "runtime_create_total": { + "samples": 500, + "mean": 177.53, + "p50": 176.16, + "p95": 190.06, + "p99": 198.61, + "min": 164.25, + "max": 252.3 + }, + "first_exec": { + "samples": 500, + "mean": 600.36, + "p50": 599.6, + "p95": 608.94, + "p99": 614.21, + "min": 592.06, + "max": 625.46 + }, + "warm_exec": { + "samples": 500, + "mean": 453.66, + "p50": 452.93, + "p95": 460.66, + "p99": 464, + "min": 446.28, + "max": 472.32 + } + } + }, + { + "scenario": "owned-sidecar", + "batchSize": 100, + "mode": "concurrent", + "iterations": 5, + "coldStart": { + "samples": 500, + "mean": 1041.19, + "p50": 1043.72, + "p95": 1120.25, + "p99": 1136.03, + "min": 874.13, + "max": 1163.67 + }, + "warmStart": { + "samples": 500, + "mean": 468.6, + "p50": 466.36, + "p95": 492.1, + "p99": 508.12, + "min": 451, + "max": 523.72 + }, + "phases": { + "filesystem_snapshot": { + "samples": 500, + "mean": 51.71, + "p50": 45.26, + "p95": 115.99, + "p99": 121.44, + "min": 0.03, + "max": 127.91 + }, + "sidecar_spawn": { + "samples": 500, + "mean": 2.85, + "p50": 2.37, + "p95": 5.17, + "p99": 6.76, + "min": 1.86, + "max": 11.4 + }, + "session_open": { + "samples": 500, + "mean": 12.93, + "p50": 10.31, + "p95": 29.16, + "p99": 79.09, + "min": 2.12, + "max": 132.03 + }, + "vm_create": { + "samples": 500, + "mean": 27.26, + "p50": 23.2, + "p95": 69.69, + "p99": 112.08, + "min": 1.2, + "max": 140.51 + }, + "vm_ready": { + "samples": 500, + "mean": 0, + "p50": 0, + "p95": 0.01, + "p99": 0.01, + "min": 0, + "max": 0.02 + }, + "vm_configure": { + "samples": 500, + "mean": 38.93, + "p50": 30.68, + "p95": 104.19, + "p99": 125.52, + "min": 0.09, + "max": 147.19 + }, + "runtime_mount_wasm": { + "samples": 500, + "mean": 391.69, + "p50": 391.04, + "p95": 462.01, + "p99": 478.64, + "min": 249.44, + "max": 494.42 + }, + "runtime_mount_node": { + "samples": 500, + "mean": 3.44, + "p50": 3.18, + "p95": 4.66, + "p99": 5.08, + "min": 2.41, + "max": 15.95 + }, + "runtime_create_total": { + "samples": 500, + "mean": 395.19, + "p50": 394.78, + "p95": 465.21, + "p99": 482.88, + "min": 252.48, + "max": 499.13 + }, + "first_exec": { + "samples": 500, + "mean": 646, + "p50": 641.96, + "p95": 684.23, + "p99": 700.88, + "min": 607.26, + "max": 719.01 + }, + "warm_exec": { + "samples": 500, + "mean": 468.6, + "p50": 466.36, + "p95": 492.1, + "p99": 508.12, + "min": 451, + "max": 523.72 + } + } + }, + { + "scenario": "owned-sidecar", + "batchSize": 200, + "mode": "sequential", + "iterations": 5, + "coldStart": { + "samples": 1000, + "mean": 778.28, + "p50": 776.94, + "p95": 793.79, + "p99": 810.69, + "min": 759.62, + "max": 841.83 + }, + "warmStart": { + "samples": 1000, + "mean": 454.03, + "p50": 453.13, + "p95": 462.01, + "p99": 468.77, + "min": 446.24, + "max": 501.19 + }, + "phases": { + "filesystem_snapshot": { + "samples": 1000, + "mean": 0.03, + "p50": 0.02, + "p95": 0.03, + "p99": 0.24, + "min": 0.02, + "max": 0.7 + }, + "sidecar_spawn": { + "samples": 1000, + "mean": 4.64, + "p50": 4.3, + "p95": 6.58, + "p99": 8.15, + "min": 3.37, + "max": 14.31 + }, + "session_open": { + "samples": 1000, + "mean": 2.49, + "p50": 2.32, + "p95": 3.77, + "p99": 4.84, + "min": 0.86, + "max": 7.37 + }, + "vm_create": { + "samples": 1000, + "mean": 1.29, + "p50": 1.2, + "p95": 1.96, + "p99": 2.63, + "min": 1, + "max": 4.02 + }, + "vm_ready": { + "samples": 1000, + "mean": 0.01, + "p50": 0, + "p95": 0.04, + "p99": 0.06, + "min": 0, + "max": 0.13 + }, + "vm_configure": { + "samples": 1000, + "mean": 0.14, + "p50": 0.1, + "p95": 0.25, + "p99": 0.96, + "min": 0.06, + "max": 1.74 + }, + "runtime_mount_wasm": { + "samples": 1000, + "mean": 174.63, + "p50": 173.7, + "p95": 187.98, + "p99": 196.17, + "min": 159.88, + "max": 235.94 + }, + "runtime_mount_node": { + "samples": 1000, + "mean": 2.84, + "p50": 2.72, + "p95": 3.68, + "p99": 4.29, + "min": 2.5, + "max": 7.88 + }, + "runtime_create_total": { + "samples": 1000, + "mean": 177.55, + "p50": 176.64, + "p95": 190.7, + "p99": 199.01, + "min": 162.73, + "max": 238.71 + }, + "first_exec": { + "samples": 1000, + "mean": 600.73, + "p50": 599.84, + "p95": 609.8, + "p99": 618.15, + "min": 589.53, + "max": 637.83 + }, + "warm_exec": { + "samples": 1000, + "mean": 454.03, + "p50": 453.13, + "p95": 462.01, + "p99": 468.77, + "min": 446.24, + "max": 501.19 + } + } + }, + { + "scenario": "owned-sidecar", + "batchSize": 200, + "mode": "concurrent", + "iterations": 5, + "coldStart": { + "samples": 1000, + "mean": 1044.03, + "p50": 1042.73, + "p95": 1124.88, + "p99": 1162.92, + "min": 925.95, + "max": 1198.97 + }, + "warmStart": { + "samples": 1000, + "mean": 468.68, + "p50": 466.89, + "p95": 488.09, + "p99": 513.54, + "min": 449.6, + "max": 520.28 + }, + "phases": { + "filesystem_snapshot": { + "samples": 1000, + "mean": 51.87, + "p50": 47.8, + "p95": 114.52, + "p99": 120.39, + "min": 0.03, + "max": 137.72 + }, + "sidecar_spawn": { + "samples": 1000, + "mean": 2.93, + "p50": 2.39, + "p95": 5.36, + "p99": 7.61, + "min": 1.86, + "max": 10.09 + }, + "session_open": { + "samples": 1000, + "mean": 13.82, + "p50": 10.58, + "p95": 37.18, + "p99": 76.75, + "min": 2, + "max": 112.27 + }, + "vm_create": { + "samples": 1000, + "mean": 26.98, + "p50": 22.78, + "p95": 77.31, + "p99": 108.9, + "min": 1.17, + "max": 231.51 + }, + "vm_ready": { + "samples": 1000, + "mean": 0, + "p50": 0, + "p95": 0.01, + "p99": 0.01, + "min": 0, + "max": 0.05 + }, + "vm_configure": { + "samples": 1000, + "mean": 39.14, + "p50": 33.55, + "p95": 104.19, + "p99": 128.42, + "min": 0.08, + "max": 146.77 + }, + "runtime_mount_wasm": { + "samples": 1000, + "mean": 394.48, + "p50": 391.19, + "p95": 470.52, + "p99": 493.46, + "min": 288.96, + "max": 547.77 + }, + "runtime_mount_node": { + "samples": 1000, + "mean": 3.45, + "p50": 3.26, + "p95": 4.63, + "p99": 5.02, + "min": 2.46, + "max": 6.2 + }, + "runtime_create_total": { + "samples": 1000, + "mean": 397.99, + "p50": 394.96, + "p95": 473.6, + "p99": 497.73, + "min": 291.81, + "max": 551.16 + }, + "first_exec": { + "samples": 1000, + "mean": 646.03, + "p50": 640.8, + "p95": 684, + "p99": 698.73, + "min": 613.51, + "max": 743.44 + }, + "warm_exec": { + "samples": 1000, + "mean": 468.68, + "p50": 466.89, + "p95": 488.09, + "p99": 513.54, + "min": 449.6, + "max": 520.28 + } + } + }, + { + "scenario": "shared-sidecar", + "batchSize": 1, + "mode": "sequential", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 4.09, + "p50": 3.94, + "p95": 5.26, + "p99": 5.26, + "min": 3.47, + "max": 5.26 + }, + "coldStart": { + "samples": 5, + "mean": 774.17, + "p50": 774.5, + "p95": 778.62, + "p99": 778.62, + "min": 767.75, + "max": 778.62 + }, + "warmStart": { + "samples": 5, + "mean": 457.19, + "p50": 452.75, + "p95": 466.31, + "p99": 466.31, + "min": 450.8, + "max": 466.31 + }, + "phases": { + "filesystem_snapshot": { + "samples": 5, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.03, + "min": 0.02, + "max": 0.03 + }, + "session_open": { + "samples": 5, + "mean": 0.37, + "p50": 0.3, + "p95": 0.71, + "p99": 0.71, + "min": 0.21, + "max": 0.71 + }, + "vm_create": { + "samples": 5, + "mean": 1.48, + "p50": 1.22, + "p95": 2.63, + "p99": 2.63, + "min": 1.08, + "max": 2.63 + }, + "vm_ready": { + "samples": 5, + "mean": 0, + "p50": 0, + "p95": 0.01, + "p99": 0.01, + "min": 0, + "max": 0.01 + }, + "vm_configure": { + "samples": 5, + "mean": 0.12, + "p50": 0.11, + "p95": 0.23, + "p99": 0.23, + "min": 0.09, + "max": 0.23 + }, + "runtime_mount_wasm": { + "samples": 5, + "mean": 169.17, + "p50": 170.47, + "p95": 175.92, + "p99": 175.92, + "min": 161.9, + "max": 175.92 + }, + "runtime_mount_node": { + "samples": 5, + "mean": 2.63, + "p50": 2.61, + "p95": 2.77, + "p99": 2.77, + "min": 2.53, + "max": 2.77 + }, + "runtime_create_total": { + "samples": 5, + "mean": 171.89, + "p50": 173.09, + "p95": 178.62, + "p99": 178.62, + "min": 164.59, + "max": 178.62 + }, + "first_exec": { + "samples": 5, + "mean": 602.27, + "p50": 604.54, + "p95": 609.91, + "p99": 609.91, + "min": 594.65, + "max": 609.91 + }, + "warm_exec": { + "samples": 5, + "mean": 457.19, + "p50": 452.75, + "p95": 466.31, + "p99": 466.31, + "min": 450.8, + "max": 466.31 + } + } + }, + { + "scenario": "shared-sidecar", + "batchSize": 1, + "mode": "concurrent", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 6.12, + "p50": 5.21, + "p95": 9.57, + "p99": 9.57, + "min": 5.06, + "max": 9.57 + }, + "coldStart": { + "samples": 5, + "mean": 775.66, + "p50": 776.97, + "p95": 780.66, + "p99": 780.66, + "min": 766.27, + "max": 780.66 + }, + "warmStart": { + "samples": 5, + "mean": 452.79, + "p50": 453.29, + "p95": 453.98, + "p99": 453.98, + "min": 451.29, + "max": 453.98 + }, + "phases": { + "filesystem_snapshot": { + "samples": 5, + "mean": 0.02, + "p50": 0.02, + "p95": 0.02, + "p99": 0.02, + "min": 0.02, + "max": 0.02 + }, + "session_open": { + "samples": 5, + "mean": 0.38, + "p50": 0.4, + "p95": 0.49, + "p99": 0.49, + "min": 0.24, + "max": 0.49 + }, + "vm_create": { + "samples": 5, + "mean": 1.28, + "p50": 1.14, + "p95": 1.87, + "p99": 1.87, + "min": 1.1, + "max": 1.87 + }, + "vm_ready": { + "samples": 5, + "mean": 0.01, + "p50": 0, + "p95": 0.02, + "p99": 0.02, + "min": 0, + "max": 0.02 + }, + "vm_configure": { + "samples": 5, + "mean": 0.09, + "p50": 0.09, + "p95": 0.12, + "p99": 0.12, + "min": 0.07, + "max": 0.12 + }, + "runtime_mount_wasm": { + "samples": 5, + "mean": 169.66, + "p50": 169.79, + "p95": 171.77, + "p99": 171.77, + "min": 166.04, + "max": 171.77 + }, + "runtime_mount_node": { + "samples": 5, + "mean": 3.18, + "p50": 2.71, + "p95": 4.19, + "p99": 4.19, + "min": 2.59, + "max": 4.19 + }, + "runtime_create_total": { + "samples": 5, + "mean": 172.93, + "p50": 173.44, + "p95": 176.06, + "p99": 176.06, + "min": 168.76, + "max": 176.06 + }, + "first_exec": { + "samples": 5, + "mean": 602.73, + "p50": 603.17, + "p95": 605.43, + "p99": 605.43, + "min": 597.5, + "max": 605.43 + }, + "warm_exec": { + "samples": 5, + "mean": 452.79, + "p50": 453.29, + "p95": 453.98, + "p99": 453.98, + "min": 451.29, + "max": 453.98 + } + } + }, + { + "scenario": "shared-sidecar", + "batchSize": 10, + "mode": "sequential", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.51, + "p50": 5.49, + "p95": 5.73, + "p99": 5.73, + "min": 5.37, + "max": 5.73 + }, + "coldStart": { + "samples": 50, + "mean": 627.58, + "p50": 613.98, + "p95": 773.01, + "p99": 783.61, + "min": 592.18, + "max": 783.61 + }, + "warmStart": { + "samples": 50, + "mean": 450.97, + "p50": 450.94, + "p95": 459.89, + "p99": 466.32, + "min": 443.57, + "max": 466.32 + }, + "phases": { + "filesystem_snapshot": { + "samples": 50, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.04, + "min": 0.02, + "max": 0.04 + }, + "session_open": { + "samples": 50, + "mean": 0.21, + "p50": 0.19, + "p95": 0.35, + "p99": 0.38, + "min": 0.14, + "max": 0.38 + }, + "vm_create": { + "samples": 50, + "mean": 1.3, + "p50": 1.17, + "p95": 1.96, + "p99": 2.3, + "min": 0.96, + "max": 2.3 + }, + "vm_ready": { + "samples": 50, + "mean": 0.01, + "p50": 0, + "p95": 0.03, + "p99": 0.03, + "min": 0, + "max": 0.03 + }, + "vm_configure": { + "samples": 50, + "mean": 0.1, + "p50": 0.09, + "p95": 0.19, + "p99": 0.21, + "min": 0.06, + "max": 0.21 + }, + "runtime_mount_wasm": { + "samples": 50, + "mean": 159.99, + "p50": 158.99, + "p95": 173.9, + "p99": 176.82, + "min": 143.47, + "max": 176.82 + }, + "runtime_mount_node": { + "samples": 50, + "mean": 2.84, + "p50": 2.68, + "p95": 3.92, + "p99": 4.16, + "min": 2.52, + "max": 4.16 + }, + "runtime_create_total": { + "samples": 50, + "mean": 162.9, + "p50": 161.69, + "p95": 177.85, + "p99": 179.59, + "min": 146.04, + "max": 179.59 + }, + "first_exec": { + "samples": 50, + "mean": 464.68, + "p50": 449.24, + "p95": 601.5, + "p99": 605.66, + "min": 444.51, + "max": 605.66 + }, + "warm_exec": { + "samples": 50, + "mean": 450.97, + "p50": 450.94, + "p95": 459.89, + "p99": 466.32, + "min": 443.57, + "max": 466.32 + } + } + }, + { + "scenario": "shared-sidecar", + "batchSize": 10, + "mode": "concurrent", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 4.94, + "p50": 5.23, + "p95": 5.88, + "p99": 5.88, + "min": 3.84, + "max": 5.88 + }, + "coldStart": { + "samples": 50, + "mean": 3493.51, + "p50": 4558.22, + "p95": 4676.49, + "p99": 6652.52, + "min": 725.69, + "max": 6652.52 + }, + "warmStart": { + "samples": 50, + "mean": 2174.08, + "p50": 2086.34, + "p95": 4965.71, + "p99": 5377.91, + "min": 438.99, + "max": 5377.91 + }, + "phases": { + "filesystem_snapshot": { + "samples": 50, + "mean": 43.92, + "p50": 25.47, + "p95": 118.61, + "p99": 126.64, + "min": 0.03, + "max": 126.64 + }, + "session_open": { + "samples": 50, + "mean": 0.61, + "p50": 0.57, + "p95": 1.17, + "p99": 1.26, + "min": 0.2, + "max": 1.26 + }, + "vm_create": { + "samples": 50, + "mean": 3.79, + "p50": 3.39, + "p95": 7.51, + "p99": 8.95, + "min": 1.13, + "max": 8.95 + }, + "vm_ready": { + "samples": 50, + "mean": 0.23, + "p50": 0, + "p95": 0.04, + "p99": 11.07, + "min": 0, + "max": 11.07 + }, + "vm_configure": { + "samples": 50, + "mean": 19.29, + "p50": 11.57, + "p95": 72.25, + "p99": 103.13, + "min": 0.18, + "max": 103.13 + }, + "runtime_mount_wasm": { + "samples": 50, + "mean": 920.58, + "p50": 1051.38, + "p95": 1149.41, + "p99": 1158.81, + "min": 270.43, + "max": 1158.81 + }, + "runtime_mount_node": { + "samples": 50, + "mean": 16.09, + "p50": 18.54, + "p95": 19.55, + "p99": 19.6, + "min": 4.69, + "max": 19.6 + }, + "runtime_create_total": { + "samples": 50, + "mean": 936.73, + "p50": 1070.66, + "p95": 1168.91, + "p99": 1178.48, + "min": 275.42, + "max": 1178.48 + }, + "first_exec": { + "samples": 50, + "mean": 2556.78, + "p50": 3507.02, + "p95": 3519.16, + "p99": 5581.86, + "min": 448.08, + "max": 5581.86 + }, + "warm_exec": { + "samples": 50, + "mean": 2174.08, + "p50": 2086.34, + "p95": 4965.71, + "p99": 5377.91, + "min": 438.99, + "max": 5377.91 + } + } + }, + { + "scenario": "shared-sidecar", + "batchSize": 50, + "mode": "sequential", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.12, + "p50": 5.09, + "p95": 5.39, + "p99": 5.39, + "min": 4.93, + "max": 5.39 + }, + "coldStart": { + "samples": 250, + "mean": 610.76, + "p50": 605.65, + "p95": 627.44, + "p99": 775.68, + "min": 591.17, + "max": 788.91 + }, + "warmStart": { + "samples": 250, + "mean": 449.02, + "p50": 448.23, + "p95": 456.81, + "p99": 464.36, + "min": 442.29, + "max": 469.3 + }, + "phases": { + "filesystem_snapshot": { + "samples": 250, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.05, + "min": 0.02, + "max": 0.07 + }, + "session_open": { + "samples": 250, + "mean": 0.26, + "p50": 0.21, + "p95": 0.41, + "p99": 1.18, + "min": 0.12, + "max": 3.85 + }, + "vm_create": { + "samples": 250, + "mean": 1.28, + "p50": 1.14, + "p95": 1.94, + "p99": 2.34, + "min": 0.93, + "max": 3.14 + }, + "vm_ready": { + "samples": 250, + "mean": 0.01, + "p50": 0, + "p95": 0.03, + "p99": 0.04, + "min": 0, + "max": 0.23 + }, + "vm_configure": { + "samples": 250, + "mean": 0.12, + "p50": 0.09, + "p95": 0.19, + "p99": 0.34, + "min": 0.06, + "max": 2.44 + }, + "runtime_mount_wasm": { + "samples": 250, + "mean": 155.28, + "p50": 153.45, + "p95": 169.89, + "p99": 187.15, + "min": 142.75, + "max": 203.16 + }, + "runtime_mount_node": { + "samples": 250, + "mean": 2.8, + "p50": 2.67, + "p95": 3.69, + "p99": 4.13, + "min": 2.48, + "max": 8.15 + }, + "runtime_create_total": { + "samples": 250, + "mean": 158.16, + "p50": 156.3, + "p95": 172.65, + "p99": 189.87, + "min": 145.32, + "max": 206.18 + }, + "first_exec": { + "samples": 250, + "mean": 452.6, + "p50": 448.83, + "p95": 459, + "p99": 599.03, + "min": 442.61, + "max": 603.9 + }, + "warm_exec": { + "samples": 250, + "mean": 449.02, + "p50": 448.23, + "p95": 456.81, + "p99": 464.36, + "min": 442.29, + "max": 469.3 + } + } + }, + { + "scenario": "shared-sidecar", + "batchSize": 50, + "mode": "concurrent", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 7.15, + "p50": 5.56, + "p95": 14.37, + "p99": 14.37, + "min": 3.96, + "max": 14.37 + }, + "coldStart": { + "samples": 250, + "mean": 3197.96, + "p50": 3299.89, + "p95": 4602.98, + "p99": 4657.39, + "min": 742.4, + "max": 6126.19 + }, + "warmStart": { + "samples": 250, + "mean": 2162.03, + "p50": 2212.06, + "p95": 3395.52, + "p99": 5415.61, + "min": 345.69, + "max": 5535.61 + }, + "phases": { + "filesystem_snapshot": { + "samples": 250, + "mean": 48.75, + "p50": 43.21, + "p95": 113.11, + "p99": 124.69, + "min": 0.03, + "max": 128.47 + }, + "session_open": { + "samples": 250, + "mean": 0.52, + "p50": 0.48, + "p95": 1.34, + "p99": 1.7, + "min": 0.15, + "max": 1.77 + }, + "vm_create": { + "samples": 250, + "mean": 4.33, + "p50": 4.2, + "p95": 8.07, + "p99": 9.82, + "min": 0.95, + "max": 10.89 + }, + "vm_ready": { + "samples": 250, + "mean": 0.4, + "p50": 0, + "p95": 0.04, + "p99": 10.69, + "min": 0, + "max": 48.48 + }, + "vm_configure": { + "samples": 250, + "mean": 21.82, + "p50": 16.13, + "p95": 55.91, + "p99": 90.5, + "min": 0.77, + "max": 92.12 + }, + "runtime_mount_wasm": { + "samples": 250, + "mean": 1061.42, + "p50": 1088.07, + "p95": 1166.25, + "p99": 1193.51, + "min": 288.93, + "max": 1210.24 + }, + "runtime_mount_node": { + "samples": 250, + "mean": 18.88, + "p50": 19.28, + "p95": 21.01, + "p99": 21.38, + "min": 4.8, + "max": 21.38 + }, + "runtime_create_total": { + "samples": 250, + "mean": 1080.36, + "p50": 1107.01, + "p95": 1184.53, + "p99": 1212.99, + "min": 294.01, + "max": 1230.75 + }, + "first_exec": { + "samples": 250, + "mean": 2117.6, + "p50": 2211.64, + "p95": 3514.23, + "p99": 3520.26, + "min": 373.91, + "max": 5044.43 + }, + "warm_exec": { + "samples": 250, + "mean": 2162.03, + "p50": 2212.06, + "p95": 3395.52, + "p99": 5415.61, + "min": 345.69, + "max": 5535.61 + } + } + }, + { + "scenario": "shared-sidecar", + "batchSize": 100, + "mode": "sequential", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.22, + "p50": 5.43, + "p95": 5.74, + "p99": 5.74, + "min": 4.48, + "max": 5.74 + }, + "coldStart": { + "samples": 500, + "mean": 619.24, + "p50": 610.78, + "p95": 656.44, + "p99": 774.55, + "min": 590.27, + "max": 970.58 + }, + "warmStart": { + "samples": 500, + "mean": 453.68, + "p50": 449.44, + "p95": 472.09, + "p99": 482.32, + "min": 442.18, + "max": 588.18 + }, + "phases": { + "filesystem_snapshot": { + "samples": 500, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.06, + "min": 0.02, + "max": 0.25 + }, + "session_open": { + "samples": 500, + "mean": 0.25, + "p50": 0.21, + "p95": 0.45, + "p99": 0.69, + "min": 0.14, + "max": 3.12 + }, + "vm_create": { + "samples": 500, + "mean": 1.27, + "p50": 1.12, + "p95": 1.98, + "p99": 2.41, + "min": 0.95, + "max": 4.51 + }, + "vm_ready": { + "samples": 500, + "mean": 0.01, + "p50": 0.01, + "p95": 0.04, + "p99": 0.23, + "min": 0, + "max": 0.26 + }, + "vm_configure": { + "samples": 500, + "mean": 0.17, + "p50": 0.1, + "p95": 0.29, + "p99": 2.49, + "min": 0.06, + "max": 3.32 + }, + "runtime_mount_wasm": { + "samples": 500, + "mean": 160.04, + "p50": 157.04, + "p95": 180.19, + "p99": 187.13, + "min": 143.95, + "max": 215.2 + }, + "runtime_mount_node": { + "samples": 500, + "mean": 3.63, + "p50": 2.84, + "p95": 3.52, + "p99": 4.54, + "min": 2.5, + "max": 348 + }, + "runtime_create_total": { + "samples": 500, + "mean": 163.74, + "p50": 159.9, + "p95": 183.45, + "p99": 190.57, + "min": 146.67, + "max": 511.74 + }, + "first_exec": { + "samples": 500, + "mean": 455.5, + "p50": 449.94, + "p95": 471.76, + "p99": 582.17, + "min": 442.77, + "max": 622.04 + }, + "warm_exec": { + "samples": 500, + "mean": 453.68, + "p50": 449.44, + "p95": 472.09, + "p99": 482.32, + "min": 442.18, + "max": 588.18 + } + } + }, + { + "scenario": "shared-sidecar", + "batchSize": 100, + "mode": "concurrent", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.09, + "p50": 5.49, + "p95": 6.49, + "p99": 6.49, + "min": 3.71, + "max": 6.49 + }, + "coldStart": { + "samples": 500, + "mean": 3187.99, + "p50": 3317.35, + "p95": 4591.89, + "p99": 4694.52, + "min": 1111.29, + "max": 6710.25 + }, + "warmStart": { + "samples": 500, + "mean": 2134.07, + "p50": 2207.08, + "p95": 2716.34, + "p99": 3940.45, + "min": 438.63, + "max": 4994.58 + }, + "phases": { + "filesystem_snapshot": { + "samples": 500, + "mean": 49.97, + "p50": 44.7, + "p95": 111.92, + "p99": 119.87, + "min": 0.03, + "max": 131.84 + }, + "session_open": { + "samples": 500, + "mean": 0.59, + "p50": 0.54, + "p95": 1.32, + "p99": 1.59, + "min": 0.13, + "max": 1.66 + }, + "vm_create": { + "samples": 500, + "mean": 4.3, + "p50": 4.24, + "p95": 7.73, + "p99": 9.48, + "min": 0.98, + "max": 10.6 + }, + "vm_ready": { + "samples": 500, + "mean": 0.55, + "p50": 0, + "p95": 0.05, + "p99": 25.38, + "min": 0, + "max": 34.38 + }, + "vm_configure": { + "samples": 500, + "mean": 19.08, + "p50": 14.62, + "p95": 50.06, + "p99": 72.85, + "min": 2.37, + "max": 81.81 + }, + "runtime_mount_wasm": { + "samples": 500, + "mean": 1092.42, + "p50": 1110.41, + "p95": 1185.79, + "p99": 1210.14, + "min": 565.86, + "max": 1221.78 + }, + "runtime_mount_node": { + "samples": 500, + "mean": 19.53, + "p50": 19.5, + "p95": 21.82, + "p99": 30.38, + "min": 9.73, + "max": 30.52 + }, + "runtime_create_total": { + "samples": 500, + "mean": 1112, + "p50": 1130.31, + "p95": 1205.87, + "p99": 1230.32, + "min": 585.93, + "max": 1242 + }, + "first_exec": { + "samples": 500, + "mean": 2075.99, + "p50": 2212.38, + "p95": 3512.6, + "p99": 3535.78, + "min": 365.84, + "max": 5608.44 + }, + "warm_exec": { + "samples": 500, + "mean": 2134.07, + "p50": 2207.08, + "p95": 2716.34, + "p99": 3940.45, + "min": 438.63, + "max": 4994.58 + } + } + }, + { + "scenario": "shared-sidecar", + "batchSize": 200, + "mode": "sequential", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.17, + "p50": 5.26, + "p95": 6.09, + "p99": 6.09, + "min": 4.06, + "max": 6.09 + }, + "coldStart": { + "samples": 1000, + "mean": 616.2, + "p50": 614.43, + "p95": 633.42, + "p99": 644.44, + "min": 592.87, + "max": 945.4 + }, + "warmStart": { + "samples": 1000, + "mean": 449.22, + "p50": 448.24, + "p95": 456.27, + "p99": 463.62, + "min": 441.39, + "max": 631.44 + }, + "phases": { + "filesystem_snapshot": { + "samples": 1000, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.05, + "min": 0.02, + "max": 0.27 + }, + "session_open": { + "samples": 1000, + "mean": 0.28, + "p50": 0.25, + "p95": 0.49, + "p99": 0.74, + "min": 0.12, + "max": 1.43 + }, + "vm_create": { + "samples": 1000, + "mean": 1.19, + "p50": 1.11, + "p95": 1.77, + "p99": 2.23, + "min": 0.95, + "max": 2.81 + }, + "vm_ready": { + "samples": 1000, + "mean": 0.01, + "p50": 0.01, + "p95": 0.03, + "p99": 0.22, + "min": 0, + "max": 0.28 + }, + "vm_configure": { + "samples": 1000, + "mean": 0.17, + "p50": 0.12, + "p95": 0.31, + "p99": 1.94, + "min": 0.07, + "max": 3.43 + }, + "runtime_mount_wasm": { + "samples": 1000, + "mean": 162.26, + "p50": 161.85, + "p95": 177.54, + "p99": 184.97, + "min": 143.69, + "max": 214.76 + }, + "runtime_mount_node": { + "samples": 1000, + "mean": 3.33, + "p50": 2.92, + "p95": 3.6, + "p99": 4.52, + "min": 2.49, + "max": 329.2 + }, + "runtime_create_total": { + "samples": 1000, + "mean": 165.66, + "p50": 164.85, + "p95": 181, + "p99": 188.95, + "min": 146.4, + "max": 489.96 + }, + "first_exec": { + "samples": 1000, + "mean": 450.54, + "p50": 448.77, + "p95": 457.83, + "p99": 473.65, + "min": 441.44, + "max": 613.1 + }, + "warm_exec": { + "samples": 1000, + "mean": 449.22, + "p50": 448.24, + "p95": 456.27, + "p99": 463.62, + "min": 441.39, + "max": 631.44 + } + } + }, + { + "scenario": "shared-sidecar", + "batchSize": 200, + "mode": "concurrent", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 3.81, + "p50": 3.79, + "p95": 3.92, + "p99": 3.92, + "min": 3.73, + "max": 3.92 + }, + "coldStart": { + "samples": 1000, + "mean": 3235.65, + "p50": 3340.36, + "p95": 3501.49, + "p99": 4639.9, + "min": 1603.72, + "max": 6731.02 + }, + "warmStart": { + "samples": 1000, + "mean": 2102.22, + "p50": 2200.24, + "p95": 2732.96, + "p99": 3665.63, + "min": 299.53, + "max": 5422.94 + }, + "phases": { + "filesystem_snapshot": { + "samples": 1000, + "mean": 49.92, + "p50": 47.71, + "p95": 111.37, + "p99": 116.24, + "min": 0.03, + "max": 123.67 + }, + "session_open": { + "samples": 1000, + "mean": 0.73, + "p50": 0.61, + "p95": 1.79, + "p99": 2.3, + "min": 0.13, + "max": 2.61 + }, + "vm_create": { + "samples": 1000, + "mean": 4.15, + "p50": 4.14, + "p95": 7.02, + "p99": 8.07, + "min": 0.96, + "max": 9.81 + }, + "vm_ready": { + "samples": 1000, + "mean": 0.12, + "p50": 0, + "p95": 0.03, + "p99": 0.05, + "min": 0, + "max": 54.89 + }, + "vm_configure": { + "samples": 1000, + "mean": 17.93, + "p50": 14.71, + "p95": 40.65, + "p99": 57.22, + "min": 1.6, + "max": 102.1 + }, + "runtime_mount_wasm": { + "samples": 1000, + "mean": 1145.4, + "p50": 1147.31, + "p95": 1239.47, + "p99": 1262.54, + "min": 996.9, + "max": 1290.99 + }, + "runtime_mount_node": { + "samples": 1000, + "mean": 20.78, + "p50": 20.37, + "p95": 22.55, + "p99": 23.87, + "min": 18.46, + "max": 61.17 + }, + "runtime_create_total": { + "samples": 1000, + "mean": 1166.23, + "p50": 1168.28, + "p95": 1260.91, + "p99": 1284.1, + "min": 1016.18, + "max": 1311.64 + }, + "first_exec": { + "samples": 1000, + "mean": 2069.42, + "p50": 2210.55, + "p95": 2262.54, + "p99": 3529.65, + "min": 369.41, + "max": 5578.1 + }, + "warm_exec": { + "samples": 1000, + "mean": 2102.22, + "p50": 2200.24, + "p95": 2732.96, + "p99": 3665.63, + "min": 299.53, + "max": 5422.94 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 1, + "mode": "sequential", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 4.81, + "p50": 4.19, + "p95": 6.14, + "p99": 6.14, + "min": 3.83, + "max": 6.14 + }, + "coldStart": { + "samples": 5, + "mean": 351, + "p50": 349.56, + "p95": 358.02, + "p99": 358.02, + "min": 347.63, + "max": 358.02 + }, + "warmStart": { + "samples": 5, + "mean": 1.27, + "p50": 1.27, + "p95": 1.32, + "p99": 1.32, + "min": 1.24, + "max": 1.32 + }, + "phases": { + "filesystem_snapshot": { + "samples": 5, + "mean": 0.02, + "p50": 0.02, + "p95": 0.02, + "p99": 0.02, + "min": 0.02, + "max": 0.02 + }, + "session_open": { + "samples": 5, + "mean": 0.33, + "p50": 0.29, + "p95": 0.43, + "p99": 0.43, + "min": 0.27, + "max": 0.43 + }, + "vm_create": { + "samples": 5, + "mean": 1.22, + "p50": 1.26, + "p95": 1.34, + "p99": 1.34, + "min": 1.12, + "max": 1.34 + }, + "vm_ready": { + "samples": 5, + "mean": 0.01, + "p50": 0.01, + "p95": 0.01, + "p99": 0.01, + "min": 0.01, + "max": 0.01 + }, + "vm_configure": { + "samples": 5, + "mean": 0.11, + "p50": 0.09, + "p95": 0.19, + "p99": 0.19, + "min": 0.09, + "max": 0.19 + }, + "runtime_mount_wasm": { + "samples": 5, + "mean": 171.79, + "p50": 170.03, + "p95": 179.45, + "p99": 179.45, + "min": 165.9, + "max": 179.45 + }, + "runtime_mount_node": { + "samples": 5, + "mean": 2.85, + "p50": 2.77, + "p95": 3.22, + "p99": 3.22, + "min": 2.58, + "max": 3.22 + }, + "runtime_create_total": { + "samples": 5, + "mean": 174.73, + "p50": 173.03, + "p95": 182.28, + "p99": 182.28, + "min": 168.77, + "max": 182.28 + }, + "resident_runner_create": { + "samples": 5, + "mean": 174.73, + "p50": 174.4, + "p95": 178.32, + "p99": 178.32, + "min": 173.04, + "max": 178.32 + }, + "resident_first_exec": { + "samples": 5, + "mean": 1.54, + "p50": 1.31, + "p95": 2.47, + "p99": 2.47, + "min": 1.25, + "max": 2.47 + }, + "resident_warm_exec": { + "samples": 5, + "mean": 1.27, + "p50": 1.27, + "p95": 1.32, + "p99": 1.32, + "min": 1.24, + "max": 1.32 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 1, + "mode": "concurrent", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 4.6, + "p50": 4.08, + "p95": 6.27, + "p99": 6.27, + "min": 3.54, + "max": 6.27 + }, + "coldStart": { + "samples": 5, + "mean": 347.91, + "p50": 346.99, + "p95": 358.2, + "p99": 358.2, + "min": 340.22, + "max": 358.2 + }, + "warmStart": { + "samples": 5, + "mean": 1.25, + "p50": 1.26, + "p95": 1.29, + "p99": 1.29, + "min": 1.19, + "max": 1.29 + }, + "phases": { + "filesystem_snapshot": { + "samples": 5, + "mean": 0.02, + "p50": 0.02, + "p95": 0.02, + "p99": 0.02, + "min": 0.02, + "max": 0.02 + }, + "session_open": { + "samples": 5, + "mean": 0.37, + "p50": 0.3, + "p95": 0.71, + "p99": 0.71, + "min": 0.22, + "max": 0.71 + }, + "vm_create": { + "samples": 5, + "mean": 1.75, + "p50": 1.21, + "p95": 3.24, + "p99": 3.24, + "min": 1.06, + "max": 3.24 + }, + "vm_ready": { + "samples": 5, + "mean": 0.01, + "p50": 0.01, + "p95": 0.01, + "p99": 0.01, + "min": 0.01, + "max": 0.01 + }, + "vm_configure": { + "samples": 5, + "mean": 0.71, + "p50": 0.1, + "p95": 3.14, + "p99": 3.14, + "min": 0.08, + "max": 3.14 + }, + "runtime_mount_wasm": { + "samples": 5, + "mean": 169.97, + "p50": 171.18, + "p95": 176.64, + "p99": 176.64, + "min": 160.07, + "max": 176.64 + }, + "runtime_mount_node": { + "samples": 5, + "mean": 2.77, + "p50": 2.67, + "p95": 3.13, + "p99": 3.13, + "min": 2.61, + "max": 3.13 + }, + "runtime_create_total": { + "samples": 5, + "mean": 172.82, + "p50": 174.09, + "p95": 179.34, + "p99": 179.34, + "min": 162.77, + "max": 179.34 + }, + "resident_runner_create": { + "samples": 5, + "mean": 173.75, + "p50": 172.53, + "p95": 177.55, + "p99": 177.55, + "min": 170.41, + "max": 177.55 + }, + "resident_first_exec": { + "samples": 5, + "mean": 1.34, + "p50": 1.32, + "p95": 1.49, + "p99": 1.49, + "min": 1.23, + "max": 1.49 + }, + "resident_warm_exec": { + "samples": 5, + "mean": 1.25, + "p50": 1.26, + "p95": 1.29, + "p99": 1.29, + "min": 1.19, + "max": 1.29 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 10, + "mode": "sequential", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.17, + "p50": 5.36, + "p95": 5.73, + "p99": 5.73, + "min": 4.08, + "max": 5.73 + }, + "coldStart": { + "samples": 50, + "mean": 202.61, + "p50": 186.16, + "p95": 350.97, + "p99": 353.52, + "min": 176.1, + "max": 353.52 + }, + "warmStart": { + "samples": 50, + "mean": 1.26, + "p50": 1.24, + "p95": 1.37, + "p99": 1.57, + "min": 1.17, + "max": 1.57 + }, + "phases": { + "filesystem_snapshot": { + "samples": 50, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.05, + "min": 0.02, + "max": 0.05 + }, + "session_open": { + "samples": 50, + "mean": 0.28, + "p50": 0.2, + "p95": 0.44, + "p99": 1.94, + "min": 0.13, + "max": 1.94 + }, + "vm_create": { + "samples": 50, + "mean": 1.4, + "p50": 1.2, + "p95": 2.49, + "p99": 2.69, + "min": 0.98, + "max": 2.69 + }, + "vm_ready": { + "samples": 50, + "mean": 0.01, + "p50": 0.01, + "p95": 0.04, + "p99": 0.05, + "min": 0, + "max": 0.05 + }, + "vm_configure": { + "samples": 50, + "mean": 0.16, + "p50": 0.09, + "p95": 0.21, + "p99": 2.67, + "min": 0.07, + "max": 2.67 + }, + "runtime_mount_wasm": { + "samples": 50, + "mean": 157.05, + "p50": 154.63, + "p95": 171.86, + "p99": 174.71, + "min": 146.87, + "max": 174.71 + }, + "runtime_mount_node": { + "samples": 50, + "mean": 2.82, + "p50": 2.7, + "p95": 3.57, + "p99": 3.8, + "min": 2.57, + "max": 3.8 + }, + "runtime_create_total": { + "samples": 50, + "mean": 159.95, + "p50": 157.63, + "p95": 175.41, + "p99": 178.08, + "min": 150.41, + "max": 178.08 + }, + "resident_runner_create": { + "samples": 50, + "mean": 41.28, + "p50": 26.24, + "p95": 174.16, + "p99": 175.12, + "min": 24.46, + "max": 175.12 + }, + "resident_first_exec": { + "samples": 50, + "mean": 1.37, + "p50": 1.3, + "p95": 2.45, + "p99": 2.5, + "min": 1.21, + "max": 2.5 + }, + "resident_warm_exec": { + "samples": 50, + "mean": 1.26, + "p50": 1.24, + "p95": 1.37, + "p99": 1.57, + "min": 1.17, + "max": 1.57 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 10, + "mode": "concurrent", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.49, + "p50": 5.51, + "p95": 5.79, + "p99": 5.79, + "min": 5.22, + "max": 5.79 + }, + "coldStart": { + "samples": 50, + "mean": 205, + "p50": 186.33, + "p95": 351.88, + "p99": 359.75, + "min": 176.52, + "max": 359.75 + }, + "warmStart": { + "samples": 50, + "mean": 1.26, + "p50": 1.25, + "p95": 1.39, + "p99": 1.42, + "min": 1.17, + "max": 1.42 + }, + "phases": { + "filesystem_snapshot": { + "samples": 50, + "mean": 0.03, + "p50": 0.02, + "p95": 0.03, + "p99": 0.23, + "min": 0.02, + "max": 0.23 + }, + "session_open": { + "samples": 50, + "mean": 0.23, + "p50": 0.2, + "p95": 0.37, + "p99": 0.41, + "min": 0.14, + "max": 0.41 + }, + "vm_create": { + "samples": 50, + "mean": 1.41, + "p50": 1.24, + "p95": 2.39, + "p99": 2.7, + "min": 1.01, + "max": 2.7 + }, + "vm_ready": { + "samples": 50, + "mean": 0.01, + "p50": 0.01, + "p95": 0.04, + "p99": 0.26, + "min": 0.01, + "max": 0.26 + }, + "vm_configure": { + "samples": 50, + "mean": 0.11, + "p50": 0.09, + "p95": 0.18, + "p99": 0.46, + "min": 0.06, + "max": 0.46 + }, + "runtime_mount_wasm": { + "samples": 50, + "mean": 158.46, + "p50": 156.45, + "p95": 179.94, + "p99": 193.32, + "min": 145.55, + "max": 193.32 + }, + "runtime_mount_node": { + "samples": 50, + "mean": 2.79, + "p50": 2.68, + "p95": 3.76, + "p99": 4.18, + "min": 2.54, + "max": 4.18 + }, + "runtime_create_total": { + "samples": 50, + "mean": 161.32, + "p50": 159.21, + "p95": 182.72, + "p99": 196.25, + "min": 148.22, + "max": 196.25 + }, + "resident_runner_create": { + "samples": 50, + "mean": 42.23, + "p50": 26.15, + "p95": 177.45, + "p99": 180.24, + "min": 24.94, + "max": 180.24 + }, + "resident_first_exec": { + "samples": 50, + "mean": 1.44, + "p50": 1.28, + "p95": 2.41, + "p99": 2.49, + "min": 1.21, + "max": 2.49 + }, + "resident_warm_exec": { + "samples": 50, + "mean": 1.26, + "p50": 1.25, + "p95": 1.39, + "p99": 1.42, + "min": 1.17, + "max": 1.42 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 50, + "mode": "sequential", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.83, + "p50": 5.55, + "p95": 6.93, + "p99": 6.93, + "min": 5.47, + "max": 6.93 + }, + "coldStart": { + "samples": 250, + "mean": 187.92, + "p50": 182.66, + "p95": 197.69, + "p99": 350.51, + "min": 172.07, + "max": 362.01 + }, + "warmStart": { + "samples": 250, + "mean": 1.26, + "p50": 1.25, + "p95": 1.37, + "p99": 1.48, + "min": 1.15, + "max": 1.67 + }, + "phases": { + "filesystem_snapshot": { + "samples": 250, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.04, + "min": 0.02, + "max": 0.23 + }, + "session_open": { + "samples": 250, + "mean": 0.23, + "p50": 0.2, + "p95": 0.44, + "p99": 0.59, + "min": 0.13, + "max": 0.68 + }, + "vm_create": { + "samples": 250, + "mean": 1.26, + "p50": 1.15, + "p95": 1.78, + "p99": 2.96, + "min": 0.95, + "max": 4.04 + }, + "vm_ready": { + "samples": 250, + "mean": 0.01, + "p50": 0.01, + "p95": 0.04, + "p99": 0.05, + "min": 0, + "max": 0.26 + }, + "vm_configure": { + "samples": 250, + "mean": 0.13, + "p50": 0.1, + "p95": 0.33, + "p99": 0.51, + "min": 0.06, + "max": 1.94 + }, + "runtime_mount_wasm": { + "samples": 250, + "mean": 153.86, + "p50": 152.45, + "p95": 165.32, + "p99": 176.7, + "min": 143.77, + "max": 182.19 + }, + "runtime_mount_node": { + "samples": 250, + "mean": 3.45, + "p50": 2.68, + "p95": 3.87, + "p99": 6.54, + "min": 2.47, + "max": 155.7 + }, + "runtime_create_total": { + "samples": 250, + "mean": 157.38, + "p50": 155.29, + "p95": 168.61, + "p99": 184.5, + "min": 146.42, + "max": 318.86 + }, + "resident_runner_create": { + "samples": 250, + "mean": 29.13, + "p50": 25.8, + "p95": 31.18, + "p99": 171.67, + "min": 24.21, + "max": 176.25 + }, + "resident_first_exec": { + "samples": 250, + "mean": 1.4, + "p50": 1.3, + "p95": 2.41, + "p99": 2.8, + "min": 1.18, + "max": 5.27 + }, + "resident_warm_exec": { + "samples": 250, + "mean": 1.26, + "p50": 1.25, + "p95": 1.37, + "p99": 1.48, + "min": 1.15, + "max": 1.67 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 50, + "mode": "concurrent", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.58, + "p50": 5.48, + "p95": 5.91, + "p99": 5.91, + "min": 5.23, + "max": 5.91 + }, + "coldStart": { + "samples": 250, + "mean": 187.45, + "p50": 183.14, + "p95": 194.96, + "p99": 346.89, + "min": 174.71, + "max": 354.51 + }, + "warmStart": { + "samples": 250, + "mean": 1.26, + "p50": 1.26, + "p95": 1.36, + "p99": 1.47, + "min": 1.14, + "max": 1.63 + }, + "phases": { + "filesystem_snapshot": { + "samples": 250, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.04, + "min": 0.02, + "max": 0.04 + }, + "session_open": { + "samples": 250, + "mean": 0.24, + "p50": 0.2, + "p95": 0.44, + "p99": 0.59, + "min": 0.14, + "max": 1.23 + }, + "vm_create": { + "samples": 250, + "mean": 1.29, + "p50": 1.16, + "p95": 1.99, + "p99": 3.13, + "min": 0.94, + "max": 3.38 + }, + "vm_ready": { + "samples": 250, + "mean": 0.01, + "p50": 0, + "p95": 0.03, + "p99": 0.04, + "min": 0, + "max": 0.04 + }, + "vm_configure": { + "samples": 250, + "mean": 0.12, + "p50": 0.09, + "p95": 0.33, + "p99": 0.5, + "min": 0.06, + "max": 0.7 + }, + "runtime_mount_wasm": { + "samples": 250, + "mean": 154.15, + "p50": 153.06, + "p95": 164.59, + "p99": 174.07, + "min": 145.22, + "max": 175.96 + }, + "runtime_mount_node": { + "samples": 250, + "mean": 2.76, + "p50": 2.67, + "p95": 3.32, + "p99": 4.14, + "min": 2.5, + "max": 4.68 + }, + "runtime_create_total": { + "samples": 250, + "mean": 156.98, + "p50": 155.91, + "p95": 167.28, + "p99": 176.83, + "min": 147.95, + "max": 178.81 + }, + "resident_runner_create": { + "samples": 250, + "mean": 29.04, + "p50": 25.71, + "p95": 29.33, + "p99": 174.02, + "min": 24.35, + "max": 176.34 + }, + "resident_first_exec": { + "samples": 250, + "mean": 1.43, + "p50": 1.3, + "p95": 2.42, + "p99": 2.68, + "min": 1.19, + "max": 4.01 + }, + "resident_warm_exec": { + "samples": 250, + "mean": 1.26, + "p50": 1.26, + "p95": 1.36, + "p99": 1.47, + "min": 1.14, + "max": 1.63 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 100, + "mode": "sequential", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.65, + "p50": 5.39, + "p95": 6.61, + "p99": 6.61, + "min": 5.27, + "max": 6.61 + }, + "coldStart": { + "samples": 500, + "mean": 186.1, + "p50": 183.49, + "p95": 196.74, + "p99": 241.08, + "min": 174.21, + "max": 356.11 + }, + "warmStart": { + "samples": 500, + "mean": 1.29, + "p50": 1.27, + "p95": 1.44, + "p99": 1.56, + "min": 1.13, + "max": 1.85 + }, + "phases": { + "filesystem_snapshot": { + "samples": 500, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.03, + "min": 0.02, + "max": 0.07 + }, + "session_open": { + "samples": 500, + "mean": 0.24, + "p50": 0.21, + "p95": 0.37, + "p99": 0.56, + "min": 0.13, + "max": 1.35 + }, + "vm_create": { + "samples": 500, + "mean": 1.22, + "p50": 1.08, + "p95": 1.89, + "p99": 2.4, + "min": 0.95, + "max": 3.1 + }, + "vm_ready": { + "samples": 500, + "mean": 0.01, + "p50": 0, + "p95": 0.03, + "p99": 0.04, + "min": 0, + "max": 0.1 + }, + "vm_configure": { + "samples": 500, + "mean": 0.12, + "p50": 0.09, + "p95": 0.31, + "p99": 0.45, + "min": 0.06, + "max": 0.83 + }, + "runtime_mount_wasm": { + "samples": 500, + "mean": 154.32, + "p50": 153.09, + "p95": 164.25, + "p99": 173.43, + "min": 144.5, + "max": 209.43 + }, + "runtime_mount_node": { + "samples": 500, + "mean": 2.79, + "p50": 2.71, + "p95": 3.29, + "p99": 4.16, + "min": 2.5, + "max": 5.19 + }, + "runtime_create_total": { + "samples": 500, + "mean": 157.18, + "p50": 155.96, + "p95": 168.01, + "p99": 177.22, + "min": 147.33, + "max": 213.6 + }, + "resident_runner_create": { + "samples": 500, + "mean": 27.46, + "p50": 25.58, + "p95": 29.87, + "p99": 34.55, + "min": 22.81, + "max": 180.7 + }, + "resident_first_exec": { + "samples": 500, + "mean": 1.45, + "p50": 1.32, + "p95": 2.53, + "p99": 2.83, + "min": 1.17, + "max": 4.15 + }, + "resident_warm_exec": { + "samples": 500, + "mean": 1.29, + "p50": 1.27, + "p95": 1.44, + "p99": 1.56, + "min": 1.13, + "max": 1.85 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 100, + "mode": "concurrent", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.63, + "p50": 5.54, + "p95": 6.12, + "p99": 6.12, + "min": 5.4, + "max": 6.12 + }, + "coldStart": { + "samples": 500, + "mean": 187.07, + "p50": 184.3, + "p95": 197.65, + "p99": 247.58, + "min": 173.96, + "max": 357.61 + }, + "warmStart": { + "samples": 500, + "mean": 1.29, + "p50": 1.28, + "p95": 1.42, + "p99": 1.53, + "min": 1.14, + "max": 1.65 + }, + "phases": { + "filesystem_snapshot": { + "samples": 500, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.03, + "min": 0.02, + "max": 0.07 + }, + "session_open": { + "samples": 500, + "mean": 0.24, + "p50": 0.21, + "p95": 0.41, + "p99": 0.59, + "min": 0.13, + "max": 1.14 + }, + "vm_create": { + "samples": 500, + "mean": 1.22, + "p50": 1.1, + "p95": 1.86, + "p99": 2.05, + "min": 0.95, + "max": 3.41 + }, + "vm_ready": { + "samples": 500, + "mean": 0.01, + "p50": 0, + "p95": 0.03, + "p99": 0.04, + "min": 0, + "max": 0.06 + }, + "vm_configure": { + "samples": 500, + "mean": 0.12, + "p50": 0.09, + "p95": 0.32, + "p99": 0.51, + "min": 0.06, + "max": 1.35 + }, + "runtime_mount_wasm": { + "samples": 500, + "mean": 155.41, + "p50": 154.31, + "p95": 166.61, + "p99": 173.9, + "min": 144.64, + "max": 217.62 + }, + "runtime_mount_node": { + "samples": 500, + "mean": 2.8, + "p50": 2.73, + "p95": 3.37, + "p99": 4.14, + "min": 2.49, + "max": 6.63 + }, + "runtime_create_total": { + "samples": 500, + "mean": 158.28, + "p50": 157.16, + "p95": 169.46, + "p99": 176.71, + "min": 147.26, + "max": 221 + }, + "resident_runner_create": { + "samples": 500, + "mean": 27.37, + "p50": 25.57, + "p95": 28.95, + "p99": 44, + "min": 22.79, + "max": 179.67 + }, + "resident_first_exec": { + "samples": 500, + "mean": 1.42, + "p50": 1.31, + "p95": 2.47, + "p99": 2.66, + "min": 1.19, + "max": 6.73 + }, + "resident_warm_exec": { + "samples": 500, + "mean": 1.29, + "p50": 1.28, + "p95": 1.42, + "p99": 1.53, + "min": 1.14, + "max": 1.65 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 200, + "mode": "sequential", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.58, + "p50": 5.51, + "p95": 5.79, + "p99": 5.79, + "min": 5.42, + "max": 5.79 + }, + "coldStart": { + "samples": 1000, + "mean": 189.76, + "p50": 187.37, + "p95": 204.16, + "p99": 215.02, + "min": 172.9, + "max": 361.82 + }, + "warmStart": { + "samples": 1000, + "mean": 1.23, + "p50": 1.29, + "p95": 1.52, + "p99": 1.66, + "min": 0.3, + "max": 3.77 + }, + "phases": { + "filesystem_snapshot": { + "samples": 1000, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.04, + "min": 0.02, + "max": 0.06 + }, + "session_open": { + "samples": 1000, + "mean": 0.27, + "p50": 0.24, + "p95": 0.52, + "p99": 0.65, + "min": 0.13, + "max": 1.31 + }, + "vm_create": { + "samples": 1000, + "mean": 1.18, + "p50": 1.1, + "p95": 1.77, + "p99": 2.29, + "min": 0.93, + "max": 3.2 + }, + "vm_ready": { + "samples": 1000, + "mean": 0.01, + "p50": 0, + "p95": 0.03, + "p99": 0.05, + "min": 0, + "max": 0.06 + }, + "vm_configure": { + "samples": 1000, + "mean": 0.15, + "p50": 0.12, + "p95": 0.33, + "p99": 0.47, + "min": 0.06, + "max": 1.02 + }, + "runtime_mount_wasm": { + "samples": 1000, + "mean": 159.24, + "p50": 157.28, + "p95": 174.65, + "p99": 178.82, + "min": 143.28, + "max": 195.21 + }, + "runtime_mount_node": { + "samples": 1000, + "mean": 2.9, + "p50": 2.79, + "p95": 3.59, + "p99": 4.24, + "min": 2.44, + "max": 7.61 + }, + "runtime_create_total": { + "samples": 1000, + "mean": 162.21, + "p50": 160.18, + "p95": 177.95, + "p99": 182.11, + "min": 145.86, + "max": 198.19 + }, + "resident_runner_create": { + "samples": 1000, + "mean": 26.07, + "p50": 25.2, + "p95": 28.4, + "p99": 33.17, + "min": 22.19, + "max": 179.58 + }, + "resident_first_exec": { + "samples": 1000, + "mean": 1.47, + "p50": 1.36, + "p95": 2.5, + "p99": 2.82, + "min": 1.07, + "max": 6.92 + }, + "resident_warm_exec": { + "samples": 1000, + "mean": 1.23, + "p50": 1.29, + "p95": 1.52, + "p99": 1.66, + "min": 0.3, + "max": 3.77 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 200, + "mode": "concurrent", + "iterations": 5, + "sidecarSetupMs": { + "samples": 5, + "mean": 5.77, + "p50": 5.61, + "p95": 6.18, + "p99": 6.18, + "min": 5.46, + "max": 6.18 + }, + "coldStart": { + "samples": 1000, + "mean": 191.33, + "p50": 189.09, + "p95": 205.41, + "p99": 218.43, + "min": 173.66, + "max": 358.92 + }, + "warmStart": { + "samples": 1000, + "mean": 1.22, + "p50": 1.29, + "p95": 1.51, + "p99": 1.64, + "min": 0.32, + "max": 2.31 + }, + "phases": { + "filesystem_snapshot": { + "samples": 1000, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.05, + "min": 0.02, + "max": 0.1 + }, + "session_open": { + "samples": 1000, + "mean": 0.27, + "p50": 0.24, + "p95": 0.52, + "p99": 0.65, + "min": 0.13, + "max": 1.4 + }, + "vm_create": { + "samples": 1000, + "mean": 1.19, + "p50": 1.1, + "p95": 1.75, + "p99": 2.45, + "min": 0.93, + "max": 7.19 + }, + "vm_ready": { + "samples": 1000, + "mean": 0.01, + "p50": 0, + "p95": 0.03, + "p99": 0.04, + "min": 0, + "max": 0.06 + }, + "vm_configure": { + "samples": 1000, + "mean": 0.14, + "p50": 0.11, + "p95": 0.32, + "p99": 0.41, + "min": 0.06, + "max": 5.13 + }, + "runtime_mount_wasm": { + "samples": 1000, + "mean": 160.77, + "p50": 158.75, + "p95": 176.08, + "p99": 181.88, + "min": 144.33, + "max": 212.94 + }, + "runtime_mount_node": { + "samples": 1000, + "mean": 2.94, + "p50": 2.85, + "p95": 3.61, + "p99": 4.18, + "min": 2.47, + "max": 5.46 + }, + "runtime_create_total": { + "samples": 1000, + "mean": 163.78, + "p50": 161.8, + "p95": 179.33, + "p99": 185.67, + "min": 146.9, + "max": 216.13 + }, + "resident_runner_create": { + "samples": 1000, + "mean": 26.08, + "p50": 25.21, + "p95": 28.18, + "p99": 34.27, + "min": 22.33, + "max": 180.21 + }, + "resident_first_exec": { + "samples": 1000, + "mean": 1.46, + "p50": 1.35, + "p95": 2.46, + "p99": 2.89, + "min": 1.08, + "max": 13.88 + }, + "resident_warm_exec": { + "samples": 1000, + "mean": 1.22, + "p50": 1.29, + "p95": 1.51, + "p99": 1.64, + "min": 0.32, + "max": 2.31 + } + } + }, + { + "scenario": "isolate-only", + "batchSize": 1, + "mode": "snapshot-restore", + "iterations": 50, + "coldStart": { + "samples": 1, + "mean": 4.5749, + "p50": 4.5749, + "p95": 4.5749, + "p99": 4.5749, + "min": 4.5749, + "max": 4.5749 + }, + "warmStart": { + "samples": 50, + "mean": 3.1503, + "p50": 3.1059, + "p95": 3.5326, + "p99": 3.8155, + "min": 2.8691, + "max": 3.8155 + }, + "phases": { + "snapshot_create": { + "samples": 1, + "mean": 123.904, + "p50": 123.904, + "p95": 123.904, + "p99": 123.904, + "min": 123.904, + "max": 123.904 + } + } + } + ] +} diff --git a/packages/benchmarks/results/coldstart-final.log b/packages/benchmarks/results/coldstart-final.log index 4b2598cf5..51c95c627 100644 --- a/packages/benchmarks/results/coldstart-final.log +++ b/packages/benchmarks/results/coldstart-final.log @@ -1,34 +1,224 @@ === Cold Start Benchmark === CPU: 12th Gen Intel(R) Core(TM) i7-12700KF -Cores: 20 | Max concurrency: 16 +Cores: 20 | Max concurrency: 16 | Max live runtimes: 8 +Resident runner live cap: 1 RAM: 62.6 GB | Node: v24.13.0 +Loadavg: 4.8, 3.73, 3.23 +MemAvailable: 26424332 kB | SwapFree: 0 kB | SwapCached: 114032 kB Iterations: 5 (+ 1 warmup) Batch sizes: 1, 10, 50, 100, 200 -Sidecar: /home/nathan/secure-exec/target/release/secure-exec-sidecar +Scenarios: owned-sidecar, shared-sidecar, resident-runner, isolate-only +Sidecar: /home/nathan/secure-exec-perf-restore/target/release/secure-exec-sidecar +WASM commands: /home/nathan/secure-exec/packages/core/commands ---- batch=1, mode=sequential --- - cold: mean=841.93ms p50=815.92ms p95=952.87ms - warm: mean=460.22ms p50=458.8ms p95=477.89ms ---- batch=1, mode=concurrent --- - cold: mean=788.79ms p50=792.08ms p95=794.85ms - warm: mean=456.3ms p50=456.46ms p95=460.45ms +--- scenario=owned-sidecar, batch=1, mode=sequential --- + cold: mean=772.88ms p50=771.96ms p95=779.42ms + warm: mean=452.68ms p50=452.37ms p95=457.13ms + phases: filesystem_snapshot.p50=0.1ms | first_exec.p50=596.26ms | runtime_create_total.p50=176.3ms | runtime_mount_node.p50=2.71ms | runtime_mount_wasm.p50=172.95ms | session_open.p50=2.34ms | sidecar_spawn.p50=2.78ms | vm_configure.p50=0.17ms | vm_create.p50=1.36ms | vm_ready.p50=0.01ms | warm_exec.p50=452.37ms ---- batch=10, mode=sequential --- - cold: mean=789.5ms p50=787.06ms p95=813.65ms - warm: mean=455.64ms p50=453.96ms p95=465.4ms +--- scenario=owned-sidecar, batch=1, mode=concurrent --- + cold: mean=776.22ms p50=775.63ms p95=782.37ms + warm: mean=449.86ms p50=449.59ms p95=452.53ms + phases: filesystem_snapshot.p50=0.26ms | first_exec.p50=599.03ms | runtime_create_total.p50=176.95ms | runtime_mount_node.p50=2.67ms | runtime_mount_wasm.p50=174.2ms | session_open.p50=2.99ms | sidecar_spawn.p50=2.79ms | vm_configure.p50=0.12ms | vm_create.p50=1.11ms | vm_ready.p50=0.01ms | warm_exec.p50=449.59ms ---- batch=10, mode=concurrent --- - cold: mean=1165.91ms p50=1159.52ms p95=1266.04ms - warm: mean=499.37ms p50=475.53ms p95=709.2ms +--- scenario=owned-sidecar, batch=10, mode=sequential --- + cold: mean=780.07ms p50=778.64ms p95=802.66ms + warm: mean=453.92ms p50=452.69ms p95=463.47ms + phases: filesystem_snapshot.p50=0.24ms | first_exec.p50=601.18ms | runtime_create_total.p50=176.21ms | runtime_mount_node.p50=2.74ms | runtime_mount_wasm.p50=173.25ms | session_open.p50=2.44ms | sidecar_spawn.p50=2.74ms | vm_configure.p50=0.11ms | vm_create.p50=1.21ms | vm_ready.p50=0.01ms | warm_exec.p50=452.69ms ---- batch=50, mode=sequential --- - cold: mean=876.98ms p50=796.45ms p95=1447.84ms - warm: mean=481.52ms p50=456.71ms p95=641.87ms +--- scenario=owned-sidecar, batch=10, mode=concurrent --- + cold: mean=1000.39ms p50=1028.29ms p95=1119.05ms + warm: mean=477.27ms p50=469.19ms p95=532.17ms + phases: filesystem_snapshot.p50=31.48ms | first_exec.p50=642.15ms | runtime_create_total.p50=373.11ms | runtime_mount_node.p50=3.08ms | runtime_mount_wasm.p50=369.84ms | session_open.p50=7.93ms | sidecar_spawn.p50=2.08ms | vm_configure.p50=22.13ms | vm_create.p50=2.54ms | vm_ready.p50=0.01ms | warm_exec.p50=469.19ms ---- batch=50, mode=concurrent --- - cold: mean=1907.44ms p50=1540.5ms p95=4991.96ms - warm: mean=542.93ms p50=523.24ms p95=741.14ms +--- scenario=owned-sidecar, batch=50, mode=sequential --- + cold: mean=777.49ms p50=775.3ms p95=797.46ms + warm: mean=453.43ms p50=452.59ms p95=461.71ms + phases: filesystem_snapshot.p50=0.03ms | first_exec.p50=599.8ms | runtime_create_total.p50=174.76ms | runtime_mount_node.p50=2.72ms | runtime_mount_wasm.p50=171.68ms | session_open.p50=2.36ms | sidecar_spawn.p50=3.13ms | vm_configure.p50=0.11ms | vm_create.p50=1.22ms | vm_ready.p50=0.01ms | warm_exec.p50=452.59ms + +--- scenario=owned-sidecar, batch=50, mode=concurrent --- + cold: mean=1041.81ms p50=1044.33ms p95=1133.77ms + warm: mean=468.05ms p50=465.77ms p95=491.47ms + phases: filesystem_snapshot.p50=45.06ms | first_exec.p50=643.04ms | runtime_create_total.p50=396.28ms | runtime_mount_node.p50=3.23ms | runtime_mount_wasm.p50=393.55ms | session_open.p50=9.75ms | sidecar_spawn.p50=2.11ms | vm_configure.p50=34.25ms | vm_create.p50=21.44ms | vm_ready.p50=0ms | warm_exec.p50=465.77ms + +--- scenario=owned-sidecar, batch=100, mode=sequential --- + cold: mean=777.89ms p50=776.54ms p95=793.79ms + warm: mean=453.66ms p50=452.93ms p95=460.66ms + phases: filesystem_snapshot.p50=0.02ms | first_exec.p50=599.6ms | runtime_create_total.p50=176.16ms | runtime_mount_node.p50=2.73ms | runtime_mount_wasm.p50=173.29ms | session_open.p50=2.33ms | sidecar_spawn.p50=4.22ms | vm_configure.p50=0.1ms | vm_create.p50=1.2ms | vm_ready.p50=0.01ms | warm_exec.p50=452.93ms + +--- scenario=owned-sidecar, batch=100, mode=concurrent --- + cold: mean=1041.19ms p50=1043.72ms p95=1120.25ms + warm: mean=468.6ms p50=466.36ms p95=492.1ms + phases: filesystem_snapshot.p50=45.26ms | first_exec.p50=641.96ms | runtime_create_total.p50=394.78ms | runtime_mount_node.p50=3.18ms | runtime_mount_wasm.p50=391.04ms | session_open.p50=10.31ms | sidecar_spawn.p50=2.37ms | vm_configure.p50=30.68ms | vm_create.p50=23.2ms | vm_ready.p50=0ms | warm_exec.p50=466.36ms + +--- scenario=owned-sidecar, batch=200, mode=sequential --- + cold: mean=778.28ms p50=776.94ms p95=793.79ms + warm: mean=454.03ms p50=453.13ms p95=462.01ms + phases: filesystem_snapshot.p50=0.02ms | first_exec.p50=599.84ms | runtime_create_total.p50=176.64ms | runtime_mount_node.p50=2.72ms | runtime_mount_wasm.p50=173.7ms | session_open.p50=2.32ms | sidecar_spawn.p50=4.3ms | vm_configure.p50=0.1ms | vm_create.p50=1.2ms | vm_ready.p50=0ms | warm_exec.p50=453.13ms + +--- scenario=owned-sidecar, batch=200, mode=concurrent --- + cold: mean=1044.03ms p50=1042.73ms p95=1124.88ms + warm: mean=468.68ms p50=466.89ms p95=488.09ms + phases: filesystem_snapshot.p50=47.8ms | first_exec.p50=640.8ms | runtime_create_total.p50=394.96ms | runtime_mount_node.p50=3.26ms | runtime_mount_wasm.p50=391.19ms | session_open.p50=10.58ms | sidecar_spawn.p50=2.39ms | vm_configure.p50=33.55ms | vm_create.p50=22.78ms | vm_ready.p50=0ms | warm_exec.p50=466.89ms + +--- scenario=shared-sidecar, batch=1, mode=sequential --- + cold: mean=774.17ms p50=774.5ms p95=778.62ms + warm: mean=457.19ms p50=452.75ms p95=466.31ms + phases: filesystem_snapshot.p50=0.02ms | first_exec.p50=604.54ms | runtime_create_total.p50=173.09ms | runtime_mount_node.p50=2.61ms | runtime_mount_wasm.p50=170.47ms | session_open.p50=0.3ms | vm_configure.p50=0.11ms | vm_create.p50=1.22ms | vm_ready.p50=0ms | warm_exec.p50=452.75ms + sidecar setup excluded: p50=3.94ms p95=5.26ms + +--- scenario=shared-sidecar, batch=1, mode=concurrent --- + cold: mean=775.66ms p50=776.97ms p95=780.66ms + warm: mean=452.79ms p50=453.29ms p95=453.98ms + phases: filesystem_snapshot.p50=0.02ms | first_exec.p50=603.17ms | runtime_create_total.p50=173.44ms | runtime_mount_node.p50=2.71ms | runtime_mount_wasm.p50=169.79ms | session_open.p50=0.4ms | vm_configure.p50=0.09ms | vm_create.p50=1.14ms | vm_ready.p50=0ms | warm_exec.p50=453.29ms + sidecar setup excluded: p50=5.21ms p95=9.57ms + +--- scenario=shared-sidecar, batch=10, mode=sequential --- + cold: mean=627.58ms p50=613.98ms p95=773.01ms + warm: mean=450.97ms p50=450.94ms p95=459.89ms + phases: filesystem_snapshot.p50=0.02ms | first_exec.p50=449.24ms | runtime_create_total.p50=161.69ms | runtime_mount_node.p50=2.68ms | runtime_mount_wasm.p50=158.99ms | session_open.p50=0.19ms | vm_configure.p50=0.09ms | vm_create.p50=1.17ms | vm_ready.p50=0ms | warm_exec.p50=450.94ms + sidecar setup excluded: p50=5.49ms p95=5.73ms + +--- scenario=shared-sidecar, batch=10, mode=concurrent --- + cold: mean=3493.51ms p50=4558.22ms p95=4676.49ms + warm: mean=2174.08ms p50=2086.34ms p95=4965.71ms + phases: filesystem_snapshot.p50=25.47ms | first_exec.p50=3507.02ms | runtime_create_total.p50=1070.66ms | runtime_mount_node.p50=18.54ms | runtime_mount_wasm.p50=1051.38ms | session_open.p50=0.57ms | vm_configure.p50=11.57ms | vm_create.p50=3.39ms | vm_ready.p50=0ms | warm_exec.p50=2086.34ms + sidecar setup excluded: p50=5.23ms p95=5.88ms + +--- scenario=shared-sidecar, batch=50, mode=sequential --- + cold: mean=610.76ms p50=605.65ms p95=627.44ms + warm: mean=449.02ms p50=448.23ms p95=456.81ms + phases: filesystem_snapshot.p50=0.02ms | first_exec.p50=448.83ms | runtime_create_total.p50=156.3ms | runtime_mount_node.p50=2.67ms | runtime_mount_wasm.p50=153.45ms | session_open.p50=0.21ms | vm_configure.p50=0.09ms | vm_create.p50=1.14ms | vm_ready.p50=0ms | warm_exec.p50=448.23ms + sidecar setup excluded: p50=5.09ms p95=5.39ms + +--- scenario=shared-sidecar, batch=50, mode=concurrent --- + cold: mean=3197.96ms p50=3299.89ms p95=4602.98ms + warm: mean=2162.03ms p50=2212.06ms p95=3395.52ms + phases: filesystem_snapshot.p50=43.21ms | first_exec.p50=2211.64ms | runtime_create_total.p50=1107.01ms | runtime_mount_node.p50=19.28ms | runtime_mount_wasm.p50=1088.07ms | session_open.p50=0.48ms | vm_configure.p50=16.13ms | vm_create.p50=4.2ms | vm_ready.p50=0ms | warm_exec.p50=2212.06ms + sidecar setup excluded: p50=5.56ms p95=14.37ms + +--- scenario=shared-sidecar, batch=100, mode=sequential --- + cold: mean=619.24ms p50=610.78ms p95=656.44ms + warm: mean=453.68ms p50=449.44ms p95=472.09ms + phases: filesystem_snapshot.p50=0.02ms | first_exec.p50=449.94ms | runtime_create_total.p50=159.9ms | runtime_mount_node.p50=2.84ms | runtime_mount_wasm.p50=157.04ms | session_open.p50=0.21ms | vm_configure.p50=0.1ms | vm_create.p50=1.12ms | vm_ready.p50=0.01ms | warm_exec.p50=449.44ms + sidecar setup excluded: p50=5.43ms p95=5.74ms + +--- scenario=shared-sidecar, batch=100, mode=concurrent --- + cold: mean=3187.99ms p50=3317.35ms p95=4591.89ms + warm: mean=2134.07ms p50=2207.08ms p95=2716.34ms + phases: filesystem_snapshot.p50=44.7ms | first_exec.p50=2212.38ms | runtime_create_total.p50=1130.31ms | runtime_mount_node.p50=19.5ms | runtime_mount_wasm.p50=1110.41ms | session_open.p50=0.54ms | vm_configure.p50=14.62ms | vm_create.p50=4.24ms | vm_ready.p50=0ms | warm_exec.p50=2207.08ms + sidecar setup excluded: p50=5.49ms p95=6.49ms + +--- scenario=shared-sidecar, batch=200, mode=sequential --- + cold: mean=616.2ms p50=614.43ms p95=633.42ms + warm: mean=449.22ms p50=448.24ms p95=456.27ms + phases: filesystem_snapshot.p50=0.02ms | first_exec.p50=448.77ms | runtime_create_total.p50=164.85ms | runtime_mount_node.p50=2.92ms | runtime_mount_wasm.p50=161.85ms | session_open.p50=0.25ms | vm_configure.p50=0.12ms | vm_create.p50=1.11ms | vm_ready.p50=0.01ms | warm_exec.p50=448.24ms + sidecar setup excluded: p50=5.26ms p95=6.09ms + +--- scenario=shared-sidecar, batch=200, mode=concurrent --- + cold: mean=3235.65ms p50=3340.36ms p95=3501.49ms + warm: mean=2102.22ms p50=2200.24ms p95=2732.96ms + phases: filesystem_snapshot.p50=47.71ms | first_exec.p50=2210.55ms | runtime_create_total.p50=1168.28ms | runtime_mount_node.p50=20.37ms | runtime_mount_wasm.p50=1147.31ms | session_open.p50=0.61ms | vm_configure.p50=14.71ms | vm_create.p50=4.14ms | vm_ready.p50=0ms | warm_exec.p50=2200.24ms + sidecar setup excluded: p50=3.79ms p95=3.92ms + +--- scenario=resident-runner, batch=1, mode=sequential --- + cold: mean=351ms p50=349.56ms p95=358.02ms + warm: mean=1.27ms p50=1.27ms p95=1.32ms + phases: filesystem_snapshot.p50=0.02ms | resident_first_exec.p50=1.31ms | resident_runner_create.p50=174.4ms | resident_warm_exec.p50=1.27ms | runtime_create_total.p50=173.03ms | runtime_mount_node.p50=2.77ms | runtime_mount_wasm.p50=170.03ms | session_open.p50=0.29ms | vm_configure.p50=0.09ms | vm_create.p50=1.26ms | vm_ready.p50=0.01ms + sidecar setup excluded: p50=4.19ms p95=6.14ms + +--- scenario=resident-runner, batch=1, mode=concurrent --- + cold: mean=347.91ms p50=346.99ms p95=358.2ms + warm: mean=1.25ms p50=1.26ms p95=1.29ms + phases: filesystem_snapshot.p50=0.02ms | resident_first_exec.p50=1.32ms | resident_runner_create.p50=172.53ms | resident_warm_exec.p50=1.26ms | runtime_create_total.p50=174.09ms | runtime_mount_node.p50=2.67ms | runtime_mount_wasm.p50=171.18ms | session_open.p50=0.3ms | vm_configure.p50=0.1ms | vm_create.p50=1.21ms | vm_ready.p50=0.01ms + sidecar setup excluded: p50=4.08ms p95=6.27ms + +--- scenario=resident-runner, batch=10, mode=sequential --- + cold: mean=202.61ms p50=186.16ms p95=350.97ms + warm: mean=1.26ms p50=1.24ms p95=1.37ms + phases: filesystem_snapshot.p50=0.02ms | resident_first_exec.p50=1.3ms | resident_runner_create.p50=26.24ms | resident_warm_exec.p50=1.24ms | runtime_create_total.p50=157.63ms | runtime_mount_node.p50=2.7ms | runtime_mount_wasm.p50=154.63ms | session_open.p50=0.2ms | vm_configure.p50=0.09ms | vm_create.p50=1.2ms | vm_ready.p50=0.01ms + sidecar setup excluded: p50=5.36ms p95=5.73ms + +--- scenario=resident-runner, batch=10, mode=concurrent --- + cold: mean=205ms p50=186.33ms p95=351.88ms + warm: mean=1.26ms p50=1.25ms p95=1.39ms + phases: filesystem_snapshot.p50=0.02ms | resident_first_exec.p50=1.28ms | resident_runner_create.p50=26.15ms | resident_warm_exec.p50=1.25ms | runtime_create_total.p50=159.21ms | runtime_mount_node.p50=2.68ms | runtime_mount_wasm.p50=156.45ms | session_open.p50=0.2ms | vm_configure.p50=0.09ms | vm_create.p50=1.24ms | vm_ready.p50=0.01ms + sidecar setup excluded: p50=5.51ms p95=5.79ms + +--- scenario=resident-runner, batch=50, mode=sequential --- + cold: mean=187.92ms p50=182.66ms p95=197.69ms + warm: mean=1.26ms p50=1.25ms p95=1.37ms + phases: filesystem_snapshot.p50=0.02ms | resident_first_exec.p50=1.3ms | resident_runner_create.p50=25.8ms | resident_warm_exec.p50=1.25ms | runtime_create_total.p50=155.29ms | runtime_mount_node.p50=2.68ms | runtime_mount_wasm.p50=152.45ms | session_open.p50=0.2ms | vm_configure.p50=0.1ms | vm_create.p50=1.15ms | vm_ready.p50=0.01ms + sidecar setup excluded: p50=5.55ms p95=6.93ms + +--- scenario=resident-runner, batch=50, mode=concurrent --- + cold: mean=187.45ms p50=183.14ms p95=194.96ms + warm: mean=1.26ms p50=1.26ms p95=1.36ms + phases: filesystem_snapshot.p50=0.02ms | resident_first_exec.p50=1.3ms | resident_runner_create.p50=25.71ms | resident_warm_exec.p50=1.26ms | runtime_create_total.p50=155.91ms | runtime_mount_node.p50=2.67ms | runtime_mount_wasm.p50=153.06ms | session_open.p50=0.2ms | vm_configure.p50=0.09ms | vm_create.p50=1.16ms | vm_ready.p50=0ms + sidecar setup excluded: p50=5.48ms p95=5.91ms + +--- scenario=resident-runner, batch=100, mode=sequential --- + cold: mean=186.1ms p50=183.49ms p95=196.74ms + warm: mean=1.29ms p50=1.27ms p95=1.44ms + phases: filesystem_snapshot.p50=0.02ms | resident_first_exec.p50=1.32ms | resident_runner_create.p50=25.58ms | resident_warm_exec.p50=1.27ms | runtime_create_total.p50=155.96ms | runtime_mount_node.p50=2.71ms | runtime_mount_wasm.p50=153.09ms | session_open.p50=0.21ms | vm_configure.p50=0.09ms | vm_create.p50=1.08ms | vm_ready.p50=0ms + sidecar setup excluded: p50=5.39ms p95=6.61ms + +--- scenario=resident-runner, batch=100, mode=concurrent --- + cold: mean=187.07ms p50=184.3ms p95=197.65ms + warm: mean=1.29ms p50=1.28ms p95=1.42ms + phases: filesystem_snapshot.p50=0.02ms | resident_first_exec.p50=1.31ms | resident_runner_create.p50=25.57ms | resident_warm_exec.p50=1.28ms | runtime_create_total.p50=157.16ms | runtime_mount_node.p50=2.73ms | runtime_mount_wasm.p50=154.31ms | session_open.p50=0.21ms | vm_configure.p50=0.09ms | vm_create.p50=1.1ms | vm_ready.p50=0ms + sidecar setup excluded: p50=5.54ms p95=6.12ms + +--- scenario=resident-runner, batch=200, mode=sequential --- + cold: mean=189.76ms p50=187.37ms p95=204.16ms + warm: mean=1.23ms p50=1.29ms p95=1.52ms + phases: filesystem_snapshot.p50=0.02ms | resident_first_exec.p50=1.36ms | resident_runner_create.p50=25.2ms | resident_warm_exec.p50=1.29ms | runtime_create_total.p50=160.18ms | runtime_mount_node.p50=2.79ms | runtime_mount_wasm.p50=157.28ms | session_open.p50=0.24ms | vm_configure.p50=0.12ms | vm_create.p50=1.1ms | vm_ready.p50=0ms + sidecar setup excluded: p50=5.51ms p95=5.79ms + +--- scenario=resident-runner, batch=200, mode=concurrent --- + cold: mean=191.33ms p50=189.09ms p95=205.41ms + warm: mean=1.22ms p50=1.29ms p95=1.51ms + phases: filesystem_snapshot.p50=0.02ms | resident_first_exec.p50=1.35ms | resident_runner_create.p50=25.21ms | resident_warm_exec.p50=1.29ms | runtime_create_total.p50=161.8ms | runtime_mount_node.p50=2.85ms | runtime_mount_wasm.p50=158.75ms | session_open.p50=0.24ms | vm_configure.p50=0.11ms | vm_create.p50=1.1ms | vm_ready.p50=0ms + sidecar setup excluded: p50=5.61ms p95=6.18ms + +--- scenario=isolate-only, mode=snapshot-restore --- + cold: mean=4.5749ms p50=4.5749ms p95=4.5749ms + warm: mean=3.1503ms p50=3.1059ms p95=3.5326ms + phases: snapshot_create.p50=123.904ms + + scenario | batch | mode | cold mean | cold p50 | cold p95 | warm mean | warm p50 | warm p95 +--------------- | ----- | ---------------- | --------- | --------- | --------- | --------- | --------- | --------- + owned-sidecar | 1 | sequential | 772.88ms | 771.96ms | 779.42ms | 452.68ms | 452.37ms | 457.13ms + owned-sidecar | 1 | concurrent | 776.22ms | 775.63ms | 782.37ms | 449.86ms | 449.59ms | 452.53ms + owned-sidecar | 10 | sequential | 780.07ms | 778.64ms | 802.66ms | 453.92ms | 452.69ms | 463.47ms + owned-sidecar | 10 | concurrent | 1000.39ms | 1028.29ms | 1119.05ms | 477.27ms | 469.19ms | 532.17ms + owned-sidecar | 50 | sequential | 777.49ms | 775.3ms | 797.46ms | 453.43ms | 452.59ms | 461.71ms + owned-sidecar | 50 | concurrent | 1041.81ms | 1044.33ms | 1133.77ms | 468.05ms | 465.77ms | 491.47ms + owned-sidecar | 100 | sequential | 777.89ms | 776.54ms | 793.79ms | 453.66ms | 452.93ms | 460.66ms + owned-sidecar | 100 | concurrent | 1041.19ms | 1043.72ms | 1120.25ms | 468.6ms | 466.36ms | 492.1ms + owned-sidecar | 200 | sequential | 778.28ms | 776.94ms | 793.79ms | 454.03ms | 453.13ms | 462.01ms + owned-sidecar | 200 | concurrent | 1044.03ms | 1042.73ms | 1124.88ms | 468.68ms | 466.89ms | 488.09ms + shared-sidecar | 1 | sequential | 774.17ms | 774.5ms | 778.62ms | 457.19ms | 452.75ms | 466.31ms + shared-sidecar | 1 | concurrent | 775.66ms | 776.97ms | 780.66ms | 452.79ms | 453.29ms | 453.98ms + shared-sidecar | 10 | sequential | 627.58ms | 613.98ms | 773.01ms | 450.97ms | 450.94ms | 459.89ms + shared-sidecar | 10 | concurrent | 3493.51ms | 4558.22ms | 4676.49ms | 2174.08ms | 2086.34ms | 4965.71ms + shared-sidecar | 50 | sequential | 610.76ms | 605.65ms | 627.44ms | 449.02ms | 448.23ms | 456.81ms + shared-sidecar | 50 | concurrent | 3197.96ms | 3299.89ms | 4602.98ms | 2162.03ms | 2212.06ms | 3395.52ms + shared-sidecar | 100 | sequential | 619.24ms | 610.78ms | 656.44ms | 453.68ms | 449.44ms | 472.09ms + shared-sidecar | 100 | concurrent | 3187.99ms | 3317.35ms | 4591.89ms | 2134.07ms | 2207.08ms | 2716.34ms + shared-sidecar | 200 | sequential | 616.2ms | 614.43ms | 633.42ms | 449.22ms | 448.24ms | 456.27ms + shared-sidecar | 200 | concurrent | 3235.65ms | 3340.36ms | 3501.49ms | 2102.22ms | 2200.24ms | 2732.96ms +resident-runner | 1 | sequential | 351ms | 349.56ms | 358.02ms | 1.27ms | 1.27ms | 1.32ms +resident-runner | 1 | concurrent | 347.91ms | 346.99ms | 358.2ms | 1.25ms | 1.26ms | 1.29ms +resident-runner | 10 | sequential | 202.61ms | 186.16ms | 350.97ms | 1.26ms | 1.24ms | 1.37ms +resident-runner | 10 | concurrent | 205ms | 186.33ms | 351.88ms | 1.26ms | 1.25ms | 1.39ms +resident-runner | 50 | sequential | 187.92ms | 182.66ms | 197.69ms | 1.26ms | 1.25ms | 1.37ms +resident-runner | 50 | concurrent | 187.45ms | 183.14ms | 194.96ms | 1.26ms | 1.26ms | 1.36ms +resident-runner | 100 | sequential | 186.1ms | 183.49ms | 196.74ms | 1.29ms | 1.27ms | 1.44ms +resident-runner | 100 | concurrent | 187.07ms | 184.3ms | 197.65ms | 1.29ms | 1.28ms | 1.42ms +resident-runner | 200 | sequential | 189.76ms | 187.37ms | 204.16ms | 1.23ms | 1.29ms | 1.52ms +resident-runner | 200 | concurrent | 191.33ms | 189.09ms | 205.41ms | 1.22ms | 1.29ms | 1.51ms + isolate-only | 1 | snapshot-restore | 4.5749ms | 4.5749ms | 4.5749ms | 3.1503ms | 3.1059ms | 3.5326ms ---- batch=100, mode=sequential --- diff --git a/packages/benchmarks/results/coldstart-resident-full-matrix-20260619.json b/packages/benchmarks/results/coldstart-resident-full-matrix-20260619.json new file mode 100644 index 000000000..98805d228 --- /dev/null +++ b/packages/benchmarks/results/coldstart-resident-full-matrix-20260619.json @@ -0,0 +1,1276 @@ + +> @secure-exec/benchmarks@ bench:coldstart /home/nathan/secure-exec-perf-restore/packages/benchmarks +> tsx coldstart.bench.ts + +{ + "hardware": { + "cpu": "12th Gen Intel(R) Core(TM) i7-12700KF", + "cores": 20, + "ram": "62.6 GB", + "node": "v24.13.0", + "os": "Linux 6.1.0-41-amd64", + "arch": "x64", + "loadAverage": [ + 2.3, + 2.56, + 4.03 + ] + }, + "scenarioSetup": { + "resident-runner": { + "sharedSidecarSetupMs": 2.91 + } + }, + "results": [ + { + "scenario": "resident-runner", + "batchSize": 1, + "mode": "sequential", + "iterations": 1, + "coldStart": { + "samples": 1, + "mean": 396.38, + "p50": 396.38, + "p95": 396.38, + "p99": 396.38, + "min": 396.38, + "max": 396.38 + }, + "warmStart": { + "samples": 1, + "mean": 1.36, + "p50": 1.36, + "p95": 1.36, + "p99": 1.36, + "min": 1.36, + "max": 1.36 + }, + "phases": { + "filesystem_snapshot": { + "samples": 1, + "mean": 0.32, + "p50": 0.32, + "p95": 0.32, + "p99": 0.32, + "min": 0.32, + "max": 0.32 + }, + "session_open": { + "samples": 1, + "mean": 6.35, + "p50": 6.35, + "p95": 6.35, + "p99": 6.35, + "min": 6.35, + "max": 6.35 + }, + "vm_create": { + "samples": 1, + "mean": 2.75, + "p50": 2.75, + "p95": 2.75, + "p99": 2.75, + "min": 2.75, + "max": 2.75 + }, + "vm_ready": { + "samples": 1, + "mean": 0.25, + "p50": 0.25, + "p95": 0.25, + "p99": 0.25, + "min": 0.25, + "max": 0.25 + }, + "vm_configure": { + "samples": 1, + "mean": 0.73, + "p50": 0.73, + "p95": 0.73, + "p99": 0.73, + "min": 0.73, + "max": 0.73 + }, + "runtime_mount_wasm": { + "samples": 1, + "mean": 220.02, + "p50": 220.02, + "p95": 220.02, + "p99": 220.02, + "min": 220.02, + "max": 220.02 + }, + "runtime_mount_node": { + "samples": 1, + "mean": 2.95, + "p50": 2.95, + "p95": 2.95, + "p99": 2.95, + "min": 2.95, + "max": 2.95 + }, + "runtime_create_total": { + "samples": 1, + "mean": 223.38, + "p50": 223.38, + "p95": 223.38, + "p99": 223.38, + "min": 223.38, + "max": 223.38 + }, + "resident_runner_create": { + "samples": 1, + "mean": 171.41, + "p50": 171.41, + "p95": 171.41, + "p99": 171.41, + "min": 171.41, + "max": 171.41 + }, + "resident_first_exec": { + "samples": 1, + "mean": 1.59, + "p50": 1.59, + "p95": 1.59, + "p99": 1.59, + "min": 1.59, + "max": 1.59 + }, + "resident_warm_exec": { + "samples": 1, + "mean": 1.36, + "p50": 1.36, + "p95": 1.36, + "p99": 1.36, + "min": 1.36, + "max": 1.36 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 1, + "mode": "concurrent", + "iterations": 1, + "coldStart": { + "samples": 1, + "mean": 204.7, + "p50": 204.7, + "p95": 204.7, + "p99": 204.7, + "min": 204.7, + "max": 204.7 + }, + "warmStart": { + "samples": 1, + "mean": 1.41, + "p50": 1.41, + "p95": 1.41, + "p99": 1.41, + "min": 1.41, + "max": 1.41 + }, + "phases": { + "filesystem_snapshot": { + "samples": 1, + "mean": 0.1, + "p50": 0.1, + "p95": 0.1, + "p99": 0.1, + "min": 0.1, + "max": 0.1 + }, + "session_open": { + "samples": 1, + "mean": 1.11, + "p50": 1.11, + "p95": 1.11, + "p99": 1.11, + "min": 1.11, + "max": 1.11 + }, + "vm_create": { + "samples": 1, + "mean": 1.26, + "p50": 1.26, + "p95": 1.26, + "p99": 1.26, + "min": 1.26, + "max": 1.26 + }, + "vm_ready": { + "samples": 1, + "mean": 0.01, + "p50": 0.01, + "p95": 0.01, + "p99": 0.01, + "min": 0.01, + "max": 0.01 + }, + "vm_configure": { + "samples": 1, + "mean": 0.14, + "p50": 0.14, + "p95": 0.14, + "p99": 0.14, + "min": 0.14, + "max": 0.14 + }, + "runtime_mount_wasm": { + "samples": 1, + "mean": 172.92, + "p50": 172.92, + "p95": 172.92, + "p99": 172.92, + "min": 172.92, + "max": 172.92 + }, + "runtime_mount_node": { + "samples": 1, + "mean": 2.82, + "p50": 2.82, + "p95": 2.82, + "p99": 2.82, + "min": 2.82, + "max": 2.82 + }, + "runtime_create_total": { + "samples": 1, + "mean": 175.85, + "p50": 175.85, + "p95": 175.85, + "p99": 175.85, + "min": 175.85, + "max": 175.85 + }, + "resident_runner_create": { + "samples": 1, + "mean": 27.31, + "p50": 27.31, + "p95": 27.31, + "p99": 27.31, + "min": 27.31, + "max": 27.31 + }, + "resident_first_exec": { + "samples": 1, + "mean": 1.54, + "p50": 1.54, + "p95": 1.54, + "p99": 1.54, + "min": 1.54, + "max": 1.54 + }, + "resident_warm_exec": { + "samples": 1, + "mean": 1.41, + "p50": 1.41, + "p95": 1.41, + "p99": 1.41, + "min": 1.41, + "max": 1.41 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 10, + "mode": "sequential", + "iterations": 1, + "coldStart": { + "samples": 10, + "mean": 197.95, + "p50": 197.93, + "p95": 211.43, + "p99": 211.43, + "min": 183.06, + "max": 211.43 + }, + "warmStart": { + "samples": 10, + "mean": 1.3, + "p50": 1.29, + "p95": 1.38, + "p99": 1.38, + "min": 1.25, + "max": 1.38 + }, + "phases": { + "filesystem_snapshot": { + "samples": 10, + "mean": 0.12, + "p50": 0.08, + "p95": 0.3, + "p99": 0.3, + "min": 0.06, + "max": 0.3 + }, + "session_open": { + "samples": 10, + "mean": 0.45, + "p50": 0.38, + "p95": 1.11, + "p99": 1.11, + "min": 0.21, + "max": 1.11 + }, + "vm_create": { + "samples": 10, + "mean": 1.49, + "p50": 1.34, + "p95": 2.41, + "p99": 2.41, + "min": 1.02, + "max": 2.41 + }, + "vm_ready": { + "samples": 10, + "mean": 0.02, + "p50": 0.01, + "p95": 0.03, + "p99": 0.03, + "min": 0.01, + "max": 0.03 + }, + "vm_configure": { + "samples": 10, + "mean": 0.76, + "p50": 0.14, + "p95": 3.93, + "p99": 3.93, + "min": 0.12, + "max": 3.93 + }, + "runtime_mount_wasm": { + "samples": 10, + "mean": 168, + "p50": 168.87, + "p95": 181.93, + "p99": 181.93, + "min": 153.02, + "max": 181.93 + }, + "runtime_mount_node": { + "samples": 10, + "mean": 2.86, + "p50": 2.76, + "p95": 4.1, + "p99": 4.1, + "min": 2.61, + "max": 4.1 + }, + "runtime_create_total": { + "samples": 10, + "mean": 170.96, + "p50": 171.63, + "p95": 184.92, + "p99": 184.92, + "min": 156.32, + "max": 184.92 + }, + "resident_runner_create": { + "samples": 10, + "mean": 25.64, + "p50": 25.27, + "p95": 28.33, + "p99": 28.33, + "min": 24.38, + "max": 28.33 + }, + "resident_first_exec": { + "samples": 10, + "mean": 1.34, + "p50": 1.32, + "p95": 1.46, + "p99": 1.46, + "min": 1.26, + "max": 1.46 + }, + "resident_warm_exec": { + "samples": 10, + "mean": 1.3, + "p50": 1.29, + "p95": 1.38, + "p99": 1.38, + "min": 1.25, + "max": 1.38 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 10, + "mode": "concurrent", + "iterations": 1, + "coldStart": { + "samples": 10, + "mean": 192.94, + "p50": 190.69, + "p95": 203.83, + "p99": 203.83, + "min": 182.62, + "max": 203.83 + }, + "warmStart": { + "samples": 10, + "mean": 1.29, + "p50": 1.28, + "p95": 1.36, + "p99": 1.36, + "min": 1.24, + "max": 1.36 + }, + "phases": { + "filesystem_snapshot": { + "samples": 10, + "mean": 0.06, + "p50": 0.03, + "p95": 0.33, + "p99": 0.33, + "min": 0.02, + "max": 0.33 + }, + "session_open": { + "samples": 10, + "mean": 0.39, + "p50": 0.33, + "p95": 0.74, + "p99": 0.74, + "min": 0.19, + "max": 0.74 + }, + "vm_create": { + "samples": 10, + "mean": 1.6, + "p50": 1.28, + "p95": 3.18, + "p99": 3.18, + "min": 1.18, + "max": 3.18 + }, + "vm_ready": { + "samples": 10, + "mean": 0.05, + "p50": 0.01, + "p95": 0.3, + "p99": 0.3, + "min": 0.01, + "max": 0.3 + }, + "vm_configure": { + "samples": 10, + "mean": 0.16, + "p50": 0.16, + "p95": 0.29, + "p99": 0.29, + "min": 0.09, + "max": 0.29 + }, + "runtime_mount_wasm": { + "samples": 10, + "mean": 163.33, + "p50": 161.48, + "p95": 174.29, + "p99": 174.29, + "min": 152.75, + "max": 174.29 + }, + "runtime_mount_node": { + "samples": 10, + "mean": 2.81, + "p50": 2.72, + "p95": 3.23, + "p99": 3.23, + "min": 2.57, + "max": 3.23 + }, + "runtime_create_total": { + "samples": 10, + "mean": 166.23, + "p50": 164.37, + "p95": 176.98, + "p99": 176.98, + "min": 155.54, + "max": 176.98 + }, + "resident_runner_create": { + "samples": 10, + "mean": 25.36, + "p50": 25.53, + "p95": 27.11, + "p99": 27.11, + "min": 24.04, + "max": 27.11 + }, + "resident_first_exec": { + "samples": 10, + "mean": 1.34, + "p50": 1.36, + "p95": 1.42, + "p99": 1.42, + "min": 1.26, + "max": 1.42 + }, + "resident_warm_exec": { + "samples": 10, + "mean": 1.29, + "p50": 1.28, + "p95": 1.36, + "p99": 1.36, + "min": 1.24, + "max": 1.36 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 50, + "mode": "sequential", + "iterations": 1, + "coldStart": { + "samples": 50, + "mean": 187.5, + "p50": 186.28, + "p95": 202.34, + "p99": 207.68, + "min": 176.54, + "max": 207.68 + }, + "warmStart": { + "samples": 50, + "mean": 1.31, + "p50": 1.3, + "p95": 1.4, + "p99": 1.54, + "min": 1.2, + "max": 1.54 + }, + "phases": { + "filesystem_snapshot": { + "samples": 50, + "mean": 0.04, + "p50": 0.03, + "p95": 0.04, + "p99": 0.37, + "min": 0.02, + "max": 0.37 + }, + "session_open": { + "samples": 50, + "mean": 0.27, + "p50": 0.2, + "p95": 0.66, + "p99": 1.07, + "min": 0.15, + "max": 1.07 + }, + "vm_create": { + "samples": 50, + "mean": 1.21, + "p50": 1.1, + "p95": 1.82, + "p99": 2.19, + "min": 0.93, + "max": 2.19 + }, + "vm_ready": { + "samples": 50, + "mean": 0.02, + "p50": 0.01, + "p95": 0.09, + "p99": 0.29, + "min": 0.01, + "max": 0.29 + }, + "vm_configure": { + "samples": 50, + "mean": 0.16, + "p50": 0.11, + "p95": 0.32, + "p99": 1.84, + "min": 0.07, + "max": 1.84 + }, + "runtime_mount_wasm": { + "samples": 50, + "mean": 157.39, + "p50": 156.58, + "p95": 172.04, + "p99": 177.36, + "min": 148.21, + "max": 177.36 + }, + "runtime_mount_node": { + "samples": 50, + "mean": 2.88, + "p50": 2.7, + "p95": 4.13, + "p99": 4.3, + "min": 2.54, + "max": 4.3 + }, + "runtime_create_total": { + "samples": 50, + "mean": 160.37, + "p50": 159.5, + "p95": 175.86, + "p99": 180.26, + "min": 150.98, + "max": 180.26 + }, + "resident_runner_create": { + "samples": 50, + "mean": 25.64, + "p50": 25.41, + "p95": 27.86, + "p99": 31.96, + "min": 24, + "max": 31.96 + }, + "resident_first_exec": { + "samples": 50, + "mean": 1.48, + "p50": 1.34, + "p95": 2.54, + "p99": 2.61, + "min": 1.23, + "max": 2.61 + }, + "resident_warm_exec": { + "samples": 50, + "mean": 1.31, + "p50": 1.3, + "p95": 1.4, + "p99": 1.54, + "min": 1.2, + "max": 1.54 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 50, + "mode": "concurrent", + "iterations": 1, + "coldStart": { + "samples": 50, + "mean": 189.8, + "p50": 188.72, + "p95": 201.12, + "p99": 206.34, + "min": 180.45, + "max": 206.34 + }, + "warmStart": { + "samples": 50, + "mean": 1.34, + "p50": 1.35, + "p95": 1.54, + "p99": 1.61, + "min": 0.33, + "max": 1.61 + }, + "phases": { + "filesystem_snapshot": { + "samples": 50, + "mean": 0.03, + "p50": 0.03, + "p95": 0.04, + "p99": 0.05, + "min": 0.02, + "max": 0.05 + }, + "session_open": { + "samples": 50, + "mean": 0.24, + "p50": 0.21, + "p95": 0.38, + "p99": 0.52, + "min": 0.14, + "max": 0.52 + }, + "vm_create": { + "samples": 50, + "mean": 1.24, + "p50": 1.13, + "p95": 1.87, + "p99": 2.11, + "min": 0.95, + "max": 2.11 + }, + "vm_ready": { + "samples": 50, + "mean": 0.03, + "p50": 0.01, + "p95": 0.06, + "p99": 0.3, + "min": 0.01, + "max": 0.3 + }, + "vm_configure": { + "samples": 50, + "mean": 0.14, + "p50": 0.12, + "p95": 0.26, + "p99": 0.3, + "min": 0.07, + "max": 0.3 + }, + "runtime_mount_wasm": { + "samples": 50, + "mean": 159.43, + "p50": 158.55, + "p95": 169.33, + "p99": 177.77, + "min": 150.65, + "max": 177.77 + }, + "runtime_mount_node": { + "samples": 50, + "mean": 2.92, + "p50": 2.76, + "p95": 3.56, + "p99": 7.39, + "min": 2.58, + "max": 7.39 + }, + "runtime_create_total": { + "samples": 50, + "mean": 162.44, + "p50": 162.01, + "p95": 172.47, + "p99": 180.66, + "min": 153.37, + "max": 180.66 + }, + "resident_runner_create": { + "samples": 50, + "mean": 25.83, + "p50": 25.36, + "p95": 28.62, + "p99": 29.19, + "min": 23.98, + "max": 29.19 + }, + "resident_first_exec": { + "samples": 50, + "mean": 1.53, + "p50": 1.41, + "p95": 2.77, + "p99": 2.91, + "min": 1.27, + "max": 2.91 + }, + "resident_warm_exec": { + "samples": 50, + "mean": 1.34, + "p50": 1.35, + "p95": 1.54, + "p99": 1.61, + "min": 0.33, + "max": 1.61 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 100, + "mode": "sequential", + "iterations": 1, + "coldStart": { + "samples": 100, + "mean": 202.61, + "p50": 200.75, + "p95": 212.41, + "p99": 218.87, + "min": 184.86, + "max": 367.94 + }, + "warmStart": { + "samples": 100, + "mean": 1.05, + "p50": 1.3, + "p95": 1.59, + "p99": 1.66, + "min": 0.37, + "max": 1.71 + }, + "phases": { + "filesystem_snapshot": { + "samples": 100, + "mean": 0.03, + "p50": 0.03, + "p95": 0.04, + "p99": 0.06, + "min": 0.03, + "max": 0.07 + }, + "session_open": { + "samples": 100, + "mean": 0.32, + "p50": 0.3, + "p95": 0.52, + "p99": 0.59, + "min": 0.19, + "max": 0.63 + }, + "vm_create": { + "samples": 100, + "mean": 1.19, + "p50": 1.15, + "p95": 1.45, + "p99": 1.54, + "min": 1.02, + "max": 1.74 + }, + "vm_ready": { + "samples": 100, + "mean": 0.01, + "p50": 0.01, + "p95": 0.03, + "p99": 0.05, + "min": 0.01, + "max": 0.06 + }, + "vm_configure": { + "samples": 100, + "mean": 0.21, + "p50": 0.16, + "p95": 0.47, + "p99": 0.72, + "min": 0.08, + "max": 1.3 + }, + "runtime_mount_wasm": { + "samples": 100, + "mean": 173.08, + "p50": 171.64, + "p95": 181.91, + "p99": 188.66, + "min": 153.84, + "max": 338.15 + }, + "runtime_mount_node": { + "samples": 100, + "mean": 3.18, + "p50": 3.16, + "p95": 3.65, + "p99": 3.95, + "min": 2.73, + "max": 4.08 + }, + "runtime_create_total": { + "samples": 100, + "mean": 176.35, + "p50": 175.13, + "p95": 185.46, + "p99": 192.36, + "min": 156.7, + "max": 341.58 + }, + "resident_runner_create": { + "samples": 100, + "mean": 24.8, + "p50": 24.23, + "p95": 28.83, + "p99": 31.04, + "min": 22.63, + "max": 33.04 + }, + "resident_first_exec": { + "samples": 100, + "mean": 1.46, + "p50": 1.41, + "p95": 1.85, + "p99": 2.16, + "min": 1.1, + "max": 2.17 + }, + "resident_warm_exec": { + "samples": 100, + "mean": 1.05, + "p50": 1.3, + "p95": 1.59, + "p99": 1.66, + "min": 0.37, + "max": 1.71 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 100, + "mode": "concurrent", + "iterations": 1, + "coldStart": { + "samples": 100, + "mean": 217.31, + "p50": 217.14, + "p95": 229.38, + "p99": 230.9, + "min": 204.32, + "max": 232.27 + }, + "warmStart": { + "samples": 100, + "mean": 0.78, + "p50": 0.73, + "p95": 1.22, + "p99": 1.35, + "min": 0.37, + "max": 1.87 + }, + "phases": { + "filesystem_snapshot": { + "samples": 100, + "mean": 0.03, + "p50": 0.03, + "p95": 0.04, + "p99": 0.06, + "min": 0.02, + "max": 0.08 + }, + "session_open": { + "samples": 100, + "mean": 0.41, + "p50": 0.35, + "p95": 0.73, + "p99": 0.87, + "min": 0.18, + "max": 0.9 + }, + "vm_create": { + "samples": 100, + "mean": 1.19, + "p50": 1.15, + "p95": 1.39, + "p99": 1.55, + "min": 0.98, + "max": 2.04 + }, + "vm_ready": { + "samples": 100, + "mean": 0.01, + "p50": 0.01, + "p95": 0.03, + "p99": 0.04, + "min": 0.01, + "max": 0.04 + }, + "vm_configure": { + "samples": 100, + "mean": 0.28, + "p50": 0.22, + "p95": 0.73, + "p99": 1.11, + "min": 0.07, + "max": 1.14 + }, + "runtime_mount_wasm": { + "samples": 100, + "mean": 187.9, + "p50": 187.62, + "p95": 199.54, + "p99": 200.84, + "min": 175.2, + "max": 204.39 + }, + "runtime_mount_node": { + "samples": 100, + "mean": 3.49, + "p50": 3.42, + "p95": 4.17, + "p99": 4.94, + "min": 2.89, + "max": 6.04 + }, + "runtime_create_total": { + "samples": 100, + "mean": 191.48, + "p50": 190.92, + "p95": 203.07, + "p99": 204.12, + "min": 178.43, + "max": 207.96 + }, + "resident_runner_create": { + "samples": 100, + "mean": 24.17, + "p50": 23.9, + "p95": 26.13, + "p99": 27.31, + "min": 22.66, + "max": 27.98 + }, + "resident_first_exec": { + "samples": 100, + "mean": 1.66, + "p50": 1.6, + "p95": 2.26, + "p99": 2.52, + "min": 1.17, + "max": 4.03 + }, + "resident_warm_exec": { + "samples": 100, + "mean": 0.78, + "p50": 0.73, + "p95": 1.22, + "p99": 1.35, + "min": 0.37, + "max": 1.87 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 200, + "mode": "sequential", + "iterations": 1, + "coldStart": { + "samples": 200, + "mean": 243.3, + "p50": 243.92, + "p95": 262.6, + "p99": 273.42, + "min": 214.29, + "max": 285.67 + }, + "warmStart": { + "samples": 200, + "mean": 0.99, + "p50": 0.92, + "p95": 1.68, + "p99": 2.01, + "min": 0.45, + "max": 2.4 + }, + "phases": { + "filesystem_snapshot": { + "samples": 200, + "mean": 0.03, + "p50": 0.03, + "p95": 0.04, + "p99": 0.07, + "min": 0.02, + "max": 0.08 + }, + "session_open": { + "samples": 200, + "mean": 0.57, + "p50": 0.51, + "p95": 1.09, + "p99": 1.45, + "min": 0.19, + "max": 1.63 + }, + "vm_create": { + "samples": 200, + "mean": 1.29, + "p50": 1.22, + "p95": 1.78, + "p99": 2.28, + "min": 0.98, + "max": 2.44 + }, + "vm_ready": { + "samples": 200, + "mean": 0.01, + "p50": 0.01, + "p95": 0.03, + "p99": 0.04, + "min": 0.01, + "max": 0.04 + }, + "vm_configure": { + "samples": 200, + "mean": 0.31, + "p50": 0.24, + "p95": 0.73, + "p99": 1.09, + "min": 0.09, + "max": 1.8 + }, + "runtime_mount_wasm": { + "samples": 200, + "mean": 212.76, + "p50": 211.89, + "p95": 230.56, + "p99": 241.01, + "min": 186.3, + "max": 255.13 + }, + "runtime_mount_node": { + "samples": 200, + "mean": 4.02, + "p50": 3.9, + "p95": 5.15, + "p99": 5.93, + "min": 3.2, + "max": 6.19 + }, + "runtime_create_total": { + "samples": 200, + "mean": 216.88, + "p50": 216.65, + "p95": 235.15, + "p99": 246.33, + "min": 189.77, + "max": 259.48 + }, + "resident_runner_create": { + "samples": 200, + "mean": 24.58, + "p50": 24.04, + "p95": 27.39, + "p99": 29.67, + "min": 22.67, + "max": 34.84 + }, + "resident_first_exec": { + "samples": 200, + "mean": 1.84, + "p50": 1.76, + "p95": 2.57, + "p99": 3.11, + "min": 1.14, + "max": 4.3 + }, + "resident_warm_exec": { + "samples": 200, + "mean": 0.99, + "p50": 0.92, + "p95": 1.68, + "p99": 2.01, + "min": 0.45, + "max": 2.4 + } + } + }, + { + "scenario": "resident-runner", + "batchSize": 200, + "mode": "concurrent", + "iterations": 1, + "coldStart": { + "samples": 200, + "mean": 283.56, + "p50": 284.31, + "p95": 302.18, + "p99": 304.27, + "min": 252.19, + "max": 306.89 + }, + "warmStart": { + "samples": 200, + "mean": 1.38, + "p50": 1.28, + "p95": 2.29, + "p99": 3.11, + "min": 0.61, + "max": 3.51 + }, + "phases": { + "filesystem_snapshot": { + "samples": 200, + "mean": 0.02, + "p50": 0.02, + "p95": 0.03, + "p99": 0.05, + "min": 0.02, + "max": 0.06 + }, + "session_open": { + "samples": 200, + "mean": 0.92, + "p50": 0.81, + "p95": 1.75, + "p99": 2.77, + "min": 0.26, + "max": 3.28 + }, + "vm_create": { + "samples": 200, + "mean": 1.44, + "p50": 1.33, + "p95": 2.1, + "p99": 2.69, + "min": 1.09, + "max": 3.32 + }, + "vm_ready": { + "samples": 200, + "mean": 0.01, + "p50": 0.01, + "p95": 0.03, + "p99": 0.04, + "min": 0, + "max": 0.06 + }, + "vm_configure": { + "samples": 200, + "mean": 0.43, + "p50": 0.3, + "p95": 1.01, + "p99": 1.28, + "min": 0.12, + "max": 2.05 + }, + "runtime_mount_wasm": { + "samples": 200, + "mean": 251.31, + "p50": 252.57, + "p95": 267.63, + "p99": 271.21, + "min": 222.52, + "max": 275.65 + }, + "runtime_mount_node": { + "samples": 200, + "mean": 4.87, + "p50": 4.67, + "p95": 6.57, + "p99": 7.09, + "min": 3.57, + "max": 7.99 + }, + "runtime_create_total": { + "samples": 200, + "mean": 256.27, + "p50": 257.46, + "p95": 273.31, + "p99": 276.15, + "min": 226.37, + "max": 280.51 + }, + "resident_runner_create": { + "samples": 200, + "mean": 25.03, + "p50": 24.63, + "p95": 28.38, + "p99": 30.39, + "min": 22.84, + "max": 30.97 + }, + "resident_first_exec": { + "samples": 200, + "mean": 2.25, + "p50": 2.16, + "p95": 3.32, + "p99": 3.97, + "min": 1.37, + "max": 4.83 + }, + "resident_warm_exec": { + "samples": 200, + "mean": 1.38, + "p50": 1.28, + "p95": 2.29, + "p99": 3.11, + "min": 0.61, + "max": 3.51 + } + } + } + ] +} diff --git a/packages/benchmarks/results/coldstart-resident-full-matrix-20260619.log b/packages/benchmarks/results/coldstart-resident-full-matrix-20260619.log new file mode 100644 index 000000000..97f0118c9 --- /dev/null +++ b/packages/benchmarks/results/coldstart-resident-full-matrix-20260619.log @@ -0,0 +1,75 @@ +=== Cold Start Benchmark === +CPU: 12th Gen Intel(R) Core(TM) i7-12700KF +Cores: 20 | Max concurrency: 16 | Max live runtimes: 8 +Resident runner live cap: 1 +RAM: 62.6 GB | Node: v24.13.0 +Loadavg: 2.3, 2.56, 4.03 +Iterations: 1 (+ 0 warmup) +Batch sizes: 1, 10, 50, 100, 200 +Scenarios: resident-runner +Sidecar: /home/nathan/secure-exec/target/release/secure-exec-sidecar + + +--- scenario=resident-runner, batch=1, mode=sequential --- + cold: mean=396.38ms p50=396.38ms p95=396.38ms + warm: mean=1.36ms p50=1.36ms p95=1.36ms + phases: filesystem_snapshot.p50=0.32ms | resident_first_exec.p50=1.59ms | resident_runner_create.p50=171.41ms | resident_warm_exec.p50=1.36ms | runtime_create_total.p50=223.38ms | runtime_mount_node.p50=2.95ms | runtime_mount_wasm.p50=220.02ms | session_open.p50=6.35ms | vm_configure.p50=0.73ms | vm_create.p50=2.75ms | vm_ready.p50=0.25ms + +--- scenario=resident-runner, batch=1, mode=concurrent --- + cold: mean=204.7ms p50=204.7ms p95=204.7ms + warm: mean=1.41ms p50=1.41ms p95=1.41ms + phases: filesystem_snapshot.p50=0.1ms | resident_first_exec.p50=1.54ms | resident_runner_create.p50=27.31ms | resident_warm_exec.p50=1.41ms | runtime_create_total.p50=175.85ms | runtime_mount_node.p50=2.82ms | runtime_mount_wasm.p50=172.92ms | session_open.p50=1.11ms | vm_configure.p50=0.14ms | vm_create.p50=1.26ms | vm_ready.p50=0.01ms + +--- scenario=resident-runner, batch=10, mode=sequential --- + cold: mean=197.95ms p50=197.93ms p95=211.43ms + warm: mean=1.3ms p50=1.29ms p95=1.38ms + phases: filesystem_snapshot.p50=0.08ms | resident_first_exec.p50=1.32ms | resident_runner_create.p50=25.27ms | resident_warm_exec.p50=1.29ms | runtime_create_total.p50=171.63ms | runtime_mount_node.p50=2.76ms | runtime_mount_wasm.p50=168.87ms | session_open.p50=0.38ms | vm_configure.p50=0.14ms | vm_create.p50=1.34ms | vm_ready.p50=0.01ms + +--- scenario=resident-runner, batch=10, mode=concurrent --- + cold: mean=192.94ms p50=190.69ms p95=203.83ms + warm: mean=1.29ms p50=1.28ms p95=1.36ms + phases: filesystem_snapshot.p50=0.03ms | resident_first_exec.p50=1.36ms | resident_runner_create.p50=25.53ms | resident_warm_exec.p50=1.28ms | runtime_create_total.p50=164.37ms | runtime_mount_node.p50=2.72ms | runtime_mount_wasm.p50=161.48ms | session_open.p50=0.33ms | vm_configure.p50=0.16ms | vm_create.p50=1.28ms | vm_ready.p50=0.01ms + +--- scenario=resident-runner, batch=50, mode=sequential --- + cold: mean=187.5ms p50=186.28ms p95=202.34ms + warm: mean=1.31ms p50=1.3ms p95=1.4ms + phases: filesystem_snapshot.p50=0.03ms | resident_first_exec.p50=1.34ms | resident_runner_create.p50=25.41ms | resident_warm_exec.p50=1.3ms | runtime_create_total.p50=159.5ms | runtime_mount_node.p50=2.7ms | runtime_mount_wasm.p50=156.58ms | session_open.p50=0.2ms | vm_configure.p50=0.11ms | vm_create.p50=1.1ms | vm_ready.p50=0.01ms + +--- scenario=resident-runner, batch=50, mode=concurrent --- + cold: mean=189.8ms p50=188.72ms p95=201.12ms + warm: mean=1.34ms p50=1.35ms p95=1.54ms + phases: filesystem_snapshot.p50=0.03ms | resident_first_exec.p50=1.41ms | resident_runner_create.p50=25.36ms | resident_warm_exec.p50=1.35ms | runtime_create_total.p50=162.01ms | runtime_mount_node.p50=2.76ms | runtime_mount_wasm.p50=158.55ms | session_open.p50=0.21ms | vm_configure.p50=0.12ms | vm_create.p50=1.13ms | vm_ready.p50=0.01ms + +--- scenario=resident-runner, batch=100, mode=sequential --- + cold: mean=202.61ms p50=200.75ms p95=212.41ms + warm: mean=1.05ms p50=1.3ms p95=1.59ms + phases: filesystem_snapshot.p50=0.03ms | resident_first_exec.p50=1.41ms | resident_runner_create.p50=24.23ms | resident_warm_exec.p50=1.3ms | runtime_create_total.p50=175.13ms | runtime_mount_node.p50=3.16ms | runtime_mount_wasm.p50=171.64ms | session_open.p50=0.3ms | vm_configure.p50=0.16ms | vm_create.p50=1.15ms | vm_ready.p50=0.01ms + +--- scenario=resident-runner, batch=100, mode=concurrent --- + cold: mean=217.31ms p50=217.14ms p95=229.38ms + warm: mean=0.78ms p50=0.73ms p95=1.22ms + phases: filesystem_snapshot.p50=0.03ms | resident_first_exec.p50=1.6ms | resident_runner_create.p50=23.9ms | resident_warm_exec.p50=0.73ms | runtime_create_total.p50=190.92ms | runtime_mount_node.p50=3.42ms | runtime_mount_wasm.p50=187.62ms | session_open.p50=0.35ms | vm_configure.p50=0.22ms | vm_create.p50=1.15ms | vm_ready.p50=0.01ms + +--- scenario=resident-runner, batch=200, mode=sequential --- + cold: mean=243.3ms p50=243.92ms p95=262.6ms + warm: mean=0.99ms p50=0.92ms p95=1.68ms + phases: filesystem_snapshot.p50=0.03ms | resident_first_exec.p50=1.76ms | resident_runner_create.p50=24.04ms | resident_warm_exec.p50=0.92ms | runtime_create_total.p50=216.65ms | runtime_mount_node.p50=3.9ms | runtime_mount_wasm.p50=211.89ms | session_open.p50=0.51ms | vm_configure.p50=0.24ms | vm_create.p50=1.22ms | vm_ready.p50=0.01ms + +--- scenario=resident-runner, batch=200, mode=concurrent --- + cold: mean=283.56ms p50=284.31ms p95=302.18ms + warm: mean=1.38ms p50=1.28ms p95=2.29ms + phases: filesystem_snapshot.p50=0.02ms | resident_first_exec.p50=2.16ms | resident_runner_create.p50=24.63ms | resident_warm_exec.p50=1.28ms | runtime_create_total.p50=257.46ms | runtime_mount_node.p50=4.67ms | runtime_mount_wasm.p50=252.57ms | session_open.p50=0.81ms | vm_configure.p50=0.3ms | vm_create.p50=1.33ms | vm_ready.p50=0.01ms + + scenario | batch | mode | cold mean | cold p50 | cold p95 | warm mean | warm p50 | warm p95 +--------------- | ----- | ---------- | --------- | -------- | -------- | --------- | -------- | -------- +resident-runner | 1 | sequential | 396.38ms | 396.38ms | 396.38ms | 1.36ms | 1.36ms | 1.36ms +resident-runner | 1 | concurrent | 204.7ms | 204.7ms | 204.7ms | 1.41ms | 1.41ms | 1.41ms +resident-runner | 10 | sequential | 197.95ms | 197.93ms | 211.43ms | 1.3ms | 1.29ms | 1.38ms +resident-runner | 10 | concurrent | 192.94ms | 190.69ms | 203.83ms | 1.29ms | 1.28ms | 1.36ms +resident-runner | 50 | sequential | 187.5ms | 186.28ms | 202.34ms | 1.31ms | 1.3ms | 1.4ms +resident-runner | 50 | concurrent | 189.8ms | 188.72ms | 201.12ms | 1.34ms | 1.35ms | 1.54ms +resident-runner | 100 | sequential | 202.61ms | 200.75ms | 212.41ms | 1.05ms | 1.3ms | 1.59ms +resident-runner | 100 | concurrent | 217.31ms | 217.14ms | 229.38ms | 0.78ms | 0.73ms | 1.22ms +resident-runner | 200 | sequential | 243.3ms | 243.92ms | 262.6ms | 0.99ms | 0.92ms | 1.68ms +resident-runner | 200 | concurrent | 283.56ms | 284.31ms | 302.18ms | 1.38ms | 1.28ms | 2.29ms + diff --git a/packages/benchmarks/run-benchmarks.sh b/packages/benchmarks/run-benchmarks.sh index e70e7ce8c..59cfb2878 100755 --- a/packages/benchmarks/run-benchmarks.sh +++ b/packages/benchmarks/run-benchmarks.sh @@ -33,12 +33,18 @@ echo "Results: $RESULTS_DIR" echo echo "=== cold + warm start ===" -npx tsx "$HERE/coldstart.bench.ts" >"$RESULTS_DIR/coldstart-$STAMP.json" +npx tsx "$HERE/coldstart.bench.ts" \ + >"$RESULTS_DIR/coldstart-$STAMP.json" \ + 2>"$RESULTS_DIR/coldstart-$STAMP.log" echo "=== memory ===" -node --expose-gc --import tsx/esm "$HERE/memory.bench.ts" >"$RESULTS_DIR/memory-$STAMP.json" +node --expose-gc --import tsx/esm "$HERE/memory.bench.ts" \ + >"$RESULTS_DIR/memory-$STAMP.json" \ + 2>"$RESULTS_DIR/memory-$STAMP.log" echo echo "Done. Results saved to:" echo " $RESULTS_DIR/coldstart-$STAMP.json" +echo " $RESULTS_DIR/coldstart-$STAMP.log" echo " $RESULTS_DIR/memory-$STAMP.json" +echo " $RESULTS_DIR/memory-$STAMP.log" diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 9e4aac760..a8c01d8dc 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -19,6 +19,8 @@ export * from "./protocol-client.js"; export * from "./protocol-frames.js"; export * from "./request-payloads.js"; export * from "./response-payloads.js"; +export { Sidecar } from "./sidecar-client.js"; +export type { SidecarSpawnOptions } from "./sidecar-client.js"; export * from "./state.js"; export * as protocol from "./generated-protocol.js"; export * from "./generated-protocol.js"; diff --git a/packages/core/src/kernel-proxy.ts b/packages/core/src/kernel-proxy.ts index 44fc6fc3a..ddd07b42b 100644 --- a/packages/core/src/kernel-proxy.ts +++ b/packages/core/src/kernel-proxy.ts @@ -20,7 +20,7 @@ import type { AuthenticatedSession, CreatedVm, GuestFilesystemStat, - NativeSidecarProcessClient, + Sidecar, SidecarMountDescriptor, SidecarProcessSnapshotEntry, SidecarSignalHandlerRegistration, @@ -310,9 +310,10 @@ interface TrackedProcessEntry { } interface NativeSidecarKernelProxyOptions { - client: NativeSidecarProcessClient; + client: Sidecar; session: AuthenticatedSession; vm: CreatedVm; + disposeClient?: boolean; env: Record; cwd: string; defaultExecCwd?: string; @@ -330,7 +331,8 @@ export class NativeSidecarKernelProxy { readonly processes = new Map(); private readonly defaultExecCwd: string | undefined; - private readonly client: NativeSidecarProcessClient; + private readonly client: Sidecar; + private readonly disposeClient: boolean; private readonly session: AuthenticatedSession; private readonly vm: CreatedVm; private readonly localMounts: LocalCompatMount[]; @@ -362,6 +364,7 @@ export class NativeSidecarKernelProxy { constructor(options: NativeSidecarKernelProxyOptions) { this.client = options.client; + this.disposeClient = options.disposeClient ?? true; this.session = options.session; this.vm = options.vm; this.env = { ...options.env }; @@ -423,7 +426,9 @@ export class NativeSidecarKernelProxy { this.finishProcess(entry, 143); } } - await this.client.dispose().catch(() => {}); + if (this.disposeClient) { + await this.client.dispose().catch(() => {}); + } await this.eventPump.catch(() => {}); await this.onDispose?.().catch(() => {}); } @@ -2268,7 +2273,7 @@ export type { AuthenticatedSession, CreatedVm, GuestFilesystemStat, - NativeSidecarSpawnOptions, + SidecarSpawnOptions, RootFilesystemEntry, SidecarEventSelector, SidecarPermissionsPolicy, @@ -2282,7 +2287,7 @@ export type { } from "./sidecar-client.js"; export { NATIVE_SIDECAR_FRAME_TIMEOUT_MS, - NativeSidecarProcessClient, + Sidecar, SidecarEventBufferOverflow, SidecarProcessError, SidecarProcessExited, diff --git a/packages/core/src/node-runtime.ts b/packages/core/src/node-runtime.ts index d860e9789..d39a43302 100644 --- a/packages/core/src/node-runtime.ts +++ b/packages/core/src/node-runtime.ts @@ -25,8 +25,10 @@ import type { ExecResult, HostToolDefinition, Kernel, + KernelBootTiming, Permissions, } from "./test-runtime.js"; +import type { Sidecar } from "./sidecar-client.js"; import { createInMemoryFileSystem, createKernel, @@ -37,6 +39,17 @@ import { export type { HostToolDefinition, HostToolExample } from "./test-runtime.js"; +export type NodeRuntimeBootTimingPhase = + | KernelBootTiming["phase"] + | "runtime_mount_wasm" + | "runtime_mount_node" + | "host_tools"; + +export interface NodeRuntimeBootTiming { + phase: NodeRuntimeBootTimingPhase; + durationMs: number; +} + /** Repository root, used to locate the in-repo WASM command build output. */ const REPO_ROOT = fileURLToPath(new URL("../../..", import.meta.url)); @@ -130,6 +143,14 @@ export interface NodeRuntimeCreateOptions { * `@secure-exec/core` package (published installs). */ commandsDir?: string; + /** + * Existing native sidecar process to use for this runtime. Omit this to use + * the default shared sidecar behavior. When provided, the runtime owns only + * its VM and leaves sidecar process disposal to the caller. + */ + sidecar?: Sidecar; + /** Receives coarse boot phase timings for benchmarks and diagnostics. */ + onBootTiming?: (timing: NodeRuntimeBootTiming) => void; /** * Files to seed into the VM's virtual filesystem before the guest runs, * keyed by absolute guest path. Parent directories are created as needed. @@ -435,6 +456,21 @@ export interface NodeRuntimeProcess { readonly exitCode: number | null; } +export interface NodeRuntimeResidentRunnerExecOptions { + /** Abort the guest eval after this many milliseconds. */ + timeout?: number; +} + +export type NodeRuntimeResidentRunnerOptions = {}; + +export interface NodeRuntimeResidentRunner { + exec( + code: string, + options?: NodeRuntimeResidentRunnerExecOptions, + ): Promise; + dispose(): Promise; +} + /** Result of {@link NodeRuntime.run}. */ export interface NodeRuntimeRunResult { /** The JSON-decoded value the guest produced, when the run succeeded. */ @@ -445,6 +481,10 @@ export interface NodeRuntimeRunResult { } let nextProgramId = 0; +let nextResidentRequestId = 0; + +const RESIDENT_READY_PREFIX = "__SECURE_EXEC_RESIDENT_READY__"; +const RESIDENT_RESULT_PREFIX = "__SECURE_EXEC_RESIDENT_RESULT__"; /** * Ergonomic, batteries-included runtime for executing guest JavaScript. @@ -526,6 +566,8 @@ export class NodeRuntime { }, env: options.env, cwd: options.cwd, + sidecar: options.sidecar, + onBootTiming: (timing) => options.onBootTiming?.(timing), loopbackExemptPorts: options.loopbackExemptPorts, }); @@ -535,13 +577,20 @@ export class NodeRuntime { // process: the kernel runs every command through a shell, so without // `sh` nothing can be spawned, including the guest `node` program we // run here and any child the guest spawns via node:child_process. - await kernel.mount(createWasmVmRuntime({ commandDirs: [commandsDir] })); - await kernel.mount(createNodeRuntime()); + await measureBootTiming("runtime_mount_wasm", options.onBootTiming, () => + kernel.mount(createWasmVmRuntime({ commandDirs: [commandsDir] })), + ); + await measureBootTiming("runtime_mount_node", options.onBootTiming, () => + kernel.mount(createNodeRuntime()), + ); // Register host tools after the runtimes are mounted so they are // installed as guest commands the moment the VM is ready. - if (options.tools && Object.keys(options.tools).length > 0) { - await kernel.registerHostTools(options.tools); + const tools = options.tools; + if (tools && Object.keys(tools).length > 0) { + await measureBootTiming("host_tools", options.onBootTiming, () => + kernel.registerHostTools(tools), + ); } } catch (error) { await kernel.dispose().catch(() => {}); @@ -551,6 +600,12 @@ export class NodeRuntime { return new NodeRuntime(kernel); } + async createResidentRunner( + _options: NodeRuntimeResidentRunnerOptions = {}, + ): Promise { + return ResidentNodeRunner.create(this); + } + /** * Run `code` as a guest Node program and capture its output. * @@ -912,6 +967,208 @@ export class NodeRuntime { } } +const RESIDENT_RUNNER_SOURCE = ` +import { Buffer } from "node:buffer"; +import { createInterface } from "node:readline"; + +const readyPrefix = ${JSON.stringify(RESIDENT_READY_PREFIX)}; +const resultPrefix = ${JSON.stringify(RESIDENT_RESULT_PREFIX)}; +console.log(readyPrefix); + +const rl = createInterface({ input: process.stdin, crlfDelay: Infinity }); +for await (const line of rl) { + let request; + try { + request = JSON.parse(line); + const source = Buffer.from(String(request.code), "utf8").toString("base64"); + await import(\`data:text/javascript;base64,\${source}#\${request.id}\`); + process.stdout.write(resultPrefix + JSON.stringify({ + id: request.id, + exitCode: 0, + stderr: "", + }) + "\\n"); + } catch (error) { + process.stdout.write(resultPrefix + JSON.stringify({ + id: request?.id, + exitCode: 1, + stderr: error instanceof Error ? (error.stack ?? error.message) : String(error), + }) + "\\n"); + } +} +`; + +class ResidentNodeRunner implements NodeRuntimeResidentRunner { + private proc: NodeRuntimeProcess | null = null; + private stdoutBuffer = ""; + private active: { + id: number; + stdout: Uint8Array[]; + stderr: Uint8Array[]; + resolve: (result: NodeRuntimeExecResult) => void; + reject: (error: Error) => void; + timer?: ReturnType; + } | null = null; + private readonly readyPromise: Promise; + private resolveReady!: () => void; + private rejectReady!: (error: Error) => void; + + private constructor() { + this.readyPromise = new Promise((resolve, reject) => { + this.resolveReady = resolve; + this.rejectReady = reject; + }); + } + + static async create(runtime: NodeRuntime): Promise { + const runner = new ResidentNodeRunner(); + runner.proc = await runtime.spawn(RESIDENT_RUNNER_SOURCE, { + onStdout: (chunk) => runner.handleStdout(chunk), + onStderr: (chunk) => runner.handleStderr(chunk), + }); + runner.proc.wait().then( + (exitCode) => { + const error = new Error( + `resident runner exited before completing request: ${exitCode}`, + ); + runner.rejectReady(error); + runner.active?.reject(error); + runner.active = null; + }, + (error) => { + const normalized = + error instanceof Error ? error : new Error(String(error)); + runner.rejectReady(normalized); + runner.active?.reject(normalized); + runner.active = null; + }, + ); + await runner.readyPromise; + return runner; + } + + async exec( + code: string, + options: NodeRuntimeResidentRunnerExecOptions = {}, + ): Promise { + await this.readyPromise; + if (!this.proc) { + throw new Error("resident runner is not running"); + } + if (this.active) { + throw new Error("resident runner supports one in-flight exec"); + } + const proc = this.proc; + const id = nextResidentRequestId++; + return new Promise((resolve, reject) => { + const active = { + id, + stdout: [], + stderr: [], + resolve, + reject, + timer: undefined as ReturnType | undefined, + }; + if (options.timeout !== undefined) { + active.timer = setTimeout(() => { + proc.kill("SIGKILL"); + this.active = null; + reject( + new Error(`resident runner timed out after ${options.timeout}ms`), + ); + }, options.timeout); + } + this.active = active; + proc.writeStdin(`${JSON.stringify({ id, code })}\n`); + }); + } + + async dispose(): Promise { + const proc = this.proc; + this.proc = null; + this.active = null; + if (!proc) { + return; + } + proc.kill("SIGTERM"); + await proc.wait().catch(() => {}); + } + + private handleStdout(chunk: Uint8Array): void { + this.stdoutBuffer += new TextDecoder().decode(chunk); + while (true) { + const newlineIndex = this.stdoutBuffer.indexOf("\n"); + if (newlineIndex < 0) { + break; + } + const rawLine = this.stdoutBuffer.slice(0, newlineIndex); + this.stdoutBuffer = this.stdoutBuffer.slice(newlineIndex + 1); + const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine; + if (line === RESIDENT_READY_PREFIX) { + this.resolveReady(); + continue; + } + if (line.startsWith(RESIDENT_RESULT_PREFIX)) { + this.finishRequest(line.slice(RESIDENT_RESULT_PREFIX.length)); + continue; + } + this.active?.stdout.push(new TextEncoder().encode(`${line}\n`)); + } + } + + private handleStderr(chunk: Uint8Array): void { + this.active?.stderr.push(chunk); + } + + private finishRequest(payload: string): void { + const active = this.active; + if (!active) { + return; + } + let parsed: { id?: number; exitCode?: number; stderr?: string }; + try { + parsed = JSON.parse(payload) as { + id?: number; + exitCode?: number; + stderr?: string; + }; + } catch (error) { + active.reject(error instanceof Error ? error : new Error(String(error))); + this.active = null; + return; + } + if (parsed.id !== active.id) { + active.reject( + new Error(`resident runner response id mismatch: ${parsed.id}`), + ); + this.active = null; + return; + } + if (active.timer !== undefined) { + clearTimeout(active.timer); + } + this.active = null; + const stderr = `${decodeChunks(active.stderr)}${parsed.stderr ?? ""}`; + active.resolve({ + stdout: decodeChunks(active.stdout), + stderr, + exitCode: parsed.exitCode ?? 1, + }); + } +} + +async function measureBootTiming( + phase: NodeRuntimeBootTimingPhase, + onBootTiming: ((timing: NodeRuntimeBootTiming) => void) | undefined, + fn: () => Promise, +): Promise { + const start = performance.now(); + try { + return await fn(); + } finally { + onBootTiming?.({ phase, durationMs: performance.now() - start }); + } +} + /** * Common POSIX signal numbers, used to translate a signal name passed to * {@link NodeRuntimeProcess.kill} into the numeric signal the kernel expects. diff --git a/packages/core/src/sidecar-client.ts b/packages/core/src/sidecar-client.ts index 95f68f954..23ad67e88 100644 --- a/packages/core/src/sidecar-client.ts +++ b/packages/core/src/sidecar-client.ts @@ -5,11 +5,11 @@ import { import type { MountConfigJsonObject } from "./descriptors.js"; import { type LiveSidecarEventSelector } from "./event-buffer.js"; import { - decodeGuestFilesystemContent, - encodeGuestFilesystemContent, - type LiveRootFilesystemEntry, - type LiveRootFilesystemEntryEncoding, - type LiveRootFilesystemLowerDescriptor, + decodeGuestFilesystemContent, + encodeGuestFilesystemContent, + type LiveRootFilesystemEntry, + type LiveRootFilesystemEntryEncoding, + type LiveRootFilesystemLowerDescriptor, } from "./filesystem.js"; import type { CreateVmConfig } from "./generated/CreateVmConfig.js"; import { @@ -56,8 +56,7 @@ export { SidecarEventBufferOverflow } from "./event-buffer.js"; const BRIDGE_CONTRACT_VERSION = 1; -export const NATIVE_SIDECAR_FRAME_TIMEOUT_MS = - DEFAULT_SIDECAR_FRAME_TIMEOUT_MS; +export const NATIVE_SIDECAR_FRAME_TIMEOUT_MS = DEFAULT_SIDECAR_FRAME_TIMEOUT_MS; type OwnershipScope = LiveOwnershipScope; @@ -162,8 +161,8 @@ type NativeTransportPayloadCodec = ProtocolFramePayloadCodec; export type SidecarRequestHandler = LiveSidecarRequestHandler; -export interface NativeSidecarSpawnOptions { - cwd: string; +export interface SidecarSpawnOptions { + cwd?: string; command?: string; args?: string[]; frameTimeoutMs?: number; @@ -237,14 +236,16 @@ export interface SidecarProjectedModuleDescriptor { entrypoint: string; } -export class NativeSidecarProcessClient { +export class Sidecar { private readonly protocolClient: StdioSidecarProtocolClient; private constructor(protocolClient: StdioSidecarProtocolClient) { this.protocolClient = protocolClient; } - static spawn(options: NativeSidecarSpawnOptions): NativeSidecarProcessClient { + static spawn( + options: SidecarSpawnOptions = {}, + ): Sidecar { const protocolClient = StdioSidecarProtocolClient.spawn({ command: options.command, args: options.args ?? [], @@ -257,7 +258,7 @@ export class NativeSidecarProcessClient { disposedErrorMessage: "native sidecar disposed", payloadCodec: options.payloadCodec ?? "bare", }); - return new NativeSidecarProcessClient(protocolClient); + return new Sidecar(protocolClient); } setSidecarRequestHandler(handler: SidecarRequestHandler | null): void { @@ -316,25 +317,25 @@ export class NativeSidecarProcessClient { }; } - async createVm( - session: AuthenticatedSession, - options: { - runtime: GuestRuntimeKind; - config: CreateVmConfig; - }, - ): Promise { + async createVm( + session: AuthenticatedSession, + options: { + runtime: GuestRuntimeKind; + config: CreateVmConfig; + }, + ): Promise { const response = await this.sendRequest({ ownership: { scope: "session", connection_id: session.connectionId, session_id: session.sessionId, }, - payload: { - type: "create_vm", - runtime: options.runtime, - config: options.config, - }, - }); + payload: { + type: "create_vm", + runtime: options.runtime, + config: options.config, + }, + }); if (response.payload.type !== "vm_created") { throw new Error( `unexpected create_vm response: ${response.payload.type}`, diff --git a/packages/core/src/test-runtime.ts b/packages/core/src/test-runtime.ts index 075e7395b..6a780c221 100644 --- a/packages/core/src/test-runtime.ts +++ b/packages/core/src/test-runtime.ts @@ -11,7 +11,7 @@ import { type LocalCompatMount, NATIVE_SIDECAR_FRAME_TIMEOUT_MS, NativeSidecarKernelProxy, - NativeSidecarProcessClient, + Sidecar, type RootFilesystemEntry, type SidecarRegisteredHostCallbackDefinition, type SidecarRequestFrame, @@ -84,6 +84,19 @@ const SIDECAR_BUILD_INPUTS = [ path.join(REPO_ROOT, "crates/kernel"), path.join(REPO_ROOT, "crates/sidecar"), ] as const; + +export type KernelBootTimingPhase = + | "filesystem_snapshot" + | "sidecar_spawn" + | "session_open" + | "vm_create" + | "vm_ready" + | "vm_configure"; + +export interface KernelBootTiming { + phase: KernelBootTimingPhase; + durationMs: number; +} let ensuredSidecarBinary: string | null = null; export type StdioChannel = "stdout" | "stderr"; @@ -2074,7 +2087,7 @@ async function snapshotFilesystemEntries( } async function materializeSnapshotEntriesIntoVm( - client: NativeSidecarProcessClient, + client: Sidecar, session: AuthenticatedSession, vm: CreatedVm, entries: RootFilesystemEntry[], @@ -2478,7 +2491,7 @@ class NativeKernel implements Kernel { readonly timerTable = {}; readonly vfs: VirtualFileSystem; - private client: NativeSidecarProcessClient | null = null; + private client: Sidecar | null = null; private session: AuthenticatedSession | null = null; private vm: CreatedVm | null = null; private proxy: NativeSidecarKernelProxy | null = null; @@ -2509,6 +2522,8 @@ class NativeKernel implements Kernel { permissions?: Permissions; env?: Record; cwd?: string; + sidecar?: Sidecar; + onBootTiming?: (timing: KernelBootTiming) => void; hostNetworkAdapter?: unknown; loopbackExemptPorts?: number[]; mounts?: Array<{ @@ -2961,14 +2976,13 @@ class NativeKernel implements Kernel { this.options.filesystem, this.pendingLocalMounts, ); - const snapshotEntries = await snapshotFilesystemEntries( - this.options.filesystem, - "/", - [], - { - passthroughDirectories: - rootPassthroughPlan.passthroughDirectories, - }, + const snapshotEntries = await this.measureBoot( + "filesystem_snapshot", + () => + snapshotFilesystemEntries(this.options.filesystem, "/", [], { + passthroughDirectories: + rootPassthroughPlan.passthroughDirectories, + }), ); this.liveFilesystemSyncRoots = collectLiveFilesystemSyncRoots(snapshotEntries); @@ -2990,34 +3004,45 @@ class NativeKernel implements Kernel { bootstrapEntries: [], }; - const client = NativeSidecarProcessClient.spawn({ - cwd: REPO_ROOT, - command: ensureNativeSidecarBinary(), - args: [], - frameTimeoutMs: NATIVE_SIDECAR_FRAME_TIMEOUT_MS, - }); - const session = await client.authenticateAndOpenSession(); - const vm = await client.createVm(session, { - runtime: "java_script", - config: { - env: createVmEnv, - rootFilesystem, - ...(bootstrapPermissions ? { permissions: bootstrapPermissions } : {}), - loopbackExemptPorts: this.loopbackExemptPorts, - }, - }); - await client.waitForEvent( - { - type: "vm_lifecycle", - ownership: { - scope: "vm", - connection_id: session.connectionId, - session_id: session.sessionId, - vm_id: vm.vmId, + const ownsClient = this.options.sidecar === undefined; + const client = + this.options.sidecar ?? + this.measureSyncBoot("sidecar_spawn", () => + Sidecar.spawn({ + cwd: REPO_ROOT, + command: ensureNativeSidecarBinary(), + args: [], + frameTimeoutMs: NATIVE_SIDECAR_FRAME_TIMEOUT_MS, + }), + ); + const session = await this.measureBoot("session_open", () => + client.authenticateAndOpenSession(), + ); + const vm = await this.measureBoot("vm_create", () => + client.createVm(session, { + runtime: "java_script", + config: { + env: createVmEnv, + rootFilesystem, + ...(bootstrapPermissions ? { permissions: bootstrapPermissions } : {}), + loopbackExemptPorts: this.loopbackExemptPorts, }, - state: "ready", - }, - 10_000, + }), + ); + await this.measureBoot("vm_ready", () => + client.waitForEvent( + { + type: "vm_lifecycle", + ownership: { + scope: "vm", + connection_id: session.connectionId, + session_id: session.sessionId, + vm_id: vm.vmId, + }, + state: "ready", + }, + 10_000, + ), ); if (requestedPermissions && snapshotEntries.length > 1) { await materializeSnapshotEntriesIntoVm( @@ -3035,35 +3060,38 @@ class NativeKernel implements Kernel { this.loopbackExemptPorts.length > 0 || requestedPermissions ) { - await client.configureVm(session, vm, { - mounts: this.pendingLocalMounts.map((mount) => - mount.fs instanceof NodeFileSystem - ? serializeMountConfigForSidecar({ - path: mount.path, - readOnly: mount.readOnly, - plugin: { - id: "host_dir", - config: { - hostPath: mount.fs.rootPath, - readOnly: mount.readOnly, + await this.measureBoot("vm_configure", () => + client.configureVm(session, vm, { + mounts: this.pendingLocalMounts.map((mount) => + mount.fs instanceof NodeFileSystem + ? serializeMountConfigForSidecar({ + path: mount.path, + readOnly: mount.readOnly, + plugin: { + id: "host_dir", + config: { + hostPath: mount.fs.rootPath, + readOnly: mount.readOnly, + }, }, - }, - }) - : serializeMountConfigForSidecar({ - path: mount.path, - driver: mount.fs, - readOnly: mount.readOnly, - }), - ), - permissions: requestedPermissions, - loopbackExemptPorts: this.loopbackExemptPorts, - }); + }) + : serializeMountConfigForSidecar({ + path: mount.path, + driver: mount.fs, + readOnly: mount.readOnly, + }), + ), + permissions: requestedPermissions, + loopbackExemptPorts: this.loopbackExemptPorts, + }), + ); } const proxy = new NativeSidecarKernelProxy({ client, session, vm, + disposeClient: ownsClient, env: this.env, cwd: this.cwd, defaultExecCwd: this.options.cwd === undefined ? "/home/user" : this.cwd, @@ -3080,6 +3108,33 @@ class NativeKernel implements Kernel { this.proxy = proxy; this.rootFilesystem = proxy.createRootView(); } + + private async measureBoot( + phase: KernelBootTimingPhase, + fn: () => Promise, + ): Promise { + const start = performance.now(); + try { + return await fn(); + } finally { + this.options.onBootTiming?.({ + phase, + durationMs: performance.now() - start, + }); + } + } + + private measureSyncBoot(phase: KernelBootTimingPhase, fn: () => T): T { + const start = performance.now(); + try { + return fn(); + } finally { + this.options.onBootTiming?.({ + phase, + durationMs: performance.now() - start, + }); + } + } } export function createKernel(options: { @@ -3087,6 +3142,8 @@ export function createKernel(options: { permissions?: Permissions; env?: Record; cwd?: string; + sidecar?: Sidecar; + onBootTiming?: (timing: KernelBootTiming) => void; maxProcesses?: number; hostNetworkAdapter?: unknown; loopbackExemptPorts?: number[]; diff --git a/packages/secure-exec/src/index.ts b/packages/secure-exec/src/index.ts index 54f9988e6..568c2cc66 100644 --- a/packages/secure-exec/src/index.ts +++ b/packages/secure-exec/src/index.ts @@ -2,6 +2,9 @@ export type { HostDirectoryMount, HostToolDefinition, HostToolExample, + SidecarSpawnOptions, + NodeRuntimeBootTiming, + NodeRuntimeBootTimingPhase, NodeRuntimeCreateOptions, NodeRuntimeExecOptions, NodeRuntimeExecResult, @@ -11,8 +14,11 @@ export type { NodeRuntimeListener, NodeRuntimeListenerQuery, NodeRuntimeProcess, + NodeRuntimeResidentRunner, + NodeRuntimeResidentRunnerExecOptions, + NodeRuntimeResidentRunnerOptions, NodeRuntimeRunResult, NodeRuntimeSpawnOptions, NodeRuntimeWaitForListenerOptions, } from "@secure-exec/core"; -export { NodeRuntime } from "@secure-exec/core"; +export { Sidecar, NodeRuntime } from "@secure-exec/core";