diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 033dcd3b5..77b1c4f45 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -722,6 +722,11 @@ jobs: with: persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + # Differential tests use fixtures recorded from this exact Node release. + node-version: "24.20.0" + - name: Install Rust run: rustup update stable --no-self-update - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -748,18 +753,25 @@ jobs: run: | pnpm install + - name: Install Puppeteer browser + if: matrix.workspace == 'rolldown-plugin-jco' + run: pnpm run test:setup:puppeteer + - name: Build workspace dependencies and package run: | if [ "${{ matrix.workspace }}" = "jco-std" ]; then - pnpm --filter "packages/jco-transpile" run build - pnpm --filter "packages/${{ matrix.workspace }}" run build + pnpm --filter "./packages/jco-transpile" --fail-if-no-match run build + pnpm --filter "./packages/${{ matrix.workspace }}" --fail-if-no-match run build else - pnpm --filter "packages/${{ matrix.workspace }}" run build + pnpm --filter "./packages/${{ matrix.workspace }}" --fail-if-no-match run build fi - name: run test on workspace + env: + TEST_PUPPETEER_LAUNCH_ARGS: --no-sandbox + TERM: xterm-256color run: | - pnpm --filter "packages/${{ matrix.workspace }}" run test + pnpm --filter "./packages/${{ matrix.workspace }}" --fail-if-no-match run test examples: needs: @@ -899,6 +911,8 @@ jobs: chmod: 0755 - uses: bytecodealliance/actions/wasmtime/setup@9152e710e9f7182e4c29ad218e4f335a7b203613 # v1.1.3 + with: + version: v50.0.0 - name: Cache node modules uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -949,7 +963,7 @@ jobs: env: TEST_PUPPETEER_LAUNCH_ARGS: --no-sandbox run: | - pnpm --filter '${{ matrix.project.workspace }}' run all + pnpm --filter './${{ matrix.project.workspace }}' --fail-if-no-match run all examples-perry: # Perry's WASI target is not released yet. Keep this compiler setup separate diff --git a/packages/jco/package.json b/packages/jco/package.json index 08eae423a..e5c60856d 100644 --- a/packages/jco/package.json +++ b/packages/jco/package.json @@ -53,6 +53,7 @@ "build:ts": "tsc -p tsconfig.json", "build:types:preview2-shim": "pnpm run build:types:preview2-shim --include-workspace-root", "build:test:components": "cargo xtask build-test-components", + "bench:componentize-backends": "node test/bench/componentize-backends.mjs", "bench:serve-isolation": "node test/bench/serve-request-isolation.js", "fmt": "oxfmt", "fmt:check": "oxfmt --check", diff --git a/packages/jco/test/README.md b/packages/jco/test/README.md index a1181ee80..d377b114a 100644 --- a/packages/jco/test/README.md +++ b/packages/jco/test/README.md @@ -2,6 +2,11 @@ This folder contains tests for Jco. +## Benchmarks + +Opt-in performance benchmarks, run instructions, and recent results are in +[`bench/README.md`](./bench/README.md). + ## Dependencies You'll need the following binaries installed in your path or specificed explicitly: diff --git a/packages/jco/test/bench/README.md b/packages/jco/test/bench/README.md new file mode 100644 index 000000000..6828fabf4 --- /dev/null +++ b/packages/jco/test/bench/README.md @@ -0,0 +1,280 @@ +# Jco benchmarks + +These opt-in benchmarks exercise the built workspace Jco CLI. Run the repository's +normal dependency setup and build first, using Node.js 24 or newer. Benchmarks are +excluded from the normal test suite and are not CI performance gates. + +Run the commands below from `packages/jco/test/bench`, or use the package scripts +from the repository root: + +| Benchmark | Command from this directory | Package script | +| ------------------------- | ----------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| Componentization backends | `node componentize-backends.mjs --perry-cli-path /path/to/perry --perry-source /path/to/perry-source` | `pnpm --filter @bytecodealliance/jco bench:componentize-backends --perry-cli-path /path/to/perry --perry-source /path/to/perry-source` | +| Serve request isolation | `node serve-request-isolation.js` | `pnpm --filter @bytecodealliance/jco bench:serve-isolation` | + +## Componentization backends + +[`componentize-backends.mjs`](./componentize-backends.mjs) compares Perry, +componentize-js (StarlingMonkey), and componentize-qjs (QuickJS). It calls +`jco componentize`, applies the same `jco opt` pass, and calls `jco transpile` +for each backend. The guest fixture reads a host file, obtains its size, and +prints both. Fixtures belong to this suite in [`fixtures/filesystem`](./fixtures/filesystem). +The benchmark does not build or run the Perry example. + +### Setup and running + +Use a WASI-enabled compiler built from Perry commit +`3945beb23ebeca54846a30d89cb4971de59b0c6c`, a clean checkout of that revision, +wasi-sdk 34, and Rust `nightly-2026-10-08` with the `wasm32-wasip2` target. +The [Perry example's installer](../../../../examples/components/perry-filesystem/README.md#prerequisites) +can provision these prerequisites; the benchmark does not install compilers. + +Every full benchmark build rebuilds Perry's runtime with **optimization level +`z`, fat LTO, and default features disabled**, retaining `keepalive-anchors` +to preserve the entry points called by generated code. +This is the smallest profile validated for this filesystem fixture. Cargo caches +unchanged builds. The benchmark selects the resulting archive automatically, +overriding `PERRY_RUNTIME_DIR`, and records the recipe, archive SHA-256, source +revision, and toolchain versions in its build metadata. The normal Perry +installation is unchanged. This feature subset is specific to the fixture; +other applications may need additional runtime features. + +| Setting | Purpose | +| ------------------------------------------ | ---------------------------------------------------------------------- | +| `--perry-cli-path` or `JCO_PERRY_CLI_PATH` | Matching WASI-enabled Perry executable or JavaScript launcher. | +| `--perry-source` or `JCO_PERRY_SOURCE` | Clean checkout of the pinned Perry revision. | +| `WASI_SDK_PATH` | wasi-sdk installation directory. | +| `CARGO_TARGET_DIR` (optional) | Reuse a Cargo build cache; defaults to the benchmark output directory. | +| `CARGO_BUILD_JOBS` (optional) | Runtime build parallelism; defaults to 2. | + +After setting `WASI_SDK_PATH`, run: + +```sh +node componentize-backends.mjs \ + --perry-cli-path /path/to/perry \ + --perry-source /path/to/perry-source +``` + +The script checks every build and runtime sample's output, prints sizes in KiB/MiB +and timings in milliseconds, and writes exact byte counts, raw timing samples, tool versions, +input hashes, and host details to +`packages/jco/test/output/bench/componentize-backends/results.json`. +Component artifacts and build metadata are retained in the same ignored directory. +Use `--output-dir /path/to/artifacts` to select another location; pass the same +location when using `--skip-build`. +Use `--samples 50 --warmup 5` to change the default sample counts, or +`--skip-build` to reuse unchanged components without requiring the toolchain. +Cached builds from the old runtime recipe are rejected and must be rebuilt. +The optional `--wasm-opt-bin` setting is passed to Jco's optimizer launcher hook. +Componentization and optimization are measured separately from runtime execution. +Compiler/engine/runtime builds and transpilation are excluded from both timing groups. + +### Compilation speed + +A full run measures **TS/JS source to a complete, usable Wasm component**, including +the JavaScript engine for StarlingMonkey and QuickJS. It does not measure a +script-only payload that requires a separately deployed engine. + +The compilation table reports three medians: + +- **TS/JS → component:** wall time around a fresh `jco componentize` process, + through successful exit after writing the component. This includes CLI startup, + TypeScript lowering and bundling, backend compilation or preinitialization, + linking/component encoding, and writing the output. Perry's CLI capability + probe is included as part of the normal Jco workflow. +- **Optimize:** wall time around a fresh `jco opt` process on that sample's + component, using the common optimization flags. +- **TS/JS → optimized component:** median of each sample's componentization plus + optimization time. This need not equal the sum of the first two medians. + +The default is **five measured builds per backend after one warmup round**. +`--build-samples` and `--build-warmup` control these independently of the runtime +`--samples` and `--warmup` settings. Builds run sequentially, rotating backend order +between rounds. Every build starts with fresh output and WIT directories; Jco's +injected WIT imports cannot accumulate between samples. Prebuilt engine binaries, +Perry's prepared runtime archive, and OS/filesystem caches are reused. These are +warm-cache application builds, not clean-machine toolchain bootstrap timings. + +The benchmark prepares its command wrapper and WIT fixture outside the timer; +the timed CLI still performs the actual TypeScript transformation and bundling. +Perry compiler installation, its one-time runtime archive preparation, dependency +installation, transpilation and validation are also outside the timer. Every +as-built **and** optimized component is transpiled and executed after the timed +commands; incorrect output fails the benchmark instead of becoming a timing +sample. The final optimized artifacts are reused for runtime measurements. + +```sh +node componentize-backends.mjs \ + --perry-cli-path /path/to/perry \ + --perry-source /path/to/perry-source \ + --build-samples 5 --build-warmup 1 \ + --samples 20 --warmup 3 +``` + +`build.json` and `results.json` retain compilation medians, every raw sample's +stage timings and output sizes, round/order, sampling settings, host details, +input hash, and measurement timestamps under `build.compilation` in the results. +`--skip-build` runs only runtime measurements and explicitly labels compilation +results as reused with their original timestamp. Older build receipts without +compilation measurements remain usable but show no compilation table. + +For environments needing a componentize-js build-time shim workaround, +`JCO_BENCH_BUILD_PRELOAD=/absolute/path/to/preload.mjs` adds Node's `--import` only +to componentization processes. The path is recorded in build metadata and its +execution cost is included in compilation time. It is not loaded during +optimization, validation, or runtime samples. Leave it unset for normal builds. + +### Compilation results (2026-10-23) + +Five measured builds per backend after one warmup, on the Linux x86_64 Intel +Xeon 2.60 GHz host with Node.js 24.21.0: + +| Backend | TS/JS → component (median ms) | Optimize (median ms) | TS/JS → optimized component (median ms) | +| -------------- | ----------------------------: | -------------------: | --------------------------------------: | +| perry | 3328.06 | 10924.11 | 14098.32 | +| starlingmonkey | 6839.72 | 23536.53 | 30376.25 | +| quickjs | 2619.39 | 3500.37 | 6119.76 | + +QuickJS was fastest for both endpoints on this fixture. These measurements use +componentize-qjs 0.4.5, componentize-js 0.23.0 without Weval AOT, the pinned Perry +0.5.1659 compiler and reduced runtime recipe described above, and native Binaryen +132 through `--wasm-opt-bin`. They do not use the experimental local Perry +specialization or QuickJS 0.5's size runtime. Jco was rebuilt from the comparison +branch for this run; package versions alone do not identify local compiler changes. + +Output artifacts were written to tmpfs; normal OS/file caches were retained. +The componentize-js 0.23.0 WIT-splicer shim needed the build-only preopen workaround +described above; its startup cost is included. All 18 builds (warmups included) +passed execution checks for both their as-built and optimized components. Results +on other source graphs, hosts, disks, or tool versions can differ; this is not a +universal compiler-throughput ranking. + +### Recent results + +Example results measured on 2026-10-23 on a shared Linux x86_64 development host +(Intel Xeon at 2.60 GHz, Node.js 24.21.0), using the bundled 26-byte input file: + +| Backend | As-built size | Optimized size | +| -------------------------------- | ------------: | -------------: | +| Perry | 6.80 MiB | 3.21 MiB | +| componentize-js (StarlingMonkey) | 15.99 MiB | 15.42 MiB | +| componentize-qjs (QuickJS) | 6.79 MiB | 6.66 MiB | + +Timings for the optimized components: + +| Backend | Run (median ms) | Load + run (median ms) | Node process (median ms) | +| -------------------------------- | --------------: | ---------------------: | -----------------------: | +| Perry | 98.80 | 153.66 | 301.14 | +| componentize-js (StarlingMonkey) | 59.58 | 228.90 | 465.38 | +| componentize-qjs (QuickJS) | 40.96 | 145.42 | 403.16 | + +Tool versions: Jco `feaa7d23` (package version 1.39.0) with this runtime build +recipe, componentize-js 0.23.0, +componentize-qjs 0.4.5, and +Perry 0.5.1659 built at commit `3945beb23ebeca54846a30d89cb4971de59b0c6c` +with LLVM 22.1.8 and wasi-sdk 34. All three components were optimized with +Binaryen 132 using `-Oz --low-memory-unused --enable-bulk-memory +--enable-reference-types --strip-debug`. These are example measurements from one +host; rerun the comparison for results on your machine. A local shim compatibility +workaround enabled filesystem access for componentize-js's build-time WIT splicer; +it was disabled for timings. + +- **Component size:** the uncompressed `.wasm` component before transpilation, + reported both as built and after optimization. MiB means 1,048,576 bytes. + Runtime timings use the optimized components; compilation measures both output stages. +- **Run:** the first command call on a fresh instance, including file read, stat, and output. +- **Load + run:** importing and instantiating the transpiled component, then running it. +- **Node process:** elapsed time from launching a fresh Node process until it exits, + including host setup, component loading, and execution. + +All three use explicit asynchronous instantiation with the same host setup. +Each backend runs in a fresh process for every sample. The default is 20 measured +samples per backend after three warmup rounds, rotating backend order between +rounds. File and OS caches are not cleared. The runtime timing group excludes +builds and transpilation; compilation is reported separately. To repeat measurements without rebuilding unchanged components: + +```sh +node componentize-backends.mjs --skip-build --samples 50 --warmup 5 +``` + +The JavaScript backends wrap the guest's statements in the WASI command export +so StarlingMonkey's build-time preinitialization cannot read the input file. +Their components include Jco's Node filesystem adapter, mapped to the same host +file through the Node filesystem capability. Perry lowers those calls directly +to WASI filesystem operations. QuickJS uses synchronous bindings for this +synchronous workload; StarlingMonkey uses its default build without Weval AOT. +These measurements compare this fixture's complete backend paths. The tiny file +and console output make them sensitive to host overhead; they do not establish +general JavaScript throughput or performance under sustained load. + +### Why Perry components are still several MiB + +The size-focused runtime reduces this fixture from **7.40 MiB to 3.21 MiB**. +The guest's original generated functions total only **1,918 bytes**; most of the +payload implements JavaScript semantics: + +- **Broad runtime defaults:** Perry's [WASI link path][perry-wasi-link] bypasses + native per-application specialization, while its [runtime builder][perry-wasi-runtime] + enables nearly all default features, including diagnostics. The native + `PERRY_SIZE_OPT` / `PERRY_SIZE_LTO` settings have no effect here, so this + benchmark explicitly rebuilds a reduced runtime at level `z` with fat LTO. +- **Reachable generic helpers:** object property reads, string coercion, + exceptions, built-ins, and GC retain general runtime code. Importing `node:fs` + also [registers streams][perry-fs-registry]. These dispatch paths keep helpers + reachable even when the fixture never calls them directly, limiting dead-code + elimination. +- **Small packaging overhead:** the 7.40 MiB baseline already had debug data + stripped; component wiring and auxiliary modules added only 19 KiB. An empty + Perry program still optimized to 6.30 MiB with the default runtime. Equivalent + C file I/O using the same SDK produced 60.9 KiB, without JS semantics. + +The reduced runtime passed changed-file, Unicode, and larger-input checks. +Hundreds of KiB will require narrower dispatch or code generation; Perry's +[tiny-program specialization][perry-tiny-program] currently excludes WASI. + +[perry-wasi-link]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/crates/perry/src/commands/compile/link/wasi.rs +[perry-wasi-runtime]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/scripts/wasi_build_runtime.py +[perry-fs-registry]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/crates/perry-runtime/src/object/native_module_registry.rs#L361-L370 +[perry-tiny-program]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/crates/perry/src/commands/compile/tiny_program/mod.rs#L204-L220 + +## Serve request isolation + +[`serve-request-isolation.js`](./serve-request-isolation.js) compares a native +Node.js HTTP handler with Jco's shared component, fresh-instance, and worker +isolation modes. It builds a minimal HTTP component, runs a local server for +each mode, checks the response, and reports requests per second and mean, +median, and p95 latency. Default counts are 100 warmups and 10,000 sequential +requests per mode; a full worker-isolation run can take a long time. + +For a smaller run, set `JCO_SERVE_BENCH_WARMUP=5` and +`JCO_SERVE_BENCH_REQUESTS=30` in your environment, then run: + +```sh +node serve-request-isolation.js +``` + +Set `JCO_SERVE_BENCH_COMPONENT_OUT` to retain a newly built component, or +`JCO_SERVE_BENCH_COMPONENT` to reuse an existing one. Componentization is excluded +from timing. Temporary server output is cleaned up automatically. + +### Most recent documented reference result + +The existing reference used Node.js 24.19.0 on a 6-vCPU Intel Xeon 2.60 GHz +virtual machine, with 10 warmup requests followed by 100 measured sequential +requests per mode. These results come from the +[development-use serve benchmark](../../../../docs/src/performance/benchmarks/development-use-jco-serve.md#reference-result); +they were not rerun as part of the componentization backend comparison. + +| Mode | Requests/s | Mean latency | Median latency | p95 latency | +| --------------------- | ---------: | -----------: | -------------: | ----------: | +| Native `node:http` | 598.13 | 1.67 ms | 1.69 ms | 2.03 ms | +| Shared component | 225.07 | 4.44 ms | 4.31 ms | 5.40 ms | +| Instance isolation | 22.88 | 43.71 ms | 43.24 ms | 47.59 ms | +| Worker, pool size 1 | 4.15 | 240.95 ms | 239.36 ms | 274.28 ms | +| Worker, pool size 50 | 13.55 | 73.82 ms | 70.14 ms | 100.00 ms | +| Worker, pool size 100 | 14.78 | 67.66 ms | 65.14 ms | 95.25 ms | + +This benchmark measures Jco's development server overhead. It does not estimate +production component-runtime capacity. Request concurrency, component size, +worker pool utilization, and host load can change both absolute and relative +results. diff --git a/packages/jco/test/bench/componentize-backends-runner.mjs b/packages/jco/test/bench/componentize-backends-runner.mjs new file mode 100644 index 000000000..099eac57c --- /dev/null +++ b/packages/jco/test/bench/componentize-backends-runner.mjs @@ -0,0 +1,34 @@ +import { WASIShim } from "@bytecodealliance/preview2-shim/instantiation"; +import { createRequire } from "node:module"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const jco = fileURLToPath(new URL("../../dist/jco.js", import.meta.url)); +const requireJco = createRequire(jco); +const host = (name) => + import(pathToFileURL(requireJco.resolve(`@bytecodealliance/jco-std/wasi/0.2.x/node/24.x.x/${name}`)).href); +const [fs, console, http] = await Promise.all([host("fs/host/node"), host("console/host/node"), host("http/host")]); +const input = fileURLToPath(new URL("./fixtures/filesystem/data/input.txt", import.meta.url)); +const mapPath = (path) => (path.tag === "text" && path.val === "/data/input.txt" ? { ...path, val: input } : path); +const wasi = new WASIShim({ + sandbox: { preopens: { "/data": fileURLToPath(new URL("./fixtures/filesystem/data/", import.meta.url)) } }, +}); +const imports = { + ...wasi.getImportObject(), + "jco:node/fs": { + ...fs, + readFile: (file, options) => + fs.readFile(file.tag === "path" ? { ...file, val: mapPath(file.val) } : file, options), + stat: (path, options) => fs.stat(mapPath(path), options), + }, + "jco:node/console": console, + "jco:node/http": http, +}; +const start = process.hrtime.bigint(); +const { instantiate } = await import(pathToFileURL(process.argv[2]).href); +const { run } = await instantiate(undefined, imports); +const runStart = process.hrtime.bigint(); +await run.run(); +const end = process.hrtime.bigint(); +const runMs = Number(end - runStart) / 1e6; +const loadAndRunMs = Number(end - start) / 1e6; +process.stderr.write(`${JSON.stringify({ runMs, loadAndRunMs })}\n`); diff --git a/packages/jco/test/bench/componentize-backends.mjs b/packages/jco/test/bench/componentize-backends.mjs new file mode 100644 index 000000000..b157a44a5 --- /dev/null +++ b/packages/jco/test/bench/componentize-backends.mjs @@ -0,0 +1,385 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { cp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { cpus, release } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { buildPerryRuntime, perryRuntimeRecipe } from './perry-runtime.mjs'; + +const root = fileURLToPath(new URL('../../', import.meta.url)); +const fixtures = fileURLToPath(new URL('./fixtures/filesystem/', import.meta.url)); +const jco = join(root, 'dist', 'jco.js'); +const requireJco = createRequire(jco); +const backends = ['perry', 'starlingmonkey', 'quickjs']; +const optimizationArgs = [ + '-Oz', + '--low-memory-unused', + '--enable-bulk-memory', + '--enable-reference-types', + '--strip-debug', +]; +const { values } = parseArgs({ + options: { + samples: { type: 'string', default: '20' }, + warmup: { type: 'string', default: '3' }, + 'skip-build': { type: 'boolean', default: false }, + 'output-dir': { type: 'string' }, + 'build-samples': { type: 'string', default: '5' }, + 'build-warmup': { type: 'string', default: '1' }, + 'wasm-opt-bin': { type: 'string' }, + 'perry-cli-path': { type: 'string' }, + 'perry-source': { type: 'string' }, + }, +}); +const output = resolve(values['output-dir'] || join(root, 'test', 'output', 'bench', 'componentize-backends')); +const samples = Number(values.samples); +const warmup = Number(values.warmup); +const buildSamples = Number(values['build-samples']); +const buildWarmup = Number(values['build-warmup']); +const compiler = values['perry-cli-path'] || process.env.JCO_PERRY_CLI_PATH; +const perryCliPath = compiler ? resolve(compiler) : null; +assert.ok(Number.isInteger(samples) && samples > 0, '--samples must be a positive integer'); +assert.ok(Number.isInteger(warmup) && warmup >= 0, '--warmup must be a nonnegative integer'); + +assert.ok(Number.isInteger(buildSamples) && buildSamples > 0, '--build-samples must be a positive integer'); +assert.ok(Number.isInteger(buildWarmup) && buildWarmup >= 0, '--build-warmup must be a nonnegative integer'); + +function command(script, args = [], env = process.env) { + const preload = args[0] === 'componentize' && process.env.JCO_BENCH_BUILD_PRELOAD; + const start = process.hrtime.bigint(); + execFileSync(process.execPath, [...(preload ? ['--import', preload] : []), script, ...args], { + cwd: root, + env, + stdio: 'inherit', + }); + return Number(process.hrtime.bigint() - start) / 1e6; +} + +function hostDetails() { + return { + node: process.version, + platform: process.platform, + arch: process.arch, + osRelease: release(), + cpu: cpus()[0]?.model, + }; +} + +function runComponent(backend, module) { + const start = process.hrtime.bigint(); + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./componentize-backends-runner.mjs', import.meta.url)), module], + { cwd: root, encoding: 'utf8', timeout: 60_000 }, + ); + const processMs = Number(process.hrtime.bigint() - start) / 1e6; + if (result.error) { + throw result.error; + } + assert.equal(result.status, 0, `${backend} failed: ${result.stderr}`); + assert.equal(result.stdout.replaceAll('\r\n', '\n'), expected, `${backend} produced different output`); + return { ...JSON.parse(result.stderr.trim()), processMs }; +} + +function transpile(component, directory) { + command(jco, [ + 'transpile', + component, + '--name', + 'component', + '--bindgen-enable-wasm-exnref', + '--instantiation', + 'async', + '-o', + directory, + ]); + return join(directory, 'component.js'); +} + +async function packageVersion(name) { + const directories = + name === '@bytecodealliance/jco' + ? [join(dirname(jco), '..')] + : (requireJco.resolve.paths(name) ?? []).map((directory) => join(directory, name)); + for (const directory of directories) { + try { + const manifest = JSON.parse(await readFile(join(directory, 'package.json'), 'utf8')); + if (manifest.name === name) { + return manifest.version; + } + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + } + throw new Error(`Cannot find the version of ${name}`); +} + +function median(numbers) { + const sorted = [...numbers].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; +} + +function componentSize(bytes) { + if (bytes < 1024) { + return `${bytes} B`; + } + if (bytes < 1024 ** 2) { + return `${(bytes / 1024).toFixed(2)} KiB`; + } + return `${(bytes / 1024 ** 2).toFixed(2)} MiB`; +} + +await mkdir(output, { recursive: true }); +const guest = await readFile(join(fixtures, 'guest.ts'), 'utf8'); +const guestSha256 = createHash('sha256').update(guest).digest('hex'); +const input = await readFile(join(fixtures, 'data', 'input.txt')); +const inputSha256 = createHash('sha256').update(input).digest('hex'); +const expected = `Perry size: ${input.length}\nPerry read: ${input.toString('utf8')}\n`.replaceAll('\r\n', '\n'); +if (!values['skip-build']) { + assert.ok(perryCliPath, 'Pass --perry-cli-path or set JCO_PERRY_CLI_PATH to a WASI-enabled Perry compiler'); + // A failed rebuild must not leave a receipt for partially replaced artifacts. + await rm(join(output, 'build.json'), { force: true }); + const perryRuntime = await buildPerryRuntime(values['perry-source'] || process.env.JCO_PERRY_SOURCE, output); + + // Keep the fixture's read/stat/print statements inside run(): StarlingMonkey + // preinitializes module-level code while building the component. + const ts = requireJco('typescript-compiler-api'); + const source = ts.createSourceFile('component.ts', guest, ts.ScriptTarget.Latest, true); + const imports = source.statements.filter(ts.isImportDeclaration); + const statements = source.statements.filter((statement) => !ts.isImportDeclaration(statement)); + const entry = join(output, 'command.ts'); + await writeFile( + entry, + `${imports.map((statement) => statement.getFullText(source)).join('\n')} +export const run = { run() { +${statements.map((statement) => statement.getFullText(source)).join('\n')} +} }; +`, + ); + + const buildMeasurements = Object.fromEntries(backends.map((backend) => [backend, []])); + const buildStartedAt = new Date().toISOString(); + // Prepare a new component for every sample; only engines/toolchains and OS caches are reused. + // Rotate the order just as for runtime samples. Builds are never concurrent. + for (let round = -buildWarmup; round < buildSamples; round++) { + const offset = (round + buildWarmup) % backends.length; + for (let index = 0; index < backends.length; index++) { + const backend = backends[(index + offset) % backends.length]; + console.log( + `\n${backend}: build ${round < 0 ? 'warmup' : 'sample'} ${round < 0 ? round + buildWarmup + 1 : round + 1}`, + ); + const directory = join(output, backend); + await rm(directory, { recursive: true, force: true }); + await mkdir(directory); + const component = join(directory, 'component.wasm'); + let componentizeMs; + if (backend === 'perry') { + componentizeMs = command( + jco, + [ + 'componentize', + join(fixtures, 'guest.ts'), + '--backend', + 'perry', + '--backend-perry-cli-path', + perryCliPath, + '-o', + component, + ], + perryRuntime.env, + ); + } else { + // Jco injects imports into this WIT world. Reset it so every build starts identically. + const wit = join(directory, 'wit'); + await cp(join(root, 'lib', 'wit', 'builtin', '0.2.12'), join(wit, 'deps'), { recursive: true }); + await writeFile( + join(wit, 'package.wit'), + 'package benchmark:filesystem-comparison;\nworld command { export wasi:cli/run@0.2.12; }\n', + ); + componentizeMs = command(jco, [ + 'componentize', + entry, + '--backend', + backend, + '--wit', + wit, + '--world-name', + 'benchmark:filesystem-comparison/command', + '-o', + component, + ...(backend === 'quickjs' ? ['--backend-qjs-disable-async'] : []), + ]); + } + const optimized = join(directory, 'component.opt.wasm'); + const optimizeMs = command(jco, [ + 'opt', + component, + '-o', + optimized, + ...(values['wasm-opt-bin'] ? ['--wasm-opt-bin', values['wasm-opt-bin']] : []), + '--', + ...optimizationArgs, + ]); + // Validate BOTH timing endpoints, after the timed commands have finished. + // This catches unusable outputs and accidental build-time execution of the guest. + const asBuiltTranspiled = join(directory, 'as-built-transpiled'); + runComponent(backend, transpile(component, asBuiltTranspiled)); + await rm(asBuiltTranspiled, { recursive: true }); + runComponent(backend, transpile(optimized, join(directory, 'transpiled'))); + if (round >= 0) { + buildMeasurements[backend].push({ + round, + order: index, + componentizeMs, + optimizeMs, + sourceToOptimizedMs: componentizeMs + optimizeMs, + componentBytes: (await stat(component)).size, + optimizedComponentBytes: (await stat(optimized)).size, + }); + } + } + } + const compilation = { + startedAt: buildStartedAt, + completedAt: new Date().toISOString(), + ...hostDetails(), + samples: buildSamples, + warmup: buildWarmup, + inputSha256, + scope: 'Fresh Jco CLI processes: TS/JS to complete engine-containing component, then jco opt', + excluded: ['toolchain/engine/runtime builds', 'fixture and WIT preparation', 'transpilation', 'validation'], + cachePolicy: 'Fresh component outputs each sample; prebuilt engines and OS/filesystem caches reused', + results: backends.map((backend) => ({ + backend, + componentizeMs: median(buildMeasurements[backend].map((sample) => sample.componentizeMs)), + optimizeMs: median(buildMeasurements[backend].map((sample) => sample.optimizeMs)), + sourceToOptimizedMs: median(buildMeasurements[backend].map((sample) => sample.sourceToOptimizedMs)), + samples: buildMeasurements[backend], + })), + }; + const jsLauncher = perryCliPath && /\.[cm]?js$/i.test(perryCliPath); + const perryVersion = perryCliPath + ? execFileSync( + jsLauncher ? process.execPath : perryCliPath, + [...(jsLauncher ? [perryCliPath] : []), '--version'], + { + cwd: root, + encoding: 'utf8', + }, + ).trim() + : null; + await writeFile( + join(output, 'build.json'), + `${JSON.stringify( + { + guestSha256, + compilation, + componentizePreload: process.env.JCO_BENCH_BUILD_PRELOAD ?? null, + jco: await packageVersion('@bytecodealliance/jco'), + componentizeJs: await packageVersion('@bytecodealliance/componentize-js'), + componentizeQjs: await packageVersion('componentize-qjs'), + optimizationArgs, + wasmOptVersion: execFileSync( + process.execPath, + [values['wasm-opt-bin'] || requireJco.resolve('binaryen/bin/wasm-opt'), '--version'], + { + cwd: root, + encoding: 'utf8', + }, + ).trim(), + wasmOptLauncher: values['wasm-opt-bin'] ?? null, + perryRef: perryRuntime.metadata.recipe.perryRef, + perryCliPath, + perryVersion, + perryRuntime: perryRuntime.metadata, + }, + null, + 2, + )}\n`, + ); +} + +const build = JSON.parse(await readFile(join(output, 'build.json'), 'utf8')); +assert.equal(build.guestSha256, guestSha256, 'Guest changed; rerun without --skip-build'); +assert.deepEqual(build.optimizationArgs, optimizationArgs, 'Optimization changed; rerun without --skip-build'); +assert.deepEqual( + build.perryRuntime?.recipe, + perryRuntimeRecipe, + 'Perry runtime recipe changed; rerun without --skip-build', +); +const measurements = Object.fromEntries(backends.map((backend) => [backend, []])); +// Rotate backend order between rounds to reduce bias from changing machine load. +for (let round = -warmup; round < samples; round++) { + const offset = (round + warmup) % backends.length; + for (let index = 0; index < backends.length; index++) { + const backend = backends[(index + offset) % backends.length]; + const module = join(output, backend, 'transpiled', 'component.js'); + const timing = runComponent(backend, module); + if (round >= 0) { + measurements[backend].push(timing); + } + } +} + +const rows = await Promise.all( + backends.map(async (backend) => ({ + backend, + componentBytes: (await stat(join(output, backend, 'component.wasm'))).size, + optimizedComponentBytes: (await stat(join(output, backend, 'component.opt.wasm'))).size, + runMs: median(measurements[backend].map((sample) => sample.runMs)), + loadAndRunMs: median(measurements[backend].map((sample) => sample.loadAndRunMs)), + processMs: median(measurements[backend].map((sample) => sample.processMs)), + samples: measurements[backend], + })), +); +const report = { + measuredAt: new Date().toISOString(), + ...hostDetails(), + samples, + warmup, + inputBytes: input.length, + inputSha256, + compilationReused: values['skip-build'], + build, + results: rows, +}; +await writeFile(join(output, 'results.json'), `${JSON.stringify(report, null, 2)}\n`); +console.log( + '\n| Backend | As-built size | Optimized size | Run (median ms) | Load + run (median ms) | Node process (median ms) |', +); +console.log('| --- | ---: | ---: | ---: | ---: | ---: |'); +for (const row of rows) { + console.log( + `| ${row.backend} | ${componentSize(row.componentBytes)} | ${componentSize(row.optimizedComponentBytes)} | ${row.runMs.toFixed(2)} | ${row.loadAndRunMs.toFixed(2)} | ${row.processMs.toFixed(2)} |`, + ); +} +console.log( + `\nOptimized components: ${samples} samples/backend after ${warmup} warmups; details: ${join(output, 'results.json')}`, +); + +if (build.compilation) { + console.log( + `\nCompilation: ${build.compilation.samples} samples/backend after ${build.compilation.warmup} warmups; recorded ${build.compilation.completedAt}${values['skip-build'] ? ' (reused; not remeasured)' : ''}.`, + ); + console.log( + '| Backend | TS/JS → component (median ms) | Optimize (median ms) | TS/JS → optimized component (median ms) |', + ); + console.log('| --- | ---: | ---: | ---: |'); + for (const row of build.compilation.results) { + console.log( + `| ${row.backend} | ${row.componentizeMs.toFixed(2)} | ${row.optimizeMs.toFixed(2)} | ${row.sourceToOptimizedMs.toFixed(2)} |`, + ); + } + console.log( + 'Complete components include their engines; prebuilt engines/toolchains are reused. Transpilation and validation are untimed.', + ); +} else { + console.log('\nNo compilation timings in this cached build; rerun without --skip-build to measure them.'); +} diff --git a/packages/jco/test/bench/fixtures/filesystem/data/input.txt b/packages/jco/test/bench/fixtures/filesystem/data/input.txt new file mode 100644 index 000000000..caad6c5df --- /dev/null +++ b/packages/jco/test/bench/fixtures/filesystem/data/input.txt @@ -0,0 +1 @@ +hello from the filesystem diff --git a/packages/jco/test/bench/fixtures/filesystem/guest.ts b/packages/jco/test/bench/fixtures/filesystem/guest.ts new file mode 100644 index 000000000..68a54a0d4 --- /dev/null +++ b/packages/jco/test/bench/fixtures/filesystem/guest.ts @@ -0,0 +1,6 @@ +import { readFileSync, statSync } from "node:fs"; + +// Perry lowers these Node-style calls to WASI P2 filesystem operations. +const input: string = readFileSync("/data/input.txt", "utf8"); +console.log(`Perry size: ${statSync("/data/input.txt").size}`); +console.log(`Perry read: ${input}`); diff --git a/packages/jco/test/bench/perry-runtime.mjs b/packages/jco/test/bench/perry-runtime.mjs new file mode 100644 index 000000000..8b16946f0 --- /dev/null +++ b/packages/jco/test/bench/perry-runtime.mjs @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { copyFile, mkdir, readFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +import pins from "../../../../examples/components/perry-filesystem/scripts/perry-toolchain.json" with { type: "json" }; + +// This feature subset covers the filesystem fixture, not arbitrary Perry apps. +export const perryRuntimeRecipe = { + perryRef: pins.perryRef, + rustToolchain: pins.rustToolchain, + target: "wasm32-wasip2", + profile: "wasi-size", + optLevel: "z", + lto: "fat", + defaultFeatures: false, + features: ["perry-runtime/keepalive-anchors"], +}; + +export async function buildPerryRuntime(source, output) { + assert.ok(source, "Pass --perry-source or set JCO_PERRY_SOURCE to the pinned Perry checkout"); + assert.ok(process.env.WASI_SDK_PATH, "Set WASI_SDK_PATH to the wasi-sdk installation"); + source = resolve(source); + const sdk = resolve(process.env.WASI_SDK_PATH); + const revision = execFileSync("git", ["rev-parse", "HEAD"], { cwd: source, encoding: "utf8" }).trim(); + assert.equal(revision, pins.perryRef, "Perry source must match the benchmark's pinned toolchain"); + execFileSync("git", ["diff", "--exit-code", "HEAD", "--"], { cwd: source, stdio: "inherit" }); + + const directory = join(output, "perry-runtime"); + await mkdir(directory, { recursive: true }); + const executable = (name) => (process.platform === "win32" ? `${name}.exe` : name); + const env = { + ...process.env, + RUSTUP_TOOLCHAIN: perryRuntimeRecipe.rustToolchain, + CARGO_TARGET_DIR: resolve(process.env.CARGO_TARGET_DIR || join(directory, "target")), + CARGO_BUILD_JOBS: process.env.CARGO_BUILD_JOBS || "2", + CC_wasm32_wasip2: join(sdk, "bin", executable("clang")), + AR_wasm32_wasip2: join(sdk, "bin", executable("llvm-ar")), + CFLAGS_wasm32_wasip2: `--target=wasm32-wasip2 --sysroot="${join(sdk, "share", "wasi-sysroot").replaceAll("\\", "/").replaceAll('"', '\\"')}"`, + CC_SHELL_ESCAPED_FLAGS: "1", + CARGO_PROFILE_WASI_SIZE_OPT_LEVEL: perryRuntimeRecipe.optLevel, + CARGO_PROFILE_WASI_SIZE_LTO: perryRuntimeRecipe.lto, + }; + const args = [ + "build", + "--locked", + "--profile", + perryRuntimeRecipe.profile, + "--config", + 'profile.wasi-size.inherits="release"', + "--config", + `profile.wasi-size.opt-level="${perryRuntimeRecipe.optLevel}"`, + // Perry overrides this package to level 3 in the inherited release profile. + "--config", + `profile.wasi-size.package.perry-runtime.opt-level="${perryRuntimeRecipe.optLevel}"`, + "--config", + `profile.wasi-size.lto="${perryRuntimeRecipe.lto}"`, + "-p", + "perry-runtime-static", + "--target", + perryRuntimeRecipe.target, + "--no-default-features", + "--features", + perryRuntimeRecipe.features.join(","), + "--message-format=json-render-diagnostics", + ]; + console.log("Building Perry's filesystem runtime with reduced features, opt-level=z and fat LTO..."); + const messages = execFileSync("cargo", args, { + cwd: source, + env, + encoding: "utf8", + stdio: ["inherit", "pipe", "inherit"], + maxBuffer: 16 * 1024 * 1024, + }); + // Observe Cargo's artifact, including cache hits, instead of assuming an old + // archive at a conventional path was rebuilt with this recipe. + const artifact = messages + .trim() + .split("\n") + .map((line) => JSON.parse(line)) + .find( + (message) => + message.reason === "compiler-artifact" && + message.target.name === "perry_runtime" && + message.target.kind.includes("staticlib"), + ); + const archive = artifact?.filenames.find((path) => path.endsWith("libperry_runtime.a")); + assert.ok(archive, "Cargo did not report a Perry runtime static archive"); + const installed = join(directory, "libperry_runtime.a"); + await copyFile(archive, installed); + const bytes = await readFile(installed); + assert.ok(bytes.length, "Perry runtime archive is empty"); + return { + env: { ...process.env, WASI_SDK_PATH: sdk, PERRY_RUNTIME_DIR: directory }, + metadata: { + recipe: perryRuntimeRecipe, + source, + archive: installed, + archiveSha256: createHash("sha256").update(bytes).digest("hex"), + rustcVersion: execFileSync("rustc", ["--version"], { env, encoding: "utf8" }).trim(), + wasiSdkPath: sdk, + clangVersion: execFileSync(join(sdk, "bin", executable("clang")), ["--version"], { + encoding: "utf8", + }).trim(), + cargoArgs: args, + }, + }; +}