From 61c7151a3bd7cf38669feaf4b34413d6a506dc91 Mon Sep 17 00:00:00 2001 From: Victor Adossi Date: Thu, 22 Oct 2026 12:23:44 +0000 Subject: [PATCH 1/6] feat(examples): compare Perry, StarlingMonkey, and QuickJS components Signed-off-by: Victor Adossi --- .../components/perry-filesystem/README.md | 55 +++++ .../components/perry-filesystem/package.json | 1 + .../scripts/compare-backends.mjs | 217 ++++++++++++++++++ .../scripts/compare-runner.mjs | 32 +++ 4 files changed, 305 insertions(+) create mode 100644 examples/components/perry-filesystem/scripts/compare-backends.mjs create mode 100644 examples/components/perry-filesystem/scripts/compare-runner.mjs diff --git a/examples/components/perry-filesystem/README.md b/examples/components/perry-filesystem/README.md index c192a1c87..3b5136d9a 100644 --- a/examples/components/perry-filesystem/README.md +++ b/examples/components/perry-filesystem/README.md @@ -209,6 +209,61 @@ the same component fails with the default shim settings. world and supports a subset of Node-style APIs. Generated guest declarations do not add custom-interface support or full Node.js compatibility. +## Comparing component size and execution speed + +After completing the prerequisites, compare Perry, componentize-js +(StarlingMonkey), and componentize-qjs (QuickJS) from this example directory: + +```sh +pnpm run compare +``` + +The [comparison script](./scripts/compare-backends.mjs) builds and transpiles all +three components, checks that they read the same host file and produce the same +output, and prints a table. It records the raw samples, tool versions, input hash, +and host details in `dist/comparison/results.json`. + +Example results measured on 2026-10-22 on a shared Linux x86_64 development host +(Intel Xeon at 2.60 GHz, Node.js 24.21.0), using the bundled 26-byte input file: + +| Backend | Component bytes | Run (median ms) | Load + run (median ms) | Node process (median ms) | +| -------------------------------- | --------------: | --------------: | ---------------------: | -----------------------: | +| Perry | 11,970,129 | 218.97 | 500.85 | 896.72 | +| componentize-js (StarlingMonkey) | 16,771,476 | 182.43 | 615.88 | 1,253.86 | +| componentize-qjs (QuickJS) | 7,122,397 | 120.39 | 430.48 | 1,140.64 | + +Tool versions: Jco `7b296602` (package version 1.39.0), componentize-js 0.23.0, +componentize-qjs 0.4.5, and +Perry 0.5.1659 built at commit `3945beb23ebeca54846a30d89cb4971de59b0c6c` +with LLVM 22.1.8 and wasi-sdk 34. These are example measurements from one host; +rerun the comparison for results on your machine. + +- **Component bytes:** the uncompressed `.wasm` component, before transpilation. +- **Run:** the first command call on a fresh instance, including file read, stat, and output. +- **Load + run:** importing and instantiating the transpiled component, then running it. +- **Node process:** elapsed time from launching a fresh Node process until it exits, + including host setup, component loading, and execution. + +All three use explicit asynchronous instantiation with the same host setup. +Each backend runs in a fresh process for every sample. The default is 20 measured +samples per backend after three warmup rounds, rotating backend order between +rounds. File and OS caches are not cleared. Builds and transpilation are excluded +from the timing. To repeat measurements without rebuilding unchanged components: + +```sh +pnpm run compare --skip-build --samples 50 --warmup 5 +``` + +The JavaScript backends wrap the guest's statements in the WASI command export +so StarlingMonkey's build-time preinitialization cannot read the input file. +Their components include Jco's Node filesystem adapter, mapped to the same host +file through the Node filesystem capability. Perry lowers those calls directly +to WASI filesystem operations. QuickJS uses synchronous bindings for this +synchronous workload; StarlingMonkey uses its default build without Weval AOT. +These measurements compare this example's complete backend paths. The tiny file +and console output make them sensitive to host overhead; they do not establish +general JavaScript throughput or performance under sustained load. + ## Installer details The example-local [installer](./scripts/install-perry.mjs): diff --git a/examples/components/perry-filesystem/package.json b/examples/components/perry-filesystem/package.json index 7a60bd93f..6a10f2338 100644 --- a/examples/components/perry-filesystem/package.json +++ b/examples/components/perry-filesystem/package.json @@ -5,6 +5,7 @@ "type": "module", "scripts": { "setup:perry": "node scripts/install-perry.mjs", + "compare": "node scripts/compare-backends.mjs", "gen:types": "node scripts/gen-types.mjs", "build:component": "node scripts/build-component.mjs", "build": "pnpm run gen:types && pnpm run build:component", diff --git a/examples/components/perry-filesystem/scripts/compare-backends.mjs b/examples/components/perry-filesystem/scripts/compare-backends.mjs new file mode 100644 index 000000000..725275edd --- /dev/null +++ b/examples/components/perry-filesystem/scripts/compare-backends.mjs @@ -0,0 +1,217 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { copyFile, cp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { cpus, release } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { localCompiler, pins } from './perry-toolchain.mjs'; + +const root = fileURLToPath(new URL('../', import.meta.url)); +const output = join(root, 'dist', 'comparison'); +const jco = fileURLToPath(new URL('jco.js', import.meta.resolve('@bytecodealliance/jco'))); +const requireJco = createRequire(jco); +const backends = ['perry', 'starlingmonkey', 'quickjs']; +const { values } = parseArgs({ + options: { + samples: { type: 'string', default: '20' }, + warmup: { type: 'string', default: '3' }, + 'skip-build': { type: 'boolean', default: false }, + }, +}); +const samples = Number(values.samples); +const warmup = Number(values.warmup); +assert.ok(Number.isInteger(samples) && samples > 0, '--samples must be a positive integer'); +assert.ok(Number.isInteger(warmup) && warmup >= 0, '--warmup must be a nonnegative integer'); + +function command(script, args = []) { + execFileSync(process.execPath, [script, ...args], { cwd: root, stdio: 'inherit' }); +} + +async function packageVersion(name) { + const directories = + name === '@bytecodealliance/jco' + ? [join(dirname(jco), '..')] + : (requireJco.resolve.paths(name) ?? []).map((directory) => join(directory, name)); + for (const directory of directories) { + try { + const manifest = JSON.parse(await readFile(join(directory, 'package.json'), 'utf8')); + if (manifest.name === name) { + return manifest.version; + } + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + } + throw new Error(`Cannot find the version of ${name}`); +} + +function median(numbers) { + const sorted = [...numbers].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; +} + +await mkdir(output, { recursive: true }); +const guest = await readFile(join(root, 'src', 'guest', 'component.ts'), 'utf8'); +const guestSha256 = createHash('sha256').update(guest).digest('hex'); +if (!values['skip-build']) { + command(join(root, 'scripts', 'gen-types.mjs')); + + // Keep the example's read/stat/print statements inside run(): StarlingMonkey + // preinitializes module-level code while building the component. + const ts = requireJco('typescript-compiler-api'); + const source = ts.createSourceFile('component.ts', guest, ts.ScriptTarget.Latest, true); + const imports = source.statements.filter(ts.isImportDeclaration); + const statements = source.statements.filter((statement) => !ts.isImportDeclaration(statement)); + const entry = join(output, 'command.ts'); + await writeFile( + entry, + `${imports.map((statement) => statement.getFullText(source)).join('\n')} +export const run = { run() { +${statements.map((statement) => statement.getFullText(source)).join('\n')} +} }; +`, + ); + + for (const backend of backends) { + const directory = join(output, backend); + await rm(directory, { recursive: true, force: true }); + await mkdir(directory); + const component = join(directory, 'component.wasm'); + if (backend === 'perry') { + command(join(root, 'scripts', 'build-component.mjs'), ['-o', component]); + } else { + const wit = join(directory, 'wit'); + await cp(join(root, 'wit', 'deps'), join(wit, 'deps'), { recursive: true }); + const cli = join(wit, 'deps', 'wasi-cli-0.2.12'); + await mkdir(cli); + await copyFile(join(root, 'wit', 'package.wit'), join(cli, 'package.wit')); + await writeFile( + join(wit, 'package.wit'), + 'package example:filesystem-comparison;\nworld command { export wasi:cli/run@0.2.12; }\n', + ); + command(jco, [ + 'componentize', + entry, + '--backend', + backend, + '--wit', + wit, + '--world-name', + 'example:filesystem-comparison/command', + '-o', + component, + ...(backend === 'quickjs' ? ['--backend-qjs-disable-async'] : []), + ]); + } + command(jco, [ + 'transpile', + component, + '--bindgen-enable-wasm-exnref', + '--instantiation', + 'async', + '-o', + join(directory, 'transpiled'), + ]); + } + const selection = await localCompiler([]); + const perryCliPath = process.env.JCO_PERRY_CLI_PATH || selection.options[1]; + const jsLauncher = perryCliPath && /\.[cm]?js$/i.test(perryCliPath); + const perryVersion = perryCliPath + ? execFileSync( + jsLauncher ? process.execPath : perryCliPath, + [...(jsLauncher ? [perryCliPath] : []), '--version'], + { + cwd: root, + env: selection.env, + encoding: 'utf8', + }, + ).trim() + : null; + await writeFile( + join(output, 'build.json'), + `${JSON.stringify( + { + guestSha256, + jco: await packageVersion('@bytecodealliance/jco'), + componentizeJs: await packageVersion('@bytecodealliance/componentize-js'), + componentizeQjs: await packageVersion('componentize-qjs'), + perryRef: !process.env.JCO_PERRY_CLI_PATH && perryCliPath ? pins.perryRef : null, + perryCliPath, + perryVersion, + }, + null, + 2, + )}\n`, + ); +} + +const build = JSON.parse(await readFile(join(output, 'build.json'), 'utf8')); +assert.equal(build.guestSha256, guestSha256, 'Guest changed; rerun without --skip-build'); +const input = await readFile(join(root, 'data', 'input.txt')); +const expected = `Perry size: ${input.length}\nPerry read: ${input.toString('utf8')}\n`.replaceAll('\r\n', '\n'); +const measurements = Object.fromEntries(backends.map((backend) => [backend, []])); +// Rotate backend order between rounds to reduce bias from changing machine load. +for (let round = -warmup; round < samples; round++) { + const offset = (round + warmup) % backends.length; + for (let index = 0; index < backends.length; index++) { + const backend = backends[(index + offset) % backends.length]; + const module = join(output, backend, 'transpiled', 'component.js'); + const start = process.hrtime.bigint(); + const result = spawnSync(process.execPath, [join(root, 'scripts', 'compare-runner.mjs'), module], { + cwd: root, + encoding: 'utf8', + timeout: 60_000, + }); + const processMs = Number(process.hrtime.bigint() - start) / 1e6; + if (result.error) { + throw result.error; + } + assert.equal(result.status, 0, `${backend} failed: ${result.stderr}`); + assert.equal(result.stdout.replaceAll('\r\n', '\n'), expected, `${backend} produced different output`); + const timing = JSON.parse(result.stderr.trim()); + if (round >= 0) { + measurements[backend].push({ ...timing, processMs }); + } + } +} + +const rows = await Promise.all( + backends.map(async (backend) => ({ + backend, + componentBytes: (await stat(join(output, backend, 'component.wasm'))).size, + runMs: median(measurements[backend].map((sample) => sample.runMs)), + loadAndRunMs: median(measurements[backend].map((sample) => sample.loadAndRunMs)), + processMs: median(measurements[backend].map((sample) => sample.processMs)), + samples: measurements[backend], + })), +); +const report = { + measuredAt: new Date().toISOString(), + node: process.version, + platform: process.platform, + arch: process.arch, + osRelease: release(), + cpu: cpus()[0]?.model, + samples, + warmup, + inputBytes: input.length, + inputSha256: createHash('sha256').update(input).digest('hex'), + build, + results: rows, +}; +await writeFile(join(output, 'results.json'), `${JSON.stringify(report, null, 2)}\n`); +console.log('\n| Backend | Component bytes | Run (median ms) | Load + run (median ms) | Node process (median ms) |'); +console.log('| --- | ---: | ---: | ---: | ---: |'); +for (const row of rows) { + console.log( + `| ${row.backend} | ${row.componentBytes} | ${row.runMs.toFixed(2)} | ${row.loadAndRunMs.toFixed(2)} | ${row.processMs.toFixed(2)} |`, + ); +} +console.log(`\n${samples} samples/backend after ${warmup} warmups; details: dist/comparison/results.json`); diff --git a/examples/components/perry-filesystem/scripts/compare-runner.mjs b/examples/components/perry-filesystem/scripts/compare-runner.mjs new file mode 100644 index 000000000..6cfd68edc --- /dev/null +++ b/examples/components/perry-filesystem/scripts/compare-runner.mjs @@ -0,0 +1,32 @@ +import { WASIShim } from '@bytecodealliance/preview2-shim/instantiation'; +import { createRequire } from 'node:module'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const jco = fileURLToPath(new URL('jco.js', import.meta.resolve('@bytecodealliance/jco'))); +const requireJco = createRequire(jco); +const host = (name) => + import(pathToFileURL(requireJco.resolve(`@bytecodealliance/jco-std/wasi/0.2.x/node/24.x.x/${name}`)).href); +const [fs, console, http] = await Promise.all([host('fs/host/node'), host('console/host/node'), host('http/host')]); +const input = fileURLToPath(new URL('../data/input.txt', import.meta.url)); +const mapPath = (path) => (path.tag === 'text' && path.val === '/data/input.txt' ? { ...path, val: input } : path); +const wasi = new WASIShim({ sandbox: { preopens: { '/data': fileURLToPath(new URL('../data/', import.meta.url)) } } }); +const imports = { + ...wasi.getImportObject(), + 'jco:node/fs': { + ...fs, + readFile: (file, options) => + fs.readFile(file.tag === 'path' ? { ...file, val: mapPath(file.val) } : file, options), + stat: (path, options) => fs.stat(mapPath(path), options), + }, + 'jco:node/console': console, + 'jco:node/http': http, +}; +const start = process.hrtime.bigint(); +const { instantiate } = await import(pathToFileURL(process.argv[2]).href); +const { run } = await instantiate(undefined, imports); +const runStart = process.hrtime.bigint(); +await run.run(); +const end = process.hrtime.bigint(); +const runMs = Number(end - runStart) / 1e6; +const loadAndRunMs = Number(end - start) / 1e6; +process.stderr.write(`${JSON.stringify({ runMs, loadAndRunMs })}\n`); From 2084a3037913013595120b51bd314e57be67847d Mon Sep 17 00:00:00 2001 From: Victor Adossi Date: Thu, 22 Oct 2026 12:52:10 +0000 Subject: [PATCH 2/6] feat(examples): compare optimized component sizes and explain Perry runtime --- .../components/perry-filesystem/README.md | 71 +++++++++++++++---- .../scripts/compare-backends.mjs | 56 +++++++++++++-- 2 files changed, 108 insertions(+), 19 deletions(-) diff --git a/examples/components/perry-filesystem/README.md b/examples/components/perry-filesystem/README.md index 3b5136d9a..81147ff4e 100644 --- a/examples/components/perry-filesystem/README.md +++ b/examples/components/perry-filesystem/README.md @@ -218,27 +218,41 @@ After completing the prerequisites, compare Perry, componentize-js pnpm run compare ``` -The [comparison script](./scripts/compare-backends.mjs) builds and transpiles all -three components, checks that they read the same host file and produce the same -output, and prints a table. It records the raw samples, tool versions, input hash, -and host details in `dist/comparison/results.json`. +The [comparison script](./scripts/compare-backends.mjs) builds all three components, +applies the same `jco opt` size-optimization pass, and transpiles the optimized +components. It checks that they read the same host file and produce the same +output, then prints sizes in KiB/MiB and timings in milliseconds. It records exact +byte counts, raw samples, tool versions, input hash, and host details in +`dist/comparison/results.json`. Example results measured on 2026-10-22 on a shared Linux x86_64 development host (Intel Xeon at 2.60 GHz, Node.js 24.21.0), using the bundled 26-byte input file: -| Backend | Component bytes | Run (median ms) | Load + run (median ms) | Node process (median ms) | -| -------------------------------- | --------------: | --------------: | ---------------------: | -----------------------: | -| Perry | 11,970,129 | 218.97 | 500.85 | 896.72 | -| componentize-js (StarlingMonkey) | 16,771,476 | 182.43 | 615.88 | 1,253.86 | -| componentize-qjs (QuickJS) | 7,122,397 | 120.39 | 430.48 | 1,140.64 | +| Backend | As-built size | Optimized size | +| --- | ---: | ---: | +| Perry | 11.42 MiB | 7.40 MiB | +| componentize-js (StarlingMonkey) | 15.99 MiB | 15.42 MiB | +| componentize-qjs (QuickJS) | 6.79 MiB | 6.66 MiB | -Tool versions: Jco `7b296602` (package version 1.39.0), componentize-js 0.23.0, +Timings for the optimized components: + +| Backend | Run (median ms) | Load + run (median ms) | Node process (median ms) | +| --- | ---: | ---: | ---: | +| Perry | 103.88 | 184.59 | 343.00 | +| componentize-js (StarlingMonkey) | 69.28 | 253.92 | 504.15 | +| componentize-qjs (QuickJS) | 43.70 | 166.15 | 441.41 | + +Tool versions: Jco `d5ae42df` (package version 1.39.0), componentize-js 0.23.0, componentize-qjs 0.4.5, and Perry 0.5.1659 built at commit `3945beb23ebeca54846a30d89cb4971de59b0c6c` -with LLVM 22.1.8 and wasi-sdk 34. These are example measurements from one host; -rerun the comparison for results on your machine. - -- **Component bytes:** the uncompressed `.wasm` component, before transpilation. +with LLVM 22.1.8 and wasi-sdk 34. All three components were optimized with +Binaryen 132 using `-Oz --low-memory-unused --enable-bulk-memory +--enable-reference-types --strip-debug`. These are example measurements from one +host; rerun the comparison for results on your machine. + +- **Component size:** the uncompressed `.wasm` component before transpilation, + reported both as built and after optimization. MiB means 1,048,576 bytes. + All timings use the optimized components. - **Run:** the first command call on a fresh instance, including file read, stat, and output. - **Load + run:** importing and instantiating the transpiled component, then running it. - **Node process:** elapsed time from launching a fresh Node process until it exits, @@ -264,6 +278,35 @@ These measurements compare this example's complete backend paths. The tiny file and console output make them sensitive to host overhead; they do not establish general JavaScript throughput or performance under sustained load. +### Why is the Perry component still large? + +AoT compilation removes the need to interpret this guest's TypeScript, but its +JavaScript semantics still require runtime support: object layouts, strings, +closures, exceptions, built-ins, and garbage collection. This pinned Perry build +links a broad runtime. Its [WASI link path][perry-wasi-link] explicitly skips the +app-specific runtime rebuild used by native targets, and the +[WASI runtime build script][perry-wasi-runtime] enables almost all default runtime +features. + +Inspecting the unoptimized core module found 9,630 defined functions and about +8.12 MiB of Wasm instructions, including broad object/built-in support, GC, +streams, regular expressions, and filesystem helpers. It also contained about +1.63 MiB of DWARF debug information and 0.75 MiB of function names. The optimized +size above removes debug information and runs size optimization and dead-code +elimination; the remaining payload is mostly runtime code. + +The emitted function names contain no SWC parser or dynamic-evaluation +interpreter functions. Perry supports an optional interpreter for runtime-created +code, but the evidence here does not attribute the size to that interpreter. +Instead, general runtime dispatch and built-in registries retain helpers that a +small statically known guest should ideally avoid. Once helpers are reachable +through those registries, ordinary linker/Binaryen dead-code elimination cannot +assume they will never be called. Further reductions would require specializing +the WASI runtime to the program's needs, then measuring the resulting component. + +[perry-wasi-link]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/crates/perry/src/commands/compile/link/wasi.rs +[perry-wasi-runtime]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/scripts/wasi_build_runtime.py + ## Installer details The example-local [installer](./scripts/install-perry.mjs): diff --git a/examples/components/perry-filesystem/scripts/compare-backends.mjs b/examples/components/perry-filesystem/scripts/compare-backends.mjs index 725275edd..9d3395e77 100644 --- a/examples/components/perry-filesystem/scripts/compare-backends.mjs +++ b/examples/components/perry-filesystem/scripts/compare-backends.mjs @@ -15,11 +15,19 @@ const output = join(root, 'dist', 'comparison'); const jco = fileURLToPath(new URL('jco.js', import.meta.resolve('@bytecodealliance/jco'))); const requireJco = createRequire(jco); const backends = ['perry', 'starlingmonkey', 'quickjs']; +const optimizationArgs = [ + '-Oz', + '--low-memory-unused', + '--enable-bulk-memory', + '--enable-reference-types', + '--strip-debug', +]; const { values } = parseArgs({ options: { samples: { type: 'string', default: '20' }, warmup: { type: 'string', default: '3' }, 'skip-build': { type: 'boolean', default: false }, + 'wasm-opt-bin': { type: 'string' }, }, }); const samples = Number(values.samples); @@ -57,6 +65,16 @@ function median(numbers) { return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; } +function componentSize(bytes) { + if (bytes < 1024) { + return `${bytes} B`; + } + if (bytes < 1024 ** 2) { + return `${(bytes / 1024).toFixed(2)} KiB`; + } + return `${(bytes / 1024 ** 2).toFixed(2)} MiB`; +} + await mkdir(output, { recursive: true }); const guest = await readFile(join(root, 'src', 'guest', 'component.ts'), 'utf8'); const guestSha256 = createHash('sha256').update(guest).digest('hex'); @@ -110,9 +128,21 @@ ${statements.map((statement) => statement.getFullText(source)).join('\n')} ...(backend === 'quickjs' ? ['--backend-qjs-disable-async'] : []), ]); } + const optimized = join(directory, 'component.opt.wasm'); command(jco, [ - 'transpile', + 'opt', component, + '-o', + optimized, + ...(values['wasm-opt-bin'] ? ['--wasm-opt-bin', values['wasm-opt-bin']] : []), + '--', + ...optimizationArgs, + ]); + command(jco, [ + 'transpile', + optimized, + '--name', + 'component', '--bindgen-enable-wasm-exnref', '--instantiation', 'async', @@ -142,6 +172,16 @@ ${statements.map((statement) => statement.getFullText(source)).join('\n')} jco: await packageVersion('@bytecodealliance/jco'), componentizeJs: await packageVersion('@bytecodealliance/componentize-js'), componentizeQjs: await packageVersion('componentize-qjs'), + optimizationArgs, + wasmOptVersion: execFileSync( + process.execPath, + [values['wasm-opt-bin'] || requireJco.resolve('binaryen/bin/wasm-opt'), '--version'], + { + cwd: root, + encoding: 'utf8', + }, + ).trim(), + wasmOptLauncher: values['wasm-opt-bin'] ?? null, perryRef: !process.env.JCO_PERRY_CLI_PATH && perryCliPath ? pins.perryRef : null, perryCliPath, perryVersion, @@ -154,6 +194,7 @@ ${statements.map((statement) => statement.getFullText(source)).join('\n')} const build = JSON.parse(await readFile(join(output, 'build.json'), 'utf8')); assert.equal(build.guestSha256, guestSha256, 'Guest changed; rerun without --skip-build'); +assert.deepEqual(build.optimizationArgs, optimizationArgs, 'Optimization changed; rerun without --skip-build'); const input = await readFile(join(root, 'data', 'input.txt')); const expected = `Perry size: ${input.length}\nPerry read: ${input.toString('utf8')}\n`.replaceAll('\r\n', '\n'); const measurements = Object.fromEntries(backends.map((backend) => [backend, []])); @@ -186,6 +227,7 @@ const rows = await Promise.all( backends.map(async (backend) => ({ backend, componentBytes: (await stat(join(output, backend, 'component.wasm'))).size, + optimizedComponentBytes: (await stat(join(output, backend, 'component.opt.wasm'))).size, runMs: median(measurements[backend].map((sample) => sample.runMs)), loadAndRunMs: median(measurements[backend].map((sample) => sample.loadAndRunMs)), processMs: median(measurements[backend].map((sample) => sample.processMs)), @@ -207,11 +249,15 @@ const report = { results: rows, }; await writeFile(join(output, 'results.json'), `${JSON.stringify(report, null, 2)}\n`); -console.log('\n| Backend | Component bytes | Run (median ms) | Load + run (median ms) | Node process (median ms) |'); -console.log('| --- | ---: | ---: | ---: | ---: |'); +console.log( + '\n| Backend | As-built size | Optimized size | Run (median ms) | Load + run (median ms) | Node process (median ms) |', +); +console.log('| --- | ---: | ---: | ---: | ---: | ---: |'); for (const row of rows) { console.log( - `| ${row.backend} | ${row.componentBytes} | ${row.runMs.toFixed(2)} | ${row.loadAndRunMs.toFixed(2)} | ${row.processMs.toFixed(2)} |`, + `| ${row.backend} | ${componentSize(row.componentBytes)} | ${componentSize(row.optimizedComponentBytes)} | ${row.runMs.toFixed(2)} | ${row.loadAndRunMs.toFixed(2)} | ${row.processMs.toFixed(2)} |`, ); } -console.log(`\n${samples} samples/backend after ${warmup} warmups; details: dist/comparison/results.json`); +console.log( + `\nOptimized components: ${samples} samples/backend after ${warmup} warmups; details: dist/comparison/results.json`, +); From ccb9ca5d1c6e9363a9209e2051c31e0d3871f384 Mon Sep 17 00:00:00 2001 From: Victor Adossi Date: Thu, 22 Oct 2026 14:34:05 +0000 Subject: [PATCH 3/6] refactor(jco): move backend comparison into Jco benchmark suite --- .../components/perry-filesystem/README.md | 98 ------- .../components/perry-filesystem/package.json | 1 - .../scripts/compare-backends.mjs | 263 ----------------- .../scripts/compare-runner.mjs | 32 --- packages/jco/package.json | 1 + packages/jco/test/README.md | 5 + packages/jco/test/bench/README.md | 178 ++++++++++++ .../bench/componentize-backends-runner.mjs | 34 +++ .../jco/test/bench/componentize-backends.mjs | 272 ++++++++++++++++++ .../bench/fixtures/filesystem/data/input.txt | 1 + .../test/bench/fixtures/filesystem/guest.ts | 6 + 11 files changed, 497 insertions(+), 394 deletions(-) delete mode 100644 examples/components/perry-filesystem/scripts/compare-backends.mjs delete mode 100644 examples/components/perry-filesystem/scripts/compare-runner.mjs create mode 100644 packages/jco/test/bench/README.md create mode 100644 packages/jco/test/bench/componentize-backends-runner.mjs create mode 100644 packages/jco/test/bench/componentize-backends.mjs create mode 100644 packages/jco/test/bench/fixtures/filesystem/data/input.txt create mode 100644 packages/jco/test/bench/fixtures/filesystem/guest.ts diff --git a/examples/components/perry-filesystem/README.md b/examples/components/perry-filesystem/README.md index 81147ff4e..c192a1c87 100644 --- a/examples/components/perry-filesystem/README.md +++ b/examples/components/perry-filesystem/README.md @@ -209,104 +209,6 @@ the same component fails with the default shim settings. world and supports a subset of Node-style APIs. Generated guest declarations do not add custom-interface support or full Node.js compatibility. -## Comparing component size and execution speed - -After completing the prerequisites, compare Perry, componentize-js -(StarlingMonkey), and componentize-qjs (QuickJS) from this example directory: - -```sh -pnpm run compare -``` - -The [comparison script](./scripts/compare-backends.mjs) builds all three components, -applies the same `jco opt` size-optimization pass, and transpiles the optimized -components. It checks that they read the same host file and produce the same -output, then prints sizes in KiB/MiB and timings in milliseconds. It records exact -byte counts, raw samples, tool versions, input hash, and host details in -`dist/comparison/results.json`. - -Example results measured on 2026-10-22 on a shared Linux x86_64 development host -(Intel Xeon at 2.60 GHz, Node.js 24.21.0), using the bundled 26-byte input file: - -| Backend | As-built size | Optimized size | -| --- | ---: | ---: | -| Perry | 11.42 MiB | 7.40 MiB | -| componentize-js (StarlingMonkey) | 15.99 MiB | 15.42 MiB | -| componentize-qjs (QuickJS) | 6.79 MiB | 6.66 MiB | - -Timings for the optimized components: - -| Backend | Run (median ms) | Load + run (median ms) | Node process (median ms) | -| --- | ---: | ---: | ---: | -| Perry | 103.88 | 184.59 | 343.00 | -| componentize-js (StarlingMonkey) | 69.28 | 253.92 | 504.15 | -| componentize-qjs (QuickJS) | 43.70 | 166.15 | 441.41 | - -Tool versions: Jco `d5ae42df` (package version 1.39.0), componentize-js 0.23.0, -componentize-qjs 0.4.5, and -Perry 0.5.1659 built at commit `3945beb23ebeca54846a30d89cb4971de59b0c6c` -with LLVM 22.1.8 and wasi-sdk 34. All three components were optimized with -Binaryen 132 using `-Oz --low-memory-unused --enable-bulk-memory ---enable-reference-types --strip-debug`. These are example measurements from one -host; rerun the comparison for results on your machine. - -- **Component size:** the uncompressed `.wasm` component before transpilation, - reported both as built and after optimization. MiB means 1,048,576 bytes. - All timings use the optimized components. -- **Run:** the first command call on a fresh instance, including file read, stat, and output. -- **Load + run:** importing and instantiating the transpiled component, then running it. -- **Node process:** elapsed time from launching a fresh Node process until it exits, - including host setup, component loading, and execution. - -All three use explicit asynchronous instantiation with the same host setup. -Each backend runs in a fresh process for every sample. The default is 20 measured -samples per backend after three warmup rounds, rotating backend order between -rounds. File and OS caches are not cleared. Builds and transpilation are excluded -from the timing. To repeat measurements without rebuilding unchanged components: - -```sh -pnpm run compare --skip-build --samples 50 --warmup 5 -``` - -The JavaScript backends wrap the guest's statements in the WASI command export -so StarlingMonkey's build-time preinitialization cannot read the input file. -Their components include Jco's Node filesystem adapter, mapped to the same host -file through the Node filesystem capability. Perry lowers those calls directly -to WASI filesystem operations. QuickJS uses synchronous bindings for this -synchronous workload; StarlingMonkey uses its default build without Weval AOT. -These measurements compare this example's complete backend paths. The tiny file -and console output make them sensitive to host overhead; they do not establish -general JavaScript throughput or performance under sustained load. - -### Why is the Perry component still large? - -AoT compilation removes the need to interpret this guest's TypeScript, but its -JavaScript semantics still require runtime support: object layouts, strings, -closures, exceptions, built-ins, and garbage collection. This pinned Perry build -links a broad runtime. Its [WASI link path][perry-wasi-link] explicitly skips the -app-specific runtime rebuild used by native targets, and the -[WASI runtime build script][perry-wasi-runtime] enables almost all default runtime -features. - -Inspecting the unoptimized core module found 9,630 defined functions and about -8.12 MiB of Wasm instructions, including broad object/built-in support, GC, -streams, regular expressions, and filesystem helpers. It also contained about -1.63 MiB of DWARF debug information and 0.75 MiB of function names. The optimized -size above removes debug information and runs size optimization and dead-code -elimination; the remaining payload is mostly runtime code. - -The emitted function names contain no SWC parser or dynamic-evaluation -interpreter functions. Perry supports an optional interpreter for runtime-created -code, but the evidence here does not attribute the size to that interpreter. -Instead, general runtime dispatch and built-in registries retain helpers that a -small statically known guest should ideally avoid. Once helpers are reachable -through those registries, ordinary linker/Binaryen dead-code elimination cannot -assume they will never be called. Further reductions would require specializing -the WASI runtime to the program's needs, then measuring the resulting component. - -[perry-wasi-link]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/crates/perry/src/commands/compile/link/wasi.rs -[perry-wasi-runtime]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/scripts/wasi_build_runtime.py - ## Installer details The example-local [installer](./scripts/install-perry.mjs): diff --git a/examples/components/perry-filesystem/package.json b/examples/components/perry-filesystem/package.json index 6a10f2338..7a60bd93f 100644 --- a/examples/components/perry-filesystem/package.json +++ b/examples/components/perry-filesystem/package.json @@ -5,7 +5,6 @@ "type": "module", "scripts": { "setup:perry": "node scripts/install-perry.mjs", - "compare": "node scripts/compare-backends.mjs", "gen:types": "node scripts/gen-types.mjs", "build:component": "node scripts/build-component.mjs", "build": "pnpm run gen:types && pnpm run build:component", diff --git a/examples/components/perry-filesystem/scripts/compare-backends.mjs b/examples/components/perry-filesystem/scripts/compare-backends.mjs deleted file mode 100644 index 9d3395e77..000000000 --- a/examples/components/perry-filesystem/scripts/compare-backends.mjs +++ /dev/null @@ -1,263 +0,0 @@ -import assert from 'node:assert/strict'; -import { execFileSync, spawnSync } from 'node:child_process'; -import { createHash } from 'node:crypto'; -import { copyFile, cp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; -import { createRequire } from 'node:module'; -import { cpus, release } from 'node:os'; -import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { parseArgs } from 'node:util'; - -import { localCompiler, pins } from './perry-toolchain.mjs'; - -const root = fileURLToPath(new URL('../', import.meta.url)); -const output = join(root, 'dist', 'comparison'); -const jco = fileURLToPath(new URL('jco.js', import.meta.resolve('@bytecodealliance/jco'))); -const requireJco = createRequire(jco); -const backends = ['perry', 'starlingmonkey', 'quickjs']; -const optimizationArgs = [ - '-Oz', - '--low-memory-unused', - '--enable-bulk-memory', - '--enable-reference-types', - '--strip-debug', -]; -const { values } = parseArgs({ - options: { - samples: { type: 'string', default: '20' }, - warmup: { type: 'string', default: '3' }, - 'skip-build': { type: 'boolean', default: false }, - 'wasm-opt-bin': { type: 'string' }, - }, -}); -const samples = Number(values.samples); -const warmup = Number(values.warmup); -assert.ok(Number.isInteger(samples) && samples > 0, '--samples must be a positive integer'); -assert.ok(Number.isInteger(warmup) && warmup >= 0, '--warmup must be a nonnegative integer'); - -function command(script, args = []) { - execFileSync(process.execPath, [script, ...args], { cwd: root, stdio: 'inherit' }); -} - -async function packageVersion(name) { - const directories = - name === '@bytecodealliance/jco' - ? [join(dirname(jco), '..')] - : (requireJco.resolve.paths(name) ?? []).map((directory) => join(directory, name)); - for (const directory of directories) { - try { - const manifest = JSON.parse(await readFile(join(directory, 'package.json'), 'utf8')); - if (manifest.name === name) { - return manifest.version; - } - } catch (error) { - if (error.code !== 'ENOENT') { - throw error; - } - } - } - throw new Error(`Cannot find the version of ${name}`); -} - -function median(numbers) { - const sorted = [...numbers].sort((a, b) => a - b); - const middle = Math.floor(sorted.length / 2); - return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; -} - -function componentSize(bytes) { - if (bytes < 1024) { - return `${bytes} B`; - } - if (bytes < 1024 ** 2) { - return `${(bytes / 1024).toFixed(2)} KiB`; - } - return `${(bytes / 1024 ** 2).toFixed(2)} MiB`; -} - -await mkdir(output, { recursive: true }); -const guest = await readFile(join(root, 'src', 'guest', 'component.ts'), 'utf8'); -const guestSha256 = createHash('sha256').update(guest).digest('hex'); -if (!values['skip-build']) { - command(join(root, 'scripts', 'gen-types.mjs')); - - // Keep the example's read/stat/print statements inside run(): StarlingMonkey - // preinitializes module-level code while building the component. - const ts = requireJco('typescript-compiler-api'); - const source = ts.createSourceFile('component.ts', guest, ts.ScriptTarget.Latest, true); - const imports = source.statements.filter(ts.isImportDeclaration); - const statements = source.statements.filter((statement) => !ts.isImportDeclaration(statement)); - const entry = join(output, 'command.ts'); - await writeFile( - entry, - `${imports.map((statement) => statement.getFullText(source)).join('\n')} -export const run = { run() { -${statements.map((statement) => statement.getFullText(source)).join('\n')} -} }; -`, - ); - - for (const backend of backends) { - const directory = join(output, backend); - await rm(directory, { recursive: true, force: true }); - await mkdir(directory); - const component = join(directory, 'component.wasm'); - if (backend === 'perry') { - command(join(root, 'scripts', 'build-component.mjs'), ['-o', component]); - } else { - const wit = join(directory, 'wit'); - await cp(join(root, 'wit', 'deps'), join(wit, 'deps'), { recursive: true }); - const cli = join(wit, 'deps', 'wasi-cli-0.2.12'); - await mkdir(cli); - await copyFile(join(root, 'wit', 'package.wit'), join(cli, 'package.wit')); - await writeFile( - join(wit, 'package.wit'), - 'package example:filesystem-comparison;\nworld command { export wasi:cli/run@0.2.12; }\n', - ); - command(jco, [ - 'componentize', - entry, - '--backend', - backend, - '--wit', - wit, - '--world-name', - 'example:filesystem-comparison/command', - '-o', - component, - ...(backend === 'quickjs' ? ['--backend-qjs-disable-async'] : []), - ]); - } - const optimized = join(directory, 'component.opt.wasm'); - command(jco, [ - 'opt', - component, - '-o', - optimized, - ...(values['wasm-opt-bin'] ? ['--wasm-opt-bin', values['wasm-opt-bin']] : []), - '--', - ...optimizationArgs, - ]); - command(jco, [ - 'transpile', - optimized, - '--name', - 'component', - '--bindgen-enable-wasm-exnref', - '--instantiation', - 'async', - '-o', - join(directory, 'transpiled'), - ]); - } - const selection = await localCompiler([]); - const perryCliPath = process.env.JCO_PERRY_CLI_PATH || selection.options[1]; - const jsLauncher = perryCliPath && /\.[cm]?js$/i.test(perryCliPath); - const perryVersion = perryCliPath - ? execFileSync( - jsLauncher ? process.execPath : perryCliPath, - [...(jsLauncher ? [perryCliPath] : []), '--version'], - { - cwd: root, - env: selection.env, - encoding: 'utf8', - }, - ).trim() - : null; - await writeFile( - join(output, 'build.json'), - `${JSON.stringify( - { - guestSha256, - jco: await packageVersion('@bytecodealliance/jco'), - componentizeJs: await packageVersion('@bytecodealliance/componentize-js'), - componentizeQjs: await packageVersion('componentize-qjs'), - optimizationArgs, - wasmOptVersion: execFileSync( - process.execPath, - [values['wasm-opt-bin'] || requireJco.resolve('binaryen/bin/wasm-opt'), '--version'], - { - cwd: root, - encoding: 'utf8', - }, - ).trim(), - wasmOptLauncher: values['wasm-opt-bin'] ?? null, - perryRef: !process.env.JCO_PERRY_CLI_PATH && perryCliPath ? pins.perryRef : null, - perryCliPath, - perryVersion, - }, - null, - 2, - )}\n`, - ); -} - -const build = JSON.parse(await readFile(join(output, 'build.json'), 'utf8')); -assert.equal(build.guestSha256, guestSha256, 'Guest changed; rerun without --skip-build'); -assert.deepEqual(build.optimizationArgs, optimizationArgs, 'Optimization changed; rerun without --skip-build'); -const input = await readFile(join(root, 'data', 'input.txt')); -const expected = `Perry size: ${input.length}\nPerry read: ${input.toString('utf8')}\n`.replaceAll('\r\n', '\n'); -const measurements = Object.fromEntries(backends.map((backend) => [backend, []])); -// Rotate backend order between rounds to reduce bias from changing machine load. -for (let round = -warmup; round < samples; round++) { - const offset = (round + warmup) % backends.length; - for (let index = 0; index < backends.length; index++) { - const backend = backends[(index + offset) % backends.length]; - const module = join(output, backend, 'transpiled', 'component.js'); - const start = process.hrtime.bigint(); - const result = spawnSync(process.execPath, [join(root, 'scripts', 'compare-runner.mjs'), module], { - cwd: root, - encoding: 'utf8', - timeout: 60_000, - }); - const processMs = Number(process.hrtime.bigint() - start) / 1e6; - if (result.error) { - throw result.error; - } - assert.equal(result.status, 0, `${backend} failed: ${result.stderr}`); - assert.equal(result.stdout.replaceAll('\r\n', '\n'), expected, `${backend} produced different output`); - const timing = JSON.parse(result.stderr.trim()); - if (round >= 0) { - measurements[backend].push({ ...timing, processMs }); - } - } -} - -const rows = await Promise.all( - backends.map(async (backend) => ({ - backend, - componentBytes: (await stat(join(output, backend, 'component.wasm'))).size, - optimizedComponentBytes: (await stat(join(output, backend, 'component.opt.wasm'))).size, - runMs: median(measurements[backend].map((sample) => sample.runMs)), - loadAndRunMs: median(measurements[backend].map((sample) => sample.loadAndRunMs)), - processMs: median(measurements[backend].map((sample) => sample.processMs)), - samples: measurements[backend], - })), -); -const report = { - measuredAt: new Date().toISOString(), - node: process.version, - platform: process.platform, - arch: process.arch, - osRelease: release(), - cpu: cpus()[0]?.model, - samples, - warmup, - inputBytes: input.length, - inputSha256: createHash('sha256').update(input).digest('hex'), - build, - results: rows, -}; -await writeFile(join(output, 'results.json'), `${JSON.stringify(report, null, 2)}\n`); -console.log( - '\n| Backend | As-built size | Optimized size | Run (median ms) | Load + run (median ms) | Node process (median ms) |', -); -console.log('| --- | ---: | ---: | ---: | ---: | ---: |'); -for (const row of rows) { - console.log( - `| ${row.backend} | ${componentSize(row.componentBytes)} | ${componentSize(row.optimizedComponentBytes)} | ${row.runMs.toFixed(2)} | ${row.loadAndRunMs.toFixed(2)} | ${row.processMs.toFixed(2)} |`, - ); -} -console.log( - `\nOptimized components: ${samples} samples/backend after ${warmup} warmups; details: dist/comparison/results.json`, -); diff --git a/examples/components/perry-filesystem/scripts/compare-runner.mjs b/examples/components/perry-filesystem/scripts/compare-runner.mjs deleted file mode 100644 index 6cfd68edc..000000000 --- a/examples/components/perry-filesystem/scripts/compare-runner.mjs +++ /dev/null @@ -1,32 +0,0 @@ -import { WASIShim } from '@bytecodealliance/preview2-shim/instantiation'; -import { createRequire } from 'node:module'; -import { fileURLToPath, pathToFileURL } from 'node:url'; - -const jco = fileURLToPath(new URL('jco.js', import.meta.resolve('@bytecodealliance/jco'))); -const requireJco = createRequire(jco); -const host = (name) => - import(pathToFileURL(requireJco.resolve(`@bytecodealliance/jco-std/wasi/0.2.x/node/24.x.x/${name}`)).href); -const [fs, console, http] = await Promise.all([host('fs/host/node'), host('console/host/node'), host('http/host')]); -const input = fileURLToPath(new URL('../data/input.txt', import.meta.url)); -const mapPath = (path) => (path.tag === 'text' && path.val === '/data/input.txt' ? { ...path, val: input } : path); -const wasi = new WASIShim({ sandbox: { preopens: { '/data': fileURLToPath(new URL('../data/', import.meta.url)) } } }); -const imports = { - ...wasi.getImportObject(), - 'jco:node/fs': { - ...fs, - readFile: (file, options) => - fs.readFile(file.tag === 'path' ? { ...file, val: mapPath(file.val) } : file, options), - stat: (path, options) => fs.stat(mapPath(path), options), - }, - 'jco:node/console': console, - 'jco:node/http': http, -}; -const start = process.hrtime.bigint(); -const { instantiate } = await import(pathToFileURL(process.argv[2]).href); -const { run } = await instantiate(undefined, imports); -const runStart = process.hrtime.bigint(); -await run.run(); -const end = process.hrtime.bigint(); -const runMs = Number(end - runStart) / 1e6; -const loadAndRunMs = Number(end - start) / 1e6; -process.stderr.write(`${JSON.stringify({ runMs, loadAndRunMs })}\n`); diff --git a/packages/jco/package.json b/packages/jco/package.json index 0f41c938c..ff7d49015 100644 --- a/packages/jco/package.json +++ b/packages/jco/package.json @@ -53,6 +53,7 @@ "build:ts": "tsc -p tsconfig.json", "build:types:preview2-shim": "pnpm run build:types:preview2-shim --include-workspace-root", "build:test:components": "cargo xtask build-test-components", + "bench:componentize-backends": "node test/bench/componentize-backends.mjs", "bench:serve-isolation": "node test/bench/serve-request-isolation.js", "fmt": "oxfmt", "fmt:check": "oxfmt --check", diff --git a/packages/jco/test/README.md b/packages/jco/test/README.md index a1181ee80..d377b114a 100644 --- a/packages/jco/test/README.md +++ b/packages/jco/test/README.md @@ -2,6 +2,11 @@ This folder contains tests for Jco. +## Benchmarks + +Opt-in performance benchmarks, run instructions, and recent results are in +[`bench/README.md`](./bench/README.md). + ## Dependencies You'll need the following binaries installed in your path or specificed explicitly: diff --git a/packages/jco/test/bench/README.md b/packages/jco/test/bench/README.md new file mode 100644 index 000000000..8fb8c52c5 --- /dev/null +++ b/packages/jco/test/bench/README.md @@ -0,0 +1,178 @@ +# Jco benchmarks + +These opt-in benchmarks exercise the built workspace Jco CLI. Run the repository's +normal dependency setup and build first, using Node.js 24 or newer. Benchmarks are +excluded from the normal test suite and are not CI performance gates. + +Run the commands below from `packages/jco/test/bench`, or use the package scripts +from the repository root: + +| Benchmark | Command from this directory | Package script | +| ------------------------- | ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Componentization backends | `node componentize-backends.mjs --perry-cli-path /absolute/path/to/perry` | `pnpm --filter @bytecodealliance/jco bench:componentize-backends --perry-cli-path /absolute/path/to/perry` | +| Serve request isolation | `node serve-request-isolation.js` | `pnpm --filter @bytecodealliance/jco bench:serve-isolation` | + +## Componentization backends + +[`componentize-backends.mjs`](./componentize-backends.mjs) compares Perry, +componentize-js (StarlingMonkey), and componentize-qjs (QuickJS). It calls +`jco componentize`, applies the same `jco opt` pass, and calls `jco transpile` +for each backend. The guest fixture reads a host file, obtains its size, and +prints both. Fixtures belong to this suite in [`fixtures/filesystem`](./fixtures/filesystem). +The benchmark does not build or run the Perry example. + +### Setup and running + +Perry must support `--target wasi`, with its matching runtime archive and +wasi-sdk configured. The [Perry example's installer](../../../../examples/components/perry-filesystem/README.md#prerequisites) +can provision a pinned source-built toolchain without sudo. Its setup is a +separate prerequisite; this benchmark does not install compilers. + +| Setting | Purpose | +| ------------------------------------------ | ----------------------------------------------------------------- | +| `--perry-cli-path` or `JCO_PERRY_CLI_PATH` | Path to the WASI-enabled Perry executable or JavaScript launcher. | +| `PERRY_RUNTIME_DIR` | Directory containing the matching `libperry_runtime.a`. | +| `WASI_SDK_PATH` | wasi-sdk installation directory. | +| `JCO_PERRY_REF` (optional) | Source revision to record alongside the compiler version. | + +After configuring the runtime and SDK in your environment: + +```sh +node componentize-backends.mjs --perry-cli-path /absolute/path/to/perry +``` + +The script checks every sample's output, prints sizes in KiB/MiB and timings in +milliseconds, and writes exact byte counts, raw timing samples, tool versions, +input hashes, and host details to +`packages/jco/test/output/bench/componentize-backends/results.json`. +Component artifacts and build metadata are retained in the same ignored directory. +Use `--samples 50 --warmup 5` to change the default sample counts, or +`--skip-build` to reuse unchanged components without requiring the compiler. +The optional `--wasm-opt-bin` setting is passed to Jco's optimizer launcher hook. +Build and transpilation time are excluded from the measurements. + +### Recent results + +Example results measured on 2026-10-22 on a shared Linux x86_64 development host +(Intel Xeon at 2.60 GHz, Node.js 24.21.0), using the bundled 26-byte input file: + +| Backend | As-built size | Optimized size | +| -------------------------------- | ------------: | -------------: | +| Perry | 11.42 MiB | 7.40 MiB | +| componentize-js (StarlingMonkey) | 15.99 MiB | 15.42 MiB | +| componentize-qjs (QuickJS) | 6.79 MiB | 6.66 MiB | + +Timings for the optimized components: + +| Backend | Run (median ms) | Load + run (median ms) | Node process (median ms) | +| -------------------------------- | --------------: | ---------------------: | -----------------------: | +| Perry | 91.77 | 168.26 | 310.67 | +| componentize-js (StarlingMonkey) | 51.03 | 212.28 | 421.66 | +| componentize-qjs (QuickJS) | 36.17 | 136.58 | 361.60 | + +Tool versions: Jco `d5ae42df` (package version 1.39.0), componentize-js 0.23.0, +componentize-qjs 0.4.5, and +Perry 0.5.1659 built at commit `3945beb23ebeca54846a30d89cb4971de59b0c6c` +with LLVM 22.1.8 and wasi-sdk 34. All three components were optimized with +Binaryen 132 using `-Oz --low-memory-unused --enable-bulk-memory +--enable-reference-types --strip-debug`. These are example measurements from one +host; rerun the comparison for results on your machine. + +- **Component size:** the uncompressed `.wasm` component before transpilation, + reported both as built and after optimization. MiB means 1,048,576 bytes. + All timings use the optimized components. +- **Run:** the first command call on a fresh instance, including file read, stat, and output. +- **Load + run:** importing and instantiating the transpiled component, then running it. +- **Node process:** elapsed time from launching a fresh Node process until it exits, + including host setup, component loading, and execution. + +All three use explicit asynchronous instantiation with the same host setup. +Each backend runs in a fresh process for every sample. The default is 20 measured +samples per backend after three warmup rounds, rotating backend order between +rounds. File and OS caches are not cleared. Builds and transpilation are excluded +from the timing. To repeat measurements without rebuilding unchanged components: + +```sh +node componentize-backends.mjs --skip-build --samples 50 --warmup 5 +``` + +The JavaScript backends wrap the guest's statements in the WASI command export +so StarlingMonkey's build-time preinitialization cannot read the input file. +Their components include Jco's Node filesystem adapter, mapped to the same host +file through the Node filesystem capability. Perry lowers those calls directly +to WASI filesystem operations. QuickJS uses synchronous bindings for this +synchronous workload; StarlingMonkey uses its default build without Weval AOT. +These measurements compare this fixture's complete backend paths. The tiny file +and console output make them sensitive to host overhead; they do not establish +general JavaScript throughput or performance under sustained load. + +### Perry component size + +AoT compilation removes the need to interpret this guest's TypeScript, but its +JavaScript semantics still require runtime support: object layouts, strings, +closures, exceptions, built-ins, and garbage collection. This pinned Perry build +links a broad runtime. Its [WASI link path][perry-wasi-link] explicitly skips the +app-specific runtime rebuild used by native targets, and the +[WASI runtime build script][perry-wasi-runtime] enables almost all default runtime +features. + +Inspecting the unoptimized core module found 9,630 defined functions and about +8.12 MiB of Wasm instructions, including broad object/built-in support, GC, +streams, regular expressions, and filesystem helpers. It also contained about +1.63 MiB of DWARF debug information and 0.75 MiB of function names. The optimized +size above removes debug information and runs size optimization and dead-code +elimination; the remaining payload is mostly runtime code. + +The emitted function names contain no SWC parser or dynamic-evaluation +interpreter functions. Perry supports an optional interpreter for runtime-created +code, but the evidence here does not attribute the size to that interpreter. +Instead, general runtime dispatch and built-in registries retain helpers that a +small statically known guest should ideally avoid. Once helpers are reachable +through those registries, ordinary linker/Binaryen dead-code elimination cannot +assume they will never be called. Further reductions would require specializing +the WASI runtime to the program's needs, then measuring the resulting component. + +[perry-wasi-link]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/crates/perry/src/commands/compile/link/wasi.rs +[perry-wasi-runtime]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/scripts/wasi_build_runtime.py + +## Serve request isolation + +[`serve-request-isolation.js`](./serve-request-isolation.js) compares a native +Node.js HTTP handler with Jco's shared component, fresh-instance, and worker +isolation modes. It builds a minimal HTTP component, runs a local server for +each mode, checks the response, and reports requests per second and mean, +median, and p95 latency. Default counts are 100 warmups and 10,000 sequential +requests per mode; a full worker-isolation run can take a long time. + +For a smaller run, set `JCO_SERVE_BENCH_WARMUP=5` and +`JCO_SERVE_BENCH_REQUESTS=30` in your environment, then run: + +```sh +node serve-request-isolation.js +``` + +Set `JCO_SERVE_BENCH_COMPONENT_OUT` to retain a newly built component, or +`JCO_SERVE_BENCH_COMPONENT` to reuse an existing one. Componentization is excluded +from timing. Temporary server output is cleaned up automatically. + +### Most recent documented reference result + +The existing reference used Node.js 24.19.0 on a 6-vCPU Intel Xeon 2.60 GHz +virtual machine, with 10 warmup requests followed by 100 measured sequential +requests per mode. These results come from the +[development-use serve benchmark](../../../../docs/src/performance/benchmarks/development-use-jco-serve.md#reference-result); +they were not rerun as part of the componentization backend comparison. + +| Mode | Requests/s | Mean latency | Median latency | p95 latency | +| --------------------- | ---------: | -----------: | -------------: | ----------: | +| Native `node:http` | 598.13 | 1.67 ms | 1.69 ms | 2.03 ms | +| Shared component | 225.07 | 4.44 ms | 4.31 ms | 5.40 ms | +| Instance isolation | 22.88 | 43.71 ms | 43.24 ms | 47.59 ms | +| Worker, pool size 1 | 4.15 | 240.95 ms | 239.36 ms | 274.28 ms | +| Worker, pool size 50 | 13.55 | 73.82 ms | 70.14 ms | 100.00 ms | +| Worker, pool size 100 | 14.78 | 67.66 ms | 65.14 ms | 95.25 ms | + +This benchmark measures Jco's development server overhead. It does not estimate +production component-runtime capacity. Request concurrency, component size, +worker pool utilization, and host load can change both absolute and relative +results. diff --git a/packages/jco/test/bench/componentize-backends-runner.mjs b/packages/jco/test/bench/componentize-backends-runner.mjs new file mode 100644 index 000000000..099eac57c --- /dev/null +++ b/packages/jco/test/bench/componentize-backends-runner.mjs @@ -0,0 +1,34 @@ +import { WASIShim } from "@bytecodealliance/preview2-shim/instantiation"; +import { createRequire } from "node:module"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const jco = fileURLToPath(new URL("../../dist/jco.js", import.meta.url)); +const requireJco = createRequire(jco); +const host = (name) => + import(pathToFileURL(requireJco.resolve(`@bytecodealliance/jco-std/wasi/0.2.x/node/24.x.x/${name}`)).href); +const [fs, console, http] = await Promise.all([host("fs/host/node"), host("console/host/node"), host("http/host")]); +const input = fileURLToPath(new URL("./fixtures/filesystem/data/input.txt", import.meta.url)); +const mapPath = (path) => (path.tag === "text" && path.val === "/data/input.txt" ? { ...path, val: input } : path); +const wasi = new WASIShim({ + sandbox: { preopens: { "/data": fileURLToPath(new URL("./fixtures/filesystem/data/", import.meta.url)) } }, +}); +const imports = { + ...wasi.getImportObject(), + "jco:node/fs": { + ...fs, + readFile: (file, options) => + fs.readFile(file.tag === "path" ? { ...file, val: mapPath(file.val) } : file, options), + stat: (path, options) => fs.stat(mapPath(path), options), + }, + "jco:node/console": console, + "jco:node/http": http, +}; +const start = process.hrtime.bigint(); +const { instantiate } = await import(pathToFileURL(process.argv[2]).href); +const { run } = await instantiate(undefined, imports); +const runStart = process.hrtime.bigint(); +await run.run(); +const end = process.hrtime.bigint(); +const runMs = Number(end - runStart) / 1e6; +const loadAndRunMs = Number(end - start) / 1e6; +process.stderr.write(`${JSON.stringify({ runMs, loadAndRunMs })}\n`); diff --git a/packages/jco/test/bench/componentize-backends.mjs b/packages/jco/test/bench/componentize-backends.mjs new file mode 100644 index 000000000..3c5f18256 --- /dev/null +++ b/packages/jco/test/bench/componentize-backends.mjs @@ -0,0 +1,272 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { cp, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { cpus, release } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +const root = fileURLToPath(new URL("../../", import.meta.url)); +const fixtures = fileURLToPath(new URL("./fixtures/filesystem/", import.meta.url)); +const output = join(root, "test", "output", "bench", "componentize-backends"); +const jco = join(root, "dist", "jco.js"); +const requireJco = createRequire(jco); +const backends = ["perry", "starlingmonkey", "quickjs"]; +const optimizationArgs = [ + "-Oz", + "--low-memory-unused", + "--enable-bulk-memory", + "--enable-reference-types", + "--strip-debug", +]; +const { values } = parseArgs({ + options: { + samples: { type: "string", default: "20" }, + warmup: { type: "string", default: "3" }, + "skip-build": { type: "boolean", default: false }, + "wasm-opt-bin": { type: "string" }, + "perry-cli-path": { type: "string" }, + }, +}); +const samples = Number(values.samples); +const warmup = Number(values.warmup); +const compiler = values["perry-cli-path"] || process.env.JCO_PERRY_CLI_PATH; +const perryCliPath = compiler ? resolve(compiler) : null; +assert.ok(Number.isInteger(samples) && samples > 0, "--samples must be a positive integer"); +assert.ok(Number.isInteger(warmup) && warmup >= 0, "--warmup must be a nonnegative integer"); + +function command(script, args = []) { + execFileSync(process.execPath, [script, ...args], { cwd: root, stdio: "inherit" }); +} + +async function packageVersion(name) { + const directories = + name === "@bytecodealliance/jco" + ? [join(dirname(jco), "..")] + : (requireJco.resolve.paths(name) ?? []).map((directory) => join(directory, name)); + for (const directory of directories) { + try { + const manifest = JSON.parse(await readFile(join(directory, "package.json"), "utf8")); + if (manifest.name === name) { + return manifest.version; + } + } catch (error) { + if (error.code !== "ENOENT") { + throw error; + } + } + } + throw new Error(`Cannot find the version of ${name}`); +} + +function median(numbers) { + const sorted = [...numbers].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; +} + +function componentSize(bytes) { + if (bytes < 1024) { + return `${bytes} B`; + } + if (bytes < 1024 ** 2) { + return `${(bytes / 1024).toFixed(2)} KiB`; + } + return `${(bytes / 1024 ** 2).toFixed(2)} MiB`; +} + +await mkdir(output, { recursive: true }); +const guest = await readFile(join(fixtures, "guest.ts"), "utf8"); +const guestSha256 = createHash("sha256").update(guest).digest("hex"); +if (!values["skip-build"]) { + assert.ok(perryCliPath, "Pass --perry-cli-path or set JCO_PERRY_CLI_PATH to a WASI-enabled Perry compiler"); + + // Keep the fixture's read/stat/print statements inside run(): StarlingMonkey + // preinitializes module-level code while building the component. + const ts = requireJco("typescript-compiler-api"); + const source = ts.createSourceFile("component.ts", guest, ts.ScriptTarget.Latest, true); + const imports = source.statements.filter(ts.isImportDeclaration); + const statements = source.statements.filter((statement) => !ts.isImportDeclaration(statement)); + const entry = join(output, "command.ts"); + await writeFile( + entry, + `${imports.map((statement) => statement.getFullText(source)).join("\n")} +export const run = { run() { +${statements.map((statement) => statement.getFullText(source)).join("\n")} +} }; +`, + ); + + for (const backend of backends) { + const directory = join(output, backend); + await rm(directory, { recursive: true, force: true }); + await mkdir(directory); + const component = join(directory, "component.wasm"); + if (backend === "perry") { + command(jco, [ + "componentize", + join(fixtures, "guest.ts"), + "--backend", + "perry", + "--backend-perry-cli-path", + perryCliPath, + "-o", + component, + ]); + } else { + const wit = join(directory, "wit"); + await cp(join(root, "lib", "wit", "builtin", "0.2.12"), join(wit, "deps"), { recursive: true }); + await writeFile( + join(wit, "package.wit"), + "package benchmark:filesystem-comparison;\nworld command { export wasi:cli/run@0.2.12; }\n", + ); + command(jco, [ + "componentize", + entry, + "--backend", + backend, + "--wit", + wit, + "--world-name", + "benchmark:filesystem-comparison/command", + "-o", + component, + ...(backend === "quickjs" ? ["--backend-qjs-disable-async"] : []), + ]); + } + const optimized = join(directory, "component.opt.wasm"); + command(jco, [ + "opt", + component, + "-o", + optimized, + ...(values["wasm-opt-bin"] ? ["--wasm-opt-bin", values["wasm-opt-bin"]] : []), + "--", + ...optimizationArgs, + ]); + command(jco, [ + "transpile", + optimized, + "--name", + "component", + "--bindgen-enable-wasm-exnref", + "--instantiation", + "async", + "-o", + join(directory, "transpiled"), + ]); + } + const jsLauncher = perryCliPath && /\.[cm]?js$/i.test(perryCliPath); + const perryVersion = perryCliPath + ? execFileSync( + jsLauncher ? process.execPath : perryCliPath, + [...(jsLauncher ? [perryCliPath] : []), "--version"], + { + cwd: root, + encoding: "utf8", + }, + ).trim() + : null; + await writeFile( + join(output, "build.json"), + `${JSON.stringify( + { + guestSha256, + jco: await packageVersion("@bytecodealliance/jco"), + componentizeJs: await packageVersion("@bytecodealliance/componentize-js"), + componentizeQjs: await packageVersion("componentize-qjs"), + optimizationArgs, + wasmOptVersion: execFileSync( + process.execPath, + [values["wasm-opt-bin"] || requireJco.resolve("binaryen/bin/wasm-opt"), "--version"], + { + cwd: root, + encoding: "utf8", + }, + ).trim(), + wasmOptLauncher: values["wasm-opt-bin"] ?? null, + perryRef: process.env.JCO_PERRY_REF ?? null, + perryCliPath, + perryVersion, + }, + null, + 2, + )}\n`, + ); +} + +const build = JSON.parse(await readFile(join(output, "build.json"), "utf8")); +assert.equal(build.guestSha256, guestSha256, "Guest changed; rerun without --skip-build"); +assert.deepEqual(build.optimizationArgs, optimizationArgs, "Optimization changed; rerun without --skip-build"); +const input = await readFile(join(fixtures, "data", "input.txt")); +const expected = `Perry size: ${input.length}\nPerry read: ${input.toString("utf8")}\n`.replaceAll("\r\n", "\n"); +const measurements = Object.fromEntries(backends.map((backend) => [backend, []])); +// Rotate backend order between rounds to reduce bias from changing machine load. +for (let round = -warmup; round < samples; round++) { + const offset = (round + warmup) % backends.length; + for (let index = 0; index < backends.length; index++) { + const backend = backends[(index + offset) % backends.length]; + const module = join(output, backend, "transpiled", "component.js"); + const start = process.hrtime.bigint(); + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL("./componentize-backends-runner.mjs", import.meta.url)), module], + { + cwd: root, + encoding: "utf8", + timeout: 60_000, + }, + ); + const processMs = Number(process.hrtime.bigint() - start) / 1e6; + if (result.error) { + throw result.error; + } + assert.equal(result.status, 0, `${backend} failed: ${result.stderr}`); + assert.equal(result.stdout.replaceAll("\r\n", "\n"), expected, `${backend} produced different output`); + const timing = JSON.parse(result.stderr.trim()); + if (round >= 0) { + measurements[backend].push({ ...timing, processMs }); + } + } +} + +const rows = await Promise.all( + backends.map(async (backend) => ({ + backend, + componentBytes: (await stat(join(output, backend, "component.wasm"))).size, + optimizedComponentBytes: (await stat(join(output, backend, "component.opt.wasm"))).size, + runMs: median(measurements[backend].map((sample) => sample.runMs)), + loadAndRunMs: median(measurements[backend].map((sample) => sample.loadAndRunMs)), + processMs: median(measurements[backend].map((sample) => sample.processMs)), + samples: measurements[backend], + })), +); +const report = { + measuredAt: new Date().toISOString(), + node: process.version, + platform: process.platform, + arch: process.arch, + osRelease: release(), + cpu: cpus()[0]?.model, + samples, + warmup, + inputBytes: input.length, + inputSha256: createHash("sha256").update(input).digest("hex"), + build, + results: rows, +}; +await writeFile(join(output, "results.json"), `${JSON.stringify(report, null, 2)}\n`); +console.log( + "\n| Backend | As-built size | Optimized size | Run (median ms) | Load + run (median ms) | Node process (median ms) |", +); +console.log("| --- | ---: | ---: | ---: | ---: | ---: |"); +for (const row of rows) { + console.log( + `| ${row.backend} | ${componentSize(row.componentBytes)} | ${componentSize(row.optimizedComponentBytes)} | ${row.runMs.toFixed(2)} | ${row.loadAndRunMs.toFixed(2)} | ${row.processMs.toFixed(2)} |`, + ); +} +console.log( + `\nOptimized components: ${samples} samples/backend after ${warmup} warmups; details: ${join(output, "results.json")}`, +); diff --git a/packages/jco/test/bench/fixtures/filesystem/data/input.txt b/packages/jco/test/bench/fixtures/filesystem/data/input.txt new file mode 100644 index 000000000..caad6c5df --- /dev/null +++ b/packages/jco/test/bench/fixtures/filesystem/data/input.txt @@ -0,0 +1 @@ +hello from the filesystem diff --git a/packages/jco/test/bench/fixtures/filesystem/guest.ts b/packages/jco/test/bench/fixtures/filesystem/guest.ts new file mode 100644 index 000000000..68a54a0d4 --- /dev/null +++ b/packages/jco/test/bench/fixtures/filesystem/guest.ts @@ -0,0 +1,6 @@ +import { readFileSync, statSync } from "node:fs"; + +// Perry lowers these Node-style calls to WASI P2 filesystem operations. +const input: string = readFileSync("/data/input.txt", "utf8"); +console.log(`Perry size: ${statSync("/data/input.txt").size}`); +console.log(`Perry read: ${input}`); From 8060028600d4f1db42a3ac7b4014f2d9a5e3c959 Mon Sep 17 00:00:00 2001 From: Victor Adossi Date: Thu, 22 Oct 2026 16:02:20 +0000 Subject: [PATCH 4/6] fix(ci): run workspace tests with required tools and path filters --- .github/workflows/main.yml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 251de0db6..2a1efda19 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -713,6 +713,11 @@ jobs: with: persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + # Differential tests use fixtures recorded from this exact Node release. + node-version: "24.20.0" + - name: Install Rust run: rustup update stable --no-self-update - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -739,18 +744,25 @@ jobs: run: | pnpm install + - name: Install Puppeteer browser + if: matrix.workspace == 'rolldown-plugin-jco' + run: pnpm run test:setup:puppeteer + - name: Build workspace dependencies and package run: | if [ "${{ matrix.workspace }}" = "jco-std" ]; then - pnpm --filter "packages/jco-transpile" run build - pnpm --filter "packages/${{ matrix.workspace }}" run build + pnpm --filter "./packages/jco-transpile" --fail-if-no-match run build + pnpm --filter "./packages/${{ matrix.workspace }}" --fail-if-no-match run build else - pnpm --filter "packages/${{ matrix.workspace }}" run build + pnpm --filter "./packages/${{ matrix.workspace }}" --fail-if-no-match run build fi - name: run test on workspace + env: + TEST_PUPPETEER_LAUNCH_ARGS: --no-sandbox + TERM: xterm-256color run: | - pnpm --filter "packages/${{ matrix.workspace }}" run test + pnpm --filter "./packages/${{ matrix.workspace }}" --fail-if-no-match run test examples: needs: @@ -897,6 +909,8 @@ jobs: chmod: 0755 - uses: bytecodealliance/actions/wasmtime/setup@9152e710e9f7182e4c29ad218e4f335a7b203613 # v1.1.3 + with: + version: v50.0.0 - name: Cache node modules uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -942,7 +956,7 @@ jobs: env: TEST_PUPPETEER_LAUNCH_ARGS: --no-sandbox run: | - pnpm --filter '${{ matrix.project.workspace }}' run all + pnpm --filter './${{ matrix.project.workspace }}' --fail-if-no-match run all examples-perry: # Perry's WASI target is not released yet. Keep this compiler setup separate From 286948ca634683a4e40c9b2a8192a33d96a68a10 Mon Sep 17 00:00:00 2001 From: Victor Adossi Date: Fri, 23 Oct 2026 04:17:54 +0000 Subject: [PATCH 5/6] perf(jco): minimize Perry runtime in backend benchmarks --- packages/jco/test/bench/README.md | 124 ++++++++++-------- .../jco/test/bench/componentize-backends.mjs | 42 ++++-- packages/jco/test/bench/perry-runtime.mjs | 109 +++++++++++++++ 3 files changed, 210 insertions(+), 65 deletions(-) create mode 100644 packages/jco/test/bench/perry-runtime.mjs diff --git a/packages/jco/test/bench/README.md b/packages/jco/test/bench/README.md index 8fb8c52c5..5e3a8dd29 100644 --- a/packages/jco/test/bench/README.md +++ b/packages/jco/test/bench/README.md @@ -7,10 +7,10 @@ excluded from the normal test suite and are not CI performance gates. Run the commands below from `packages/jco/test/bench`, or use the package scripts from the repository root: -| Benchmark | Command from this directory | Package script | -| ------------------------- | ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| Componentization backends | `node componentize-backends.mjs --perry-cli-path /absolute/path/to/perry` | `pnpm --filter @bytecodealliance/jco bench:componentize-backends --perry-cli-path /absolute/path/to/perry` | -| Serve request isolation | `node serve-request-isolation.js` | `pnpm --filter @bytecodealliance/jco bench:serve-isolation` | +| Benchmark | Command from this directory | Package script | +| ------------------------- | ----------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| Componentization backends | `node componentize-backends.mjs --perry-cli-path /path/to/perry --perry-source /path/to/perry-source` | `pnpm --filter @bytecodealliance/jco bench:componentize-backends --perry-cli-path /path/to/perry --perry-source /path/to/perry-source` | +| Serve request isolation | `node serve-request-isolation.js` | `pnpm --filter @bytecodealliance/jco bench:serve-isolation` | ## Componentization backends @@ -23,22 +23,36 @@ The benchmark does not build or run the Perry example. ### Setup and running -Perry must support `--target wasi`, with its matching runtime archive and -wasi-sdk configured. The [Perry example's installer](../../../../examples/components/perry-filesystem/README.md#prerequisites) -can provision a pinned source-built toolchain without sudo. Its setup is a -separate prerequisite; this benchmark does not install compilers. - -| Setting | Purpose | -| ------------------------------------------ | ----------------------------------------------------------------- | -| `--perry-cli-path` or `JCO_PERRY_CLI_PATH` | Path to the WASI-enabled Perry executable or JavaScript launcher. | -| `PERRY_RUNTIME_DIR` | Directory containing the matching `libperry_runtime.a`. | -| `WASI_SDK_PATH` | wasi-sdk installation directory. | -| `JCO_PERRY_REF` (optional) | Source revision to record alongside the compiler version. | - -After configuring the runtime and SDK in your environment: +Use a WASI-enabled compiler built from Perry commit +`3945beb23ebeca54846a30d89cb4971de59b0c6c`, a clean checkout of that revision, +wasi-sdk 34, and Rust `nightly-2026-10-08` with the `wasm32-wasip2` target. +The [Perry example's installer](../../../../examples/components/perry-filesystem/README.md#prerequisites) +can provision these prerequisites; the benchmark does not install compilers. + +Every full benchmark build rebuilds Perry's runtime with **optimization level +`z`, fat LTO, and default features disabled**, retaining `keepalive-anchors` +to preserve the entry points called by generated code. +This is the smallest profile validated for this filesystem fixture. Cargo caches +unchanged builds. The benchmark selects the resulting archive automatically, +overriding `PERRY_RUNTIME_DIR`, and records the recipe, archive SHA-256, source +revision, and toolchain versions in its build metadata. The normal Perry +installation is unchanged. This feature subset is specific to the fixture; +other applications may need additional runtime features. + +| Setting | Purpose | +| ------------------------------------------ | ---------------------------------------------------------------------- | +| `--perry-cli-path` or `JCO_PERRY_CLI_PATH` | Matching WASI-enabled Perry executable or JavaScript launcher. | +| `--perry-source` or `JCO_PERRY_SOURCE` | Clean checkout of the pinned Perry revision. | +| `WASI_SDK_PATH` | wasi-sdk installation directory. | +| `CARGO_TARGET_DIR` (optional) | Reuse a Cargo build cache; defaults to the benchmark output directory. | +| `CARGO_BUILD_JOBS` (optional) | Runtime build parallelism; defaults to 2. | + +After setting `WASI_SDK_PATH`, run: ```sh -node componentize-backends.mjs --perry-cli-path /absolute/path/to/perry +node componentize-backends.mjs \ + --perry-cli-path /path/to/perry \ + --perry-source /path/to/perry-source ``` The script checks every sample's output, prints sizes in KiB/MiB and timings in @@ -47,18 +61,20 @@ input hashes, and host details to `packages/jco/test/output/bench/componentize-backends/results.json`. Component artifacts and build metadata are retained in the same ignored directory. Use `--samples 50 --warmup 5` to change the default sample counts, or -`--skip-build` to reuse unchanged components without requiring the compiler. +`--skip-build` to reuse unchanged components without requiring the toolchain. +Cached builds from the old runtime recipe are rejected and must be rebuilt. The optional `--wasm-opt-bin` setting is passed to Jco's optimizer launcher hook. -Build and transpilation time are excluded from the measurements. +Runtime compilation, componentization, optimization, and transpilation time are +excluded from the measurements. ### Recent results -Example results measured on 2026-10-22 on a shared Linux x86_64 development host +Example results measured on 2026-10-23 on a shared Linux x86_64 development host (Intel Xeon at 2.60 GHz, Node.js 24.21.0), using the bundled 26-byte input file: | Backend | As-built size | Optimized size | | -------------------------------- | ------------: | -------------: | -| Perry | 11.42 MiB | 7.40 MiB | +| Perry | 6.80 MiB | 3.21 MiB | | componentize-js (StarlingMonkey) | 15.99 MiB | 15.42 MiB | | componentize-qjs (QuickJS) | 6.79 MiB | 6.66 MiB | @@ -66,17 +82,20 @@ Timings for the optimized components: | Backend | Run (median ms) | Load + run (median ms) | Node process (median ms) | | -------------------------------- | --------------: | ---------------------: | -----------------------: | -| Perry | 91.77 | 168.26 | 310.67 | -| componentize-js (StarlingMonkey) | 51.03 | 212.28 | 421.66 | -| componentize-qjs (QuickJS) | 36.17 | 136.58 | 361.60 | +| Perry | 98.80 | 153.66 | 301.14 | +| componentize-js (StarlingMonkey) | 59.58 | 228.90 | 465.38 | +| componentize-qjs (QuickJS) | 40.96 | 145.42 | 403.16 | -Tool versions: Jco `d5ae42df` (package version 1.39.0), componentize-js 0.23.0, +Tool versions: Jco `feaa7d23` (package version 1.39.0) with this runtime build +recipe, componentize-js 0.23.0, componentize-qjs 0.4.5, and Perry 0.5.1659 built at commit `3945beb23ebeca54846a30d89cb4971de59b0c6c` with LLVM 22.1.8 and wasi-sdk 34. All three components were optimized with Binaryen 132 using `-Oz --low-memory-unused --enable-bulk-memory --enable-reference-types --strip-debug`. These are example measurements from one -host; rerun the comparison for results on your machine. +host; rerun the comparison for results on your machine. A local shim compatibility +workaround enabled filesystem access for componentize-js's build-time WIT splicer; +it was disabled for timings. - **Component size:** the uncompressed `.wasm` component before transpilation, reported both as built and after optimization. MiB means 1,048,576 bytes. @@ -106,34 +125,35 @@ These measurements compare this fixture's complete backend paths. The tiny file and console output make them sensitive to host overhead; they do not establish general JavaScript throughput or performance under sustained load. -### Perry component size - -AoT compilation removes the need to interpret this guest's TypeScript, but its -JavaScript semantics still require runtime support: object layouts, strings, -closures, exceptions, built-ins, and garbage collection. This pinned Perry build -links a broad runtime. Its [WASI link path][perry-wasi-link] explicitly skips the -app-specific runtime rebuild used by native targets, and the -[WASI runtime build script][perry-wasi-runtime] enables almost all default runtime -features. - -Inspecting the unoptimized core module found 9,630 defined functions and about -8.12 MiB of Wasm instructions, including broad object/built-in support, GC, -streams, regular expressions, and filesystem helpers. It also contained about -1.63 MiB of DWARF debug information and 0.75 MiB of function names. The optimized -size above removes debug information and runs size optimization and dead-code -elimination; the remaining payload is mostly runtime code. - -The emitted function names contain no SWC parser or dynamic-evaluation -interpreter functions. Perry supports an optional interpreter for runtime-created -code, but the evidence here does not attribute the size to that interpreter. -Instead, general runtime dispatch and built-in registries retain helpers that a -small statically known guest should ideally avoid. Once helpers are reachable -through those registries, ordinary linker/Binaryen dead-code elimination cannot -assume they will never be called. Further reductions would require specializing -the WASI runtime to the program's needs, then measuring the resulting component. +### Why Perry components are still several MiB + +The size-focused runtime reduces this fixture from **7.40 MiB to 3.21 MiB**. +The guest's original generated functions total only **1,918 bytes**; most of the +payload implements JavaScript semantics: + +- **Broad runtime defaults:** Perry's [WASI link path][perry-wasi-link] bypasses + native per-application specialization, while its [runtime builder][perry-wasi-runtime] + enables nearly all default features, including diagnostics. The native + `PERRY_SIZE_OPT` / `PERRY_SIZE_LTO` settings have no effect here, so this + benchmark explicitly rebuilds a reduced runtime at level `z` with fat LTO. +- **Reachable generic helpers:** object property reads, string coercion, + exceptions, built-ins, and GC retain general runtime code. Importing `node:fs` + also [registers streams][perry-fs-registry]. These dispatch paths keep helpers + reachable even when the fixture never calls them directly, limiting dead-code + elimination. +- **Small packaging overhead:** the 7.40 MiB baseline already had debug data + stripped; component wiring and auxiliary modules added only 19 KiB. An empty + Perry program still optimized to 6.30 MiB with the default runtime. Equivalent + C file I/O using the same SDK produced 60.9 KiB, without JS semantics. + +The reduced runtime passed changed-file, Unicode, and larger-input checks. +Hundreds of KiB will require narrower dispatch or code generation; Perry's +[tiny-program specialization][perry-tiny-program] currently excludes WASI. [perry-wasi-link]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/crates/perry/src/commands/compile/link/wasi.rs [perry-wasi-runtime]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/scripts/wasi_build_runtime.py +[perry-fs-registry]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/crates/perry-runtime/src/object/native_module_registry.rs#L361-L370 +[perry-tiny-program]: https://github.com/PerryTS/perry/blob/3945beb23ebeca54846a30d89cb4971de59b0c6c/crates/perry/src/commands/compile/tiny_program/mod.rs#L204-L220 ## Serve request isolation diff --git a/packages/jco/test/bench/componentize-backends.mjs b/packages/jco/test/bench/componentize-backends.mjs index 3c5f18256..cc6f73cd6 100644 --- a/packages/jco/test/bench/componentize-backends.mjs +++ b/packages/jco/test/bench/componentize-backends.mjs @@ -8,6 +8,8 @@ import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; +import { buildPerryRuntime, perryRuntimeRecipe } from "./perry-runtime.mjs"; + const root = fileURLToPath(new URL("../../", import.meta.url)); const fixtures = fileURLToPath(new URL("./fixtures/filesystem/", import.meta.url)); const output = join(root, "test", "output", "bench", "componentize-backends"); @@ -28,6 +30,7 @@ const { values } = parseArgs({ "skip-build": { type: "boolean", default: false }, "wasm-opt-bin": { type: "string" }, "perry-cli-path": { type: "string" }, + "perry-source": { type: "string" }, }, }); const samples = Number(values.samples); @@ -37,8 +40,8 @@ const perryCliPath = compiler ? resolve(compiler) : null; assert.ok(Number.isInteger(samples) && samples > 0, "--samples must be a positive integer"); assert.ok(Number.isInteger(warmup) && warmup >= 0, "--warmup must be a nonnegative integer"); -function command(script, args = []) { - execFileSync(process.execPath, [script, ...args], { cwd: root, stdio: "inherit" }); +function command(script, args = [], env = process.env) { + execFileSync(process.execPath, [script, ...args], { cwd: root, env, stdio: "inherit" }); } async function packageVersion(name) { @@ -82,6 +85,9 @@ const guest = await readFile(join(fixtures, "guest.ts"), "utf8"); const guestSha256 = createHash("sha256").update(guest).digest("hex"); if (!values["skip-build"]) { assert.ok(perryCliPath, "Pass --perry-cli-path or set JCO_PERRY_CLI_PATH to a WASI-enabled Perry compiler"); + // A failed rebuild must not leave a receipt for partially replaced artifacts. + await rm(join(output, "build.json"), { force: true }); + const perryRuntime = await buildPerryRuntime(values["perry-source"] || process.env.JCO_PERRY_SOURCE, output); // Keep the fixture's read/stat/print statements inside run(): StarlingMonkey // preinitializes module-level code while building the component. @@ -105,16 +111,20 @@ ${statements.map((statement) => statement.getFullText(source)).join("\n")} await mkdir(directory); const component = join(directory, "component.wasm"); if (backend === "perry") { - command(jco, [ - "componentize", - join(fixtures, "guest.ts"), - "--backend", - "perry", - "--backend-perry-cli-path", - perryCliPath, - "-o", - component, - ]); + command( + jco, + [ + "componentize", + join(fixtures, "guest.ts"), + "--backend", + "perry", + "--backend-perry-cli-path", + perryCliPath, + "-o", + component, + ], + perryRuntime.env, + ); } else { const wit = join(directory, "wit"); await cp(join(root, "lib", "wit", "builtin", "0.2.12"), join(wit, "deps"), { recursive: true }); @@ -187,9 +197,10 @@ ${statements.map((statement) => statement.getFullText(source)).join("\n")} }, ).trim(), wasmOptLauncher: values["wasm-opt-bin"] ?? null, - perryRef: process.env.JCO_PERRY_REF ?? null, + perryRef: perryRuntime.metadata.recipe.perryRef, perryCliPath, perryVersion, + perryRuntime: perryRuntime.metadata, }, null, 2, @@ -200,6 +211,11 @@ ${statements.map((statement) => statement.getFullText(source)).join("\n")} const build = JSON.parse(await readFile(join(output, "build.json"), "utf8")); assert.equal(build.guestSha256, guestSha256, "Guest changed; rerun without --skip-build"); assert.deepEqual(build.optimizationArgs, optimizationArgs, "Optimization changed; rerun without --skip-build"); +assert.deepEqual( + build.perryRuntime?.recipe, + perryRuntimeRecipe, + "Perry runtime recipe changed; rerun without --skip-build", +); const input = await readFile(join(fixtures, "data", "input.txt")); const expected = `Perry size: ${input.length}\nPerry read: ${input.toString("utf8")}\n`.replaceAll("\r\n", "\n"); const measurements = Object.fromEntries(backends.map((backend) => [backend, []])); diff --git a/packages/jco/test/bench/perry-runtime.mjs b/packages/jco/test/bench/perry-runtime.mjs new file mode 100644 index 000000000..8b16946f0 --- /dev/null +++ b/packages/jco/test/bench/perry-runtime.mjs @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { copyFile, mkdir, readFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +import pins from "../../../../examples/components/perry-filesystem/scripts/perry-toolchain.json" with { type: "json" }; + +// This feature subset covers the filesystem fixture, not arbitrary Perry apps. +export const perryRuntimeRecipe = { + perryRef: pins.perryRef, + rustToolchain: pins.rustToolchain, + target: "wasm32-wasip2", + profile: "wasi-size", + optLevel: "z", + lto: "fat", + defaultFeatures: false, + features: ["perry-runtime/keepalive-anchors"], +}; + +export async function buildPerryRuntime(source, output) { + assert.ok(source, "Pass --perry-source or set JCO_PERRY_SOURCE to the pinned Perry checkout"); + assert.ok(process.env.WASI_SDK_PATH, "Set WASI_SDK_PATH to the wasi-sdk installation"); + source = resolve(source); + const sdk = resolve(process.env.WASI_SDK_PATH); + const revision = execFileSync("git", ["rev-parse", "HEAD"], { cwd: source, encoding: "utf8" }).trim(); + assert.equal(revision, pins.perryRef, "Perry source must match the benchmark's pinned toolchain"); + execFileSync("git", ["diff", "--exit-code", "HEAD", "--"], { cwd: source, stdio: "inherit" }); + + const directory = join(output, "perry-runtime"); + await mkdir(directory, { recursive: true }); + const executable = (name) => (process.platform === "win32" ? `${name}.exe` : name); + const env = { + ...process.env, + RUSTUP_TOOLCHAIN: perryRuntimeRecipe.rustToolchain, + CARGO_TARGET_DIR: resolve(process.env.CARGO_TARGET_DIR || join(directory, "target")), + CARGO_BUILD_JOBS: process.env.CARGO_BUILD_JOBS || "2", + CC_wasm32_wasip2: join(sdk, "bin", executable("clang")), + AR_wasm32_wasip2: join(sdk, "bin", executable("llvm-ar")), + CFLAGS_wasm32_wasip2: `--target=wasm32-wasip2 --sysroot="${join(sdk, "share", "wasi-sysroot").replaceAll("\\", "/").replaceAll('"', '\\"')}"`, + CC_SHELL_ESCAPED_FLAGS: "1", + CARGO_PROFILE_WASI_SIZE_OPT_LEVEL: perryRuntimeRecipe.optLevel, + CARGO_PROFILE_WASI_SIZE_LTO: perryRuntimeRecipe.lto, + }; + const args = [ + "build", + "--locked", + "--profile", + perryRuntimeRecipe.profile, + "--config", + 'profile.wasi-size.inherits="release"', + "--config", + `profile.wasi-size.opt-level="${perryRuntimeRecipe.optLevel}"`, + // Perry overrides this package to level 3 in the inherited release profile. + "--config", + `profile.wasi-size.package.perry-runtime.opt-level="${perryRuntimeRecipe.optLevel}"`, + "--config", + `profile.wasi-size.lto="${perryRuntimeRecipe.lto}"`, + "-p", + "perry-runtime-static", + "--target", + perryRuntimeRecipe.target, + "--no-default-features", + "--features", + perryRuntimeRecipe.features.join(","), + "--message-format=json-render-diagnostics", + ]; + console.log("Building Perry's filesystem runtime with reduced features, opt-level=z and fat LTO..."); + const messages = execFileSync("cargo", args, { + cwd: source, + env, + encoding: "utf8", + stdio: ["inherit", "pipe", "inherit"], + maxBuffer: 16 * 1024 * 1024, + }); + // Observe Cargo's artifact, including cache hits, instead of assuming an old + // archive at a conventional path was rebuilt with this recipe. + const artifact = messages + .trim() + .split("\n") + .map((line) => JSON.parse(line)) + .find( + (message) => + message.reason === "compiler-artifact" && + message.target.name === "perry_runtime" && + message.target.kind.includes("staticlib"), + ); + const archive = artifact?.filenames.find((path) => path.endsWith("libperry_runtime.a")); + assert.ok(archive, "Cargo did not report a Perry runtime static archive"); + const installed = join(directory, "libperry_runtime.a"); + await copyFile(archive, installed); + const bytes = await readFile(installed); + assert.ok(bytes.length, "Perry runtime archive is empty"); + return { + env: { ...process.env, WASI_SDK_PATH: sdk, PERRY_RUNTIME_DIR: directory }, + metadata: { + recipe: perryRuntimeRecipe, + source, + archive: installed, + archiveSha256: createHash("sha256").update(bytes).digest("hex"), + rustcVersion: execFileSync("rustc", ["--version"], { env, encoding: "utf8" }).trim(), + wasiSdkPath: sdk, + clangVersion: execFileSync(join(sdk, "bin", executable("clang")), ["--version"], { + encoding: "utf8", + }).trim(), + cargoArgs: args, + }, + }; +} From e419b86e7cb1496e8d5fb646469a68698ec8358a Mon Sep 17 00:00:00 2001 From: Victor Adossi Date: Fri, 23 Oct 2026 09:52:28 +0000 Subject: [PATCH 6/6] feat(jco): benchmark complete-component compilation speed --- packages/jco/test/bench/README.md | 96 ++++- .../jco/test/bench/componentize-backends.mjs | 395 +++++++++++------- 2 files changed, 335 insertions(+), 156 deletions(-) diff --git a/packages/jco/test/bench/README.md b/packages/jco/test/bench/README.md index 5e3a8dd29..6828fabf4 100644 --- a/packages/jco/test/bench/README.md +++ b/packages/jco/test/bench/README.md @@ -55,17 +55,99 @@ node componentize-backends.mjs \ --perry-source /path/to/perry-source ``` -The script checks every sample's output, prints sizes in KiB/MiB and timings in -milliseconds, and writes exact byte counts, raw timing samples, tool versions, +The script checks every build and runtime sample's output, prints sizes in KiB/MiB +and timings in milliseconds, and writes exact byte counts, raw timing samples, tool versions, input hashes, and host details to `packages/jco/test/output/bench/componentize-backends/results.json`. Component artifacts and build metadata are retained in the same ignored directory. +Use `--output-dir /path/to/artifacts` to select another location; pass the same +location when using `--skip-build`. Use `--samples 50 --warmup 5` to change the default sample counts, or `--skip-build` to reuse unchanged components without requiring the toolchain. Cached builds from the old runtime recipe are rejected and must be rebuilt. The optional `--wasm-opt-bin` setting is passed to Jco's optimizer launcher hook. -Runtime compilation, componentization, optimization, and transpilation time are -excluded from the measurements. +Componentization and optimization are measured separately from runtime execution. +Compiler/engine/runtime builds and transpilation are excluded from both timing groups. + +### Compilation speed + +A full run measures **TS/JS source to a complete, usable Wasm component**, including +the JavaScript engine for StarlingMonkey and QuickJS. It does not measure a +script-only payload that requires a separately deployed engine. + +The compilation table reports three medians: + +- **TS/JS → component:** wall time around a fresh `jco componentize` process, + through successful exit after writing the component. This includes CLI startup, + TypeScript lowering and bundling, backend compilation or preinitialization, + linking/component encoding, and writing the output. Perry's CLI capability + probe is included as part of the normal Jco workflow. +- **Optimize:** wall time around a fresh `jco opt` process on that sample's + component, using the common optimization flags. +- **TS/JS → optimized component:** median of each sample's componentization plus + optimization time. This need not equal the sum of the first two medians. + +The default is **five measured builds per backend after one warmup round**. +`--build-samples` and `--build-warmup` control these independently of the runtime +`--samples` and `--warmup` settings. Builds run sequentially, rotating backend order +between rounds. Every build starts with fresh output and WIT directories; Jco's +injected WIT imports cannot accumulate between samples. Prebuilt engine binaries, +Perry's prepared runtime archive, and OS/filesystem caches are reused. These are +warm-cache application builds, not clean-machine toolchain bootstrap timings. + +The benchmark prepares its command wrapper and WIT fixture outside the timer; +the timed CLI still performs the actual TypeScript transformation and bundling. +Perry compiler installation, its one-time runtime archive preparation, dependency +installation, transpilation and validation are also outside the timer. Every +as-built **and** optimized component is transpiled and executed after the timed +commands; incorrect output fails the benchmark instead of becoming a timing +sample. The final optimized artifacts are reused for runtime measurements. + +```sh +node componentize-backends.mjs \ + --perry-cli-path /path/to/perry \ + --perry-source /path/to/perry-source \ + --build-samples 5 --build-warmup 1 \ + --samples 20 --warmup 3 +``` + +`build.json` and `results.json` retain compilation medians, every raw sample's +stage timings and output sizes, round/order, sampling settings, host details, +input hash, and measurement timestamps under `build.compilation` in the results. +`--skip-build` runs only runtime measurements and explicitly labels compilation +results as reused with their original timestamp. Older build receipts without +compilation measurements remain usable but show no compilation table. + +For environments needing a componentize-js build-time shim workaround, +`JCO_BENCH_BUILD_PRELOAD=/absolute/path/to/preload.mjs` adds Node's `--import` only +to componentization processes. The path is recorded in build metadata and its +execution cost is included in compilation time. It is not loaded during +optimization, validation, or runtime samples. Leave it unset for normal builds. + +### Compilation results (2026-10-23) + +Five measured builds per backend after one warmup, on the Linux x86_64 Intel +Xeon 2.60 GHz host with Node.js 24.21.0: + +| Backend | TS/JS → component (median ms) | Optimize (median ms) | TS/JS → optimized component (median ms) | +| -------------- | ----------------------------: | -------------------: | --------------------------------------: | +| perry | 3328.06 | 10924.11 | 14098.32 | +| starlingmonkey | 6839.72 | 23536.53 | 30376.25 | +| quickjs | 2619.39 | 3500.37 | 6119.76 | + +QuickJS was fastest for both endpoints on this fixture. These measurements use +componentize-qjs 0.4.5, componentize-js 0.23.0 without Weval AOT, the pinned Perry +0.5.1659 compiler and reduced runtime recipe described above, and native Binaryen +132 through `--wasm-opt-bin`. They do not use the experimental local Perry +specialization or QuickJS 0.5's size runtime. Jco was rebuilt from the comparison +branch for this run; package versions alone do not identify local compiler changes. + +Output artifacts were written to tmpfs; normal OS/file caches were retained. +The componentize-js 0.23.0 WIT-splicer shim needed the build-only preopen workaround +described above; its startup cost is included. All 18 builds (warmups included) +passed execution checks for both their as-built and optimized components. Results +on other source graphs, hosts, disks, or tool versions can differ; this is not a +universal compiler-throughput ranking. ### Recent results @@ -99,7 +181,7 @@ it was disabled for timings. - **Component size:** the uncompressed `.wasm` component before transpilation, reported both as built and after optimization. MiB means 1,048,576 bytes. - All timings use the optimized components. + Runtime timings use the optimized components; compilation measures both output stages. - **Run:** the first command call on a fresh instance, including file read, stat, and output. - **Load + run:** importing and instantiating the transpiled component, then running it. - **Node process:** elapsed time from launching a fresh Node process until it exits, @@ -108,8 +190,8 @@ it was disabled for timings. All three use explicit asynchronous instantiation with the same host setup. Each backend runs in a fresh process for every sample. The default is 20 measured samples per backend after three warmup rounds, rotating backend order between -rounds. File and OS caches are not cleared. Builds and transpilation are excluded -from the timing. To repeat measurements without rebuilding unchanged components: +rounds. File and OS caches are not cleared. The runtime timing group excludes +builds and transpilation; compilation is reported separately. To repeat measurements without rebuilding unchanged components: ```sh node componentize-backends.mjs --skip-build --samples 50 --warmup 5 diff --git a/packages/jco/test/bench/componentize-backends.mjs b/packages/jco/test/bench/componentize-backends.mjs index cc6f73cd6..b157a44a5 100644 --- a/packages/jco/test/bench/componentize-backends.mjs +++ b/packages/jco/test/bench/componentize-backends.mjs @@ -1,62 +1,118 @@ -import assert from "node:assert/strict"; -import { execFileSync, spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; -import { cp, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises"; -import { createRequire } from "node:module"; -import { cpus, release } from "node:os"; -import { dirname, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { parseArgs } from "node:util"; +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { cp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { cpus, release } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; -import { buildPerryRuntime, perryRuntimeRecipe } from "./perry-runtime.mjs"; +import { buildPerryRuntime, perryRuntimeRecipe } from './perry-runtime.mjs'; -const root = fileURLToPath(new URL("../../", import.meta.url)); -const fixtures = fileURLToPath(new URL("./fixtures/filesystem/", import.meta.url)); -const output = join(root, "test", "output", "bench", "componentize-backends"); -const jco = join(root, "dist", "jco.js"); +const root = fileURLToPath(new URL('../../', import.meta.url)); +const fixtures = fileURLToPath(new URL('./fixtures/filesystem/', import.meta.url)); +const jco = join(root, 'dist', 'jco.js'); const requireJco = createRequire(jco); -const backends = ["perry", "starlingmonkey", "quickjs"]; +const backends = ['perry', 'starlingmonkey', 'quickjs']; const optimizationArgs = [ - "-Oz", - "--low-memory-unused", - "--enable-bulk-memory", - "--enable-reference-types", - "--strip-debug", + '-Oz', + '--low-memory-unused', + '--enable-bulk-memory', + '--enable-reference-types', + '--strip-debug', ]; const { values } = parseArgs({ options: { - samples: { type: "string", default: "20" }, - warmup: { type: "string", default: "3" }, - "skip-build": { type: "boolean", default: false }, - "wasm-opt-bin": { type: "string" }, - "perry-cli-path": { type: "string" }, - "perry-source": { type: "string" }, + samples: { type: 'string', default: '20' }, + warmup: { type: 'string', default: '3' }, + 'skip-build': { type: 'boolean', default: false }, + 'output-dir': { type: 'string' }, + 'build-samples': { type: 'string', default: '5' }, + 'build-warmup': { type: 'string', default: '1' }, + 'wasm-opt-bin': { type: 'string' }, + 'perry-cli-path': { type: 'string' }, + 'perry-source': { type: 'string' }, }, }); +const output = resolve(values['output-dir'] || join(root, 'test', 'output', 'bench', 'componentize-backends')); const samples = Number(values.samples); const warmup = Number(values.warmup); -const compiler = values["perry-cli-path"] || process.env.JCO_PERRY_CLI_PATH; +const buildSamples = Number(values['build-samples']); +const buildWarmup = Number(values['build-warmup']); +const compiler = values['perry-cli-path'] || process.env.JCO_PERRY_CLI_PATH; const perryCliPath = compiler ? resolve(compiler) : null; -assert.ok(Number.isInteger(samples) && samples > 0, "--samples must be a positive integer"); -assert.ok(Number.isInteger(warmup) && warmup >= 0, "--warmup must be a nonnegative integer"); +assert.ok(Number.isInteger(samples) && samples > 0, '--samples must be a positive integer'); +assert.ok(Number.isInteger(warmup) && warmup >= 0, '--warmup must be a nonnegative integer'); + +assert.ok(Number.isInteger(buildSamples) && buildSamples > 0, '--build-samples must be a positive integer'); +assert.ok(Number.isInteger(buildWarmup) && buildWarmup >= 0, '--build-warmup must be a nonnegative integer'); function command(script, args = [], env = process.env) { - execFileSync(process.execPath, [script, ...args], { cwd: root, env, stdio: "inherit" }); + const preload = args[0] === 'componentize' && process.env.JCO_BENCH_BUILD_PRELOAD; + const start = process.hrtime.bigint(); + execFileSync(process.execPath, [...(preload ? ['--import', preload] : []), script, ...args], { + cwd: root, + env, + stdio: 'inherit', + }); + return Number(process.hrtime.bigint() - start) / 1e6; +} + +function hostDetails() { + return { + node: process.version, + platform: process.platform, + arch: process.arch, + osRelease: release(), + cpu: cpus()[0]?.model, + }; +} + +function runComponent(backend, module) { + const start = process.hrtime.bigint(); + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./componentize-backends-runner.mjs', import.meta.url)), module], + { cwd: root, encoding: 'utf8', timeout: 60_000 }, + ); + const processMs = Number(process.hrtime.bigint() - start) / 1e6; + if (result.error) { + throw result.error; + } + assert.equal(result.status, 0, `${backend} failed: ${result.stderr}`); + assert.equal(result.stdout.replaceAll('\r\n', '\n'), expected, `${backend} produced different output`); + return { ...JSON.parse(result.stderr.trim()), processMs }; +} + +function transpile(component, directory) { + command(jco, [ + 'transpile', + component, + '--name', + 'component', + '--bindgen-enable-wasm-exnref', + '--instantiation', + 'async', + '-o', + directory, + ]); + return join(directory, 'component.js'); } async function packageVersion(name) { const directories = - name === "@bytecodealliance/jco" - ? [join(dirname(jco), "..")] + name === '@bytecodealliance/jco' + ? [join(dirname(jco), '..')] : (requireJco.resolve.paths(name) ?? []).map((directory) => join(directory, name)); for (const directory of directories) { try { - const manifest = JSON.parse(await readFile(join(directory, "package.json"), "utf8")); + const manifest = JSON.parse(await readFile(join(directory, 'package.json'), 'utf8')); if (manifest.name === name) { return manifest.version; } } catch (error) { - if (error.code !== "ENOENT") { + if (error.code !== 'ENOENT') { throw error; } } @@ -81,122 +137,164 @@ function componentSize(bytes) { } await mkdir(output, { recursive: true }); -const guest = await readFile(join(fixtures, "guest.ts"), "utf8"); -const guestSha256 = createHash("sha256").update(guest).digest("hex"); -if (!values["skip-build"]) { - assert.ok(perryCliPath, "Pass --perry-cli-path or set JCO_PERRY_CLI_PATH to a WASI-enabled Perry compiler"); +const guest = await readFile(join(fixtures, 'guest.ts'), 'utf8'); +const guestSha256 = createHash('sha256').update(guest).digest('hex'); +const input = await readFile(join(fixtures, 'data', 'input.txt')); +const inputSha256 = createHash('sha256').update(input).digest('hex'); +const expected = `Perry size: ${input.length}\nPerry read: ${input.toString('utf8')}\n`.replaceAll('\r\n', '\n'); +if (!values['skip-build']) { + assert.ok(perryCliPath, 'Pass --perry-cli-path or set JCO_PERRY_CLI_PATH to a WASI-enabled Perry compiler'); // A failed rebuild must not leave a receipt for partially replaced artifacts. - await rm(join(output, "build.json"), { force: true }); - const perryRuntime = await buildPerryRuntime(values["perry-source"] || process.env.JCO_PERRY_SOURCE, output); + await rm(join(output, 'build.json'), { force: true }); + const perryRuntime = await buildPerryRuntime(values['perry-source'] || process.env.JCO_PERRY_SOURCE, output); // Keep the fixture's read/stat/print statements inside run(): StarlingMonkey // preinitializes module-level code while building the component. - const ts = requireJco("typescript-compiler-api"); - const source = ts.createSourceFile("component.ts", guest, ts.ScriptTarget.Latest, true); + const ts = requireJco('typescript-compiler-api'); + const source = ts.createSourceFile('component.ts', guest, ts.ScriptTarget.Latest, true); const imports = source.statements.filter(ts.isImportDeclaration); const statements = source.statements.filter((statement) => !ts.isImportDeclaration(statement)); - const entry = join(output, "command.ts"); + const entry = join(output, 'command.ts'); await writeFile( entry, - `${imports.map((statement) => statement.getFullText(source)).join("\n")} + `${imports.map((statement) => statement.getFullText(source)).join('\n')} export const run = { run() { -${statements.map((statement) => statement.getFullText(source)).join("\n")} +${statements.map((statement) => statement.getFullText(source)).join('\n')} } }; `, ); - for (const backend of backends) { - const directory = join(output, backend); - await rm(directory, { recursive: true, force: true }); - await mkdir(directory); - const component = join(directory, "component.wasm"); - if (backend === "perry") { - command( - jco, - [ - "componentize", - join(fixtures, "guest.ts"), - "--backend", - "perry", - "--backend-perry-cli-path", - perryCliPath, - "-o", - component, - ], - perryRuntime.env, - ); - } else { - const wit = join(directory, "wit"); - await cp(join(root, "lib", "wit", "builtin", "0.2.12"), join(wit, "deps"), { recursive: true }); - await writeFile( - join(wit, "package.wit"), - "package benchmark:filesystem-comparison;\nworld command { export wasi:cli/run@0.2.12; }\n", + const buildMeasurements = Object.fromEntries(backends.map((backend) => [backend, []])); + const buildStartedAt = new Date().toISOString(); + // Prepare a new component for every sample; only engines/toolchains and OS caches are reused. + // Rotate the order just as for runtime samples. Builds are never concurrent. + for (let round = -buildWarmup; round < buildSamples; round++) { + const offset = (round + buildWarmup) % backends.length; + for (let index = 0; index < backends.length; index++) { + const backend = backends[(index + offset) % backends.length]; + console.log( + `\n${backend}: build ${round < 0 ? 'warmup' : 'sample'} ${round < 0 ? round + buildWarmup + 1 : round + 1}`, ); - command(jco, [ - "componentize", - entry, - "--backend", - backend, - "--wit", - wit, - "--world-name", - "benchmark:filesystem-comparison/command", - "-o", + const directory = join(output, backend); + await rm(directory, { recursive: true, force: true }); + await mkdir(directory); + const component = join(directory, 'component.wasm'); + let componentizeMs; + if (backend === 'perry') { + componentizeMs = command( + jco, + [ + 'componentize', + join(fixtures, 'guest.ts'), + '--backend', + 'perry', + '--backend-perry-cli-path', + perryCliPath, + '-o', + component, + ], + perryRuntime.env, + ); + } else { + // Jco injects imports into this WIT world. Reset it so every build starts identically. + const wit = join(directory, 'wit'); + await cp(join(root, 'lib', 'wit', 'builtin', '0.2.12'), join(wit, 'deps'), { recursive: true }); + await writeFile( + join(wit, 'package.wit'), + 'package benchmark:filesystem-comparison;\nworld command { export wasi:cli/run@0.2.12; }\n', + ); + componentizeMs = command(jco, [ + 'componentize', + entry, + '--backend', + backend, + '--wit', + wit, + '--world-name', + 'benchmark:filesystem-comparison/command', + '-o', + component, + ...(backend === 'quickjs' ? ['--backend-qjs-disable-async'] : []), + ]); + } + const optimized = join(directory, 'component.opt.wasm'); + const optimizeMs = command(jco, [ + 'opt', component, - ...(backend === "quickjs" ? ["--backend-qjs-disable-async"] : []), + '-o', + optimized, + ...(values['wasm-opt-bin'] ? ['--wasm-opt-bin', values['wasm-opt-bin']] : []), + '--', + ...optimizationArgs, ]); + // Validate BOTH timing endpoints, after the timed commands have finished. + // This catches unusable outputs and accidental build-time execution of the guest. + const asBuiltTranspiled = join(directory, 'as-built-transpiled'); + runComponent(backend, transpile(component, asBuiltTranspiled)); + await rm(asBuiltTranspiled, { recursive: true }); + runComponent(backend, transpile(optimized, join(directory, 'transpiled'))); + if (round >= 0) { + buildMeasurements[backend].push({ + round, + order: index, + componentizeMs, + optimizeMs, + sourceToOptimizedMs: componentizeMs + optimizeMs, + componentBytes: (await stat(component)).size, + optimizedComponentBytes: (await stat(optimized)).size, + }); + } } - const optimized = join(directory, "component.opt.wasm"); - command(jco, [ - "opt", - component, - "-o", - optimized, - ...(values["wasm-opt-bin"] ? ["--wasm-opt-bin", values["wasm-opt-bin"]] : []), - "--", - ...optimizationArgs, - ]); - command(jco, [ - "transpile", - optimized, - "--name", - "component", - "--bindgen-enable-wasm-exnref", - "--instantiation", - "async", - "-o", - join(directory, "transpiled"), - ]); } + const compilation = { + startedAt: buildStartedAt, + completedAt: new Date().toISOString(), + ...hostDetails(), + samples: buildSamples, + warmup: buildWarmup, + inputSha256, + scope: 'Fresh Jco CLI processes: TS/JS to complete engine-containing component, then jco opt', + excluded: ['toolchain/engine/runtime builds', 'fixture and WIT preparation', 'transpilation', 'validation'], + cachePolicy: 'Fresh component outputs each sample; prebuilt engines and OS/filesystem caches reused', + results: backends.map((backend) => ({ + backend, + componentizeMs: median(buildMeasurements[backend].map((sample) => sample.componentizeMs)), + optimizeMs: median(buildMeasurements[backend].map((sample) => sample.optimizeMs)), + sourceToOptimizedMs: median(buildMeasurements[backend].map((sample) => sample.sourceToOptimizedMs)), + samples: buildMeasurements[backend], + })), + }; const jsLauncher = perryCliPath && /\.[cm]?js$/i.test(perryCliPath); const perryVersion = perryCliPath ? execFileSync( jsLauncher ? process.execPath : perryCliPath, - [...(jsLauncher ? [perryCliPath] : []), "--version"], + [...(jsLauncher ? [perryCliPath] : []), '--version'], { cwd: root, - encoding: "utf8", + encoding: 'utf8', }, ).trim() : null; await writeFile( - join(output, "build.json"), + join(output, 'build.json'), `${JSON.stringify( { guestSha256, - jco: await packageVersion("@bytecodealliance/jco"), - componentizeJs: await packageVersion("@bytecodealliance/componentize-js"), - componentizeQjs: await packageVersion("componentize-qjs"), + compilation, + componentizePreload: process.env.JCO_BENCH_BUILD_PRELOAD ?? null, + jco: await packageVersion('@bytecodealliance/jco'), + componentizeJs: await packageVersion('@bytecodealliance/componentize-js'), + componentizeQjs: await packageVersion('componentize-qjs'), optimizationArgs, wasmOptVersion: execFileSync( process.execPath, - [values["wasm-opt-bin"] || requireJco.resolve("binaryen/bin/wasm-opt"), "--version"], + [values['wasm-opt-bin'] || requireJco.resolve('binaryen/bin/wasm-opt'), '--version'], { cwd: root, - encoding: "utf8", + encoding: 'utf8', }, ).trim(), - wasmOptLauncher: values["wasm-opt-bin"] ?? null, + wasmOptLauncher: values['wasm-opt-bin'] ?? null, perryRef: perryRuntime.metadata.recipe.perryRef, perryCliPath, perryVersion, @@ -208,42 +306,24 @@ ${statements.map((statement) => statement.getFullText(source)).join("\n")} ); } -const build = JSON.parse(await readFile(join(output, "build.json"), "utf8")); -assert.equal(build.guestSha256, guestSha256, "Guest changed; rerun without --skip-build"); -assert.deepEqual(build.optimizationArgs, optimizationArgs, "Optimization changed; rerun without --skip-build"); +const build = JSON.parse(await readFile(join(output, 'build.json'), 'utf8')); +assert.equal(build.guestSha256, guestSha256, 'Guest changed; rerun without --skip-build'); +assert.deepEqual(build.optimizationArgs, optimizationArgs, 'Optimization changed; rerun without --skip-build'); assert.deepEqual( build.perryRuntime?.recipe, perryRuntimeRecipe, - "Perry runtime recipe changed; rerun without --skip-build", + 'Perry runtime recipe changed; rerun without --skip-build', ); -const input = await readFile(join(fixtures, "data", "input.txt")); -const expected = `Perry size: ${input.length}\nPerry read: ${input.toString("utf8")}\n`.replaceAll("\r\n", "\n"); const measurements = Object.fromEntries(backends.map((backend) => [backend, []])); // Rotate backend order between rounds to reduce bias from changing machine load. for (let round = -warmup; round < samples; round++) { const offset = (round + warmup) % backends.length; for (let index = 0; index < backends.length; index++) { const backend = backends[(index + offset) % backends.length]; - const module = join(output, backend, "transpiled", "component.js"); - const start = process.hrtime.bigint(); - const result = spawnSync( - process.execPath, - [fileURLToPath(new URL("./componentize-backends-runner.mjs", import.meta.url)), module], - { - cwd: root, - encoding: "utf8", - timeout: 60_000, - }, - ); - const processMs = Number(process.hrtime.bigint() - start) / 1e6; - if (result.error) { - throw result.error; - } - assert.equal(result.status, 0, `${backend} failed: ${result.stderr}`); - assert.equal(result.stdout.replaceAll("\r\n", "\n"), expected, `${backend} produced different output`); - const timing = JSON.parse(result.stderr.trim()); + const module = join(output, backend, 'transpiled', 'component.js'); + const timing = runComponent(backend, module); if (round >= 0) { - measurements[backend].push({ ...timing, processMs }); + measurements[backend].push(timing); } } } @@ -251,8 +331,8 @@ for (let round = -warmup; round < samples; round++) { const rows = await Promise.all( backends.map(async (backend) => ({ backend, - componentBytes: (await stat(join(output, backend, "component.wasm"))).size, - optimizedComponentBytes: (await stat(join(output, backend, "component.opt.wasm"))).size, + componentBytes: (await stat(join(output, backend, 'component.wasm'))).size, + optimizedComponentBytes: (await stat(join(output, backend, 'component.opt.wasm'))).size, runMs: median(measurements[backend].map((sample) => sample.runMs)), loadAndRunMs: median(measurements[backend].map((sample) => sample.loadAndRunMs)), processMs: median(measurements[backend].map((sample) => sample.processMs)), @@ -261,28 +341,45 @@ const rows = await Promise.all( ); const report = { measuredAt: new Date().toISOString(), - node: process.version, - platform: process.platform, - arch: process.arch, - osRelease: release(), - cpu: cpus()[0]?.model, + ...hostDetails(), samples, warmup, inputBytes: input.length, - inputSha256: createHash("sha256").update(input).digest("hex"), + inputSha256, + compilationReused: values['skip-build'], build, results: rows, }; -await writeFile(join(output, "results.json"), `${JSON.stringify(report, null, 2)}\n`); +await writeFile(join(output, 'results.json'), `${JSON.stringify(report, null, 2)}\n`); console.log( - "\n| Backend | As-built size | Optimized size | Run (median ms) | Load + run (median ms) | Node process (median ms) |", + '\n| Backend | As-built size | Optimized size | Run (median ms) | Load + run (median ms) | Node process (median ms) |', ); -console.log("| --- | ---: | ---: | ---: | ---: | ---: |"); +console.log('| --- | ---: | ---: | ---: | ---: | ---: |'); for (const row of rows) { console.log( `| ${row.backend} | ${componentSize(row.componentBytes)} | ${componentSize(row.optimizedComponentBytes)} | ${row.runMs.toFixed(2)} | ${row.loadAndRunMs.toFixed(2)} | ${row.processMs.toFixed(2)} |`, ); } console.log( - `\nOptimized components: ${samples} samples/backend after ${warmup} warmups; details: ${join(output, "results.json")}`, + `\nOptimized components: ${samples} samples/backend after ${warmup} warmups; details: ${join(output, 'results.json')}`, ); + +if (build.compilation) { + console.log( + `\nCompilation: ${build.compilation.samples} samples/backend after ${build.compilation.warmup} warmups; recorded ${build.compilation.completedAt}${values['skip-build'] ? ' (reused; not remeasured)' : ''}.`, + ); + console.log( + '| Backend | TS/JS → component (median ms) | Optimize (median ms) | TS/JS → optimized component (median ms) |', + ); + console.log('| --- | ---: | ---: | ---: |'); + for (const row of build.compilation.results) { + console.log( + `| ${row.backend} | ${row.componentizeMs.toFixed(2)} | ${row.optimizeMs.toFixed(2)} | ${row.sourceToOptimizedMs.toFixed(2)} |`, + ); + } + console.log( + 'Complete components include their engines; prebuilt engines/toolchains are reused. Transpilation and validation are untimed.', + ); +} else { + console.log('\nNo compilation timings in this cached build; rerun without --skip-build to measure them.'); +}