diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..a44a243 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,27 @@ +name: 📊 Benchmarks +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] +jobs: + benchmark: + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: aiken-lang/setup-aiken@v1 + with: + version: v1.1.17 + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + - name: Runner unit tests + run: deno test -A benchmarks/ + - name: Benchmark regression check + run: deno run -A benchmarks/run.ts --check + - uses: actions/upload-artifact@v4 + if: always() + with: + name: benchmark-results + path: benchmarks/results.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f338754..c41b701 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,6 +12,6 @@ jobs: - uses: actions/checkout@v4 - uses: aiken-lang/setup-aiken@v1 with: - version: v1.1.17+c3a7fba + version: v1.1.17 - run: aiken fmt --check - run: aiken check \ No newline at end of file diff --git a/.gitignore b/.gitignore index f3385c3..5b15814 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ Thumbs.db build/ blueprint.json plutus.json + +# Benchmark artifacts (baseline.json stays tracked) +benchmarks/results.json diff --git a/aiken.toml b/aiken.toml index 8b5e3c0..9af9e3e 100644 --- a/aiken.toml +++ b/aiken.toml @@ -1,5 +1,6 @@ name = "uverify/validators" version = "0.1.0" +compiler = "v1.1.17" description = "Aiken contracts for project 'uverify/validators'" [repository] diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..0bbc04d --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,64 @@ +# Validator Benchmarks + +Measures mem units, cpu steps, and a rough fee estimate for the `uverify_state` +validator across representative certificate workloads. Guards against cost +regressions when upgrading Aiken or changing validator rules. + +## Usage + +Requires [Deno](https://deno.land) and the Aiken version pinned in +`.github/workflows/test.yml`. + + deno run -A benchmarks/run.ts # measure and print the table + deno run -A benchmarks/run.ts --check # compare against baseline.json, exit 1 on regression + deno run -A benchmarks/run.ts --update-baseline # rewrite the baseline for the versions that ran + deno run -A benchmarks/run.ts --version v1 # restrict to one validator version + deno test -A benchmarks/ # runner unit tests + +## Scenarios + +Two payload presets frozen in `lib/benchmarks/scenarios.ak`: + +| Preset | Extra metadata | Modeled after | +|---|---|---| +| standard | 120 bytes | default template payload | +| rich | 1479 bytes | digital product passport payload | + +Fixed scenarios run 1, 5, and 10 certificates per transaction on the +MINT_STATE and UPDATE_STATE paths. Sweeps sample 1 through 31 certificates +and report the batch ceiling, the largest batch that fits the mainnet +budget (14M mem, 10G cpu steps, 16384 bytes tx size). + +## What the numbers mean + +The measurements run the validator against an in-script mock transaction. +They exclude ScriptContext deserialization, which a real node pays, so +absolute values sit below on-chain ExUnits. Treat them as a consistent +regression signal, not ground truth. + +The fee column is an estimate: +`txFeeFixed + txFeePerByte * estimatedTxSize + priceMem * mem + priceStep * cpu` +with an assumed fixed transaction overhead (`txSize.overheadBytes` in +`config.json`). It tracks relative change well and absolute cost roughly. + +## Updating the baseline + +`baseline.json` is committed. CI fails when mem or cpu regresses more than +`tolerancePercent` (default 5) or a batch ceiling drops. For an intentional +change (new validator rule, Aiken upgrade): + +1. Run `deno run -A benchmarks/run.ts --check` and review the delta table. +2. Run `deno run -A benchmarks/run.ts --update-baseline`. +3. Commit `baseline.json` in the same pull request as the change, so the + cost delta is visible in review. + +## Adding a validator version + +1. Create `validators/benchmarks/uverify_state_v2_bench.ak` mirroring the + v1 module, importing the shared scenarios and calling the v2 handler. + Name every block with the `bench_v2_` prefix. +2. Run `deno run -A benchmarks/run.ts --update-baseline --version v2`. +3. Commit the new module and the baseline. + +The runner discovers versions from the `bench_v_` prefix. No runner +changes are needed. diff --git a/benchmarks/baseline.json b/benchmarks/baseline.json new file mode 100644 index 0000000..3c8acf7 --- /dev/null +++ b/benchmarks/baseline.json @@ -0,0 +1,747 @@ +{ + "aikenVersion": "aiken v1.1.17+c3a7fba", + "versions": { + "v1": { + "scenarios": { + "mint_standard_single": { + "mem": 513286, + "cpu": 190669253, + "estimatedTxSizeBytes": 1703, + "estimatedFeeLovelace": 273677, + "estimatedFeeAda": 0.273677 + }, + "mint_standard_batch_5": { + "mem": 729658, + "cpu": 309247597, + "estimatedTxSizeBytes": 2515, + "estimatedFeeLovelace": 330440, + "estimatedFeeAda": 0.33044 + }, + "mint_standard_batch_10": { + "mem": 1001173, + "cpu": 457686059, + "estimatedTxSizeBytes": 3530, + "estimatedFeeLovelace": 401468, + "estimatedFeeAda": 0.401468 + }, + "mint_rich_single": { + "mem": 532934, + "cpu": 319011433, + "estimatedTxSizeBytes": 3062, + "estimatedFeeLovelace": 343861, + "estimatedFeeAda": 0.343861 + }, + "mint_rich_batch_5": { + "mem": 831294, + "cpu": 951546005, + "estimatedTxSizeBytes": 9310, + "estimatedFeeLovelace": 681594, + "estimatedFeeAda": 0.681594 + }, + "mint_rich_batch_10": { + "mem": 1212941, + "cpu": 1743707507, + "estimatedTxSizeBytes": 17120, + "estimatedFeeLovelace": 1104370, + "estimatedFeeAda": 1.10437 + }, + "update_standard_single": { + "mem": 417188, + "cpu": 159139509, + "estimatedTxSizeBytes": 1703, + "estimatedFeeLovelace": 265859, + "estimatedFeeAda": 0.265859 + }, + "update_standard_batch_5": { + "mem": 633560, + "cpu": 277717853, + "estimatedTxSizeBytes": 2515, + "estimatedFeeLovelace": 322621, + "estimatedFeeAda": 0.322621 + }, + "update_standard_batch_10": { + "mem": 905075, + "cpu": 426156315, + "estimatedTxSizeBytes": 3530, + "estimatedFeeLovelace": 393650, + "estimatedFeeAda": 0.39365 + }, + "update_rich_single": { + "mem": 436836, + "cpu": 287481689, + "estimatedTxSizeBytes": 3062, + "estimatedFeeLovelace": 336042, + "estimatedFeeAda": 0.336042 + }, + "update_rich_batch_5": { + "mem": 735196, + "cpu": 920016261, + "estimatedTxSizeBytes": 9310, + "estimatedFeeLovelace": 673775, + "estimatedFeeAda": 0.673775 + }, + "update_rich_batch_10": { + "mem": 1116843, + "cpu": 1712177763, + "estimatedTxSizeBytes": 17120, + "estimatedFeeLovelace": 1096551, + "estimatedFeeAda": 1.096551 + } + }, + "sweeps": { + "mint_standard": { + "points": [ + { + "count": 1, + "mem": 457423, + "cpu": 176304501 + }, + { + "count": 2, + "mem": 509343, + "cpu": 205361536 + }, + { + "count": 3, + "mem": 561309, + "cpu": 234471705 + }, + { + "count": 4, + "mem": 613323, + "cpu": 263545002 + }, + { + "count": 5, + "mem": 665383, + "cpu": 292626257 + }, + { + "count": 6, + "mem": 717489, + "cpu": 321760646 + }, + { + "count": 7, + "mem": 769643, + "cpu": 350858163 + }, + { + "count": 8, + "mem": 821843, + "cpu": 379963638 + }, + { + "count": 9, + "mem": 874089, + "cpu": 409122247 + }, + { + "count": 10, + "mem": 926383, + "cpu": 438243984 + }, + { + "count": 11, + "mem": 978723, + "cpu": 467418855 + }, + { + "count": 12, + "mem": 1031111, + "cpu": 496556854 + }, + { + "count": 13, + "mem": 1083545, + "cpu": 525702811 + }, + { + "count": 14, + "mem": 1136025, + "cpu": 554901902 + }, + { + "count": 15, + "mem": 1188553, + "cpu": 584064121 + }, + { + "count": 16, + "mem": 1241127, + "cpu": 613234298 + }, + { + "count": 17, + "mem": 1293747, + "cpu": 642457609 + }, + { + "count": 18, + "mem": 1346415, + "cpu": 671644048 + }, + { + "count": 19, + "mem": 1399129, + "cpu": 700883621 + }, + { + "count": 20, + "mem": 1451891, + "cpu": 730086322 + }, + { + "count": 21, + "mem": 1504699, + "cpu": 759296981 + }, + { + "count": 22, + "mem": 1557553, + "cpu": 788560774 + }, + { + "count": 23, + "mem": 1610455, + "cpu": 817787695 + }, + { + "count": 24, + "mem": 1663403, + "cpu": 847022574 + }, + { + "count": 25, + "mem": 1716397, + "cpu": 876310587 + }, + { + "count": 26, + "mem": 1769439, + "cpu": 905561728 + }, + { + "count": 27, + "mem": 1822527, + "cpu": 934866003 + }, + { + "count": 28, + "mem": 1875663, + "cpu": 964133406 + }, + { + "count": 29, + "mem": 1928845, + "cpu": 993408767 + }, + { + "count": 30, + "mem": 1982073, + "cpu": 1022737262 + }, + { + "count": 31, + "mem": 2035349, + "cpu": 1052028885 + } + ], + "ceiling": { + "value": 31, + "saturated": true + } + }, + "mint_rich": { + "points": [ + { + "count": 1, + "mem": 477071, + "cpu": 304646681 + }, + { + "count": 2, + "mem": 548979, + "cpu": 462104716 + }, + { + "count": 3, + "mem": 621273, + "cpu": 619629529 + }, + { + "count": 4, + "mem": 693953, + "cpu": 777221120 + }, + { + "count": 5, + "mem": 767019, + "cpu": 934924665 + }, + { + "count": 6, + "mem": 840473, + "cpu": 1092650158 + }, + { + "count": 7, + "mem": 914313, + "cpu": 1250442429 + }, + { + "count": 8, + "mem": 988539, + "cpu": 1408301478 + }, + { + "count": 9, + "mem": 1063151, + "cpu": 1566272481 + }, + { + "count": 10, + "mem": 1138151, + "cpu": 1724265432 + }, + { + "count": 11, + "mem": 1213537, + "cpu": 1882325161 + }, + { + "count": 12, + "mem": 1289309, + "cpu": 2040451668 + }, + { + "count": 13, + "mem": 1365467, + "cpu": 2198690129 + }, + { + "count": 14, + "mem": 1442013, + "cpu": 2356950538 + }, + { + "count": 15, + "mem": 1518945, + "cpu": 2515277725 + }, + { + "count": 16, + "mem": 1596263, + "cpu": 2673671690 + }, + { + "count": 17, + "mem": 1673967, + "cpu": 2832177609 + }, + { + "count": 18, + "mem": 1752059, + "cpu": 2990705476 + }, + { + "count": 19, + "mem": 1830537, + "cpu": 3149300121 + }, + { + "count": 20, + "mem": 1909401, + "cpu": 3307961544 + }, + { + "count": 21, + "mem": 1988651, + "cpu": 3466734921 + }, + { + "count": 22, + "mem": 2068289, + "cpu": 3625530246 + }, + { + "count": 23, + "mem": 2148313, + "cpu": 3784392349 + }, + { + "count": 24, + "mem": 2228723, + "cpu": 3943321230 + }, + { + "count": 25, + "mem": 2309519, + "cpu": 4102362065 + }, + { + "count": 26, + "mem": 2390703, + "cpu": 4261424848 + }, + { + "count": 27, + "mem": 2472273, + "cpu": 4420554409 + }, + { + "count": 28, + "mem": 2554229, + "cpu": 4579750748 + }, + { + "count": 29, + "mem": 2636571, + "cpu": 4739059041 + }, + { + "count": 30, + "mem": 2719301, + "cpu": 4898389282 + }, + { + "count": 31, + "mem": 2802417, + "cpu": 5057786301 + } + ], + "ceiling": { + "value": 9, + "saturated": false + } + }, + "update_standard": { + "points": [ + { + "count": 1, + "mem": 355534, + "cpu": 143272699 + }, + { + "count": 2, + "mem": 407454, + "cpu": 172329734 + }, + { + "count": 3, + "mem": 459420, + "cpu": 201439903 + }, + { + "count": 4, + "mem": 511434, + "cpu": 230513200 + }, + { + "count": 5, + "mem": 563494, + "cpu": 259594455 + }, + { + "count": 6, + "mem": 615600, + "cpu": 288728844 + }, + { + "count": 7, + "mem": 667754, + "cpu": 317826361 + }, + { + "count": 8, + "mem": 719954, + "cpu": 346931836 + }, + { + "count": 9, + "mem": 772200, + "cpu": 376090445 + }, + { + "count": 10, + "mem": 824494, + "cpu": 405212182 + }, + { + "count": 11, + "mem": 876834, + "cpu": 434387053 + }, + { + "count": 12, + "mem": 929222, + "cpu": 463525052 + }, + { + "count": 13, + "mem": 981656, + "cpu": 492671009 + }, + { + "count": 14, + "mem": 1034136, + "cpu": 521870100 + }, + { + "count": 15, + "mem": 1086664, + "cpu": 551032319 + }, + { + "count": 16, + "mem": 1139238, + "cpu": 580202496 + }, + { + "count": 17, + "mem": 1191858, + "cpu": 609425807 + }, + { + "count": 18, + "mem": 1244526, + "cpu": 638612246 + }, + { + "count": 19, + "mem": 1297240, + "cpu": 667851819 + }, + { + "count": 20, + "mem": 1350002, + "cpu": 697054520 + }, + { + "count": 21, + "mem": 1402810, + "cpu": 726265179 + }, + { + "count": 22, + "mem": 1455664, + "cpu": 755528972 + }, + { + "count": 23, + "mem": 1508566, + "cpu": 784755893 + }, + { + "count": 24, + "mem": 1561514, + "cpu": 813990772 + }, + { + "count": 25, + "mem": 1614508, + "cpu": 843278785 + }, + { + "count": 26, + "mem": 1667550, + "cpu": 872529926 + }, + { + "count": 27, + "mem": 1720638, + "cpu": 901834201 + }, + { + "count": 28, + "mem": 1773774, + "cpu": 931101604 + }, + { + "count": 29, + "mem": 1826956, + "cpu": 960376965 + }, + { + "count": 30, + "mem": 1880184, + "cpu": 989705460 + }, + { + "count": 31, + "mem": 1933460, + "cpu": 1018997083 + } + ], + "ceiling": { + "value": 31, + "saturated": true + } + }, + "update_rich": { + "points": [ + { + "count": 1, + "mem": 375182, + "cpu": 271614879 + }, + { + "count": 2, + "mem": 447090, + "cpu": 429072914 + }, + { + "count": 3, + "mem": 519384, + "cpu": 586597727 + }, + { + "count": 4, + "mem": 592064, + "cpu": 744189318 + }, + { + "count": 5, + "mem": 665130, + "cpu": 901892863 + }, + { + "count": 6, + "mem": 738584, + "cpu": 1059618356 + }, + { + "count": 7, + "mem": 812424, + "cpu": 1217410627 + }, + { + "count": 8, + "mem": 886650, + "cpu": 1375269676 + }, + { + "count": 9, + "mem": 961262, + "cpu": 1533240679 + }, + { + "count": 10, + "mem": 1036262, + "cpu": 1691233630 + }, + { + "count": 11, + "mem": 1111648, + "cpu": 1849293359 + }, + { + "count": 12, + "mem": 1187420, + "cpu": 2007419866 + }, + { + "count": 13, + "mem": 1263578, + "cpu": 2165658327 + }, + { + "count": 14, + "mem": 1340124, + "cpu": 2323918736 + }, + { + "count": 15, + "mem": 1417056, + "cpu": 2482245923 + }, + { + "count": 16, + "mem": 1494374, + "cpu": 2640639888 + }, + { + "count": 17, + "mem": 1572078, + "cpu": 2799145807 + }, + { + "count": 18, + "mem": 1650170, + "cpu": 2957673674 + }, + { + "count": 19, + "mem": 1728648, + "cpu": 3116268319 + }, + { + "count": 20, + "mem": 1807512, + "cpu": 3274929742 + }, + { + "count": 21, + "mem": 1886762, + "cpu": 3433703119 + }, + { + "count": 22, + "mem": 1966400, + "cpu": 3592498444 + }, + { + "count": 23, + "mem": 2046424, + "cpu": 3751360547 + }, + { + "count": 24, + "mem": 2126834, + "cpu": 3910289428 + }, + { + "count": 25, + "mem": 2207630, + "cpu": 4069330263 + }, + { + "count": 26, + "mem": 2288814, + "cpu": 4228393046 + }, + { + "count": 27, + "mem": 2370384, + "cpu": 4387522607 + }, + { + "count": 28, + "mem": 2452340, + "cpu": 4546718946 + }, + { + "count": 29, + "mem": 2534682, + "cpu": 4706027239 + }, + { + "count": 30, + "mem": 2617412, + "cpu": 4865357480 + }, + { + "count": 31, + "mem": 2700528, + "cpu": 5024754499 + } + ], + "ceiling": { + "value": 9, + "saturated": false + } + } + } + } + } +} diff --git a/benchmarks/config.json b/benchmarks/config.json new file mode 100644 index 0000000..e449077 --- /dev/null +++ b/benchmarks/config.json @@ -0,0 +1,23 @@ +{ + "tolerancePercent": 5, + "maxSweepSize": 30, + "budget": { + "maxMem": 14000000, + "maxCpu": 10000000000, + "maxTxSizeBytes": 16384 + }, + "feeParams": { + "txFeeFixed": 155381, + "txFeePerByte": 44, + "priceMem": 0.0577, + "priceStep": 0.0000721 + }, + "txSize": { + "overheadBytes": 1500, + "certificateOverheadBytes": 16, + "presetExtraBytes": { + "standard": 120, + "rich": 1479 + } + } +} diff --git a/benchmarks/deno.json b/benchmarks/deno.json new file mode 100644 index 0000000..c278355 --- /dev/null +++ b/benchmarks/deno.json @@ -0,0 +1,8 @@ +{ + "tasks": { + "measure": "deno run -A run.ts", + "check": "deno run -A run.ts --check", + "update-baseline": "deno run -A run.ts --update-baseline", + "test": "deno test -A" + } +} diff --git a/benchmarks/deno.lock b/benchmarks/deno.lock new file mode 100644 index 0000000..fc5ef0e --- /dev/null +++ b/benchmarks/deno.lock @@ -0,0 +1,18 @@ +{ + "version": "5", + "specifiers": { + "jsr:@std/internal@^1.0.12": "1.0.12", + "jsr:@std/path@1": "1.1.4" + }, + "jsr": { + "@std/internal@1.0.12": { + "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027" + }, + "@std/path@1.1.4": { + "integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5", + "dependencies": [ + "jsr:@std/internal" + ] + } + } +} diff --git a/benchmarks/lib/ceiling.ts b/benchmarks/lib/ceiling.ts new file mode 100644 index 0000000..ea2f1e0 --- /dev/null +++ b/benchmarks/lib/ceiling.ts @@ -0,0 +1,33 @@ +import type { SweepPoint } from "./parse.ts"; + +export interface Budget { + maxMem: number; + maxCpu: number; + maxTxSizeBytes: number; +} + +export interface Ceiling { + value: number; + saturated: boolean; +} + +export function computeCeiling( + points: SweepPoint[], + budget: Budget, + txSizeForCount: (count: number) => number, +): Ceiling { + const sorted = [...points].sort((a, b) => a.count - b.count); + let value = 0; + let saturated = sorted.length > 0; + for (const point of sorted) { + const fits = point.mem <= budget.maxMem && + point.cpu <= budget.maxCpu && + txSizeForCount(point.count) <= budget.maxTxSizeBytes; + if (!fits) { + saturated = false; + break; + } + value = point.count; + } + return { value, saturated }; +} diff --git a/benchmarks/lib/compare.ts b/benchmarks/lib/compare.ts new file mode 100644 index 0000000..9c65aa8 --- /dev/null +++ b/benchmarks/lib/compare.ts @@ -0,0 +1,121 @@ +import type { SweepPoint } from "./parse.ts"; +import type { Ceiling } from "./ceiling.ts"; + +export interface ScenarioResult { + mem: number; + cpu: number; + estimatedTxSizeBytes: number; + estimatedFeeLovelace: number; + estimatedFeeAda: number; +} + +export interface SweepResult { + points: SweepPoint[]; + ceiling: Ceiling; +} + +export interface VersionResults { + scenarios: Record; + sweeps: Record; +} + +export interface BenchmarkResults { + aikenVersion: string; + versions: Record; +} + +export function compareResults( + baseline: BenchmarkResults, + current: BenchmarkResults, + tolerancePercent: number, + versionFilter: string[], +): string[] { + const errors: string[] = []; + const versions = Object.keys(baseline.versions).filter( + (version) => versionFilter.length === 0 || versionFilter.includes(version), + ); + for (const version of versions) { + const baselineVersion = baseline.versions[version]; + const currentVersion = current.versions[version]; + if (!currentVersion) { + errors.push( + `${version}: present in the baseline but produced no results.`, + ); + continue; + } + for (const [name, base] of Object.entries(baselineVersion.scenarios)) { + const now = currentVersion.scenarios[name]; + if (!now) { + errors.push( + `${version}/${name}: scenario missing from results (renamed or deleted test?).`, + ); + continue; + } + for (const metric of ["mem", "cpu"] as const) { + const limit = base[metric] * (1 + tolerancePercent / 100); + if (now[metric] > limit) { + const delta = ((now[metric] - base[metric]) / base[metric] * 100) + .toFixed(1); + errors.push( + `${version}/${name}: ${metric} regressed ${delta}% ` + + `(${base[metric]} -> ${now[metric]}, tolerance ${tolerancePercent}%).`, + ); + } + } + } + for (const [name, base] of Object.entries(baselineVersion.sweeps)) { + const now = currentVersion.sweeps[name]; + if (!now) { + errors.push(`${version}/${name}: sweep missing from results.`); + continue; + } + const baseFloor = base.ceiling.saturated ? Infinity : base.ceiling.value; + const nowFloor = now.ceiling.saturated ? Infinity : now.ceiling.value; + if (nowFloor < baseFloor) { + errors.push( + `${version}/${name}: batch ceiling dropped ` + + `(${describeCeiling(base.ceiling)} -> ${ + describeCeiling(now.ceiling) + }).`, + ); + } + } + } + return errors; +} + +export function describeCeiling(ceiling: Ceiling): string { + return ceiling.saturated ? `>=${ceiling.value}` : `${ceiling.value}`; +} + +export function renderSummaryTable( + baseline: BenchmarkResults | null, + current: BenchmarkResults, +): string { + const lines: string[] = []; + for (const [version, results] of Object.entries(current.versions)) { + lines.push(`### ${version} (aiken ${current.aikenVersion})`, ""); + lines.push("| Scenario | Mem | Δ Mem | CPU | Δ CPU | Fee est. (ADA) |"); + lines.push("|---|---|---|---|---|---|"); + for (const [name, scenario] of Object.entries(results.scenarios)) { + const base = baseline?.versions[version]?.scenarios[name]; + lines.push( + `| ${name} | ${scenario.mem} | ${delta(base?.mem, scenario.mem)} ` + + `| ${scenario.cpu} | ${delta(base?.cpu, scenario.cpu)} ` + + `| ${scenario.estimatedFeeAda.toFixed(6)} |`, + ); + } + lines.push("", "| Sweep | Batch ceiling |", "|---|---|"); + for (const [name, sweep] of Object.entries(results.sweeps)) { + lines.push(`| ${name} | ${describeCeiling(sweep.ceiling)} |`); + } + lines.push(""); + } + return lines.join("\n"); +} + +function delta(base: number | undefined, current: number): string { + if (base === undefined || base === 0) return "n/a"; + const percent = (current - base) / base * 100; + return `${percent >= 0 ? "+" : ""}${percent.toFixed(1)}%`; +} diff --git a/benchmarks/lib/fee.ts b/benchmarks/lib/fee.ts new file mode 100644 index 0000000..de08321 --- /dev/null +++ b/benchmarks/lib/fee.ts @@ -0,0 +1,42 @@ +export interface FeeParams { + txFeeFixed: number; + txFeePerByte: number; + priceMem: number; + priceStep: number; +} + +export interface TxSizeModel { + overheadBytes: number; + certificateOverheadBytes: number; + presetExtraBytes: Record; +} + +const HASH_BYTES = 32; +const ALGORITHM_BYTES = 7; +const ISSUER_BYTES = 28; + +export function estimateTxSizeBytes( + model: TxSizeModel, + preset: string, + certificateCount: number, +): number { + const extraBytes = model.presetExtraBytes[preset]; + if (extraBytes === undefined) { + throw new Error(`Unknown payload preset: ${preset}`); + } + const perCertificate = HASH_BYTES + ALGORITHM_BYTES + ISSUER_BYTES + + extraBytes + model.certificateOverheadBytes; + return model.overheadBytes + perCertificate * certificateCount; +} + +export function estimateFeeLovelace( + params: FeeParams, + txSizeBytes: number, + mem: number, + cpu: number, +): number { + return Math.ceil( + params.txFeeFixed + params.txFeePerByte * txSizeBytes + + params.priceMem * mem + params.priceStep * cpu, + ); +} diff --git a/benchmarks/lib/parse.ts b/benchmarks/lib/parse.ts new file mode 100644 index 0000000..8e67d06 --- /dev/null +++ b/benchmarks/lib/parse.ts @@ -0,0 +1,90 @@ +export interface ScenarioMeasurement { + version: string; + path: string; + scenario: string; + mem: number; + cpu: number; +} + +export interface SweepPoint { + count: number; + mem: number; + cpu: number; +} + +export interface Sweep { + version: string; + path: string; + preset: string; + points: SweepPoint[]; +} + +const TEST_NAME = /^bench_(v\d+)_(mint|update)_(.+)$/; +const BENCH_NAME = /^bench_(v\d+)_(mint|update)_([a-z0-9]+)_sweep$/; + +export function parseCheckOutput(raw: string): ScenarioMeasurement[] { + const data = parseJson(raw, "aiken check"); + const measurements: ScenarioMeasurement[] = []; + for (const module of data.modules ?? []) { + for (const test of module.tests ?? []) { + const match = TEST_NAME.exec(test.title); + if (!match) continue; + if (test.status !== "pass") { + throw new Error( + `Benchmark test ${test.title} failed. Fix the scenario before measuring.`, + ); + } + measurements.push({ + version: match[1], + path: match[2], + scenario: match[3], + mem: test.execution_units.mem, + cpu: test.execution_units.cpu, + }); + } + } + if (measurements.length === 0) { + throw new Error( + `No bench_ tests found in aiken check output:\n${raw.slice(0, 2000)}`, + ); + } + return measurements; +} + +export function parseBenchOutput(raw: string): Sweep[] { + const data = parseJson(raw, "aiken bench"); + const sweeps: Sweep[] = []; + for (const benchmark of data.benchmarks ?? []) { + const match = BENCH_NAME.exec(benchmark.name); + if (!match) continue; + sweeps.push({ + version: match[1], + path: match[2], + preset: match[3], + points: benchmark.measures.map( + (measure: { size: number; memory: number; cpu: number }) => ({ + count: measure.size + 1, + mem: measure.memory, + cpu: measure.cpu, + }), + ), + }); + } + if (sweeps.length === 0) { + throw new Error( + `No bench_ sweeps found in aiken bench output:\n${raw.slice(0, 2000)}`, + ); + } + return sweeps; +} + +// deno-lint-ignore no-explicit-any +function parseJson(raw: string, source: string): any { + try { + return JSON.parse(raw); + } catch { + throw new Error( + `${source} did not produce JSON output:\n${raw.slice(0, 2000)}`, + ); + } +} diff --git a/benchmarks/run.ts b/benchmarks/run.ts new file mode 100644 index 0000000..c89bfa8 --- /dev/null +++ b/benchmarks/run.ts @@ -0,0 +1,188 @@ +import { dirname, fromFileUrl, join } from "jsr:@std/path@1"; +import { parseBenchOutput, parseCheckOutput } from "./lib/parse.ts"; +import { + estimateFeeLovelace, + estimateTxSizeBytes, + type FeeParams, + type TxSizeModel, +} from "./lib/fee.ts"; +import { type Budget, computeCeiling } from "./lib/ceiling.ts"; +import { + type BenchmarkResults, + compareResults, + renderSummaryTable, + type VersionResults, +} from "./lib/compare.ts"; + +interface Config { + tolerancePercent: number; + maxSweepSize: number; + budget: Budget; + feeParams: FeeParams; + txSize: TxSizeModel; +} + +const benchmarksDir = dirname(fromFileUrl(import.meta.url)); +const repoRoot = dirname(benchmarksDir); +const baselinePath = join(benchmarksDir, "baseline.json"); +const resultsPath = join(benchmarksDir, "results.json"); + +function exitWith(message: string): never { + console.error(message); + Deno.exit(1); +} + +function parseArgs(args: string[]): { + mode: "measure" | "check" | "update-baseline"; + versions: string[]; +} { + const versions: string[] = []; + let mode: "measure" | "check" | "update-baseline" = "measure"; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "--check") mode = "check"; + else if (arg === "--update-baseline") mode = "update-baseline"; + else if (arg === "--version") { + const value = args[++i]; + if (!value || !/^v\d+$/.test(value)) { + exitWith( + `--version expects a value like v1, got: ${value ?? "nothing"}`, + ); + } + versions.push(value); + } else exitWith(`Unknown argument: ${arg}`); + } + return { mode, versions }; +} + +async function runAiken(args: string[]): Promise { + const command = new Deno.Command("aiken", { + args, + cwd: repoRoot, + stdout: "piped", + stderr: "piped", + }); + const output = await command.output(); + const stdout = new TextDecoder().decode(output.stdout); + const stderr = new TextDecoder().decode(output.stderr); + if (output.code !== 0) { + exitWith(`aiken ${args.join(" ")} failed:\n${stderr}\n${stdout}`); + } + return stdout; +} + +async function readBaseline(): Promise { + try { + return JSON.parse(await Deno.readTextFile(baselinePath)); + } catch (error) { + if (error instanceof Deno.errors.NotFound) return null; + throw error; + } +} + +const config: Config = JSON.parse( + await Deno.readTextFile(join(benchmarksDir, "config.json")), +); +const { mode, versions } = parseArgs(Deno.args); + +const matchPrefix = versions.length === 1 ? `bench_${versions[0]}` : "bench_"; +const checkRaw = await runAiken(["check", "-m", matchPrefix]); +const benchRaw = await runAiken([ + "bench", + "-m", + matchPrefix, + "--max-size", + String(config.maxSweepSize), +]); +const aikenVersion = (await runAiken(["--version"])).trim(); + +let scenarioMeasurements = parseCheckOutput(checkRaw); +let sweeps = parseBenchOutput(benchRaw); +if (versions.length > 0) { + scenarioMeasurements = scenarioMeasurements.filter((m) => + versions.includes(m.version) + ); + sweeps = sweeps.filter((s) => versions.includes(s.version)); +} + +const results: BenchmarkResults = { aikenVersion, versions: {} }; + +for (const measurement of scenarioMeasurements) { + const versionResults: VersionResults = results.versions[ + measurement.version + ] ??= { scenarios: {}, sweeps: {} }; + const preset = measurement.scenario.split("_")[0]; + const batchMatch = /batch_(\d+)$/.exec(measurement.scenario); + const count = batchMatch ? Number(batchMatch[1]) : 1; + const txSizeBytes = estimateTxSizeBytes(config.txSize, preset, count); + const feeLovelace = estimateFeeLovelace( + config.feeParams, + txSizeBytes, + measurement.mem, + measurement.cpu, + ); + versionResults.scenarios[`${measurement.path}_${measurement.scenario}`] = { + mem: measurement.mem, + cpu: measurement.cpu, + estimatedTxSizeBytes: txSizeBytes, + estimatedFeeLovelace: feeLovelace, + estimatedFeeAda: feeLovelace / 1_000_000, + }; +} + +for (const sweep of sweeps) { + const versionResults: VersionResults = results.versions[sweep.version] ??= { + scenarios: {}, + sweeps: {}, + }; + versionResults.sweeps[`${sweep.path}_${sweep.preset}`] = { + points: sweep.points, + ceiling: computeCeiling( + sweep.points, + config.budget, + (count) => estimateTxSizeBytes(config.txSize, sweep.preset, count), + ), + }; +} + +await Deno.writeTextFile(resultsPath, JSON.stringify(results, null, 2) + "\n"); + +if (mode === "update-baseline") { + const baseline = await readBaseline() ?? { aikenVersion, versions: {} }; + baseline.aikenVersion = aikenVersion; + for (const [version, versionResults] of Object.entries(results.versions)) { + baseline.versions[version] = versionResults; + } + await Deno.writeTextFile( + baselinePath, + JSON.stringify(baseline, null, 2) + "\n", + ); + console.log( + `Baseline updated for: ${Object.keys(results.versions).join(", ")}`, + ); +} else if (mode === "check") { + const baseline = await readBaseline(); + if (!baseline) { + exitWith("No baseline.json found. Run with --update-baseline first."); + } + const errors = compareResults( + baseline, + results, + config.tolerancePercent, + versions, + ); + const summary = renderSummaryTable(baseline, results); + console.log(summary); + const summaryFile = Deno.env.get("GITHUB_STEP_SUMMARY"); + if (summaryFile) { + await Deno.writeTextFile(summaryFile, summary + "\n", { append: true }); + } + if (errors.length > 0) { + console.error("\nRegression check failed:"); + for (const error of errors) console.error(` - ${error}`); + Deno.exit(1); + } + console.log("\nRegression check passed."); +} else { + console.log(renderSummaryTable(await readBaseline(), results)); +} diff --git a/benchmarks/tests/ceiling_test.ts b/benchmarks/tests/ceiling_test.ts new file mode 100644 index 0000000..f72a7c5 --- /dev/null +++ b/benchmarks/tests/ceiling_test.ts @@ -0,0 +1,40 @@ +import { assertEquals } from "jsr:@std/assert@1"; +import { computeCeiling } from "../lib/ceiling.ts"; + +const points = [ + { count: 1, mem: 100, cpu: 1000 }, + { count: 2, mem: 200, cpu: 2000 }, + { count: 3, mem: 900, cpu: 9000 }, +]; + +Deno.test("computeCeiling returns the largest fitting count", () => { + const budget = { maxMem: 500, maxCpu: 100000, maxTxSizeBytes: 16384 }; + assertEquals(computeCeiling(points, budget, () => 100), { + value: 2, + saturated: false, + }); +}); + +Deno.test("computeCeiling reports saturation when every sample fits", () => { + const budget = { maxMem: 10000, maxCpu: 100000, maxTxSizeBytes: 16384 }; + assertEquals(computeCeiling(points, budget, () => 100), { + value: 3, + saturated: true, + }); +}); + +Deno.test("computeCeiling applies the transaction size limit", () => { + const budget = { maxMem: 10000, maxCpu: 100000, maxTxSizeBytes: 16384 }; + assertEquals(computeCeiling(points, budget, (count) => count * 8000), { + value: 2, + saturated: false, + }); +}); + +Deno.test("computeCeiling handles an empty sweep", () => { + const budget = { maxMem: 10000, maxCpu: 100000, maxTxSizeBytes: 16384 }; + assertEquals(computeCeiling([], budget, () => 100), { + value: 0, + saturated: false, + }); +}); diff --git a/benchmarks/tests/compare_test.ts b/benchmarks/tests/compare_test.ts new file mode 100644 index 0000000..a699eab --- /dev/null +++ b/benchmarks/tests/compare_test.ts @@ -0,0 +1,98 @@ +import { assertEquals } from "jsr:@std/assert@1"; +import { + type BenchmarkResults, + compareResults, + describeCeiling, + renderSummaryTable, +} from "../lib/compare.ts"; + +function makeResults( + mem: number, + cpu: number, + ceiling: { value: number; saturated: boolean }, +): BenchmarkResults { + return { + aikenVersion: "v1.1.17+c3a7fba", + versions: { + v1: { + scenarios: { + mint_standard_single: { + mem, + cpu, + estimatedTxSizeBytes: 1700, + estimatedFeeLovelace: 250000, + estimatedFeeAda: 0.25, + }, + }, + sweeps: { + mint_standard: { points: [], ceiling }, + }, + }, + }, + }; +} + +const baseline = makeResults(100000, 1000000, { value: 30, saturated: true }); + +Deno.test("compareResults passes within tolerance", () => { + const current = makeResults(104000, 1040000, { value: 30, saturated: true }); + assertEquals(compareResults(baseline, current, 5, []), []); +}); + +Deno.test("compareResults flags mem regression beyond tolerance", () => { + const current = makeResults(106000, 1000000, { value: 30, saturated: true }); + const errors = compareResults(baseline, current, 5, []); + assertEquals(errors.length, 1); + if (!errors[0].includes("mem regressed")) { + throw new Error(`unexpected message: ${errors[0]}`); + } +}); + +Deno.test("compareResults flags a batch ceiling drop", () => { + const current = makeResults(100000, 1000000, { value: 25, saturated: false }); + const errors = compareResults(baseline, current, 5, []); + assertEquals(errors.length, 1); + if (!errors[0].includes("ceiling dropped")) { + throw new Error(`unexpected message: ${errors[0]}`); + } +}); + +Deno.test("compareResults flags a missing scenario", () => { + const current = makeResults(100000, 1000000, { value: 30, saturated: true }); + current.versions.v1.scenarios = {}; + const errors = compareResults(baseline, current, 5, []); + assertEquals(errors.length, 1); + if (!errors[0].includes("missing")) { + throw new Error(`unexpected message: ${errors[0]}`); + } +}); + +Deno.test("compareResults flags a missing version", () => { + const current = makeResults(100000, 1000000, { value: 30, saturated: true }); + current.versions = {}; + const errors = compareResults(baseline, current, 5, []); + assertEquals(errors.length, 1); +}); + +Deno.test("compareResults respects the version filter", () => { + const current = makeResults(999999999, 999999999, { + value: 1, + saturated: false, + }); + assertEquals(compareResults(baseline, current, 5, ["v2"]), []); +}); + +Deno.test("describeCeiling marks saturated ceilings as lower bounds", () => { + assertEquals(describeCeiling({ value: 31, saturated: true }), ">=31"); + assertEquals(describeCeiling({ value: 12, saturated: false }), "12"); +}); + +Deno.test("renderSummaryTable includes scenarios, deltas, and ceilings", () => { + const current = makeResults(110000, 1000000, { value: 30, saturated: true }); + const table = renderSummaryTable(baseline, current); + if (!table.includes("mint_standard_single")) { + throw new Error("scenario row missing"); + } + if (!table.includes("+10.0%")) throw new Error("delta missing"); + if (!table.includes(">=30")) throw new Error("ceiling missing"); +}); diff --git a/benchmarks/tests/fee_test.ts b/benchmarks/tests/fee_test.ts new file mode 100644 index 0000000..f0eae09 --- /dev/null +++ b/benchmarks/tests/fee_test.ts @@ -0,0 +1,34 @@ +import { assertEquals, assertThrows } from "jsr:@std/assert@1"; +import { estimateFeeLovelace, estimateTxSizeBytes } from "../lib/fee.ts"; + +Deno.test("estimateTxSizeBytes adds overhead plus per-certificate bytes", () => { + const model = { + overheadBytes: 1000, + certificateOverheadBytes: 10, + presetExtraBytes: { standard: 100 }, + }; + // per certificate: 32 hash + 7 algorithm + 28 issuer + 100 extra + 10 framing = 177 + assertEquals(estimateTxSizeBytes(model, "standard", 1), 1177); + assertEquals(estimateTxSizeBytes(model, "standard", 5), 1885); +}); + +Deno.test("estimateTxSizeBytes rejects unknown presets", () => { + const model = { + overheadBytes: 1000, + certificateOverheadBytes: 10, + presetExtraBytes: { standard: 100 }, + }; + assertThrows(() => estimateTxSizeBytes(model, "luxurious", 1), Error, "preset"); +}); + +Deno.test("estimateFeeLovelace applies the mainnet formula and rounds up", () => { + const params = { + txFeeFixed: 155381, + txFeePerByte: 44, + priceMem: 0.0577, + priceStep: 0.0000721, + }; + // 155381 + 44 * 2000 + 0.0577 * 400000 + 0.0000721 * 150000000 + // = 155381 + 88000 + 23080 + 10815 = 277276 + assertEquals(estimateFeeLovelace(params, 2000, 400000, 150000000), 277276); +}); diff --git a/benchmarks/tests/fixtures/bench_v1.json b/benchmarks/tests/fixtures/bench_v1.json new file mode 100644 index 0000000..48eec37 --- /dev/null +++ b/benchmarks/tests/fixtures/bench_v1.json @@ -0,0 +1,149 @@ +{ + "benchmarks": [ + { + "name": "bench_v1_mint_standard_sweep", + "module": "benchmarks/uverify_state_v1_bench", + "measures": [ + { + "size": 0, + "memory": 457423, + "cpu": 176304501 + }, + { + "size": 1, + "memory": 509343, + "cpu": 205361536 + }, + { + "size": 2, + "memory": 561309, + "cpu": 234471705 + }, + { + "size": 3, + "memory": 613323, + "cpu": 263545002 + }, + { + "size": 4, + "memory": 665383, + "cpu": 292626257 + }, + { + "size": 5, + "memory": 717489, + "cpu": 321760646 + } + ] + }, + { + "name": "bench_v1_mint_rich_sweep", + "module": "benchmarks/uverify_state_v1_bench", + "measures": [ + { + "size": 0, + "memory": 477071, + "cpu": 304646681 + }, + { + "size": 1, + "memory": 548979, + "cpu": 462104716 + }, + { + "size": 2, + "memory": 621273, + "cpu": 619629529 + }, + { + "size": 3, + "memory": 693953, + "cpu": 777221120 + }, + { + "size": 4, + "memory": 767019, + "cpu": 934924665 + }, + { + "size": 5, + "memory": 840473, + "cpu": 1092650158 + } + ] + }, + { + "name": "bench_v1_update_standard_sweep", + "module": "benchmarks/uverify_state_v1_bench", + "measures": [ + { + "size": 0, + "memory": 355534, + "cpu": 143272699 + }, + { + "size": 1, + "memory": 407454, + "cpu": 172329734 + }, + { + "size": 2, + "memory": 459420, + "cpu": 201439903 + }, + { + "size": 3, + "memory": 511434, + "cpu": 230513200 + }, + { + "size": 4, + "memory": 563494, + "cpu": 259594455 + }, + { + "size": 5, + "memory": 615600, + "cpu": 288728844 + } + ] + }, + { + "name": "bench_v1_update_rich_sweep", + "module": "benchmarks/uverify_state_v1_bench", + "measures": [ + { + "size": 0, + "memory": 375182, + "cpu": 271614879 + }, + { + "size": 1, + "memory": 447090, + "cpu": 429072914 + }, + { + "size": 2, + "memory": 519384, + "cpu": 586597727 + }, + { + "size": 3, + "memory": 592064, + "cpu": 744189318 + }, + { + "size": 4, + "memory": 665130, + "cpu": 901892863 + }, + { + "size": 5, + "memory": 738584, + "cpu": 1059618356 + } + ] + } + ], + "seed": 835164720 +} diff --git a/benchmarks/tests/fixtures/check_v1.json b/benchmarks/tests/fixtures/check_v1.json new file mode 100644 index 0000000..3f3d9da --- /dev/null +++ b/benchmarks/tests/fixtures/check_v1.json @@ -0,0 +1,136 @@ +{ + "seed": 1621899022, + "summary": { + "total": 12, + "passed": 12, + "failed": 0, + "kind": { + "unit": 12, + "property": 0 + } + }, + "modules": [ + { + "name": "benchmarks/uverify_state_v1_bench", + "summary": { + "total": 12, + "passed": 12, + "failed": 0, + "kind": { + "unit": 12, + "property": 0 + } + }, + "tests": [ + { + "title": "bench_v1_mint_standard_single", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 513286, + "cpu": 190669253 + } + }, + { + "title": "bench_v1_mint_standard_batch_5", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 729658, + "cpu": 309247597 + } + }, + { + "title": "bench_v1_mint_standard_batch_10", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 1001173, + "cpu": 457686059 + } + }, + { + "title": "bench_v1_mint_rich_single", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 532934, + "cpu": 319011433 + } + }, + { + "title": "bench_v1_mint_rich_batch_5", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 831294, + "cpu": 951546005 + } + }, + { + "title": "bench_v1_mint_rich_batch_10", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 1212941, + "cpu": 1743707507 + } + }, + { + "title": "bench_v1_update_standard_single", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 417188, + "cpu": 159139509 + } + }, + { + "title": "bench_v1_update_standard_batch_5", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 633560, + "cpu": 277717853 + } + }, + { + "title": "bench_v1_update_standard_batch_10", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 905075, + "cpu": 426156315 + } + }, + { + "title": "bench_v1_update_rich_single", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 436836, + "cpu": 287481689 + } + }, + { + "title": "bench_v1_update_rich_batch_5", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 735196, + "cpu": 920016261 + } + }, + { + "title": "bench_v1_update_rich_batch_10", + "status": "pass", + "on_failure": "fail_immediately", + "execution_units": { + "mem": 1116843, + "cpu": 1712177763 + } + } + ] + } + ] +} diff --git a/benchmarks/tests/parse_test.ts b/benchmarks/tests/parse_test.ts new file mode 100644 index 0000000..803dee2 --- /dev/null +++ b/benchmarks/tests/parse_test.ts @@ -0,0 +1,101 @@ +import { assertEquals, assertThrows } from "jsr:@std/assert@1"; +import { parseBenchOutput, parseCheckOutput } from "../lib/parse.ts"; + +const checkFixture = await Deno.readTextFile( + new URL("./fixtures/check_v1.json", import.meta.url), +); +const benchFixture = await Deno.readTextFile( + new URL("./fixtures/bench_v1.json", import.meta.url), +); + +Deno.test("parseCheckOutput extracts all 12 v1 scenarios", () => { + const measurements = parseCheckOutput(checkFixture); + assertEquals(measurements.length, 12); + const single = measurements.find( + (m) => m.path === "mint" && m.scenario === "standard_single", + ); + if (!single) throw new Error("mint standard_single not found"); + assertEquals(single.version, "v1"); + if (single.mem <= 0 || single.cpu <= 0) { + throw new Error("expected positive execution units"); + } +}); + +Deno.test("parseCheckOutput ignores tests without the bench_ prefix", () => { + const raw = JSON.stringify({ + modules: [ + { + name: "tests/uverify_state_test", + tests: [ + { + title: "should_mint_a_state_token", + status: "pass", + execution_units: { mem: 1, cpu: 1 }, + }, + { + title: "bench_v1_mint_standard_single", + status: "pass", + execution_units: { mem: 2, cpu: 3 }, + }, + ], + }, + ], + }); + const measurements = parseCheckOutput(raw); + assertEquals(measurements.length, 1); + assertEquals(measurements[0].mem, 2); +}); + +Deno.test("parseCheckOutput throws when a bench test failed", () => { + const raw = JSON.stringify({ + modules: [ + { + name: "benchmarks/uverify_state_v1_bench", + tests: [ + { + title: "bench_v1_mint_standard_single", + status: "fail", + execution_units: { mem: 2, cpu: 3 }, + }, + ], + }, + ], + }); + assertThrows(() => parseCheckOutput(raw), Error, "failed"); +}); + +Deno.test("parseCheckOutput throws on non-JSON input", () => { + assertThrows( + () => parseCheckOutput("Compiling uverify/validators..."), + Error, + "did not produce JSON", + ); +}); + +Deno.test("parseCheckOutput throws when nothing matches", () => { + assertThrows( + () => parseCheckOutput(JSON.stringify({ modules: [] })), + Error, + "No bench_", + ); +}); + +Deno.test("parseBenchOutput extracts sweeps and maps size to count", () => { + const sweeps = parseBenchOutput(benchFixture); + assertEquals(sweeps.length, 4); + const mintStandard = sweeps.find( + (s) => s.path === "mint" && s.preset === "standard", + ); + if (!mintStandard) throw new Error("mint standard sweep not found"); + assertEquals(mintStandard.version, "v1"); + assertEquals(mintStandard.points[0].count, 1); + assertEquals(mintStandard.points.length, 6); +}); + +Deno.test("parseBenchOutput throws when nothing matches", () => { + assertThrows( + () => parseBenchOutput(JSON.stringify({ benchmarks: [] })), + Error, + "No bench_", + ); +}); diff --git a/lib/benchmarks/scenarios.ak b/lib/benchmarks/scenarios.ak new file mode 100644 index 0000000..52174a1 --- /dev/null +++ b/lib/benchmarks/scenarios.ak @@ -0,0 +1,251 @@ +use aiken/collection/list +use aiken/crypto.{sha2_256} +use aiken/primitive/bytearray +use aiken/primitive/string +use cardano/address.{from_script, from_verification_key} +use cardano/assets.{PolicyId, from_asset, from_lovelace} +use cardano/transaction.{ + InlineDatum, Input, NoDatum, Output, Transaction, placeholder, +} +use common.{certificates_to_bytearray} +use mocktail/virgin_key_hash.{mock_policy_id, mock_pub_key_hash} +use mocktail/virgin_output_reference.{mock_utxo_ref} +use mocktail/virgin_validity_range.{mock_interval} +use types.{ + BootstrapDatum, MINT_STATE, StateDatum, UPDATE_STATE, UVerifyCertificate, + UVerifyStateRedeemer, +} + +pub type BenchScenario { + policy_id: PolicyId, + redeemer: UVerifyStateRedeemer, + transaction: Transaction, +} + +const bootstrap_token_name = #"6461746166726f73745f76305f305f32" + +const certificate_hash = + #"a601a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e" + +// batch_size must stay above the largest sweep count (max-size 30 -> 31 certificates) +const bench_batch_size = 100 + +fn bench_user_credential() { + mock_pub_key_hash(3) +} + +pub fn standard_extra() -> List { + [ + @"{\"uverify_template_id\":\"default\",\"title\":\"Certificate of Completion\",\"issuer_name\":\"UVerify Benchmarks\",\"language\":\"en\"}", + ] +} + +pub fn rich_extra() -> List { + [ + @"{\"uverify_template_id\":\"digitalProductPassport\",\"product_name\":\"EcoCharge Pro Wireless Charging Station\",\"manufacturer\":\"GreenTech Solutions AG\",\"manufacturer_address\":\"Industriestrasse 42, 8304 Wallisellen, Switzerland\",\"gtin\":\"04012345678901\",\"serial_number\":\"EC200-2024-SN-0084731\",\"model\":\"EC200\",\"production_date\":\"2024-03-15\",\"production_site\":\"Plant 2, Rotkreuz, Switzerland\",\"weight_grams\":\"412\",\"dimensions_mm\":\"120x120x18\",\"color\":\"graphite\",\"warranty_months\":\"36\",", + @"\"materials\":{\"aluminum_housing\":\"45 percent\",\"recycled_abs_plastic\":\"38 percent\",\"copper_coil\":\"9 percent\",\"neodymium_magnets\":\"3 percent\",\"silicone_pad\":\"2 percent\",\"other_components\":\"3 percent\"},\"recycled_content_total\":\"41 percent\",\"hazardous_substances\":\"none above regulatory threshold\",\"reach_svhc_declaration\":\"no substances of very high concern present\",\"rohs_status\":\"compliant\",\"battery_included\":\"false\",\"repairability_index\":\"8.1 of 10\",\"spare_parts_available_until\":\"2032-03-15\",", + @"\"certifications\":{\"ce\":\"CE Marking 2014-30-EU\",\"fcc\":\"FCC Part 15 Class B\",\"qi\":\"Qi v2.0 Certified WPC-2024-8841\",\"energy_star\":\"not applicable\"},\"sustainability\":{\"carbon_footprint_kg_co2e\":\"14.2\",\"energy_efficiency_class\":\"A\",\"recyclability_rate\":\"87 percent\",\"packaging\":\"FSC certified cardboard, plastic free\"},\"end_of_life\":{\"disposal\":\"WEEE collection point\",\"disassembly_manual\":\"https://greentech.example/ec200/disassembly\"},\"notes\":\"Digital product passport issued under EU ESPR pilot program batch 7\"}", + ] +} + +pub fn make_certificates( + count: Int, + extra: List, +) -> List { + list.map( + list.range(1, count), + fn(_index) { + UVerifyCertificate { + hash: certificate_hash, + algorithm: @"SHA-256", + issuer: bench_user_credential(), + extra, + } + }, + ) +} + +pub fn mint_state_scenario( + certificates: List, +) -> BenchScenario { + let user_credential = bench_user_credential() + let policy_id = mock_policy_id(1) + + let bootstrap_datum = + BootstrapDatum { + allowed_credentials: [], + token_name: bootstrap_token_name, + fee: 0, + fee_interval: 0, + fee_receivers: [], + ttl: 1833689900000, + transaction_limit: 1000, + batch_size: bench_batch_size, + } + + let utxo = mock_utxo_ref(1, 0) + let state_id = + sha2_256( + bytearray.concat( + left: utxo.transaction_id, + right: bytearray.from_int_little_endian(0, 2), + ), + ) + + let state_datum = + StateDatum { + id: state_id, + owner: user_credential, + fee: 0, + fee_interval: 0, + fee_receivers: [], + ttl: 1833689900000, + countdown: 999, + batch_size: bench_batch_size, + keep_as_oracle: False, + certificate_data_hash: sha2_256(certificates_to_bytearray(certificates)), + bootstrap_datum_name: bootstrap_token_name, + } + + let transaction = + Transaction { + ..placeholder, + inputs: [ + Input { + output: Output { + address: from_verification_key(user_credential), + value: from_lovelace(2000000), + datum: NoDatum, + reference_script: None, + }, + output_reference: utxo, + }, + ], + outputs: [ + Output { + address: from_script(mock_policy_id(1)), + value: from_asset(policy_id, state_id, 1), + datum: InlineDatum(state_datum), + reference_script: None, + }, + ], + mint: from_asset(policy_id, state_id, 1), + extra_signatories: [user_credential], + reference_inputs: [ + Input { + output: Output { + address: from_script(mock_policy_id(1)), + value: from_asset(policy_id, bootstrap_token_name, 1), + datum: InlineDatum(bootstrap_datum), + reference_script: None, + }, + output_reference: mock_utxo_ref(2, 0), + }, + ], + validity_range: mock_interval(Some(1756203803000), Some(1756203804000)), + } + + BenchScenario { + policy_id, + redeemer: UVerifyStateRedeemer { purpose: MINT_STATE, certificates }, + transaction, + } +} + +pub fn update_state_scenario( + certificates: List, +) -> BenchScenario { + let user_credential = bench_user_credential() + let policy_id = mock_policy_id(1) + + let utxo = mock_utxo_ref(1, 0) + let state_id = + sha2_256( + bytearray.concat( + left: utxo.transaction_id, + right: bytearray.from_int_little_endian(0, 2), + ), + ) + + let current_state_datum = + StateDatum { + id: state_id, + owner: user_credential, + fee: 0, + fee_interval: 0, + fee_receivers: [], + ttl: 1833689900000, + countdown: 999, + batch_size: bench_batch_size, + keep_as_oracle: False, + certificate_data_hash: #"0000000000000000000000000000000000000000000000000000000000000000", + bootstrap_datum_name: bootstrap_token_name, + } + + let next_state_datum = + StateDatum { + ..current_state_datum, + countdown: 998, + certificate_data_hash: sha2_256(certificates_to_bytearray(certificates)), + } + + let transaction = + Transaction { + ..placeholder, + inputs: [ + Input( + mock_utxo_ref(2, 1), + Output { + address: from_script(mock_policy_id(1)), + value: from_asset(policy_id, state_id, 1), + datum: InlineDatum(current_state_datum), + reference_script: None, + }, + ), + ], + outputs: [ + Output { + datum: InlineDatum(next_state_datum), + value: from_asset(policy_id, state_id, 1), + address: from_script(mock_policy_id(1)), + reference_script: None, + }, + ], + extra_signatories: [user_credential], + validity_range: mock_interval(Some(1756203803000), Some(1756203804000)), + } + + BenchScenario { + policy_id, + redeemer: UVerifyStateRedeemer { purpose: UPDATE_STATE, certificates }, + transaction, + } +} + +fn extra_byte_length(extra: List) -> Int { + list.reduce( + extra, + 0, + fn(total, part) { total + bytearray.length(string.to_bytearray(part)) }, + ) +} + +test standard_extra_has_realistic_size() { + let size = extra_byte_length(standard_extra()) + and { + size >= 100, + size <= 160, + } +} + +test rich_extra_has_realistic_size() { + let size = extra_byte_length(rich_extra()) + and { + size >= 1300, + size <= 1700, + } +} + +test make_certificates_returns_requested_count() { + list.length(make_certificates(5, standard_extra())) == 5 +} diff --git a/validators/benchmarks/uverify_state_v1_bench.ak b/validators/benchmarks/uverify_state_v1_bench.ak new file mode 100644 index 0000000..7d10e2d --- /dev/null +++ b/validators/benchmarks/uverify_state_v1_bench.ak @@ -0,0 +1,85 @@ +use benchmarks/scenarios.{ + BenchScenario, make_certificates, mint_state_scenario, rich_extra, + standard_extra, update_state_scenario, +} +use mocktail/virgin_address.{mock_script_credential} +use uverify_v1 + +fn run_v1(scenario: BenchScenario) -> Bool { + uverify_v1.uverify_state.withdraw( + scenario.policy_id, + "test_proxy_token", + scenario.redeemer, + mock_script_credential(0), + scenario.transaction, + ) +} + +// Maps bench size n to n + 1 certificates, so size 0 measures a single certificate +fn certificate_count_sampler() -> Sampler { + fn(n: Int) { fn(prng: PRNG) { Some((prng, n + 1)) } } +} + +test bench_v1_mint_standard_single() { + run_v1(mint_state_scenario(make_certificates(1, standard_extra()))) +} + +test bench_v1_mint_standard_batch_5() { + run_v1(mint_state_scenario(make_certificates(5, standard_extra()))) +} + +test bench_v1_mint_standard_batch_10() { + run_v1(mint_state_scenario(make_certificates(10, standard_extra()))) +} + +test bench_v1_mint_rich_single() { + run_v1(mint_state_scenario(make_certificates(1, rich_extra()))) +} + +test bench_v1_mint_rich_batch_5() { + run_v1(mint_state_scenario(make_certificates(5, rich_extra()))) +} + +test bench_v1_mint_rich_batch_10() { + run_v1(mint_state_scenario(make_certificates(10, rich_extra()))) +} + +test bench_v1_update_standard_single() { + run_v1(update_state_scenario(make_certificates(1, standard_extra()))) +} + +test bench_v1_update_standard_batch_5() { + run_v1(update_state_scenario(make_certificates(5, standard_extra()))) +} + +test bench_v1_update_standard_batch_10() { + run_v1(update_state_scenario(make_certificates(10, standard_extra()))) +} + +test bench_v1_update_rich_single() { + run_v1(update_state_scenario(make_certificates(1, rich_extra()))) +} + +test bench_v1_update_rich_batch_5() { + run_v1(update_state_scenario(make_certificates(5, rich_extra()))) +} + +test bench_v1_update_rich_batch_10() { + run_v1(update_state_scenario(make_certificates(10, rich_extra()))) +} + +bench bench_v1_mint_standard_sweep(count via certificate_count_sampler()) { + run_v1(mint_state_scenario(make_certificates(count, standard_extra()))) +} + +bench bench_v1_mint_rich_sweep(count via certificate_count_sampler()) { + run_v1(mint_state_scenario(make_certificates(count, rich_extra()))) +} + +bench bench_v1_update_standard_sweep(count via certificate_count_sampler()) { + run_v1(update_state_scenario(make_certificates(count, standard_extra()))) +} + +bench bench_v1_update_rich_sweep(count via certificate_count_sampler()) { + run_v1(update_state_scenario(make_certificates(count, rich_extra()))) +}