From 5a82634a80cb1be32a567fca22ffde20c271e4ca Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 9 Aug 2026 22:20:07 -0700 Subject: [PATCH 01/59] engine: profile harness fidelity -- mimalloc backing and an LTM mode The clearn_profile harness backed its counting allocator with the system allocator, but every native binary that embeds the engine (simlin-cli, simlin-serve, simlin-mcp, and libsimlin under the mimalloc feature that pysimlin's build turns on) installs mimalloc. Since the compile path is allocation-bound, that measured an allocator no shipped native build runs and over-credited any change that only moves malloc traffic: on C-LEARN the same compile is 194 ms on system malloc vs 171 ms on mimalloc. Allocation counts remain the allocator-independent metric, and the one that carries over to the wasm bundle, which links neither. CLEARN_LTM=1 compiles with Loops That Matter enabled so the LTM-augmented compile and run are measurable through the same per-phase accounting as the ordinary path, rather than only through ltm_full_bench (which stops at compile and never simulates). --- Cargo.lock | 1 + src/simlin-engine/Cargo.toml | 12 +++++++++ src/simlin-engine/examples/clearn_profile.rs | 27 +++++++++++++++----- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ceac32df8..32ebefdd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3311,6 +3311,7 @@ dependencies = [ "ed25519-dalek", "indexmap", "jsonschema", + "mimalloc", "proptest", "prost", "quick-xml 0.41.0", diff --git a/src/simlin-engine/Cargo.toml b/src/simlin-engine/Cargo.toml index 264606d82..55b018e60 100644 --- a/src/simlin-engine/Cargo.toml +++ b/src/simlin-engine/Cargo.toml @@ -169,4 +169,16 @@ harness = false name = "rapidhash_bench" harness = false +# Every native binary that embeds this engine (simlin-cli, simlin-serve, +# simlin-mcp, and libsimlin's `mimalloc` feature, which pysimlin's build turns +# on) installs mimalloc as its global allocator. The profiling examples and +# criterion benches therefore back their allocator with mimalloc too: the +# compile path is allocation-bound, so a harness on system malloc measures an +# allocator no shipped native build actually runs, and over-credits any change +# that only moves malloc traffic. Allocation *counts* stay the +# allocator-independent metric (and the one that carries over to the wasm +# bundle, which links neither mimalloc nor this dependency). +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +mimalloc = "0.1" + [build-dependencies] diff --git a/src/simlin-engine/examples/clearn_profile.rs b/src/simlin-engine/examples/clearn_profile.rs index 7fc0f2297..b8c65c6b4 100644 --- a/src/simlin-engine/examples/clearn_profile.rs +++ b/src/simlin-engine/examples/clearn_profile.rs @@ -19,16 +19,26 @@ //! //! Environment: //! CLEARN_MODEL override the .mdl path +//! CLEARN_LTM "1" to compile with Loops That Matter enabled //! CLEARN_COMPILE_ITERS extra compile-only iterations (default 0) //! CLEARN_RUN_ITERS extra run-only iterations (default 0) //! CLEARN_PROFILE "compile" | "run" | "both" (default both) -- which //! extra-iteration loop(s) to execute -use std::alloc::{GlobalAlloc, Layout, System as Backing}; +use std::alloc::{GlobalAlloc, Layout}; + +// Back the counting allocator with mimalloc, which is what every native binary +// embedding the engine installs (simlin-cli, simlin-serve, simlin-mcp, and +// libsimlin under its `mimalloc` feature, which pysimlin's build turns on). +// The compile path is allocation-bound, so profiling against system malloc +// measures an allocator no shipped native build runs. +use mimalloc::MiMalloc as Backing; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Instant; -use simlin_engine::db::{SimlinDb, compile_project_incremental, sync_from_datamodel_incremental}; +use simlin_engine::db::{ + SimlinDb, compile_project_incremental, set_project_ltm_enabled, sync_from_datamodel_incremental, +}; use simlin_engine::{CompiledSimulation, Vm, open_vensim}; // --- Counting allocator ----------------------------------------------------- @@ -145,9 +155,12 @@ fn model_path() -> String { ) } -fn compile_once(datamodel: &simlin_engine::datamodel::Project) -> CompiledSimulation { +fn compile_once(datamodel: &simlin_engine::datamodel::Project, ltm: bool) -> CompiledSimulation { let mut db = SimlinDb::default(); let sync = sync_from_datamodel_incremental(&mut db, datamodel, None); + if ltm { + set_project_ltm_enabled(&mut db, sync.project, true); + } compile_project_incremental(&db, sync.project, "main").unwrap() } @@ -163,11 +176,13 @@ fn main() { let compile_iters = env_usize("CLEARN_COMPILE_ITERS", 0); let run_iters = env_usize("CLEARN_RUN_ITERS", 0); let which = std::env::var("CLEARN_PROFILE").unwrap_or_else(|_| "both".to_string()); + let ltm = std::env::var("CLEARN_LTM").is_ok_and(|v| v != "0"); if std::env::var("CLEARN_COUNT_ALLOCS").is_ok_and(|v| v != "0") { COUNTING_ON.store(true, Ordering::Relaxed); } println!("model: {path}"); + println!("ltm: {ltm}"); let contents = phase("read_file", || { std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("failed to read {path}: {e}")) @@ -186,7 +201,7 @@ fn main() { datamodel.dimensions.len() ); - let compiled = phase("compile (salsa)", || compile_once(&datamodel)); + let compiled = phase("compile (salsa)", || compile_once(&datamodel, ltm)); println!(" n_slots (root): {}", compiled.n_slots()); let prof = compiled.bytecode_profile(); @@ -255,14 +270,14 @@ fn main() { if compile_iters > 0 && do_compile { let t0 = Instant::now(); for _ in 0..compile_iters { - std::hint::black_box(compile_once(&datamodel)); + std::hint::black_box(compile_once(&datamodel, ltm)); } let per = t0.elapsed().as_secs_f64() * 1000.0 / compile_iters as f64; println!("compile x{compile_iters}: {per:.2} ms/iter"); } if run_iters > 0 && do_run { - let compiled = compile_once(&datamodel); + let compiled = compile_once(&datamodel, ltm); let t0 = Instant::now(); for _ in 0..run_iters { let mut vm = Vm::new(compiled.clone()).unwrap(); From 766472967e3a28dca5f121f2639aabbb41b559f0 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 9 Aug 2026 23:31:24 -0700 Subject: [PATCH 02/59] engine: fuse SetCond;If[;AssignCurr] into a conditional-select opcode `compiler::codegen`'s `Expr::If` arm emits `SetCond` and `If` in one breath and is the sole producer of either, so the pair is adjacent by construction -- measured on C-LEARN as exactly equal executed counts (1,874,169 each, 6.38% of dispatches apiece). Neither `peephole_optimize` nor `fuse_three_address` can separate them: both only ever REPLACE an adjacent run, and `SetCond` is neither a leaf load nor a combiner, so no fusion window can absorb it. An `AssignCurr` follows ~91% of executed `If`s (LBR-sampled, corrected for window truncation against the structurally-known 100% SetCond->If pair), so the three-opcode form is the dominant shape and gets its own fused opcode. The pass lives in `ByteCode::fuse_three_address` (Vm-local) rather than the symbolic layer. The rule that decides this is worth stating in its reusable form: a fusion may live in the symbolic layer iff the fused opcode has a `SymbolicOpcode` form, because `CompiledSimulation` must stay the pure resolution of the salsa-cached symbolic fragments. That is why `peephole_optimize` can fuse LoadConstant+AssignCurr and Op2+AssignCurr (both have symbolic twins) while the 3-address family cannot. A symbolic home was possible here -- add `SymbolicOpcode::SelectIf` -- but it would rewrite every golden in `src/db/fragment_char_golden/`, change the cached artifact, and need `resolve` + wasmgen arms, all for the same dispatch reduction. The wasm backend therefore does not inherit this win; it already documents that late-fusion superinstructions never reach it and returns a loud `WasmGenError::Unsupported` if one ever did, so the premise fails noisily rather than mis-lowering. Three properties checked before adding opcodes: `fuse_three_address` runs only on the Vm's private `flows`/`stocks` copies, initials are left unfused, and both `invariant_flow_offsets` and `collect_constant_info` read the PRE-fusion cached bytecode -- so neither the GH #712 invariance oracle nor the constant-override set can be weakened by a new fused form. Stack effects are asserted, not assumed: `SelectIf` is (3,1) and `SelectIfAssignCurr` is (3,0), identical in net effect and peak to the `SetCond`(1,0) + `If`(2,1) [+ `AssignCurr`(1,0)] sequences they replace, which is what keeps `resolve_bytecode`'s fixed-stack safety proof valid. Measured (perf stat, 26 runs/side, one build pair): C-LEARN instructions -4.00%, branches -6.98%, cycles -6.26%, IPC 3.04->3.12 WORLD3 instructions -2.24%, branches -5.87%, cycles +0.62% (flat) One expectation this refutes. I predicted the removed dispatches would also remove indirect-branch mispredicts (GH #604's threshold hypothesis). They did not: C-LEARN branch-misses were flat (+0.7%) while branches fell 7.0%, so the miss RATE rose slightly. In hindsight this is what should have been predicted -- `SetCond`'s dispatch always jumps to `If`'s arm, making it among the most perfectly-predicted indirect branches in the program. Fusing away predictable dispatches buys instructions and branches, not mispredicts. The remaining mispredicts sit in the genuinely-unpredictable dispatches, which this class of superinstruction cannot reach. Behaviour-preserving: 5473 engine lib tests and 631 integration tests pass, including `clearn_residual_exactness` (which pins the exact SET of residual variables, not just a tolerance), `oracle_clearn`, and `vdf_parity`. --- src/simlin-engine/src/bytecode.rs | 182 ++++++++++++++++++++++++++++++ src/simlin-engine/src/vm.rs | 20 ++++ 2 files changed, 202 insertions(+) diff --git a/src/simlin-engine/src/bytecode.rs b/src/simlin-engine/src/bytecode.rs index 0aa73b071..109613d6d 100644 --- a/src/simlin-engine/src/bytecode.rs +++ b/src/simlin-engine/src/bytecode.rs @@ -759,6 +759,26 @@ pub(crate) enum Opcode { off: VariableOffset, }, + // === CONDITIONAL SELECT (R3) === + // `compiler::codegen`'s `Expr::If` arm emits `SetCond` and `If` in one + // breath and is the SOLE producer of either, so the pair is adjacent BY + // CONSTRUCTION -- measured on C-LEARN as exactly equal executed counts + // (1,874,169 each, 6.38% of dispatches apiece). Neither `peephole_optimize` + // nor `fuse_three_address` can separate them: both only ever REPLACE an + // adjacent run, and `SetCond` is neither a leaf load nor a combiner, so no + // fusion window can absorb it. + // + // Folding the pair removes a dispatch AND the `condition` round trip; the + // trailing `AssignCurr` (which follows ~91% of executed `If`s) folds in too. + // Created only by the late `fuse_three_address` pass, like the 3-address + // forms below -- they never enter the symbolic/incremental layer. + /// Pop `cond`, `f`, `t`; push `t` if `cond` is truthy else `f`. + SelectIf {}, + /// Pop `cond`, `f`, `t`; `curr[module_off + off] = if cond { t } else { f }`. + SelectIfAssignCurr { + off: VariableOffset, + }, + // === 3-ADDRESS BINARY OPS (R2) === // Fold the leaf operand load(s) of a binary op into the op itself, so a // subexpression `a op b` dispatches once instead of 3 (two loads + Op2) or @@ -1361,6 +1381,13 @@ impl Opcode { Opcode::BinOpAssignCurr { .. } => (2, 0), // pops 2, assigns directly Opcode::BinOpAssignNext { .. } => (2, 0), // pops 2, assigns directly + // Conditional select: the fusions of `SetCond`(1,0)+`If`(2,1) and of + // that pair plus `AssignCurr`(1,0). Net effect is identical to the + // sequence they replace, which is what keeps the fixed-stack safety + // proof in `resolve_bytecode` valid across the pass. + Opcode::SelectIf {} => (3, 1), // pops cond+false+true, pushes result + Opcode::SelectIfAssignCurr { .. } => (3, 0), // same, assigns directly + // 3-address binops: the *Var/*Const forms read both operands from // curr/literals and push (0 pops, 1 push); the Stack* forms pop the // lhs and push the result (1 pop, 1 push). @@ -1515,6 +1542,8 @@ impl Opcode { Opcode::BinConstConst { .. } => "BinConstConst", Opcode::BinOpAssignCurr { .. } => "BinOpAssignCurr", Opcode::BinOpAssignNext { .. } => "BinOpAssignNext", + Opcode::SelectIf {} => "SelectIf", + Opcode::SelectIfAssignCurr { .. } => "SelectIfAssignCurr", Opcode::AssignAddVarVarCurr { .. } => "AssignAddVarVarCurr", Opcode::AssignSubVarVarCurr { .. } => "AssignSubVarVarCurr", Opcode::AssignMulVarVarCurr { .. } => "AssignMulVarVarCurr", @@ -1974,6 +2003,44 @@ impl ByteCode { while i < self.code.len() { let new_pc = optimized.len(); + // Conditional select: `SetCond; If[; AssignCurr]`. Tried before the + // leaf windows because `SetCond` matches none of them (it is neither + // a leaf load nor a combiner), so the two rule sets are disjoint and + // the order is a readability choice, not a precedence one. The + // 3-window is tried first for the same reason the leaf-assign forms + // are: it collapses the store too (3->1 rather than 2->1 plus a + // separate store), and ~91% of executed `If`s are followed by one. + if matches!(self.code[i], Opcode::SetCond {}) { + let if_at = i + 1; + let pair_ok = if_at < self.code.len() + && matches!(self.code[if_at], Opcode::If {}) + && !jump_targets[if_at]; + if pair_ok { + let assign_at = i + 2; + let fused_assign = if assign_at < self.code.len() && !jump_targets[assign_at] { + match &self.code[assign_at] { + Opcode::AssignCurr { off } => Some(*off), + _ => None, + } + } else { + None + }; + if let Some(off) = fused_assign { + optimized.push(Opcode::SelectIfAssignCurr { off }); + pc_map.push(new_pc); // old i + pc_map.push(new_pc); // old i+1 + pc_map.push(new_pc); // old i+2 + i += 3; + continue; + } + optimized.push(Opcode::SelectIf {}); + pc_map.push(new_pc); // old i + pc_map.push(new_pc); // old i+1 + i += 2; + continue; + } + } + // 3-window: [leaf load, leaf load, ] where the combiner is // either an `Op2` (a pushing subexpression) or a `BinOpAssign{Curr| // Next}` (a leaf assignment, post-peephole). Both absorbed @@ -3695,6 +3762,121 @@ mod tests { )); } + // === Conditional-select fusion (SetCond;If[;AssignCurr]) === + // + // `compiler::codegen`'s `Expr::If` arm is the SOLE producer of both opcodes + // and pushes them in one breath, so the pair is adjacent BY CONSTRUCTION -- + // measured on C-LEARN as exactly equal executed counts (1,874,169 each). + // Neither `peephole_optimize` nor this pass can separate them: both only + // ever REPLACE an adjacent run, and `SetCond` is neither a leaf load nor a + // combiner, so no window can absorb it. These tests pin the fusion, both + // jump-target guards, and the stack-depth effect. + + #[test] + fn test_fuse_setcond_if_pair() { + // `IF c THEN t ELSE f` as a pushing subexpression: codegen emits + // t; f; c; SetCond; If. The trailing pair collapses to one dispatch. + let mut bc = ByteCode { + literals: vec![], + code: vec![ + Opcode::LoadVar { off: 0 }, // t + Opcode::LoadVar { off: 1 }, // f + Opcode::LoadVar { off: 2 }, // c + Opcode::SetCond {}, + Opcode::If {}, + ], + }; + bc.fuse_three_address(); + // The leading loads are not a fusible window (no combiner follows the + // pair), so only the SetCond;If tail collapses: 5 -> 4. + assert_eq!(bc.code.len(), 4); + assert!(matches!(bc.code[3], Opcode::SelectIf {})); + } + + #[test] + fn test_fuse_setcond_if_assign_triple() { + // `x = IF c THEN t ELSE f`: the whole tail is one dispatch. This is the + // dominant shape -- ~91% of executed `If`s are followed by `AssignCurr`. + let mut bc = ByteCode { + literals: vec![], + code: vec![ + Opcode::LoadVar { off: 0 }, + Opcode::LoadVar { off: 1 }, + Opcode::LoadVar { off: 2 }, + Opcode::SetCond {}, + Opcode::If {}, + Opcode::AssignCurr { off: 9 }, + ], + }; + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 4); + assert!(matches!(bc.code[3], Opcode::SelectIfAssignCurr { off: 9 })); + } + + #[test] + fn test_fuse_setcond_if_blocked_when_if_is_jump_target() { + // A jump landing on the `If` means the pair is not a unit: fusing would + // make the jump land mid-fusion. Leave both alone. + let mut bc = ByteCode { + literals: vec![], + code: vec![ + Opcode::SetCond {}, // [0] + Opcode::If {}, // [1] <- jump target + Opcode::NextIterOrJump { jump_back: -1 }, // [2] -> [1] + ], + }; + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 3); + assert!(matches!(bc.code[0], Opcode::SetCond {})); + assert!(matches!(bc.code[1], Opcode::If {})); + } + + #[test] + fn test_fuse_setcond_if_assign_falls_back_to_pair_when_assign_is_jump_target() { + // The 3-window is blocked because a jump targets the AssignCurr, but the + // SetCond;If pair is still a unit and must still fuse. + let mut bc = ByteCode { + literals: vec![], + code: vec![ + Opcode::SetCond {}, // [0] + Opcode::If {}, // [1] + Opcode::AssignCurr { off: 3 }, // [2] <- jump target + Opcode::NextIterOrJump { jump_back: -1 }, // [3] -> [2] + ], + }; + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 3); + assert!(matches!(bc.code[0], Opcode::SelectIf {})); + assert!(matches!(bc.code[1], Opcode::AssignCurr { off: 3 })); + // The jump must have been retargeted onto the AssignCurr's new pc. + assert!(matches!( + bc.code[2], + Opcode::NextIterOrJump { jump_back: -1 } + )); + } + + #[test] + fn test_fuse_setcond_if_preserves_max_stack_depth() { + // `SetCond` is (1,0) and `If` is (2,1); the fused `SelectIf` is (3,1) and + // `SelectIfAssignCurr` is (3,0). Net effect and peak must be unchanged -- + // the VM's fixed-stack safety proof is discharged against these numbers. + let mut bc = ByteCode { + literals: vec![], + code: vec![ + Opcode::LoadVar { off: 0 }, + Opcode::LoadVar { off: 1 }, + Opcode::LoadVar { off: 2 }, + Opcode::SetCond {}, + Opcode::If {}, + Opcode::AssignCurr { off: 9 }, + ], + }; + let before = bc.max_stack_depth().unwrap(); + assert_eq!(before, 3); + bc.fuse_three_address(); + assert_eq!(bc.max_stack_depth().unwrap(), 3); + } + // === 3-address fusion with GLOBAL operands and two-constant operands === // // Globals (TIME/DT/...) load via `LoadGlobalVar`; the fusion now folds them diff --git a/src/simlin-engine/src/vm.rs b/src/simlin-engine/src/vm.rs index ab63b3d6c..ed9a5d57f 100644 --- a/src/simlin-engine/src/vm.rs +++ b/src/simlin-engine/src/vm.rs @@ -2157,6 +2157,26 @@ impl Vm { next[module_off + *off as usize] = eval_op2(*op, l, r); debug_assert_eq!(0, stack.len()); } + // === CONDITIONAL SELECT (R3) === + // The fused `SetCond; If[; AssignCurr]`. Codegen pushes the true + // arm, then the false arm, then the condition, so these pop in + // the order cond, false, true -- exactly the order the three + // separate arms performed them in. Selecting between two + // already-evaluated operands is what `If` did; nothing about + // branch evaluation changes here. + Opcode::SelectIf {} => { + let cond = stack.pop(); + let f = stack.pop(); + let t = stack.pop(); + stack.push(if is_truthy(cond) { t } else { f }); + } + Opcode::SelectIfAssignCurr { off } => { + let cond = stack.pop(); + let f = stack.pop(); + let t = stack.pop(); + curr[module_off + *off as usize] = if is_truthy(cond) { t } else { f }; + debug_assert_eq!(0, stack.len()); + } // === 3-ADDRESS BINARY OPS (R2) === // Operands are read straight from curr[]/literals; the *Stack* // forms take the lhs from the arithmetic stack. Each pushes the From 2d34ba48f9c80a3d362938c074e65a7150d74642 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 9 Aug 2026 23:43:41 -0700 Subject: [PATCH 03/59] engine: give builtins a real arity instead of padding Apply to three operands `Opcode::Apply` unconditionally popped three operands, so codegen pushed `LoadConstant(0.0)` padding for every shorter call: two pads for the 14 single-operand builtins, one for the four two-operand ones. `vm::apply` then discarded them. Measured on C-LEARN that is 0.73 wasted pads per executed `Apply` -- 583k dispatches per run, 2.0% of all dispatches -- and on WORLD3 the padding is a smaller share only because WORLD3 barely uses builtins. The arity turns out to be a property of the BUILTIN, not of the call site. I had assumed otherwise (that `PULSE`/`RAMP`/`SAFEDIV` with an omitted third argument would need a per-site count) but reading `vm::apply` arm by arm settles it: codegen substitutes a real VALUE for those three, not a pad -- `PULSE`'s third defaults to `0` and `apply` reads it, `SAFEDIV`'s third IS the divide-by-zero result, and `RAMP`'s defaults to `final_time` via `LoadGlobalVar`. So they are genuinely 3-operand and the count is a pure function of `BuiltinId`. No opcode payload is needed. `BuiltinId::arity()` is that one table, and its three consumers all read it -- codegen (how many to push), `Opcode::stack_effect` (how many the opcode pops, which is what keeps `max_stack_depth` and `resolve_bytecode`'s fixed-stack proof in step with what is actually pushed), and the `Apply` arms in `vm.rs` and `wasmgen::lower`. Keeping it in one place is what makes the three unable to disagree; the match is exhaustive with no `_`, so a new builtin cannot be added without deciding its arity. Two test fixtures hand-built the padded shape and had to be rebuilt from the arity, which is the point of the repo's rule that a fixture must construct what production supplies: `apply_eval` pushed three `LoadConstant`s unconditionally, and `apply_inside_if_does_not_clobber_condition` spelled out two pads. Both now derive the operand count, so they exercise a stream codegen can actually emit -- before, they built one it cannot, and the wasm module failed validation with `EndInvalidValueStack` rather than telling us anything about padding. Two LTM goldens regenerate. The regeneration was inspected rather than blessed: computing the net change per opcode mnemonic across both files, the ONLY non-zero entry is `LoadConstant` (-6 in ltm_loop_exhaustive, -10 in ltm_loop_discovery). Every other opcode's count is unchanged; the rest of the diff is position shift. That is exactly the padding and nothing else. Measured cumulatively with the preceding SetCond;If fusion (perf stat, 26 runs/side, one build pair; instructions and branches are the layout-insensitive numbers, cycles bounce several percent between builds): C-LEARN instructions -4.25%, branches -7.72%, cycles -5.38% WORLD3 instructions -2.35%, branches -6.12%, cycles -7.58% Behaviour-preserving: 5475 engine lib tests and 631 integration tests pass, including `clearn_residual_exactness`, `oracle_clearn`, and `vdf_parity`. --- src/simlin-engine/src/bytecode.rs | 108 ++++++++++++- src/simlin-engine/src/compiler/codegen.rs | 13 +- .../ltm_loop_discovery.txt | 148 ++++++++---------- .../ltm_loop_exhaustive.txt | 86 +++++----- src/simlin-engine/src/vm.rs | 12 +- src/simlin-engine/src/wasmgen/lower.rs | 22 ++- src/simlin-engine/src/wasmgen/lower_tests.rs | 34 ++-- 7 files changed, 262 insertions(+), 161 deletions(-) diff --git a/src/simlin-engine/src/bytecode.rs b/src/simlin-engine/src/bytecode.rs index 109613d6d..0af917f0b 100644 --- a/src/simlin-engine/src/bytecode.rs +++ b/src/simlin-engine/src/bytecode.rs @@ -619,6 +619,52 @@ pub(crate) enum BuiltinId { Tan, } +impl BuiltinId { + /// How many operands `vm::apply` actually READS for this builtin. + /// + /// This is the single statement of that fact, and its three consumers -- + /// `compiler::codegen` (how many operands to push), `Opcode::stack_effect` + /// (how many the fused opcode pops), and the `Opcode::Apply` arms in + /// `vm.rs` and `wasmgen::lower` -- all read it, so they cannot disagree. + /// Before it existed, `Apply` unconditionally popped 3 and codegen padded + /// every shorter call with `LoadConstant(0.0)` pushes that `apply` then + /// discarded: 0.73 wasted pads per executed `Apply` on C-LEARN (583k + /// dispatches/run, 2.0% of all dispatches). + /// + /// The arms are derived from `vm::apply`'s body -- which operands each match + /// arm names -- and the match is exhaustive with no `_`, so a new builtin + /// cannot be added without deciding its arity here. + /// + /// `Inf`/`Pi` are 0: codegen returns early for both (they lower to a + /// `LoadConstant`), so no `Apply` opcode carrying them is ever emitted. + /// The three genuinely-3-operand builtins whose LAST operand is optional in + /// the source language stay 3, because codegen substitutes a real value + /// rather than a pad: `PULSE`'s third defaults to `0` and `apply` reads it, + /// `SAFEDIV`'s third IS the divide-by-zero result, and `RAMP`'s third + /// defaults to `final_time` via `LoadGlobalVar`. + pub(crate) fn arity(self) -> u8 { + match self { + BuiltinId::Abs + | BuiltinId::Arccos + | BuiltinId::Arcsin + | BuiltinId::Arctan + | BuiltinId::Cos + | BuiltinId::Exp + | BuiltinId::Int + | BuiltinId::Ln + | BuiltinId::Log10 + | BuiltinId::Round + | BuiltinId::Sign + | BuiltinId::Sin + | BuiltinId::Sqrt + | BuiltinId::Tan => 1, + BuiltinId::Max | BuiltinId::Min | BuiltinId::Quantum | BuiltinId::Step => 2, + BuiltinId::Pulse | BuiltinId::Ramp | BuiltinId::SafeDiv | BuiltinId::Sshape => 3, + BuiltinId::Inf | BuiltinId::Pi => 0, + } + } +} + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum Op2 { Add, @@ -1372,7 +1418,9 @@ impl Opcode { Opcode::AssignCurr { .. } => (1, 0), // Builtins always take 3 args (actual + padding), push 1 result - Opcode::Apply { .. } => (3, 1), + // Builtins pop exactly the operands `vm::apply` reads (see + // `BuiltinId::arity`), not a fixed 3 with discarded padding. + Opcode::Apply { func } => (func.arity(), 1), // Lookup pops element_offset and lookup_index, pushes result Opcode::Lookup { .. } => (2, 1), @@ -3762,6 +3810,64 @@ mod tests { )); } + // === Builtin arity (P2) === + + /// Every `BuiltinId`, with the arity `vm::apply` actually reads. Derived + /// from `apply`'s body arm by arm rather than sampled, so a builtin whose + /// operand use changes without its arity being revisited fails here. The + /// list is exhaustive over the enum: adding a variant without adding a row + /// makes `BuiltinId::arity`'s no-`_` match a compile error, and omitting the + /// row here makes the count assertion fail. + #[test] + fn builtin_arity_matches_what_apply_reads() { + let rows: &[(BuiltinId, u8)] = &[ + (BuiltinId::Abs, 1), + (BuiltinId::Arccos, 1), + (BuiltinId::Arcsin, 1), + (BuiltinId::Arctan, 1), + (BuiltinId::Cos, 1), + (BuiltinId::Exp, 1), + (BuiltinId::Int, 1), + (BuiltinId::Ln, 1), + (BuiltinId::Log10, 1), + (BuiltinId::Round, 1), + (BuiltinId::Sign, 1), + (BuiltinId::Sin, 1), + (BuiltinId::Sqrt, 1), + (BuiltinId::Tan, 1), + (BuiltinId::Max, 2), + (BuiltinId::Min, 2), + (BuiltinId::Quantum, 2), + (BuiltinId::Step, 2), + (BuiltinId::Pulse, 3), + (BuiltinId::Ramp, 3), + (BuiltinId::SafeDiv, 3), + (BuiltinId::Sshape, 3), + (BuiltinId::Inf, 0), + (BuiltinId::Pi, 0), + ]; + // 24 = every variant of BuiltinId. A new builtin must add a row. + assert_eq!(rows.len(), 24); + for (id, want) in rows { + assert_eq!(id.arity(), *want, "arity of {id:?}"); + } + } + + /// `Apply`'s stack effect must be its arity, not a fixed 3 -- this is what + /// keeps `max_stack_depth` (and so `resolve_bytecode`'s fixed-stack safety + /// proof) in step with what codegen actually pushes. + #[test] + fn apply_stack_effect_follows_arity() { + for (id, want) in [ + (BuiltinId::Abs, 1u8), + (BuiltinId::Max, 2), + (BuiltinId::Pulse, 3), + ] { + let op = Opcode::Apply { func: id }; + assert_eq!(op.stack_effect(), (want, 1), "stack effect of {id:?}"); + } + } + // === Conditional-select fusion (SetCond;If[;AssignCurr]) === // // `compiler::codegen`'s `Expr::If` arm is the SOLE producer of both opcodes diff --git a/src/simlin-engine/src/compiler/codegen.rs b/src/simlin-engine/src/compiler/codegen.rs index f30153069..58c28cadd 100644 --- a/src/simlin-engine/src/compiler/codegen.rs +++ b/src/simlin-engine/src/compiler/codegen.rs @@ -1304,23 +1304,18 @@ impl<'module> Compiler<'module> { | BuiltinFn::Sin(a) | BuiltinFn::Sqrt(a) | BuiltinFn::Tan(a) => { + // No operand padding: `Apply` pops exactly + // `BuiltinId::arity()`, which for this family is 1. self.walk_expr(a)?.unwrap(); - let id = self.curr_code.intern_literal(0.0); - self.push(SymbolicOpcode::LoadConstant { id }); - self.push(SymbolicOpcode::LoadConstant { id }); } BuiltinFn::Step(a, b) => { self.walk_expr(a)?.unwrap(); self.walk_expr(b)?.unwrap(); - let id = self.curr_code.intern_literal(0.0); - self.push(SymbolicOpcode::LoadConstant { id }); } BuiltinFn::Max(a, b) => { if let Some(b) = b { self.walk_expr(a)?.unwrap(); self.walk_expr(b)?.unwrap(); - let id = self.curr_code.intern_literal(0.0); - self.push(SymbolicOpcode::LoadConstant { id }); } else { return self.emit_array_reduce(a, SymbolicOpcode::ArrayMax {}); } @@ -1329,8 +1324,6 @@ impl<'module> Compiler<'module> { if let Some(b) = b { self.walk_expr(a)?.unwrap(); self.walk_expr(b)?.unwrap(); - let id = self.curr_code.intern_literal(0.0); - self.push(SymbolicOpcode::LoadConstant { id }); } else { return self.emit_array_reduce(a, SymbolicOpcode::ArrayMin {}); } @@ -1338,8 +1331,6 @@ impl<'module> Compiler<'module> { BuiltinFn::Quantum(a, b) => { self.walk_expr(a)?.unwrap(); self.walk_expr(b)?.unwrap(); - let id = self.curr_code.intern_literal(0.0); - self.push(SymbolicOpcode::LoadConstant { id }); } BuiltinFn::Pulse(a, b, c) => { self.walk_expr(a)?.unwrap(); diff --git a/src/simlin-engine/src/db/fragment_char_golden/ltm_loop_discovery.txt b/src/simlin-engine/src/db/fragment_char_golden/ltm_loop_discovery.txt index 5198f08a3..9537d0992 100644 --- a/src/simlin-engine/src/db/fragment_char_golden/ltm_loop_discovery.txt +++ b/src/simlin-engine/src/db/fragment_char_golden/ltm_loop_discovery.txt @@ -116,21 +116,19 @@ 0017 Op2 Sub 0018 LoadConstant #0 (=0.0) 0019 Apply SafeDiv - 0020 LoadConstant #0 (=0.0) - 0021 LoadConstant #0 (=0.0) - 0022 Apply Abs - 0023 LoadGlobalVar off=0 (time) + 0020 Apply Abs + 0021 LoadGlobalVar off=0 (time) + 0022 LoadGlobalVar off=2 (initial_time) + 0023 Op2 Eq 0024 LoadGlobalVar off=2 (initial_time) - 0025 Op2 Eq + 0025 LoadPrev $⁚$⁚ltm⁚link_score⁚growth→level⁚0⁚arg0@0 0026 LoadGlobalVar off=2 (initial_time) - 0027 LoadPrev $⁚$⁚ltm⁚link_score⁚growth→level⁚0⁚arg0@0 - 0028 LoadGlobalVar off=2 (initial_time) - 0029 Op2 Eq - 0030 Op2 Or - 0031 SetCond - 0032 If - 0033 AssignCurr $⁚ltm⁚link_score⁚growth→level@0 - 0034 Ret + 0027 Op2 Eq + 0028 Op2 Or + 0029 SetCond + 0030 If + 0031 AssignCurr $⁚ltm⁚link_score⁚growth→level@0 + 0032 Ret stock: == main::$⁚ltm⁚link_score⁚level→growth [ltm-synthetic] : flow == initial: @@ -155,41 +153,37 @@ 0010 LoadConstant #0 (=0.0) 0011 LoadPrev growth@0 0012 Op2 Sub - 0013 LoadConstant #0 (=0.0) + 0013 Apply Abs 0014 LoadConstant #0 (=0.0) - 0015 Apply Abs - 0016 LoadConstant #0 (=0.0) - 0017 Apply SafeDiv - 0018 LoadVar level@0 - 0019 LoadConstant #0 (=0.0) - 0020 LoadPrev level@0 - 0021 Op2 Sub - 0022 LoadConstant #0 (=0.0) + 0015 Apply SafeDiv + 0016 LoadVar level@0 + 0017 LoadConstant #0 (=0.0) + 0018 LoadPrev level@0 + 0019 Op2 Sub + 0020 Apply Sign + 0021 Op2 Mul + 0022 LoadVar growth@0 0023 LoadConstant #0 (=0.0) - 0024 Apply Sign - 0025 Op2 Mul - 0026 LoadVar growth@0 - 0027 LoadConstant #0 (=0.0) - 0028 LoadPrev growth@0 - 0029 Op2 Sub - 0030 LoadConstant #0 (=0.0) - 0031 Op2 Eq - 0032 LoadVar level@0 - 0033 LoadConstant #0 (=0.0) - 0034 LoadPrev level@0 - 0035 Op2 Sub - 0036 LoadConstant #0 (=0.0) - 0037 Op2 Eq - 0038 Op2 Or - 0039 SetCond - 0040 If - 0041 LoadGlobalVar off=0 (time) - 0042 LoadGlobalVar off=2 (initial_time) - 0043 Op2 Eq - 0044 SetCond - 0045 If - 0046 AssignCurr $⁚ltm⁚link_score⁚level→growth@0 - 0047 Ret + 0024 LoadPrev growth@0 + 0025 Op2 Sub + 0026 LoadConstant #0 (=0.0) + 0027 Op2 Eq + 0028 LoadVar level@0 + 0029 LoadConstant #0 (=0.0) + 0030 LoadPrev level@0 + 0031 Op2 Sub + 0032 LoadConstant #0 (=0.0) + 0033 Op2 Eq + 0034 Op2 Or + 0035 SetCond + 0036 If + 0037 LoadGlobalVar off=0 (time) + 0038 LoadGlobalVar off=2 (initial_time) + 0039 Op2 Eq + 0040 SetCond + 0041 If + 0042 AssignCurr $⁚ltm⁚link_score⁚level→growth@0 + 0043 Ret stock: == main::$⁚ltm⁚link_score⁚rate→growth [ltm-synthetic] : flow == initial: @@ -214,41 +208,37 @@ 0010 LoadConstant #0 (=0.0) 0011 LoadPrev growth@0 0012 Op2 Sub - 0013 LoadConstant #0 (=0.0) + 0013 Apply Abs 0014 LoadConstant #0 (=0.0) - 0015 Apply Abs - 0016 LoadConstant #0 (=0.0) - 0017 Apply SafeDiv - 0018 LoadVar rate@0 - 0019 LoadConstant #0 (=0.0) - 0020 LoadPrev rate@0 - 0021 Op2 Sub - 0022 LoadConstant #0 (=0.0) + 0015 Apply SafeDiv + 0016 LoadVar rate@0 + 0017 LoadConstant #0 (=0.0) + 0018 LoadPrev rate@0 + 0019 Op2 Sub + 0020 Apply Sign + 0021 Op2 Mul + 0022 LoadVar growth@0 0023 LoadConstant #0 (=0.0) - 0024 Apply Sign - 0025 Op2 Mul - 0026 LoadVar growth@0 - 0027 LoadConstant #0 (=0.0) - 0028 LoadPrev growth@0 - 0029 Op2 Sub - 0030 LoadConstant #0 (=0.0) - 0031 Op2 Eq - 0032 LoadVar rate@0 - 0033 LoadConstant #0 (=0.0) - 0034 LoadPrev rate@0 - 0035 Op2 Sub - 0036 LoadConstant #0 (=0.0) - 0037 Op2 Eq - 0038 Op2 Or - 0039 SetCond - 0040 If - 0041 LoadGlobalVar off=0 (time) - 0042 LoadGlobalVar off=2 (initial_time) - 0043 Op2 Eq - 0044 SetCond - 0045 If - 0046 AssignCurr $⁚ltm⁚link_score⁚rate→growth@0 - 0047 Ret + 0024 LoadPrev growth@0 + 0025 Op2 Sub + 0026 LoadConstant #0 (=0.0) + 0027 Op2 Eq + 0028 LoadVar rate@0 + 0029 LoadConstant #0 (=0.0) + 0030 LoadPrev rate@0 + 0031 Op2 Sub + 0032 LoadConstant #0 (=0.0) + 0033 Op2 Eq + 0034 Op2 Or + 0035 SetCond + 0036 If + 0037 LoadGlobalVar off=0 (time) + 0038 LoadGlobalVar off=2 (initial_time) + 0039 Op2 Eq + 0040 SetCond + 0041 If + 0042 AssignCurr $⁚ltm⁚link_score⁚rate→growth@0 + 0043 Ret stock: == main::growth [explicit] : flow == initial: diff --git a/src/simlin-engine/src/db/fragment_char_golden/ltm_loop_exhaustive.txt b/src/simlin-engine/src/db/fragment_char_golden/ltm_loop_exhaustive.txt index 1678cab78..adf624feb 100644 --- a/src/simlin-engine/src/db/fragment_char_golden/ltm_loop_exhaustive.txt +++ b/src/simlin-engine/src/db/fragment_char_golden/ltm_loop_exhaustive.txt @@ -116,21 +116,19 @@ 0017 Op2 Sub 0018 LoadConstant #0 (=0.0) 0019 Apply SafeDiv - 0020 LoadConstant #0 (=0.0) - 0021 LoadConstant #0 (=0.0) - 0022 Apply Abs - 0023 LoadGlobalVar off=0 (time) + 0020 Apply Abs + 0021 LoadGlobalVar off=0 (time) + 0022 LoadGlobalVar off=2 (initial_time) + 0023 Op2 Eq 0024 LoadGlobalVar off=2 (initial_time) - 0025 Op2 Eq + 0025 LoadPrev $⁚$⁚ltm⁚link_score⁚growth→level⁚0⁚arg0@0 0026 LoadGlobalVar off=2 (initial_time) - 0027 LoadPrev $⁚$⁚ltm⁚link_score⁚growth→level⁚0⁚arg0@0 - 0028 LoadGlobalVar off=2 (initial_time) - 0029 Op2 Eq - 0030 Op2 Or - 0031 SetCond - 0032 If - 0033 AssignCurr $⁚ltm⁚link_score⁚growth→level@0 - 0034 Ret + 0027 Op2 Eq + 0028 Op2 Or + 0029 SetCond + 0030 If + 0031 AssignCurr $⁚ltm⁚link_score⁚growth→level@0 + 0032 Ret stock: == main::$⁚ltm⁚link_score⁚level→growth [ltm-synthetic] : flow == initial: @@ -155,41 +153,37 @@ 0010 LoadConstant #0 (=0.0) 0011 LoadPrev growth@0 0012 Op2 Sub - 0013 LoadConstant #0 (=0.0) + 0013 Apply Abs 0014 LoadConstant #0 (=0.0) - 0015 Apply Abs - 0016 LoadConstant #0 (=0.0) - 0017 Apply SafeDiv - 0018 LoadVar level@0 - 0019 LoadConstant #0 (=0.0) - 0020 LoadPrev level@0 - 0021 Op2 Sub - 0022 LoadConstant #0 (=0.0) + 0015 Apply SafeDiv + 0016 LoadVar level@0 + 0017 LoadConstant #0 (=0.0) + 0018 LoadPrev level@0 + 0019 Op2 Sub + 0020 Apply Sign + 0021 Op2 Mul + 0022 LoadVar growth@0 0023 LoadConstant #0 (=0.0) - 0024 Apply Sign - 0025 Op2 Mul - 0026 LoadVar growth@0 - 0027 LoadConstant #0 (=0.0) - 0028 LoadPrev growth@0 - 0029 Op2 Sub - 0030 LoadConstant #0 (=0.0) - 0031 Op2 Eq - 0032 LoadVar level@0 - 0033 LoadConstant #0 (=0.0) - 0034 LoadPrev level@0 - 0035 Op2 Sub - 0036 LoadConstant #0 (=0.0) - 0037 Op2 Eq - 0038 Op2 Or - 0039 SetCond - 0040 If - 0041 LoadGlobalVar off=0 (time) - 0042 LoadGlobalVar off=2 (initial_time) - 0043 Op2 Eq - 0044 SetCond - 0045 If - 0046 AssignCurr $⁚ltm⁚link_score⁚level→growth@0 - 0047 Ret + 0024 LoadPrev growth@0 + 0025 Op2 Sub + 0026 LoadConstant #0 (=0.0) + 0027 Op2 Eq + 0028 LoadVar level@0 + 0029 LoadConstant #0 (=0.0) + 0030 LoadPrev level@0 + 0031 Op2 Sub + 0032 LoadConstant #0 (=0.0) + 0033 Op2 Eq + 0034 Op2 Or + 0035 SetCond + 0036 If + 0037 LoadGlobalVar off=0 (time) + 0038 LoadGlobalVar off=2 (initial_time) + 0039 Op2 Eq + 0040 SetCond + 0041 If + 0042 AssignCurr $⁚ltm⁚link_score⁚level→growth@0 + 0043 Ret stock: == main::$⁚ltm⁚loop_score⁚r1 [ltm-synthetic] : flow == initial: diff --git a/src/simlin-engine/src/vm.rs b/src/simlin-engine/src/vm.rs index ed9a5d57f..40e29929e 100644 --- a/src/simlin-engine/src/vm.rs +++ b/src/simlin-engine/src/vm.rs @@ -2412,9 +2412,15 @@ impl Vm { Opcode::Apply { func } => { let time = curr[TIME_OFF]; let dt = curr[DT_OFF]; - let c = stack.pop(); - let b = stack.pop(); - let a = stack.pop(); + // Pop exactly the operands this builtin reads. Codegen + // pushes `BuiltinId::arity()` of them and no padding, so an + // unread operand is never on the stack to begin with; the + // value handed to `apply` for an unread position is + // arbitrary, and 0.0 keeps it deterministic. + let arity = func.arity(); + let c = if arity >= 3 { stack.pop() } else { 0.0 }; + let b = if arity >= 2 { stack.pop() } else { 0.0 }; + let a = if arity >= 1 { stack.pop() } else { 0.0 }; stack.push(apply(*func, time, dt, a, b, c)); } diff --git a/src/simlin-engine/src/wasmgen/lower.rs b/src/simlin-engine/src/wasmgen/lower.rs index ad0d7b8b8..f663dbf55 100644 --- a/src/simlin-engine/src/wasmgen/lower.rs +++ b/src/simlin-engine/src/wasmgen/lower.rs @@ -2448,11 +2448,23 @@ fn emit_apply(func: BuiltinId, ctx: &EmitCtx, f: &mut Function) { use Instruction as Ins; let [a, b, c] = ctx.apply_locals; - // Pop the three padded operands. The stack top is `c`, so set c, then b, - // then a (the VM pops in the same order). - f.instruction(&Ins::LocalSet(c)); - f.instruction(&Ins::LocalSet(b)); - f.instruction(&Ins::LocalSet(a)); + // Pop exactly `BuiltinId::arity()` operands -- the same count codegen + // pushes and the same count the VM's `Apply` arm pops, all three reading + // the one shared table so they cannot disagree. The wasm stack top is `c`, + // so set c, then b, then a (the VM pops in the same order). Locals for + // positions this builtin does not read keep whatever a previous `Apply` + // left in them and are never read back: each `match` arm below touches only + // the locals its own arity covers. + let arity = func.arity(); + if arity >= 3 { + f.instruction(&Ins::LocalSet(c)); + } + if arity >= 2 { + f.instruction(&Ins::LocalSet(b)); + } + if arity >= 1 { + f.instruction(&Ins::LocalSet(a)); + } let get = |f: &mut Function, l: u32| { f.instruction(&Ins::LocalGet(l)); diff --git a/src/simlin-engine/src/wasmgen/lower_tests.rs b/src/simlin-engine/src/wasmgen/lower_tests.rs index 9629fb6c7..00fce0f63 100644 --- a/src/simlin-engine/src/wasmgen/lower_tests.rs +++ b/src/simlin-engine/src/wasmgen/lower_tests.rs @@ -1752,17 +1752,20 @@ fn setcond_if_uses_approx_eq_truthiness() { // ── Apply: per-builtin parity with the VM's apply() ─────────────────── -/// Run `Apply{func}` over the three operands `(a, b, c)` with `time`/`dt` -/// seeded into the reserved global slots (TIME at byte 0, DT at byte 8 of -/// `curr`). The program pushes a, b, c then `Apply`, so `c` is on top -- -/// matching the VM's pop order. +/// Run `Apply{func}` over the operands `(a, b, c)` with `time`/`dt` seeded into +/// the reserved global slots (TIME at byte 0, DT at byte 8 of `curr`). +/// +/// The operand pushes are derived from `BuiltinId::arity()` rather than fixed +/// at three, because that is what `compiler::codegen` emits: a builtin is +/// pushed exactly the operands `vm::apply` reads, with no padding. Hard-coding +/// three here would build a stream production cannot produce and would leave +/// the extra values stranded on the wasm stack. Operands past the arity are +/// ignored, so callers may keep passing 0.0 for them. fn apply_eval(func: BuiltinId, a: f64, b: f64, c: f64, time: f64, dt: f64) -> f64 { - let code = vec![ - Opcode::LoadConstant { id: 0 }, - Opcode::LoadConstant { id: 1 }, - Opcode::LoadConstant { id: 2 }, - Opcode::Apply { func }, - ]; + let code: Vec = (0..func.arity() as u16) + .map(|id| Opcode::LoadConstant { id }) + .chain(std::iter::once(Opcode::Apply { func })) + .collect(); // Seed TIME (slot 0 -> byte 0) and DT (slot 1 -> byte 8) of curr. value(code, vec![a, b, c], &[(0, time), (8, dt)]) } @@ -2001,12 +2004,11 @@ fn apply_inf_pi() { #[test] fn apply_inside_if_does_not_clobber_condition() { // An `Apply` in an If arm shares the function with the condition local; - // the dedicated apply locals must not collide. Build (codegen-padded - // Apply operands): `if cond then ABS(a) else f`, cond truthy. - let padded = vec![ + // the dedicated apply locals must not collide. `ABS` has arity 1, so + // codegen pushes exactly one operand -- no padding (see + // `BuiltinId::arity`). Build `if cond then ABS(a) else f`, cond truthy. + let ops = vec![ Opcode::LoadConstant { id: 1 }, // a = -4 (the `then` operand) - Opcode::LoadConstant { id: 3 }, // pad b = 0 - Opcode::LoadConstant { id: 3 }, // pad c = 0 Opcode::Apply { func: BuiltinId::Abs, }, // ABS(-4) = 4 -> the `then` value @@ -2016,7 +2018,7 @@ fn apply_inside_if_does_not_clobber_condition() { Opcode::If {}, ]; let got = run( - &bc(vec![1.0, -4.0, 99.0, 0.0], padded), + &bc(vec![1.0, -4.0, 99.0, 0.0], ops), &ctx_with_cond_depth(1), true, 1, From 4c68ef3314dd26270487e43128feeeedb1404182 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 9 Aug 2026 23:50:39 -0700 Subject: [PATCH 04/59] engine: memoize the direct LTM fragment compile `compile_ltm_synthetic_fragment` routed only the scalar `Bare` from->to link score through a memoized query; every element-pinned, aggregate-touching, A2A and loop score took the plain-function `compile_direct` path. Two walkers over the LTM variable list -- `assemble_module`'s pass 3 and `model_ltm_fragment_diagnostics` -- therefore compiled those fragments twice, independently. On C-LEARN that is 5,985 of 7,125 variables, whose compilation callgrind measures at ~9.98e9 instructions, roughly half of a full LTM compile stage. The duplication is paid in shipped flows, not just in principle: libsimlin's `simlin_project_get_errors` (and so pysimlin's `Project.get_errors`) runs the diagnostic pass after `simlin_sim_new` has already assembled, MCP `read_model` runs it once, and MCP `edit_model` runs it on a pre-edit and a post-edit database and then assembles again. Both walkers now reach a fragment through `compile_ltm_fragment_at`, keyed by the variable's index into `model_ltm_variables(..).vars`. The index is what both walkers already hold, and it keeps the query a salsa firewall: it reads the whole-model variable list so it re-executes on any edit, but its value is one fragment, so salsa backdates it whenever that fragment is unchanged and assembly is not re-run -- the same shape as `reconstruct_named_variable` over `reconstruct_model_variables`. Behaviour-preserving: C-LEARN's LTM bytecode is byte-identical at 1,238,728 opcodes and its slot count is unchanged, and WORLD3's LTM compile and run are unmoved. The cost is retention -- roughly 7k symbolic fragments now stay live in the database rather than being dropped after assembly, measured at +57 MiB peak on C-LEARN (438.8 -> 496.3 MiB) for +0.07% allocations. The compile-only path gains nothing from this change and pays one fragment clone per variable; the win is entirely in the second walk. `FragmentExecKind::LtmBody` records the fragment-compile body wherever it runs, which is what lets the reuse be tested at all: a memo hit is invisible to a timing but so is a cache miss, and pointer equality cannot see it because salsa backdates a re-executed query whose value compares equal. --- src/simlin-engine/src/db/assemble.rs | 18 ++-- src/simlin-engine/src/db/fragment_compile.rs | 6 ++ src/simlin-engine/src/db/ltm/compile.rs | 45 +++++++- src/simlin-engine/src/db/ltm/mod.rs | 9 +- src/simlin-engine/src/db/ltm_tests.rs | 105 +++++++++++++++++++ 5 files changed, 171 insertions(+), 12 deletions(-) diff --git a/src/simlin-engine/src/db/assemble.rs b/src/simlin-engine/src/db/assemble.rs index 269fa54f5..f6e870209 100644 --- a/src/simlin-engine/src/db/assemble.rs +++ b/src/simlin-engine/src/db/assemble.rs @@ -1481,16 +1481,18 @@ pub fn assemble_module<'db>( // `assemble_simulation`. let ltm_vars = model_ltm_variables(db, model, project); - for ltm_var in <m_vars.vars { + for (ltm_index, ltm_var) in ltm_vars.vars.iter().enumerate() { let ltm_var_canonical = canonicalize(<m_var.name).into_owned(); - // Select and compile this LTM var's fragment. The - // selection logic (salsa-cached `(from, to)` path vs. - // direct compilation of the prepared equation) lives in - // `compile_ltm_synthetic_fragment` so the diagnostic pass - // (`model_ltm_fragment_diagnostics`) detects the exact same - // compile failures this assembly pass would silently drop. - let fragment_result = compile_ltm_synthetic_fragment(db, ltm_var, model, project); + // Select and compile this LTM var's fragment. The selection logic + // (salsa-cached `(from, to)` path vs. direct compilation of the + // prepared equation) lives in `compile_ltm_synthetic_fragment` so + // the diagnostic pass (`model_ltm_fragment_diagnostics`) detects the + // exact same compile failures this assembly pass would silently + // drop. Both walkers reach it through the memoized per-index query, + // so the diagnostic pass reuses these fragments instead of + // recompiling the ones the direct path does not otherwise cache. + let fragment_result = compile_ltm_fragment_at(db, model, project, ltm_index).clone(); if let Some(result) = fragment_result { // Drop LTM fragments whose symbolic variable references can't diff --git a/src/simlin-engine/src/db/fragment_compile.rs b/src/simlin-engine/src/db/fragment_compile.rs index 194d47515..12e86698c 100644 --- a/src/simlin-engine/src/db/fragment_compile.rs +++ b/src/simlin-engine/src/db/fragment_compile.rs @@ -57,6 +57,12 @@ pub(crate) enum FragmentExecKind { Implicit, /// `compile_ltm_var_fragment` -- salsa-tracked, keyed by `(from, to)` link. Ltm, + /// `compile_ltm_equation_fragment` -- the LTM fragment-compile BODY, + /// recorded wherever it runs. Every LTM path funnels through it (the + /// `(from, to)`-keyed one and the per-index `compile_ltm_fragment_at` one), + /// so this counts real compiles rather than cache lookups -- which is what + /// makes "the diagnostic pass reuses assembly's work" measurable at all. + LtmBody, } #[cfg(test)] diff --git a/src/simlin-engine/src/db/ltm/compile.rs b/src/simlin-engine/src/db/ltm/compile.rs index e694a609f..6e8264c4d 100644 --- a/src/simlin-engine/src/db/ltm/compile.rs +++ b/src/simlin-engine/src/db/ltm/compile.rs @@ -979,6 +979,9 @@ pub(crate) fn compile_ltm_equation_fragment( ) -> Option { use crate::compiler::symbolic::{CompiledVarFragment, PerVarBytecodes}; + #[cfg(test)] + crate::db::note_fragment_execution(crate::db::FragmentExecKind::LtmBody, var_name); + // Project-global dims (datamodel form, used to resolve the equation's // dimension names) plus the canonicalized context + converted dims, all // from the salsa-cached queries rather than rebuilt per LTM fragment. @@ -1906,6 +1909,42 @@ pub(crate) fn compile_ltm_synthetic_fragment( } } +/// The salsa-memoized entry point for one LTM synthetic variable's fragment, +/// keyed by its INDEX into `model_ltm_variables(..).vars`. +/// +/// [`compile_ltm_synthetic_fragment`] routes only the scalar `Bare` `from->to` +/// score through a memoized query ([`compile_ltm_var_fragment`], keyed by the +/// link); every element-pinned, aggregate-touching, A2A or loop score takes the +/// plain-function `compile_direct` path. Both walkers over the variable list -- +/// `assemble_module`'s pass 3 and [`model_ltm_fragment_diagnostics`] -- then +/// compiled those from scratch, independently. On C-LEARN that is 5,985 of +/// 7,125 variables, roughly half of a full compile stage, paid a second time on +/// every `simlin_project_get_errors` / MCP `read_model`, and twice more on every +/// MCP `edit_model` (which runs a pre- and a post-edit diagnostic pass). +/// +/// Keyed by INDEX rather than by name because the index is what both walkers +/// already have, and because it keeps this a salsa FIREWALL: the query reads +/// the whole-model `model_ltm_variables`, so it re-executes on any edit, but its +/// VALUE is one fragment -- so salsa backdates it whenever that variable's +/// fragment is unchanged and `assemble_module` is not re-run. Same shape, and +/// the same reason, as `reconstruct_named_variable` over +/// `reconstruct_model_variables`. +/// +/// An out-of-range index yields `None`, which is also what a variable whose +/// fragment failed to compile yields; callers treat both as "no fragment", +/// exactly as they treated a `None` from the direct path. +#[salsa::tracked(returns(ref))] +pub(crate) fn compile_ltm_fragment_at( + db: &dyn Db, + model: SourceModel, + project: SourceProject, + index: usize, +) -> Option { + let ltm_vars = model_ltm_variables(db, model, project); + let ltm_var = ltm_vars.vars.get(index)?; + compile_ltm_synthetic_fragment(db, ltm_var, model, project) +} + #[cfg(test)] thread_local! { /// Test-only forced-failure pattern for @@ -1995,8 +2034,10 @@ pub fn model_ltm_fragment_diagnostics(db: &dyn Db, model: SourceModel, project: use crate::db::{CompilationDiagnostic, Diagnostic, DiagnosticError, DiagnosticSeverity}; let ltm_vars = model_ltm_variables(db, model, project); - for ltm_var in <m_vars.vars { - let fragment = compile_ltm_synthetic_fragment(db, ltm_var, model, project); + for (index, ltm_var) in ltm_vars.vars.iter().enumerate() { + // Through the memoized per-index query, so this pass READS assembly's + // fragments rather than compiling its own copies. + let fragment = compile_ltm_fragment_at(db, model, project, index); // A fragment is usable only if it compiled *and* produced // flow-phase bytecodes. `compile_ltm_equation_fragment` returns // `Some(_)` with `flow_bytecodes: None` when the synthetic diff --git a/src/simlin-engine/src/db/ltm/mod.rs b/src/simlin-engine/src/db/ltm/mod.rs index a0c159dfe..e7e2d32bd 100644 --- a/src/simlin-engine/src/db/ltm/mod.rs +++ b/src/simlin-engine/src/db/ltm/mod.rs @@ -51,9 +51,14 @@ pub use equation::{LtmArm, LtmEquation}; pub(crate) use compile::ForcePartialEquationErrorGuard; pub use compile::{ShapedLinkScore, compile_ltm_var_fragment, link_score_equation_text_shaped}; pub(crate) use compile::{ - compile_ltm_implicit_var_fragment, compile_ltm_synthetic_fragment, - model_ltm_fragment_diagnostics, + compile_ltm_fragment_at, compile_ltm_implicit_var_fragment, model_ltm_fragment_diagnostics, }; +// Production reaches an LTM fragment only through the memoized +// `compile_ltm_fragment_at`; the unmemoized selector below it is re-exported +// for the fragment characterization/determinism tests, which drive one +// variable's compile directly rather than through a whole-model walk. +#[cfg(test)] +pub(crate) use compile::compile_ltm_synthetic_fragment; pub(crate) use link_scores::emit_ltm_partial_equation_warning; #[cfg(test)] pub(crate) use link_scores::ltm_partial_equation_warning_message; diff --git a/src/simlin-engine/src/db/ltm_tests.rs b/src/simlin-engine/src/db/ltm_tests.rs index 58a3ddd00..ca77088e7 100644 --- a/src/simlin-engine/src/db/ltm_tests.rs +++ b/src/simlin-engine/src/db/ltm_tests.rs @@ -2067,3 +2067,108 @@ fn the_completeness_guard_holds_on_the_per_element_emitter() { .collect::>() ); } + +/// A `Wide`-dimensioned per-element link-score fixture: a per-element flow +/// whose element `e` reads only `pop[e]`, so the emitted scores are the +/// element-pinned and A2A shapes that `compile_ltm_synthetic_fragment` routes +/// down its uncached `compile_direct` path -- which is what makes it a fixture +/// for the fragment-reuse tests below. +fn per_element_zero_slot_project(n: usize) -> datamodel::Project { + let elems: Vec = (0..n).map(|i| format!("e{i}")).collect(); + let elem_refs: Vec<&str> = elems.iter().map(String::as_str).collect(); + let eqns: Vec<(String, String)> = elems + .iter() + .map(|e| (e.clone(), format!("pop[{e}] * rate * 0.01"))) + .collect(); + let eqn_refs: Vec<(&str, &str)> = eqns.iter().map(|(e, q)| (e.as_str(), q.as_str())).collect(); + + TestProject::new("per_element_zero_slots") + .named_dimension("Wide", &elem_refs) + .aux("rate", "1", None) + .array_flow_with_ranges("growth[Wide]", eqn_refs) + .array_stock("pop[Wide]", "10", &["growth"], &[], None) + .build_datamodel() +} + +/// The diagnostic pass must REUSE assembly's compiled LTM fragments, not +/// recompile them. +/// +/// `assemble_module` and `model_ltm_fragment_diagnostics` each walk every LTM +/// synthetic variable and ask for its fragment. Only the scalar `Bare` +/// `from->to` score went through a salsa-memoized query; every element-pinned, +/// aggregate-touching or A2A score took a plain-function path, so the second +/// walk recompiled it from scratch. On C-LEARN that is 5,985 of 7,125 variables +/// -- about half a full compile stage -- paid again on every +/// `simlin_project_get_errors`, every MCP `read_model`, and twice more on every +/// MCP `edit_model`. +/// +/// The measurement is a body-entry count, not a timing: `LtmBody` is recorded +/// inside `compile_ltm_equation_fragment`, which every LTM path funnels +/// through, so a cache hit is invisible to it and a real compile is not. +#[test] +fn the_ltm_diagnostic_pass_does_not_recompile_assembly_fragments() { + let project = per_element_zero_slot_project(4); + let mut db = SimlinDb::default(); + let (source_project, model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + use salsa::Setter; + source_project.set_ltm_enabled(&mut db).to(true); + + // Assembly first: this is the walk that legitimately compiles every + // fragment. Priming it here is what makes the measured region below a + // second walk rather than a first one. + let compiled = crate::db::compile_project_incremental(&db, source_project, "main"); + assert!( + compiled.is_ok(), + "the fixture must compile with LTM enabled: {:?}", + compiled.err() + ); + + // Control: the recorder is armed and the fixture really does generate LTM + // fragments, so a zero below cannot be an empty model or a dead counter. + crate::db::reset_fragment_executions(); + let _ = crate::db::ltm::model_ltm_fragment_diagnostics(&db, model, source_project); + let after_first = crate::db::fragment_executions(); + let ltm_bodies: Vec<&str> = after_first + .iter() + .filter(|(kind, _)| *kind == crate::db::FragmentExecKind::LtmBody) + .map(|(_, name)| name.as_str()) + .collect(); + + assert!( + ltm_bodies.is_empty(), + "the diagnostic pass recompiled {} LTM fragment(s) that assembly had \ + already compiled: {ltm_bodies:?}", + ltm_bodies.len() + ); +} + +/// The control for the test above: the recorder DOES see LTM fragment compiles +/// when they genuinely happen, so an empty log there is evidence of reuse +/// rather than of a counter that never fires. +#[test] +fn the_ltm_fragment_body_counter_observes_a_cold_compile() { + let project = per_element_zero_slot_project(4); + let mut db = SimlinDb::default(); + let source_project = { + let sync = sync_from_datamodel(&db, &project); + sync.project + }; + use salsa::Setter; + source_project.set_ltm_enabled(&mut db).to(true); + + crate::db::reset_fragment_executions(); + let _ = crate::db::compile_project_incremental(&db, source_project, "main"); + let execs = crate::db::fragment_executions(); + let n_ltm = execs + .iter() + .filter(|(kind, _)| *kind == crate::db::FragmentExecKind::LtmBody) + .count(); + assert!( + n_ltm > 0, + "a cold LTM compile must record LtmBody entries, or the reuse assertion \ + in the sibling test proves nothing; got: {execs:?}" + ); +} From c556957673a3ded169760792bb4b1a06e9300d1c Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 9 Aug 2026 23:56:42 -0700 Subject: [PATCH 05/59] engine: fuse leaf stores and give module inputs a fusible leaf form `AssignCurr` is 10.68% of executed dispatches on C-LEARN, and the measured bigrams account for essentially all of it: `If` 5.64% (taken by the preceding conditional-select fusion), `LoadModuleInput` 1.48%, `LoadVar` 1.41%, `LoadInitial` 1.34%, `Apply` 0.85%. The three leaf loads that feed a store get a fused register-style form here -- `AssignVarCurr` (a slot-to-slot copy, which is what an alias or pass-through variable compiles to), `AssignInitialCurr`, and `AssignModInputCurr`. `Apply; AssignCurr` is deliberately NOT fused. The `Apply` arm inlines every builtin body, so duplicating it to fold a store would be the largest code growth in the hot function for the smallest member of the set, and `eval_bytecode` is already >= 66 KB against a 32 KB L1i. Separately, `LoadModuleInput` was not a fusible leaf at all, despite being 4.68% of C-LEARN dispatches and 5.8% of WORLD3's -- the 2-window handled `LoadVar`/`LoadConstant`/`LoadGlobalVar` and stopped there. It now joins them with `BinStackModInput` and `AssignStackModInputCurr`. Module inputs stay 2-window only: 3-window leaf forms would need one opcode per (leaf x leaf) pairing for a measured minority of the bigrams, and each new arm costs icache in the hot function permanently. `LoadConstant; AssignCurr` is absent from the leaf-store set because it never reaches this pass -- the symbolic `peephole_optimize` already folds it into `AssignConstCurr`. Stack effects are asserted: the three stores are (0,0), exactly the net of the `LoadX`(0,1) + `AssignCurr`(1,0) they replace, so a program's peak depth cannot move; `BinStackModInput` is (1,1) and `AssignStackModInputCurr` is (1,0), mirroring their var/const twins. A test drives all three stores through `max_stack_depth` and pins 1 before, 0 after. Operand order is pinned for the non-commutative Sub and Div, where a swapped encoding would be a silent miscompile rather than a loud failure. Measured for the whole Stage A bundle against the pre-bundle baseline (perf stat, 26 runs/side, two interleaved rounds; instructions and branches are the layout-insensitive numbers, cycles moved several percent between builds of identical source so they are reported but not leaned on): C-LEARN instructions -6.22%, branches -6.34%, cycles -5.20% WORLD3 instructions -4.07%, branches -3.67%, cycles -0.82% This commit is the largest single contributor: it took C-LEARN from -4.25% to -6.22% and WORLD3 from -2.35% to -4.07%. Behaviour-preserving: 5482 engine lib tests and 631 integration tests pass, including `clearn_residual_exactness`, `oracle_clearn`, and `vdf_parity`. --- src/simlin-engine/src/bytecode.rs | 259 ++++++++++++++++++++++++++++++ src/simlin-engine/src/vm.rs | 35 ++++ 2 files changed, 294 insertions(+) diff --git a/src/simlin-engine/src/bytecode.rs b/src/simlin-engine/src/bytecode.rs index 0af917f0b..b33950869 100644 --- a/src/simlin-engine/src/bytecode.rs +++ b/src/simlin-engine/src/bytecode.rs @@ -825,6 +825,49 @@ pub(crate) enum Opcode { off: VariableOffset, }, + // === LEAF STORES AND MODULE-INPUT OPERANDS (R3) === + // `AssignCurr` is 10.68% of executed dispatches on C-LEARN, and the measured + // bigrams account for essentially all of it. The conditional-select forms + // above take the `If` share; these take the three leaf loads that feed a + // store directly. `Apply; AssignCurr` is deliberately left unfused -- the + // `Apply` arm inlines every builtin body, so duplicating it to fold a store + // would be the largest code growth in the hot function for the smallest + // member of the set. + // + // `LoadConstant; AssignCurr` is absent because it never reaches this pass: + // the symbolic `peephole_optimize` already folds it into `AssignConstCurr`. + /// `curr[module_off + dst] = curr[module_off + src]` -- a slot-to-slot copy + /// (alias / pass-through variables). + AssignVarCurr { + src: VariableOffset, + dst: VariableOffset, + }, + /// `curr[module_off + dst] = `. Reads `curr` during + /// the initials phase and `initial_values` afterwards, exactly as + /// `LoadInitial` does. + AssignInitialCurr { + src: VariableOffset, + dst: VariableOffset, + }, + /// `curr[module_off + dst] = module_inputs[input]`. + AssignModInputCurr { + input: ModuleInputOffset, + dst: VariableOffset, + }, + /// Pop `lhs`; push `lhs op module_inputs[r_input]`. `LoadModuleInput` was + /// not a fusible leaf at all before this, despite being 4.68% of C-LEARN + /// dispatches and 5.8% of WORLD3's. + BinStackModInput { + r_input: ModuleInputOffset, + op: Op2, + }, + /// Pop `lhs`; `curr[module_off + dst] = lhs op module_inputs[b_input]`. + AssignStackModInputCurr { + dst: VariableOffset, + b_input: ModuleInputOffset, + op: Op2, + }, + // === 3-ADDRESS BINARY OPS (R2) === // Fold the leaf operand load(s) of a binary op into the op itself, so a // subexpression `a op b` dispatches once instead of 3 (two loads + Op2) or @@ -1436,6 +1479,15 @@ impl Opcode { Opcode::SelectIf {} => (3, 1), // pops cond+false+true, pushes result Opcode::SelectIfAssignCurr { .. } => (3, 0), // same, assigns directly + // Leaf stores: exactly the net of the `LoadX`(0,1) + `AssignCurr`(1,0) + // they replace, so a program's peak depth cannot move. + Opcode::AssignVarCurr { .. } + | Opcode::AssignInitialCurr { .. } + | Opcode::AssignModInputCurr { .. } => (0, 0), + // Module-input operand forms mirror their var/const twins. + Opcode::BinStackModInput { .. } => (1, 1), + Opcode::AssignStackModInputCurr { .. } => (1, 0), + // 3-address binops: the *Var/*Const forms read both operands from // curr/literals and push (0 pops, 1 push); the Stack* forms pop the // lhs and push the result (1 pop, 1 push). @@ -1592,6 +1644,11 @@ impl Opcode { Opcode::BinOpAssignNext { .. } => "BinOpAssignNext", Opcode::SelectIf {} => "SelectIf", Opcode::SelectIfAssignCurr { .. } => "SelectIfAssignCurr", + Opcode::AssignVarCurr { .. } => "AssignVarCurr", + Opcode::AssignInitialCurr { .. } => "AssignInitialCurr", + Opcode::AssignModInputCurr { .. } => "AssignModInputCurr", + Opcode::BinStackModInput { .. } => "BinStackModInput", + Opcode::AssignStackModInputCurr { .. } => "AssignStackModInputCurr", Opcode::AssignAddVarVarCurr { .. } => "AssignAddVarVarCurr", Opcode::AssignSubVarVarCurr { .. } => "AssignSubVarVarCurr", Opcode::AssignMulVarVarCurr { .. } => "AssignMulVarVarCurr", @@ -2089,6 +2146,34 @@ impl ByteCode { } } + // Leaf store: `LoadX; AssignCurr` -> one register-style store, for + // the three leaf loads the measured bigrams show feeding a store. + // Tried before the leaf windows for the same reason as the select + // above: `AssignCurr` is not a combiner in either of them, so the + // rule sets are disjoint. + if i + 1 < self.code.len() + && !jump_targets[i + 1] + && let Opcode::AssignCurr { off: dst } = self.code[i + 1] + { + let fused_store = match self.code[i] { + Opcode::LoadVar { off: src } => Some(Opcode::AssignVarCurr { src, dst }), + Opcode::LoadInitial { off: src } => { + Some(Opcode::AssignInitialCurr { src, dst }) + } + Opcode::LoadModuleInput { input } => { + Some(Opcode::AssignModInputCurr { input, dst }) + } + _ => None, + }; + if let Some(op) = fused_store { + optimized.push(op); + pc_map.push(new_pc); // old i + pc_map.push(new_pc); // old i+1 + i += 2; + continue; + } + } + // 3-window: [leaf load, leaf load, ] where the combiner is // either an `Op2` (a pushing subexpression) or a `BinOpAssign{Curr| // Next}` (a leaf assignment, post-peephole). Both absorbed @@ -2220,6 +2305,22 @@ impl ByteCode { Opcode::LoadGlobalVar { off: b } => { push2.map(|op| Opcode::BinStackGlobal { r_global: *b, op }) } + // `(lhs on stack) op module_input`. Both combiners are + // handled, mirroring the var/const leaves above. Module + // inputs are 2-window only: giving them 3-window leaf forms + // would need one opcode per (leaf x leaf) pairing for a + // measured minority of the bigrams. + Opcode::LoadModuleInput { input: b } => assign2 + .and_then(|(op, dst, n)| { + // No `next[]` module-input store form: a stock update + // never reads a module input as its trailing leaf. + (!n).then_some(Opcode::AssignStackModInputCurr { + dst, + b_input: *b, + op, + }) + }) + .or_else(|| push2.map(|op| Opcode::BinStackModInput { r_input: *b, op })), _ => None, } } else { @@ -3983,6 +4084,164 @@ mod tests { assert_eq!(bc.max_stack_depth().unwrap(), 3); } + // === Leaf-store and module-input fusion (P5) === + // + // `AssignCurr` is 10.68% of executed dispatches on C-LEARN and the measured + // bigrams account for essentially all of it: `If` 5.64% (taken by the + // conditional-select fusion above), `LoadModuleInput` 1.48%, `LoadVar` + // 1.41%, `LoadInitial` 1.34%, `Apply` 0.85%. The first three get a fused + // store here. `Apply;AssignCurr` is deliberately NOT fused: the `Apply` arm + // inlines every builtin body, so duplicating it for a store would be the + // largest code growth in the hot function for the smallest member of the + // set. + // + // `LoadModuleInput` was additionally not a fusible LEAF at all, though it is + // 4.68% of C-LEARN dispatches and 5.8% of WORLD3's, so it joins the 2-window + // alongside LoadVar/LoadConstant/LoadGlobalVar. + // + // `LoadConstant; AssignCurr` never reaches this pass -- the symbolic + // `peephole_optimize` already folds it into `AssignConstCurr`. + + #[test] + fn test_fuse_load_var_assign_is_a_slot_copy() { + let mut bc = ByteCode { + literals: vec![], + code: vec![Opcode::LoadVar { off: 5 }, Opcode::AssignCurr { off: 9 }], + }; + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 1); + assert!(matches!( + bc.code[0], + Opcode::AssignVarCurr { src: 5, dst: 9 } + )); + } + + #[test] + fn test_fuse_load_initial_assign() { + let mut bc = ByteCode { + literals: vec![], + code: vec![ + Opcode::LoadInitial { off: 2 }, + Opcode::AssignCurr { off: 7 }, + ], + }; + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 1); + assert!(matches!( + bc.code[0], + Opcode::AssignInitialCurr { src: 2, dst: 7 } + )); + } + + #[test] + fn test_fuse_load_module_input_assign() { + let mut bc = ByteCode { + literals: vec![], + code: vec![ + Opcode::LoadModuleInput { input: 3 }, + Opcode::AssignCurr { off: 4 }, + ], + }; + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 1); + assert!(matches!( + bc.code[0], + Opcode::AssignModInputCurr { input: 3, dst: 4 } + )); + } + + #[test] + fn test_fuse_module_input_as_binop_rhs_preserves_operand_order() { + // `(lhs on stack) - module_input[2]`. Sub is non-commutative, so a + // swapped encoding would be a silent miscompile. + let mut bc = ByteCode { + literals: vec![], + code: vec![ + Opcode::LoadVar { off: 1 }, + Opcode::LoadModuleInput { input: 2 }, + Opcode::Op2 { op: Op2::Sub }, + ], + }; + bc.fuse_three_address(); + // LoadVar;LoadModuleInput is not a 3-window leaf pair (module inputs are + // 2-window only), so the LoadVar stays and the rhs+op fuse. + assert_eq!(bc.code.len(), 2); + assert!(matches!(bc.code[0], Opcode::LoadVar { off: 1 })); + assert!(matches!( + bc.code[1], + Opcode::BinStackModInput { + r_input: 2, + op: Op2::Sub + } + )); + } + + #[test] + fn test_fuse_module_input_stack_leaf_assign_preserves_operand_order() { + // `dst = (lhs on stack) / module_input[6]`, post-peephole. + let mut bc = ByteCode { + literals: vec![], + code: vec![ + Opcode::LoadVar { off: 1 }, + Opcode::LoadModuleInput { input: 6 }, + Opcode::BinOpAssignCurr { + op: Op2::Div, + off: 8, + }, + ], + }; + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 2); + assert!(matches!(bc.code[0], Opcode::LoadVar { off: 1 })); + assert!(matches!( + bc.code[1], + Opcode::AssignStackModInputCurr { + dst: 8, + b_input: 6, + op: Op2::Div + } + )); + } + + #[test] + fn test_fuse_leaf_store_blocked_by_jump_target() { + // A jump targets the AssignCurr the pair would absorb. + let mut bc = ByteCode { + literals: vec![], + code: vec![ + Opcode::LoadVar { off: 0 }, // [0] + Opcode::AssignCurr { off: 1 }, // [1] <- jump target + Opcode::NextIterOrJump { jump_back: -1 }, // [2] -> [1] + ], + }; + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 3); + assert!(matches!(bc.code[0], Opcode::LoadVar { off: 0 })); + assert!(matches!(bc.code[1], Opcode::AssignCurr { off: 1 })); + } + + #[test] + fn test_fuse_leaf_store_preserves_max_stack_depth() { + // Each fused store is (0,0), exactly the net of the `LoadX`(0,1) + + // `AssignCurr`(1,0) it replaces, so the program's peak cannot move. + let mut bc = ByteCode { + literals: vec![], + code: vec![ + Opcode::LoadVar { off: 0 }, + Opcode::AssignCurr { off: 1 }, + Opcode::LoadInitial { off: 2 }, + Opcode::AssignCurr { off: 3 }, + Opcode::LoadModuleInput { input: 0 }, + Opcode::AssignCurr { off: 4 }, + ], + }; + let before = bc.max_stack_depth().unwrap(); + assert_eq!(before, 1); + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 3); + assert_eq!(bc.max_stack_depth().unwrap(), 0); + } + // === 3-address fusion with GLOBAL operands and two-constant operands === // // Globals (TIME/DT/...) load via `LoadGlobalVar`; the fusion now folds them diff --git a/src/simlin-engine/src/vm.rs b/src/simlin-engine/src/vm.rs index 40e29929e..40ff4a459 100644 --- a/src/simlin-engine/src/vm.rs +++ b/src/simlin-engine/src/vm.rs @@ -2177,6 +2177,41 @@ impl Vm { curr[module_off + *off as usize] = if is_truthy(cond) { t } else { f }; debug_assert_eq!(0, stack.len()); } + // === LEAF STORES AND MODULE-INPUT OPERANDS (R3) === + // Each reads its leaf from the region `LoadVar` / `LoadInitial` + // / `LoadModuleInput` would have read and writes `curr` + // directly, touching the arithmetic stack not at all. + Opcode::AssignVarCurr { src, dst } => { + curr[module_off + *dst as usize] = curr[module_off + *src as usize]; + debug_assert_eq!(0, stack.len()); + } + Opcode::AssignInitialCurr { src, dst } => { + // Mirrors `LoadInitial`: during the initials phase the + // snapshot does not exist yet, so read the row being built. + let abs_src = module_off + *src as usize; + let value = if part == StepPart::Initials { + curr[abs_src] + } else { + initial_values[abs_src] + }; + curr[module_off + *dst as usize] = value; + debug_assert_eq!(0, stack.len()); + } + Opcode::AssignModInputCurr { input, dst } => { + curr[module_off + *dst as usize] = module_inputs[*input as usize]; + debug_assert_eq!(0, stack.len()); + } + Opcode::BinStackModInput { r_input, op } => { + let lv = stack.pop(); + let rv = module_inputs[*r_input as usize]; + stack.push(eval_op2(*op, lv, rv)); + } + Opcode::AssignStackModInputCurr { dst, b_input, op } => { + let lhs = stack.pop(); + let rhs = module_inputs[*b_input as usize]; + curr[module_off + *dst as usize] = eval_op2(*op, lhs, rhs); + debug_assert_eq!(0, stack.len()); + } // === 3-ADDRESS BINARY OPS (R2) === // Operands are read straight from curr[]/literals; the *Stack* // forms take the lhs from the arithmetic stack. Each pushes the From fb8220995e45ec22767d6e699c906651e2a60838 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:03:49 -0700 Subject: [PATCH 06/59] engine: skip the no-op canonicalize in is_dimension_name `compiler::context`'s `is_dimension_name` canonicalized every declared dimension's name on every call, to compare it against the canonicalized subscript. A `Dimension`'s name is a `CanonicalDimensionName`, canonical at every site that builds one -- both arms of `From<&datamodel::Dimension>` and every other production construction go through `CanonicalDimensionName::from_raw` -- so that inner call could not change its input, and `canonicalize` still scanned the whole string to decide that. The predicate runs once per bare-identifier subscript per reference (`IndexExpr3::from_index_expr2` asks it whether the index names a dimension), so the redundant scan was paid once per DECLARED DIMENSION per subscript. On C-LEARN, whose project declares 126 dimensions, that one call site was 1.09M of the compiler's 2.15M `canonicalize` calls; removing it measures -5.0% of a cold compile's instructions (interleaved A/B, three rounds: 8.528G -> 8.102G over four compiles). The premise is what makes this safe rather than merely faster, so it is pinned directly: `dimension_name_is_canonical_for_every_constructor` asserts `Dimension::name()` equals `canonicalize(raw)` over both the Named and the Indexed arm (they canonicalize at separate call sites) for the shapes canonicalization actually changes -- case, interior whitespace, padding, and a dotted name. A constructor that stopped canonicalizing would red that test rather than silently making this predicate miss a dimension. The compiled artifact is unchanged: C-LEARN still assembles 5215 slots and 58291 opcodes (31525 flow + 1477 stock + 25289 initial), 2196 literals, 162 graphical functions over 37065 points, 28 temp slots, 126 dimensions, 643 static views, 371 names and 7 modules. --- src/simlin-engine/src/compiler/context.rs | 15 ++++++-- src/simlin-engine/src/dimensions.rs | 42 +++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/simlin-engine/src/compiler/context.rs b/src/simlin-engine/src/compiler/context.rs index ac642034c..579828559 100644 --- a/src/simlin-engine/src/compiler/context.rs +++ b/src/simlin-engine/src/compiler/context.rs @@ -1127,11 +1127,20 @@ impl Expr3LowerContext for Context<'_> { var_metadata.var.get_dimensions() } + /// Only `ident` needs canonicalizing: a `Dimension`'s name is a + /// `CanonicalDimensionName`, canonical by construction at every site that + /// builds one (`dimensions::dimension_name_is_canonical_for_every_constructor` + /// pins that), so canonicalizing it again could not change it -- and + /// `canonicalize` still scans the whole string to decide that. + /// + /// This runs once per bare-identifier subscript per reference + /// (`ast::expr3::IndexExpr3::from_index_expr2`), so the redundant scan was + /// paid once per DECLARED DIMENSION per subscript. On a model with 126 + /// dimensions it was half of every `canonicalize` call the compiler made + /// and ~5% of a whole compile. fn is_dimension_name(&self, ident: &str) -> bool { let canonical = canonicalize(ident); - self.dimensions - .iter() - .any(|dim| *canonicalize(dim.name()) == *canonical) + self.dimensions.iter().any(|dim| dim.name() == &*canonical) } } diff --git a/src/simlin-engine/src/dimensions.rs b/src/simlin-engine/src/dimensions.rs index 3e9d9ca58..32595e17e 100644 --- a/src/simlin-engine/src/dimensions.rs +++ b/src/simlin-engine/src/dimensions.rs @@ -2492,6 +2492,48 @@ mod tests { assert!(ctx.get(&unknown).is_none()); } + /// `Dimension::name()` is canonical for EVERY constructor, which is what + /// lets a caller comparing against it skip re-canonicalizing: both arms of + /// `From<&datamodel::Dimension>` build the name with + /// `CanonicalDimensionName::from_raw`, and so does every other production + /// construction of the two variants. `compiler::context`'s + /// `is_dimension_name` relies on this to compare a canonicalized subscript + /// against `dim.name()` directly; re-canonicalizing there was a provable + /// no-op that still scanned the string once per declared dimension per + /// reference, and on a 126-dimension model it was ~5% of a compile. + /// + /// The rows are the shapes canonicalization actually changes -- case, + /// interior whitespace, a leading/trailing pad, and a dotted name (the + /// period becomes the module-separator middle dot) -- over both the Named + /// and the Indexed arm, since they canonicalize at separate call sites. + #[test] + fn dimension_name_is_canonical_for_every_constructor() { + for raw in [ + "Region", + "My Region", + " Padded Region ", + "MIXED.Case", + "already_canonical", + ] { + let named = Dimension::from(&datamodel::Dimension::named( + raw.to_string(), + vec!["North".to_string()], + )); + let indexed = Dimension::from(&datamodel::Dimension::indexed(raw.to_string(), 3)); + let expected = crate::common::canonicalize(raw); + assert_eq!( + named.name(), + &*expected, + "Named dimension name not canonical for {raw:?}" + ); + assert_eq!( + indexed.name(), + &*expected, + "Indexed dimension name not canonical for {raw:?}" + ); + } + } + #[test] fn test_indexed_dimension_with_maps_to_is_ignored() { // Indexed dimensions should not have maps_to - this test verifies From 9a9934ec1da39659a565c147f997f985b0ca1cc6 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:05:51 -0700 Subject: [PATCH 07/59] engine: share LTM equation ASTs between the shaped memo and emission Every emitted link score is cloned out of the `link_score_equation_text_shaped` memo into `model_ltm_variables`' own variable list (`db/ltm/link_scores.rs`), so each generated equation's parsed tree was retained TWICE for the life of the database. On C-LEARN the generation stage retains +273 MiB for 12.78 MB of equation text, and the ASTs dominate that. `LtmArm::expr` is now an `Arc`, so that clone is a refcount bump and one copy is retained. Measured on C-LEARN: peak live bytes 496.3 -> 440.7 MiB (-55.6 MiB) and 1.3M fewer allocations, with the LTM bytecode byte-identical at 1,238,728 opcodes and the root slot count unchanged. `Arc` still compares BY VALUE. That is load-bearing rather than incidental: salsa backdates a re-executed query whose value compares equal, and that backdating is what lets an unrelated edit reuse the expensive downstream fragment (GH #981). The existing NaN-equality test pins it, and the new sharing test asserts value equality alongside pointer identity so a future change cannot buy sharing by weakening comparison. Building an `Ast` in `to_flow_ast` still unshares, because that type owns its tree. That is the right split: the result is consumed by the fragment compile and dropped, while the arm is retained for the life of the database -- so sharing bounds retention rather than avoiding a transient copy. Pointer identity is the only way to observe this, since both copies compare equal either way, so a value assertion would pass on a deep copy. The new test was verified to constrain the code by temporarily reintroducing the deep copy in `retarget_dims` and confirming it fails. --- src/simlin-engine/src/db/ltm/equation.rs | 36 ++++++++++-- src/simlin-engine/src/db/ltm_tests.rs | 71 ++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/src/simlin-engine/src/db/ltm/equation.rs b/src/simlin-engine/src/db/ltm/equation.rs index 304ed2922..d2e0c34d4 100644 --- a/src/simlin-engine/src/db/ltm/equation.rs +++ b/src/simlin-engine/src/db/ltm/equation.rs @@ -24,6 +24,8 @@ use std::collections::HashMap; +use std::sync::Arc; + use crate::ast::{Ast, Expr0}; use crate::common::{CanonicalElementName, EquationError}; use crate::lexer::LexerType; @@ -51,7 +53,20 @@ pub struct LtmArm { /// diagnostics; never re-parsed to compile. pub text: String, /// The authoritative compiled AST (`Expr0::new(text)`). - pub expr: Option, + /// + /// Behind an `Arc` because every emitted link score is cloned out of the + /// `link_score_equation_text_shaped` memo (`db/ltm/link_scores.rs`) into + /// `model_ltm_variables`' own list, so the tree would otherwise be retained + /// TWICE for the whole life of the database -- on C-LEARN, two copies of + /// 12.78 MB of equations, whose ASTs dominate that query's ~273 MiB. Sharing + /// makes that clone a refcount bump and retains one copy. + /// + /// `Arc` still compares BY VALUE, which is load-bearing: salsa + /// backdates a re-executed query whose value compares equal, and that is + /// what lets an unrelated edit reuse the expensive downstream fragment (GH + /// #981). Pointer equality would be an optimization on top, never a + /// substitute. + pub expr: Option>, /// `Some` iff `text` FAILED to parse -- never merely because it was empty. /// Preserved (rather than discarded at construction) so the arm that failed /// can reject its whole equation; see the type docs. @@ -88,7 +103,7 @@ impl LtmArm { // strictly worse than a diagnostic, and libsimlin release builds are // panic=abort. let (expr, parse_error) = match Expr0::new(&text, LexerType::Equation) { - Ok(expr) => (expr, None), + Ok(expr) => (expr.map(Arc::new), None), // `Expr0::new` reports every position it found; keep the first as // the failure's provenance (see the field docs). Err(errs) => (None, errs.into_iter().next()), @@ -314,12 +329,20 @@ impl LtmEquation { if !parse_errors.is_empty() { return (None, parse_errors); } + // The arms' ASTs are shared (`Arc`), but `Ast` owns its tree, so + // building one unshares. That is the right trade: the result is consumed + // by the fragment compile and dropped, whereas the arm itself is retained + // for the life of the database -- so the sharing is what bounds RETENTION, + // not what avoids this transient copy. match self { - LtmEquation::Scalar(arm) => (arm.expr.clone().map(Ast::Scalar), vec![]), + LtmEquation::Scalar(arm) => (arm.expr.as_deref().cloned().map(Ast::Scalar), vec![]), LtmEquation::ApplyToAll(dims, arm) => { match crate::variable::get_dimensions(dimensions, dims) { Ok(resolved) => ( - arm.expr.clone().map(|e| Ast::ApplyToAll(resolved, e)), + arm.expr + .as_deref() + .cloned() + .map(|e| Ast::ApplyToAll(resolved, e)), vec![], ), Err(err) => (None, vec![err]), @@ -338,11 +361,12 @@ impl LtmEquation { .iter() .filter_map(|(subscript, arm)| { arm.expr - .clone() + .as_deref() + .cloned() .map(|e| (CanonicalElementName::from_raw(subscript), e)) }) .collect(); - let default_expr = default.as_ref().and_then(|a| a.expr.clone()); + let default_expr = default.as_ref().and_then(|a| a.expr.as_deref().cloned()); match crate::variable::get_dimensions(dimensions, dims) { Ok(resolved) => ( Some(Ast::Arrayed( diff --git a/src/simlin-engine/src/db/ltm_tests.rs b/src/simlin-engine/src/db/ltm_tests.rs index ca77088e7..92904a2ef 100644 --- a/src/simlin-engine/src/db/ltm_tests.rs +++ b/src/simlin-engine/src/db/ltm_tests.rs @@ -2172,3 +2172,74 @@ fn the_ltm_fragment_body_counter_observes_a_cold_compile() { in the sibling test proves nothing; got: {execs:?}" ); } + +/// The first arm's shared AST of an LTM equation, whatever its shape. +fn first_arm_expr(eq: &crate::db::LtmEquation) -> &std::sync::Arc { + use crate::db::LtmEquation; + let arm = match eq { + LtmEquation::Scalar(arm) | LtmEquation::ApplyToAll(_, arm) => arm, + LtmEquation::Arrayed { elements, .. } => { + &elements + .first() + .expect("an arrayed equation must have an arm") + .1 + } + }; + arm.expr.as_ref().expect("the fixture's arm must parse") +} + +/// `model_ltm_variables` must SHARE each emitted score's parsed AST with the +/// `link_score_equation_text_shaped` memo it came from, not deep-copy it. +/// +/// The emission loop clones the shaped result out of the memo for every score, +/// so before the ASTs were shared each equation was retained twice for the life +/// of the database. On C-LEARN the generation stage retains +273 MiB for 12.78 MB +/// of equation text, and roughly half of that is the second copy. +/// +/// Pointer identity is the only way to see this: both copies compare EQUAL by +/// value either way -- which they must, since salsa backdates on value equality +/// and that is what lets an unrelated edit reuse the expensive downstream +/// fragment (GH #981). A value assertion here would pass on a deep copy. +#[test] +fn an_emitted_link_score_shares_its_ast_with_the_shaped_memo() { + let project = per_element_zero_slot_project(4); + let mut db = SimlinDb::default(); + let (source_project, model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + use salsa::Setter; + source_project.set_ltm_enabled(&mut db).to(true); + + let ltm = crate::db::model_ltm_variables(&db, model, source_project); + let emitted = ltm + .vars + .iter() + .find(|v| v.name == "$\u{205A}ltm\u{205A}link_score\u{205A}growth\u{2192}pop") + .unwrap_or_else(|| { + panic!( + "fixture must emit the growth->pop link score; got: {:?}", + ltm.vars.iter().map(|v| &v.name).collect::>() + ) + }); + + let link_id = LtmLinkId::new(&db, "growth".to_string(), "pop".to_string()); + let shaped = + link_score_equation_text_shaped(&db, link_id, RefShape::Bare, model, source_project); + let ShapedLinkScore::Scored { var: memo_var, .. } = shaped else { + panic!("the growth->pop edge must be scored; got: {shaped:?}"); + }; + + let memo_expr = first_arm_expr(&memo_var.equation); + let emitted_expr = first_arm_expr(&emitted.equation); + + // The control: they must still be EQUAL, or salsa backdating breaks. + assert_eq!( + memo_expr, emitted_expr, + "the emitted score and the memo must compare equal by value" + ); + assert!( + std::sync::Arc::ptr_eq(memo_expr, emitted_expr), + "the emitted score must SHARE the memo's AST, not hold a deep copy" + ); +} From 6b27decc1ead585f7fd9a15fb65e0fab9b52297a Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:12:40 -0700 Subject: [PATCH 08/59] diagram: share one font database across PNG renders `svg_to_png` built a fresh `usvg::fontdb::Database` on every call, copying the 170 KB embedded Roboto Light with `to_vec()` and re-parsing the face each time. The database holds exactly one face and never changes, so every render after the first re-derived an identical immutable value; a `simlin_project_render_png` FFI caller rendering N diagrams paid it N times. This is a shipped-path cleanup, not a test-speed change, and the measurement says so: the 16 `render_png` unit tests take 2.51 s before and 2.53 s across three alternating runs -- indistinguishable. Those tests are dominated by SVG filter rasterization (`perf`: ~80% in `resvg::filter::morphology::apply`), so the font parse was never a visible share of them. What the change buys is bounded by that same measurement: under 3 ms and one 170 KB allocation per render. `usvg::Options::fontdb` is already an `Arc`, so sharing costs a refcount bump and no caller can observe the difference. --- src/simlin-engine/src/diagram/render_png.rs | 24 +++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/simlin-engine/src/diagram/render_png.rs b/src/simlin-engine/src/diagram/render_png.rs index 216aae505..bfd4c87ac 100644 --- a/src/simlin-engine/src/diagram/render_png.rs +++ b/src/simlin-engine/src/diagram/render_png.rs @@ -9,6 +9,8 @@ //! the SVG with resvg. The Roboto Light font is embedded into the binary //! so that text renders identically across all platforms and environments. +use std::sync::{Arc, OnceLock}; + use resvg::tiny_skia; use resvg::usvg; @@ -17,6 +19,23 @@ use crate::datamodel; /// Roboto Light font data, embedded at compile time. static ROBOTO_LIGHT: &[u8] = include_bytes!("fonts/Roboto-Light.ttf"); +/// The font database every render shares. +/// +/// It holds exactly one face and never changes, but building it parses the +/// embedded TTF, so doing it per call made font parsing a fixed tax on every +/// render rather than a startup cost paid once. `usvg::Options::fontdb` is an +/// `Arc` already, so sharing costs a refcount bump and callers cannot observe +/// the difference. +fn roboto_light_db() -> Arc { + static DB: OnceLock> = OnceLock::new(); + DB.get_or_init(|| { + let mut fontdb = usvg::fontdb::Database::new(); + fontdb.load_font_data(ROBOTO_LIGHT.to_vec()); + Arc::new(fontdb) + }) + .clone() +} + /// Options controlling PNG rendering output. #[derive(Default)] pub struct PngRenderOpts { @@ -48,12 +67,9 @@ pub fn render_png( /// Exposed separately so callers that already have an SVG string (e.g. /// from a different rendering path) can convert it directly. pub fn svg_to_png(svg_str: &str, opts: &PngRenderOpts) -> Result, String> { - let mut fontdb = usvg::fontdb::Database::new(); - fontdb.load_font_data(ROBOTO_LIGHT.to_vec()); - let usvg_opts = usvg::Options { font_family: "Roboto Light".to_string(), - fontdb: std::sync::Arc::new(fontdb), + fontdb: roboto_light_db(), ..usvg::Options::default() }; From f47a73c9e466244065d0b23b36e1990014bc2477 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:15:55 -0700 Subject: [PATCH 09/59] engine: resolve a lookup's constant element offset at compile time `compiler::codegen` emits a `LoadConstant` for a lookup's element offset before the index expression, and for a scalar table that constant is always a literal 0 -- a push the VM immediately pops and range-checks. It is 429k dispatches per C-LEARN run and 5.1% of WORLD3's, where roughly 70% of ALL constant loads are these zeros. The push is not adjacent to its `Lookup` (the index expression sits between), so no peephole can remove it; codegen has to not emit it. `LookupDirect` carries the resolved element instead. `const_element_offset` accepts only a non-negative integral constant strictly inside `[0, table_count)` that fits `u8`, and each condition is load-bearing: the VM's runtime path truncates with `as usize` after rejecting negatives, so a fractional or negative constant would fold to a different table than the runtime rule picks; an out-of-range constant must keep the general form so the VM still yields its documented NaN; and `u8` is the width the 8-byte `Opcode` budget leaves, so an arrayed GF with 256+ elements keeps the runtime push. Every rejected shape falls back to the unchanged `Lookup`. THE DURABLE PART OF THIS COMMIT IS THE MERGE TEST, not the 1-3.6pp. `gf_blocks_of_fragment` reconstructs each fragment's GF block layout by scanning its opcodes for `(base_gf, table_count)` runs, and its match ends in `_ => continue`. A lookup-family opcode it does not know about is therefore skipped SILENTLY: the referenced runs stop being seen, collapse into one maximal un-referenced GAP block, and the de-duplicated table layout comes out wrong with no diagnostic anywhere -- wrong numbers, not an error. That is a defect in its own right and it is why this change needed an audit rather than a patch. `test_gf_block_scan_sees_lookup_direct_runs` is built to fail on exactly that. Two distinct single-table blocks in one fragment, both read through `LookupDirect`, merged with a fragment holding only the second table's content. With the scan correct there are two deduped tables; with `LookupDirect` unknown the two runs collapse into one gap block keyed by its whole content, the shared table stops matching, and the merge yields three. Asserting the deduped COUNT is what makes it discriminating -- a single-block fixture dedups identically either way and would pin nothing. The `_ => continue` now carries the obligation in a comment naming this test. Nine `base_gf` sites, audited by hand because that one is silent. Eight are exhaustive matches and two of them caught the omission at compile time -- `db::fragment_char_tests`' opcode renderer and `symbolic_merge_proptest`'s shrinker -- which is the mechanism working as intended. The ninth, `per_element_gf_tests`' nameless-opcode enumeration, is a test whose claim `LookupDirect` strengthens rather than weakens: it asserts the per-element reorder is materialized at compile time and the hot path does no name lookup, and this opcode resolves the element at compile time too. The wasm backend splices the constant in beneath the index and reuses `emit_lookup` rather than growing a second copy of the directory-read plus helper-call sequence. It passes `table_count = elem + 1`, making that lowering's range check vacuously true -- sound rather than a fudge, since codegen only emits the opcode when `elem < table_count`, so the check was already discharged at emit time. Two goldens regenerate and the regeneration is fully explained: net per opcode mnemonic is `Lookup` -5 / `LoadConstant` -5 / `LookupDirect` +5 in graphical_functions.txt and -2/-2/+2 in lookup_only_table.txt. Every lookup became a direct lookup and its paired constant push vanished 1:1; no other opcode moved. Stage A bundle measured against the pre-bundle baseline (perf stat, 26 runs/side, two interleaved rounds). Instructions and branches are the layout-insensitive numbers; the machine was heavily contended for round 2, where the SAME baseline binary measured 17.95e9 cycles against 14.24e9 in round 1, so cycles are reported and not leaned on: C-LEARN instructions -7.22%, branches -6.49% WORLD3 instructions -7.66%, branches -5.69% This commit contributed +1.0pp on C-LEARN and +3.6pp on WORLD3 -- well above the ~1.4pp I projected for WORLD3, because the projection counted only the dispatch and not the popped-and-discarded operand work it takes with it. Behaviour-preserving: 5483 engine lib tests and 631 integration tests pass, including `clearn_residual_exactness`, `oracle_clearn`, `vdf_parity`, and the wasm parity corpus. --- src/simlin-engine/src/bytecode.rs | 20 +++ src/simlin-engine/src/compiler/codegen.rs | 55 ++++++++ src/simlin-engine/src/compiler/symbolic.rs | 132 ++++++++++++++++++ .../src/compiler/symbolic_merge_proptest.rs | 11 ++ .../graphical_functions.txt | 47 +++---- .../lookup_only_table.txt | 22 ++- .../src/db/fragment_char_tests.rs | 8 ++ src/simlin-engine/src/per_element_gf_tests.rs | 8 ++ src/simlin-engine/src/vm.rs | 18 +++ src/simlin-engine/src/wasmgen/lower.rs | 21 +++ 10 files changed, 304 insertions(+), 38 deletions(-) diff --git a/src/simlin-engine/src/bytecode.rs b/src/simlin-engine/src/bytecode.rs index b33950869..f2f5d5213 100644 --- a/src/simlin-engine/src/bytecode.rs +++ b/src/simlin-engine/src/bytecode.rs @@ -776,6 +776,22 @@ pub(crate) enum Opcode { mode: LookupMode, }, + /// `Lookup` whose element offset was resolved at COMPILE time, so it pops + /// only the index. Codegen emits this whenever a lookup's element-offset + /// expression is a constant in range -- which is every scalar table, where + /// the offset is a literal 0. The bounds check `Lookup` performs at runtime + /// is discharged at emit time (`elem < table_count`), so the table index is + /// `base_gf + elem` unconditionally. + /// + /// `table_count` is deliberately absent: it exists on `Lookup` for the + /// runtime range check and on the SYMBOLIC twin for GF block extents, and + /// neither applies once the element is fixed. + LookupDirect { + base_gf: GraphicalFunctionId, + elem: u8, + mode: LookupMode, + }, + // === SUPERINSTRUCTIONS (fused opcodes for common patterns) === /// Fused LoadConstant + AssignCurr. /// curr[module_off + off] = literals[literal_id]; stack unchanged. @@ -1466,6 +1482,9 @@ impl Opcode { Opcode::Apply { func } => (func.arity(), 1), // Lookup pops element_offset and lookup_index, pushes result Opcode::Lookup { .. } => (2, 1), + // LookupDirect's element offset is baked into the opcode, so only + // the index is popped. + Opcode::LookupDirect { .. } => (1, 1), // Superinstructions Opcode::AssignConstCurr { .. } => (0, 0), // reads literal directly @@ -1627,6 +1646,7 @@ impl Opcode { Opcode::AssignCurr { .. } => "AssignCurr", Opcode::Apply { .. } => "Apply", Opcode::Lookup { .. } => "Lookup", + Opcode::LookupDirect { .. } => "LookupDirect", Opcode::AssignConstCurr { .. } => "AssignConstCurr", Opcode::BinVarVar { .. } => "BinVarVar", Opcode::BinVarConst { .. } => "BinVarConst", diff --git a/src/simlin-engine/src/compiler/codegen.rs b/src/simlin-engine/src/compiler/codegen.rs index 58c28cadd..6357740d4 100644 --- a/src/simlin-engine/src/compiler/codegen.rs +++ b/src/simlin-engine/src/compiler/codegen.rs @@ -1129,6 +1129,19 @@ impl<'module> Compiler<'module> { .map(|tables| tables.len() as u16) .unwrap_or(1); + // A constant, in-range element offset is resolved here so + // no `LoadConstant` push is emitted for it (every scalar + // table takes this path, its offset being a literal 0). + if let Some(elem) = const_element_offset(&element_offset_expr, table_count) { + self.walk_expr(index)?.unwrap(); + self.push(SymbolicOpcode::LookupDirect { + base_gf, + table_count, + elem, + mode: LookupMode::Interpolate, + }); + return Ok(Some(())); + } // Emit: push element_offset, push lookup_index, Lookup { base_gf, table_count, mode } self.walk_expr(&element_offset_expr)?.unwrap(); self.walk_expr(index)?.unwrap(); @@ -1166,6 +1179,16 @@ impl<'module> Compiler<'module> { .map(|tables| tables.len() as u16) .unwrap_or(1); + if let Some(elem) = const_element_offset(&element_offset_expr, table_count) { + self.walk_expr(index)?.unwrap(); + self.push(SymbolicOpcode::LookupDirect { + base_gf, + table_count, + elem, + mode, + }); + return Ok(Some(())); + } self.walk_expr(&element_offset_expr)?.unwrap(); self.walk_expr(index)?.unwrap(); self.push(SymbolicOpcode::Lookup { @@ -2090,6 +2113,38 @@ impl<'module> Compiler<'module> { } } +/// Resolve a lookup's element-offset expression to a constant slot within the +/// variable's table block, or `None` if it must stay a runtime push. +/// +/// Accepts only a non-negative integral constant strictly inside +/// `[0, table_count)` that also fits `u8`. Each condition is load-bearing: +/// +/// - INTEGRAL and NON-NEGATIVE, because the VM's runtime path truncates with +/// `element_offset as usize` after rejecting negatives, and a fractional or +/// negative constant would fold to a different table than the runtime rule +/// picks. Those spellings keep the general `Lookup`. +/// - IN RANGE, because `LookupDirect` carries no `table_count` and performs no +/// runtime check; an out-of-range constant must keep the general form so the +/// VM still yields its documented NaN. +/// - FITS `u8`, because that is the field width the 8-byte `Opcode` budget +/// leaves. An arrayed GF with 256+ elements simply keeps the runtime push. +fn const_element_offset(expr: &Expr, table_count: u16) -> Option { + let Expr::Const(value, _) = expr else { + return None; + }; + let value = *value; + // `is_finite` rejects NaN and the infinities explicitly rather than leaning + // on the `floor` comparison to catch them incidentally. + if !value.is_finite() || value < 0.0 || value.floor() != value { + return None; + } + let elem = value as usize; + if elem >= table_count as usize { + return None; + } + u8::try_from(elem).ok() +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/simlin-engine/src/compiler/symbolic.rs b/src/simlin-engine/src/compiler/symbolic.rs index e40f4a0fa..0c88fb35e 100644 --- a/src/simlin-engine/src/compiler/symbolic.rs +++ b/src/simlin-engine/src/compiler/symbolic.rs @@ -118,6 +118,26 @@ pub(crate) enum SymbolicOpcode { table_count: u16, mode: LookupMode, }, + /// `Lookup` with the element offset resolved at COMPILE time. + /// + /// `compiler::codegen` pushes a `LoadConstant` for a lookup's element + /// offset before the index expression, and for a scalar table that + /// constant is always 0 -- 429k dispatches per C-LEARN run and 5.1% of + /// WORLD3's, spent pushing a zero the VM immediately pops and range-checks. + /// The push is not adjacent to the `Lookup` (the index expression sits + /// between), so no peephole can remove it; it has to not be emitted. + /// + /// `base_gf`/`table_count` still describe the variable's WHOLE table block, + /// exactly as on `Lookup`, because `gf_blocks_of_fragment` reads block + /// extents off these two fields. `elem` is the resolved offset WITHIN that + /// block, bounds-checked at emit time (codegen only emits this form when + /// `elem < table_count`), so the VM needs no runtime range check. + LookupDirect { + base_gf: GraphicalFunctionId, + table_count: u16, + elem: u8, + mode: LookupMode, + }, // === SUPERINSTRUCTIONS === AssignConstCurr { @@ -949,6 +969,16 @@ pub(crate) fn resolve_opcode( table_count: *table_count, mode: *mode, }), + SymbolicOpcode::LookupDirect { + base_gf, + elem, + mode, + .. + } => Ok(Opcode::LookupDirect { + base_gf: *base_gf, + elem: *elem, + mode: *mode, + }), SymbolicOpcode::PushTempView { temp_id, dim_list_id, @@ -1840,7 +1870,18 @@ fn gf_blocks_of_fragment(frag: &PerVarBytecodes) -> Result, base_gf, table_count, .. + } + | SymbolicOpcode::LookupDirect { + base_gf, + table_count, + .. } => (*base_gf as usize, *table_count as usize), + // OBLIGATION: every lookup-family opcode that carries a `base_gf` + // MUST be listed above. This arm is silent -- an unlisted one is + // skipped with no diagnostic, its block collapses into an + // un-referenced gap, and the de-duplicated table layout is wrong + // with no error anywhere. `test_gf_block_scan_sees_lookup_direct_runs` + // is the tripwire; extend it when adding a lookup opcode. _ => continue, }; if count == 0 { @@ -2516,6 +2557,17 @@ pub(crate) fn renumber_opcode( table_count: *table_count, mode: *mode, }, + SymbolicOpcode::LookupDirect { + base_gf, + table_count, + elem, + mode, + } => SymbolicOpcode::LookupDirect { + base_gf: remap_gf(*base_gf, gf_remap)?, + table_count: *table_count, + elem: *elem, + mode: *mode, + }, SymbolicOpcode::EvalModule { id, n_inputs } => SymbolicOpcode::EvalModule { id: checked_add_u16(*id, mod_off, "ModuleId")?, n_inputs: *n_inputs, @@ -4508,6 +4560,86 @@ mod tests { } } + /// Two SEPARATE single-table GF blocks in one fragment, each read by a + /// `LookupDirect`, merged with a fragment holding only the second table's + /// content. + /// + /// This is the pin on `gf_blocks_of_fragment`'s opcode scan, and it is + /// built to FAIL if a lookup-family opcode is added without teaching that + /// scan about it. The scan ends in a `_ => continue`, so an unknown + /// lookup opcode is skipped SILENTLY -- the two referenced runs stop being + /// seen as runs and collapse into one maximal un-referenced GAP block + /// `[0, 2)`. A gap block is keyed for de-duplication by its whole content, + /// so the shared second table no longer matches the other fragment's copy + /// and the merge yields three tables instead of two, with the interior + /// `base_gf` remapped off the wrong block base. + /// + /// Asserting the deduped COUNT is what makes the test discriminating: a + /// fixture with a single block would dedup identically whether or not the + /// opcode were known, and would pin nothing. + #[test] + fn test_gf_block_scan_sees_lookup_direct_runs() { + let table_a = vec![(0.0, 1.0), (1.0, 2.0)]; + let table_b = vec![(0.0, 5.0), (1.0, 6.0)]; + + // One fragment, two distinct single-table blocks, both read through + // the constant-element-offset form. + let two_blocks = PerVarBytecodes { + symbolic: SymbolicByteCode { + literals: vec![], + code: vec![ + SymbolicOpcode::LookupDirect { + base_gf: 0, + table_count: 1, + elem: 0, + mode: LookupMode::Interpolate, + }, + SymbolicOpcode::LookupDirect { + base_gf: 1, + table_count: 1, + elem: 0, + mode: LookupMode::Interpolate, + }, + SymbolicOpcode::Ret, + ], + }, + graphical_functions: vec![table_a.clone(), table_b.clone()], + module_decls: vec![], + static_views: vec![], + temp_sizes: vec![], + dim_lists: vec![], + }; + // A second fragment holding ONLY table_b, so a correct scan lets the + // two copies of table_b dedup to one slot. + let shares_b = gf_lookup_frag(table_b.clone()); + + let no_base = ContextResourceCounts::default(); + let merged = concatenate_fragments(&[&two_blocks, &shares_b], &no_base).unwrap(); + + assert_eq!( + merged.graphical_functions.len(), + 2, + "the two LookupDirect runs must be seen as separate blocks so the \ + shared table de-duplicates; 3 means `gf_blocks_of_fragment` did \ + not recognise LookupDirect and collapsed them into one gap block" + ); + assert!(merged.graphical_functions.contains(&table_a)); + assert!(merged.graphical_functions.contains(&table_b)); + + // And every emitted lookup must still address a real table. + for op in &merged.bytecode.code { + let base = match op { + SymbolicOpcode::LookupDirect { base_gf, .. } + | SymbolicOpcode::Lookup { base_gf, .. } => *base_gf as usize, + _ => continue, + }; + assert!( + base < merged.graphical_functions.len(), + "remapped base_gf {base} is past the merged table list" + ); + } + } + #[test] fn test_concatenate_dedups_identical_gf_tables_under_u8_capacity() { // 300 consumer fragments, each referencing the SAME dependency GF diff --git a/src/simlin-engine/src/compiler/symbolic_merge_proptest.rs b/src/simlin-engine/src/compiler/symbolic_merge_proptest.rs index 10cd030cb..9d4d33036 100644 --- a/src/simlin-engine/src/compiler/symbolic_merge_proptest.rs +++ b/src/simlin-engine/src/compiler/symbolic_merge_proptest.rs @@ -503,6 +503,17 @@ fn blank_resource_ids(op: &SymbolicOpcode) -> SymbolicOpcode { table_count: *table_count, mode: *mode, }, + SymbolicOpcode::LookupDirect { + table_count, + elem, + mode, + .. + } => SymbolicOpcode::LookupDirect { + base_gf: 0, + table_count: *table_count, + elem: *elem, + mode: *mode, + }, SymbolicOpcode::LookupArray { table_count, mode, .. } => SymbolicOpcode::LookupArray { diff --git a/src/simlin-engine/src/db/fragment_char_golden/graphical_functions.txt b/src/simlin-engine/src/db/fragment_char_golden/graphical_functions.txt index 0d02f76e8..4eee2a4e1 100644 --- a/src/simlin-engine/src/db/fragment_char_golden/graphical_functions.txt +++ b/src/simlin-engine/src/db/fragment_char_golden/graphical_functions.txt @@ -9,7 +9,7 @@ == main::curve [explicit] : flow == initial: flow: - literals: [0.0] + literals: [] temp_sizes: [] dim_lists: [] graphical_functions: @@ -17,11 +17,10 @@ module_decls: [] static_views: [] code: - 0000 LoadConstant #0 (=0.0) - 0001 LoadVar drive@0 - 0002 Lookup base_gf=0 table_count=1 mode=Interpolate - 0003 AssignCurr curve@0 - 0004 Ret + 0000 LoadVar drive@0 + 0001 LookupDirect base_gf=0 table_count=1 elem=0 mode=Interpolate + 0002 AssignCurr curve@0 + 0003 Ret stock: == main::drive [explicit] : flow == initial: @@ -40,7 +39,7 @@ == main::g [explicit] : flow == initial: flow: - literals: [0.0, 1.0] + literals: [] temp_sizes: [] dim_lists: [] graphical_functions: @@ -49,15 +48,13 @@ module_decls: [] static_views: [] code: - 0000 LoadConstant #0 (=0.0) - 0001 LoadGlobalVar off=0 (time) - 0002 Lookup base_gf=0 table_count=2 mode=Interpolate - 0003 AssignCurr g@0 - 0004 LoadConstant #1 (=1.0) - 0005 LoadGlobalVar off=0 (time) - 0006 Lookup base_gf=0 table_count=2 mode=Interpolate - 0007 AssignCurr g@1 - 0008 Ret + 0000 LoadGlobalVar off=0 (time) + 0001 LookupDirect base_gf=0 table_count=2 elem=0 mode=Interpolate + 0002 AssignCurr g@0 + 0003 LoadGlobalVar off=0 (time) + 0004 LookupDirect base_gf=0 table_count=2 elem=1 mode=Interpolate + 0005 AssignCurr g@1 + 0006 Ret stock: == main::gtotal [explicit] : flow == initial: @@ -86,7 +83,7 @@ == main::out [explicit] : flow == initial: flow: - literals: [0.0, 1.0] + literals: [] temp_sizes: [] dim_lists: [] graphical_functions: @@ -95,15 +92,13 @@ module_decls: [] static_views: [] code: - 0000 LoadConstant #0 (=0.0) - 0001 LoadGlobalVar off=0 (time) - 0002 Lookup base_gf=0 table_count=2 mode=Interpolate - 0003 AssignCurr out@0 - 0004 LoadConstant #1 (=1.0) - 0005 LoadGlobalVar off=0 (time) - 0006 Lookup base_gf=0 table_count=2 mode=Interpolate - 0007 AssignCurr out@1 - 0008 Ret + 0000 LoadGlobalVar off=0 (time) + 0001 LookupDirect base_gf=0 table_count=2 elem=0 mode=Interpolate + 0002 AssignCurr out@0 + 0003 LoadGlobalVar off=0 (time) + 0004 LookupDirect base_gf=0 table_count=2 elem=1 mode=Interpolate + 0005 AssignCurr out@1 + 0006 Ret stock: ########## runtime ########## step 0: diff --git a/src/simlin-engine/src/db/fragment_char_golden/lookup_only_table.txt b/src/simlin-engine/src/db/fragment_char_golden/lookup_only_table.txt index 7d4f15634..359c47584 100644 --- a/src/simlin-engine/src/db/fragment_char_golden/lookup_only_table.txt +++ b/src/simlin-engine/src/db/fragment_char_golden/lookup_only_table.txt @@ -7,7 +7,7 @@ == main::at_half [explicit] : flow == initial: flow: - literals: [0.0, 0.5] + literals: [0.5] temp_sizes: [] dim_lists: [] graphical_functions: @@ -15,16 +15,15 @@ module_decls: [] static_views: [] code: - 0000 LoadConstant #0 (=0.0) - 0001 LoadConstant #1 (=0.5) - 0002 Lookup base_gf=0 table_count=1 mode=Interpolate - 0003 AssignCurr at_half@0 - 0004 Ret + 0000 LoadConstant #0 (=0.5) + 0001 LookupDirect base_gf=0 table_count=1 elem=0 mode=Interpolate + 0002 AssignCurr at_half@0 + 0003 Ret stock: == main::at_time [explicit] : flow == initial: flow: - literals: [0.0] + literals: [] temp_sizes: [] dim_lists: [] graphical_functions: @@ -32,11 +31,10 @@ module_decls: [] static_views: [] code: - 0000 LoadConstant #0 (=0.0) - 0001 LoadGlobalVar off=0 (time) - 0002 Lookup base_gf=0 table_count=1 mode=Interpolate - 0003 AssignCurr at_time@0 - 0004 Ret + 0000 LoadGlobalVar off=0 (time) + 0001 LookupDirect base_gf=0 table_count=1 elem=0 mode=Interpolate + 0002 AssignCurr at_time@0 + 0003 Ret stock: == main::table [explicit] : none == initial: diff --git a/src/simlin-engine/src/db/fragment_char_tests.rs b/src/simlin-engine/src/db/fragment_char_tests.rs index 26b45f447..e82c37f98 100644 --- a/src/simlin-engine/src/db/fragment_char_tests.rs +++ b/src/simlin-engine/src/db/fragment_char_tests.rs @@ -290,6 +290,14 @@ fn render_opcode(op: &SymbolicOpcode, literals: &[f64]) -> String { format!("PushSubscriptIndex bounds={bounds}") } SymbolicOpcode::LoadSubscript { var } => format!("LoadSubscript {}", render_var_ref(var)), + SymbolicOpcode::LookupDirect { + base_gf, + table_count, + elem, + mode, + } => format!( + "LookupDirect base_gf={base_gf} table_count={table_count} elem={elem} mode={mode:?}" + ), SymbolicOpcode::SetCond {} => "SetCond".to_string(), SymbolicOpcode::If {} => "If".to_string(), SymbolicOpcode::Ret => "Ret".to_string(), diff --git a/src/simlin-engine/src/per_element_gf_tests.rs b/src/simlin-engine/src/per_element_gf_tests.rs index fa6994649..5e3953762 100644 --- a/src/simlin-engine/src/per_element_gf_tests.rs +++ b/src/simlin-engine/src/per_element_gf_tests.rs @@ -397,6 +397,14 @@ fn per_element_gf_reorder_is_compile_time_with_nameless_opcode() { mode: _, write_temp_id: _, } => Some(*base_gf), + // The constant-element form belongs to the same family and makes + // this claim stronger, not weaker: it resolves the element offset + // at COMPILE time, so the hot path does not even push it. + Opcode::LookupDirect { + base_gf, + elem: _, + mode: _, + } => Some(*base_gf), _ => None, }) .collect(); diff --git a/src/simlin-engine/src/vm.rs b/src/simlin-engine/src/vm.rs index 40ff4a459..1eb97bef3 100644 --- a/src/simlin-engine/src/vm.rs +++ b/src/simlin-engine/src/vm.rs @@ -2481,6 +2481,24 @@ impl Vm { stack.push(result); } } + // The element offset was resolved and bounds-checked at emit + // time, so this reads `graphical_functions[base_gf + elem]` + // with no pop and no range check -- the two things the general + // `Lookup` arm above spends its extra dispatch on. + Opcode::LookupDirect { + base_gf, + elem, + mode, + } => { + let lookup_index = stack.pop(); + let gf = &context.graphical_functions[*base_gf as usize + *elem as usize]; + let result = match mode { + LookupMode::Interpolate => lookup(gf, lookup_index), + LookupMode::Forward => lookup_forward(gf, lookup_index), + LookupMode::Backward => lookup_backward(gf, lookup_index), + }; + stack.push(result); + } Opcode::Ret => { break; } diff --git a/src/simlin-engine/src/wasmgen/lower.rs b/src/simlin-engine/src/wasmgen/lower.rs index f663dbf55..0c504d37b 100644 --- a/src/simlin-engine/src/wasmgen/lower.rs +++ b/src/simlin-engine/src/wasmgen/lower.rs @@ -1427,6 +1427,27 @@ fn emit_ops( table_count, mode, } => emit_lookup(*base_gf, *table_count, *mode, ctx, f), + // The constant element offset is not on the wasm stack (codegen + // never emitted a push for it), so splice it in beneath the index + // and reuse the one lowering rather than growing a second copy of + // the directory-read + helper-call sequence. + // + // `table_count` is passed as `elem + 1`, which makes + // `emit_lookup`'s range check vacuously true. That is sound rather + // than a fudge: `compiler::codegen::const_element_offset` only + // emits this opcode when `elem < table_count`, so the check it + // replaces was already discharged at emit time -- which is the + // whole point of the opcode. + Opcode::LookupDirect { + base_gf, + elem, + mode, + } => { + f.instruction(&Instruction::LocalSet(ctx.scratch_local)); + f.instruction(&f64_const(*elem as f64)); + f.instruction(&Instruction::LocalGet(ctx.scratch_local)); + emit_lookup(*base_gf, *elem as u16 + 1, *mode, ctx, f) + } // `LoadPrev` mirrors the VM (`vm.rs:1320-1328`): a fallback is // already on the stack (codegen pushes it just before this opcode); // yield it while `use_prev_fallback` is set, otherwise read From 6e3334b83e3760f8dd2b58cd9a676a8122a9bf14 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:18:08 -0700 Subject: [PATCH 10/59] engine: derive variable_dimensions without a second parse `variable_dimensions` asked for a parse under an EMPTY `ModuleIdentContext`, noting that the module context does not affect dimension extraction. That is true, and it was also the problem: the context is part of the parse's salsa cache key, and this query takes no `model`, so the empty context is the only one it could name. Every variable was therefore parsed a second time under a key nothing else in the pipeline uses -- on C-LEARN, 1,910 executions of `parse_source_variable_with_module_context` for 934 variables. The declared dimensions are available without parsing anything: they are the dimension-name list on `datamodel::Equation` itself, which `variable_relevant_dimensions` already reads, resolved through the same `variable::get_dimensions` the parse calls. The derivation mirrors `parse_source_variable_impl`'s narrowed dimension context exactly -- the relevant names widened by `expand_maps_to_chains` and filtered out of `project_datamodel_dims` -- so a name resolves here iff it resolves there. Keeping that narrowing rather than reading the whole-project `project_dimensions_context` is what preserves dimension-granularity invalidation: a scalar takes the early return and never depends on the project's dimensions at all. Measured -3.5% of a cold C-LEARN compile and -6.7% on WORLD3 (interleaved A/B, three rounds), the larger share on WORLD3 because it declares no dimensions and so gains nothing from the sibling `is_dimension_name` change. ONE arm deliberately differs from the parse. `parse_equation` builds an A2A as `ast.map(|ast| Ast::ApplyToAll(dims, ast))`, so a variable whose equation does not parse produced no `Ast` and reported no dimensions -- giving it a `variable_size` of 1 despite a declared extent. This reports the declared shape. The divergence is confined to a project that already fails to assemble (the parse error still reaches `compile_var_fragment`, which drops the fragment and accumulates the diagnostic), and it moves the reported size from a wrong 1 toward the declaration, so nothing that compiled before reads a different slot. `Arrayed` is unchanged in both its failure modes, and so is the unresolvable-dimension-name arm. The tests assert against the previous implementation as an ORACLE rather than against hand-written expectations, which is what makes the agreement claim mean anything: the first draft's hand-written row for a canonically spelled reference to a `DimA`-cased dimension was wrong in a way only the oracle caught. Both paths share a pre-filter that seeds `expanded` with the equation's RAW dimension names and then filters by display name, so such a reference resolves to nothing on either path -- a property of the shared narrowing rather than of either implementation, and left exactly as it was. The compiled artifact is unchanged: C-LEARN still assembles 5215 slots and 58291 opcodes (31525 flow + 1477 stock + 25289 initial) with the same literal, graphical-function, temp, dimension, view, name and module counts. --- src/simlin-engine/src/db.rs | 2 + src/simlin-engine/src/db/query.rs | 65 +++- .../src/db/variable_dimensions_tests.rs | 322 ++++++++++++++++++ 3 files changed, 382 insertions(+), 7 deletions(-) create mode 100644 src/simlin-engine/src/db/variable_dimensions_tests.rs diff --git a/src/simlin-engine/src/db.rs b/src/simlin-engine/src/db.rs index f57c3e5ca..2e24be3f5 100644 --- a/src/simlin-engine/src/db.rs +++ b/src/simlin-engine/src/db.rs @@ -1446,4 +1446,6 @@ mod stages_tests; #[cfg(test)] mod tests; #[cfg(test)] +mod variable_dimensions_tests; +#[cfg(test)] mod vm_verification_tests; diff --git a/src/simlin-engine/src/db/query.rs b/src/simlin-engine/src/db/query.rs index 72e2306d9..0aa9da974 100644 --- a/src/simlin-engine/src/db/query.rs +++ b/src/simlin-engine/src/db/query.rs @@ -779,20 +779,71 @@ pub fn variable_relevant_dimensions(db: &dyn Db, var: SourceVariable) -> BTreeSe } } +/// A variable's DECLARED dimensions, resolved against the project. +/// +/// Derived straight from `var.equation(db)`'s dimension-name list rather than +/// from a parse, and that is the whole point: the parse is keyed on a +/// `ModuleIdentContext`, so asking for one here under the empty context -- +/// which is the only context this query could name, since it takes no `model` +/// -- minted a SECOND full parse of every variable under a key nothing else +/// uses. On C-LEARN that was 1,910 executions of +/// `parse_source_variable_with_module_context` for 934 variables (~2.05x), and +/// removing it measures -3.5% of a cold compile there and -6.7% on WORLD3. +/// +/// The derivation mirrors `parse_source_variable_impl`'s own narrowed +/// dimension context exactly -- `variable_relevant_dimensions` widened by +/// `expand_maps_to_chains` and filtered out of `project_datamodel_dims` -- so +/// a name resolves here iff it resolves there. Keeping the narrowing (rather +/// than reading the whole-project `project_dimensions_context`) is what +/// preserves dimension-granularity invalidation: a scalar variable takes the +/// early return and never depends on the project's dimensions at all +/// (`db::dimension_invalidation_tests`). +/// +/// **One arm differs from the parse, deliberately.** The parse builds +/// `Ast::ApplyToAll` as `ast.map(|ast| Ast::ApplyToAll(dims, ast))`, so an A2A +/// variable whose EQUATION does not parse yields no `Ast` and therefore +/// reported no dimensions; this reports its declared ones. Both the +/// unresolvable-dimension-name arm (`[]` on either path) and the `Arrayed` arm +/// (which the parse builds unconditionally once its dims resolve, however many +/// element equations failed) are unchanged. The divergence is confined to a +/// project that already fails to assemble -- the parse error still reaches +/// `compile_var_fragment`, which drops the fragment and accumulates the +/// diagnostic -- and it moves the reported size from a wrong 1 toward the +/// declared extent, so nothing that compiled before reads a different slot. +/// Every arm is enumerated in `db::variable_dimensions_tests`. #[salsa::tracked(returns(ref))] pub fn variable_dimensions( db: &dyn Db, var: SourceVariable, project: SourceProject, ) -> Vec { - // Module context doesn't affect dimension extraction, so an empty - // context is correct here. - let empty_context = ModuleIdentContext::new(db, vec![]); - let parsed = parse_source_variable_with_module_context(db, var, project, empty_context); - match parsed.variable.get_dimensions() { - Some(dims) => dims.to_vec(), - None => Vec::new(), + let dimension_names: &[String] = match var.equation(db) { + datamodel::Equation::Scalar(_) => return Vec::new(), + datamodel::Equation::ApplyToAll(dim_names, _) => dim_names, + datamodel::Equation::Arrayed(dim_names, _, _, _) => dim_names, + }; + // A module variable carries a synthesized equation but has no array shape + // of its own (the parse's `Variable::Module` has no `ast` for + // `get_dimensions` to read), so it must report none. + if var.kind(db) == SourceVariableKind::Module { + return Vec::new(); + } + if dimension_names.is_empty() { + return Vec::new(); } + let expanded = expand_maps_to_chains( + variable_relevant_dimensions(db, var), + project.dimensions(db), + ); + let dims: Vec = project_datamodel_dims(db, project) + .iter() + .filter(|d| expanded.contains(&d.name)) + .cloned() + .collect(); + let dim_ctx = crate::dimensions::DimensionsContext::from(&dims); + // `Err` is an unresolvable dimension name, which the parse also turns into + // "no dimensions" (it pushes a `BadDimensionName` and drops the `Ast`). + crate::variable::get_dimensions(&dim_ctx, dimension_names).unwrap_or_default() } #[salsa::tracked(returns(clone))] diff --git a/src/simlin-engine/src/db/variable_dimensions_tests.rs b/src/simlin-engine/src/db/variable_dimensions_tests.rs new file mode 100644 index 000000000..8ecefa756 --- /dev/null +++ b/src/simlin-engine/src/db/variable_dimensions_tests.rs @@ -0,0 +1,322 @@ +// Copyright 2026 The Simlin Authors. All rights reserved. +// Use of this source code is governed by the Apache License, +// Version 2.0, that can be found in the LICENSE file. + +//! The `variable_dimensions` decision, arm by arm. +//! +//! `variable_dimensions` derives a variable's declared dimensions from its +//! `datamodel::Equation` instead of demanding a parse. The rows here are +//! derived from the enumeration that decision ranges over -- the three +//! `datamodel::Equation` variants, crossed with the two ways resolution can +//! fail (an unresolvable dimension name, and an equation that does not parse), +//! plus the `Module` kind, which carries an equation but has no array shape. +//! +//! Every row states what the PARSE-backed implementation answered, because +//! this query is a behavioural mirror of it in all but one cell, and that cell +//! is the reason the file exists: an A2A variable whose equation does not +//! parse reported no dimensions and now reports its declared ones. A test that +//! covered only the healthy rows would pass under an implementation that got +//! that cell wrong in either direction. + +use super::*; +use crate::datamodel; + +fn dims() -> Vec { + vec![ + datamodel::Dimension::named( + "DimA".to_string(), + vec!["a1".to_string(), "a2".to_string(), "a3".to_string()], + ), + datamodel::Dimension::named("DimB".to_string(), vec!["b1".to_string(), "b2".to_string()]), + ] +} + +fn aux(ident: &str, equation: datamodel::Equation) -> datamodel::Variable { + datamodel::Variable::Aux(datamodel::Aux { + ident: ident.to_string(), + equation, + documentation: String::new(), + units: None, + gf: None, + ai_state: None, + uid: None, + compat: datamodel::Compat::default(), + }) +} + +fn arrayed(dim_names: &[&str], elements: &[(&str, &str)]) -> datamodel::Equation { + datamodel::Equation::Arrayed( + dim_names.iter().map(|d| d.to_string()).collect(), + elements + .iter() + .map(|(e, eqn)| (e.to_string(), eqn.to_string(), None, None)) + .collect(), + None, + false, + ) +} + +fn a2a(dim_names: &[&str], eqn: &str) -> datamodel::Equation { + datamodel::Equation::ApplyToAll( + dim_names.iter().map(|d| d.to_string()).collect(), + eqn.to_string(), + ) +} + +/// The implementation `variable_dimensions` replaced, kept verbatim as the +/// ORACLE: parse the variable under the empty module-ident context and read +/// the shape off the resulting `Ast`. +/// +/// Asserting against this rather than against hand-written expectations is +/// what makes the agreement claim mean anything. Writing the rows out by hand +/// got the cased-dimension row wrong in the first draft of this file -- the +/// parse's pre-filter seeds `expanded` with the equation's RAW dimension names +/// and then filters `project_datamodel_dims` by display name, so a reference +/// spelled `dima` against a dimension declared `DimA` never reaches +/// `variable::get_dimensions`' canonical matching and resolves to nothing on +/// BOTH paths. That is a property of the shared narrowing, not of either +/// implementation, and only an oracle catches it. +fn oracle_dimension_names(db: &dyn Db, var: SourceVariable, project: SourceProject) -> Vec { + let empty_context = ModuleIdentContext::new(db, vec![]); + let parsed = parse_source_variable_with_module_context(db, var, project, empty_context); + match parsed.variable.get_dimensions() { + Some(dims) => dims.iter().map(|d| d.name().to_string()).collect(), + None => Vec::new(), + } +} + +/// Every arm of the enumeration, checked against the parse-backed oracle. +/// +/// The rows are the three `datamodel::Equation` variants crossed with the two +/// ways resolution can fail, plus the spellings that exercise the shared +/// narrowing. `broken_a2a` is the one row the two implementations are expected +/// to DISAGREE on and is asserted separately below; every other row must agree +/// with the oracle exactly. +#[test] +fn variable_dimensions_matches_the_parse_on_every_agreeing_arm() { + let variables = vec![ + // Scalar: no declared dimensions. + aux("scalar", datamodel::Equation::Scalar("1 + 1".to_string())), + // A2A, one and two dimensions, resolvable and parseable. + aux("a2a_1d", a2a(&["DimA"], "1")), + aux("a2a_2d", a2a(&["DimA", "DimB"], "1")), + // A2A naming a dimension the project does not declare. + aux("a2a_bad_dim", a2a(&["NoSuchDim"], "1")), + // Arrayed, resolvable and parseable. + aux( + "arrayed_ok", + arrayed(&["DimB"], &[("b1", "1"), ("b2", "2")]), + ), + // Arrayed naming a dimension the project does not declare. + aux("arrayed_bad_dim", arrayed(&["NoSuchDim"], &[("b1", "1")])), + // Arrayed whose element equations do not parse: the parse builds the + // `Ast::Arrayed` anyway once its dims resolve, dropping the elements. + aux( + "arrayed_bad_eqn", + arrayed(&["DimB"], &[("b1", "1 +"), ("b2", ")(")]), + ), + // A reference spelled canonically against a dimension declared with + // original casing. Both paths share the raw-name pre-filter, so both + // resolve nothing -- the row exists to hold that agreement, not to + // claim the resolution succeeds. + aux("a2a_cased", a2a(&["dima"], "1")), + ]; + + let mut db = SimlinDb::default(); + let project = datamodel::Project { + name: "vardims".to_string(), + sim_specs: datamodel::SimSpecs::default(), + dimensions: dims(), + units: vec![], + models: vec![datamodel::Model { + name: "main".to_string(), + sim_specs: None, + variables: variables.clone(), + views: vec![], + loop_metadata: vec![], + groups: vec![], + macro_spec: None, + }], + source: None, + ai_information: None, + }; + let state = sync_from_datamodel_incremental(&mut db, &project, None); + let sync = state.to_sync_result(); + let model = &sync.models["main"]; + + let mut checked = 0usize; + let mut idents: Vec<&String> = model.variables.keys().collect(); + idents.sort_unstable(); + for ident in idents { + let sv = model.variables[ident].source; + let derived: Vec = crate::db::query::variable_dimensions(&db, sv, sync.project) + .iter() + .map(|d| d.name().to_string()) + .collect(); + let oracle = oracle_dimension_names(&db, sv, sync.project); + assert_eq!( + derived, oracle, + "variable_dimensions disagrees with the parse for {ident}" + ); + checked += 1; + } + assert_eq!( + checked, + variables.len(), + "every declared fixture variable must have been compared" + ); +} + +/// The ONE arm that changed, pinned in the direction it changed to. +/// +/// The parse builds `Ast::ApplyToAll` as `ast.map(|ast| ApplyToAll(dims, ast))`, +/// so an unparseable A2A equation yielded no `Ast` and hence no dimensions -- +/// which gave the variable a `variable_size` of 1 despite being declared over +/// a 3-element dimension. The derivation reports the declared shape. +/// +/// This is only reachable on a project that already fails to assemble (the +/// parse error still reaches `compile_var_fragment`, which drops the fragment +/// and accumulates the diagnostic), so no compiling model can observe it. The +/// assertion below is the record of that decision; a future change that wants +/// the old answer must restate it here rather than silently flip it. +#[test] +fn an_unparseable_a2a_equation_reports_its_declared_dimensions() { + let mut db = SimlinDb::default(); + let project = datamodel::Project { + name: "vardims_diverge".to_string(), + sim_specs: datamodel::SimSpecs::default(), + dimensions: dims(), + units: vec![], + models: vec![datamodel::Model { + name: "main".to_string(), + sim_specs: None, + variables: vec![aux("broken", a2a(&["DimA"], "1 +"))], + views: vec![], + loop_metadata: vec![], + groups: vec![], + macro_spec: None, + }], + source: None, + ai_information: None, + }; + let state = sync_from_datamodel_incremental(&mut db, &project, None); + let sync = state.to_sync_result(); + let sv = sync.models["main"].variables["broken"].source; + + let derived: Vec = crate::db::query::variable_dimensions(&db, sv, sync.project) + .iter() + .map(|d| d.name().to_string()) + .collect(); + assert_eq!( + derived, + vec!["dima".to_string()], + "an A2A variable's declared shape is a property of its declaration, \ + not of whether its equation parses" + ); + // Both halves are asserted so the divergence is a recorded decision rather + // than a coincidence: the parse really did answer differently here. + assert_eq!( + oracle_dimension_names(&db, sv, sync.project), + Vec::::new(), + "the parse-backed oracle is expected to answer with no dimensions here" + ); + assert_eq!( + crate::db::query::variable_size(&db, sv, sync.project), + 3, + "the declared extent follows the declared shape" + ); +} + +/// The same fixture still fails to compile, which is what confines the arm +/// above to projects that were already rejected. +#[test] +fn an_unparseable_a2a_equation_still_fails_to_compile() { + let mut db = SimlinDb::default(); + let project = datamodel::Project { + name: "vardims_broken".to_string(), + sim_specs: datamodel::SimSpecs::default(), + dimensions: dims(), + units: vec![], + models: vec![datamodel::Model { + name: "main".to_string(), + sim_specs: None, + variables: vec![aux("broken", a2a(&["DimA"], "1 +"))], + views: vec![], + loop_metadata: vec![], + groups: vec![], + macro_spec: None, + }], + source: None, + ai_information: None, + }; + let state = sync_from_datamodel_incremental(&mut db, &project, None); + let diagnostics = collect_all_diagnostics(&db, state.project); + assert!( + diagnostics + .iter() + .any(|d| d.variable.as_deref() == Some("broken")), + "the parse error must still be reported: {diagnostics:?}" + ); +} + +/// A module variable carries a synthesized equation but has no array shape of +/// its own -- the parse's `Variable::Module` has no `ast` for `get_dimensions` +/// to read, so it answered `None`. Derived from the equation alone this needs +/// an explicit kind check, which is why it is a row rather than a corollary. +#[test] +fn a_module_variable_reports_no_dimensions() { + let mut db = SimlinDb::default(); + let project = datamodel::Project { + name: "vardims_module".to_string(), + sim_specs: datamodel::SimSpecs::default(), + dimensions: dims(), + units: vec![], + models: vec![ + datamodel::Model { + name: "main".to_string(), + sim_specs: None, + variables: vec![ + aux("driver", datamodel::Equation::Scalar("3".to_string())), + datamodel::Variable::Module(datamodel::Module { + ident: "inst".to_string(), + model_name: "sub".to_string(), + documentation: String::new(), + units: None, + references: vec![datamodel::ModuleReference { + src: "driver".to_string(), + dst: "inst.input".to_string(), + }], + ai_state: None, + uid: None, + compat: datamodel::Compat::default(), + }), + ], + views: vec![], + loop_metadata: vec![], + groups: vec![], + macro_spec: None, + }, + datamodel::Model { + name: "sub".to_string(), + sim_specs: None, + variables: vec![ + aux("input", datamodel::Equation::Scalar("0".to_string())), + aux("out", datamodel::Equation::Scalar("input * 2".to_string())), + ], + views: vec![], + loop_metadata: vec![], + groups: vec![], + macro_spec: None, + }, + ], + source: None, + ai_information: None, + }; + let state = sync_from_datamodel_incremental(&mut db, &project, None); + let sync = state.to_sync_result(); + let inst = sync.models["main"].variables["inst"].source; + assert!( + crate::db::query::variable_dimensions(&db, inst, sync.project).is_empty(), + "a module instance has no array shape of its own" + ); +} From 819d23d0fc026e527a250a5b28737a80bf51b62d Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:18:37 -0700 Subject: [PATCH 11/59] build: optimize the resvg rasterization stack in dev builds The `diagram::render_png` tests rasterize a real diagram through resvg, and at opt-level 0 that is almost entirely SVG filter code: `perf` on the slowest of them attributes ~80% to `resvg::filter::morphology::apply`, with the un-inlined `::max` and `core::cmp::max::` inside it accounting for ~35% between them. None of that is engine code, so no amount of engine work moves it. Pinning the stack to opt-level 3 takes the 16 `render_png` tests from 2.54s to 0.33s (-87%, three alternating runs each) and takes the engine's lib-test binary's longest single test from 2.35s to 1.55s. That second number is the structural one: a test binary's parallel wall is max(longest test, cpu_sum/threads), so the PNG test was setting a floor no amount of parallelism could get under. It no longer is -- the vdf truncation sweep is now the longest test. Whole-binary CPU drops 61.5s to 57.2s, which is what a 4-core CI runner (throughput-bound, not floor-bound) actually collects. The cost is one rebuild of these crates and nothing thereafter: a dependency is not recompiled by an edit to a workspace crate, so unlike an opt-level on a workspace crate this never touches the edit-compile-test loop. Measured: a cold `cargo test -p simlin-engine --no-run` is 61.3s unpinned vs 58.7s pinned, inside run-to-run noise. All nine crates are reached through libsimlin's default `png_render` feature, so a workspace build always has them. A build that does not -- `cargo check -p simlin-engine --lib --no-default-features` and the `--no-default-features` wasm32 bundle build -- emits no unmatched-spec warning; both were checked. --- Cargo.toml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 3f709b4d0..3595cd6f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,19 @@ strip = false # --extract-json payload (with float_roundtrip) on every `cargo test`. # - wasm-interpreter/checked: every wasmgen parity test executes an emitted # blob under this pure-Rust interpreter. +# - the resvg rasterization stack: the `diagram::render_png` tests rasterize +# a diagram through resvg, and unoptimized SVG filter code dominates them +# (~80% of the slowest one is `resvg::filter::morphology::apply`, per +# `perf`, with the un-inlined `u8::max` inside it alone at ~35%). Pinning +# the stack takes those 16 tests from 2.52s to 0.33s and drops the engine's +# lib-test binary below its previous one-test parallel floor. +# +# The asymmetry that makes these worth it: a dependency is not recompiled by an +# edit to a workspace crate, so a pin costs one rebuild of that dependency (and +# whatever sits above it) and then nothing at all on the edit-compile-test loop. +# Measured on the render stack: no change to a cold `cargo test -p simlin-engine +# --no-run` (61.3s vs 58.7s, inside run-to-run noise). Pinning a WORKSPACE crate +# is a different trade and is deliberately not done here. [profile.dev.package.serde_json] opt-level = 3 @@ -43,3 +56,30 @@ opt-level = 3 [profile.dev.package.checked] opt-level = 3 + +[profile.dev.package.resvg] +opt-level = 3 + +[profile.dev.package.usvg] +opt-level = 3 + +[profile.dev.package.tiny-skia] +opt-level = 3 + +[profile.dev.package.tiny-skia-path] +opt-level = 3 + +[profile.dev.package.png] +opt-level = 3 + +[profile.dev.package.rustybuzz] +opt-level = 3 + +[profile.dev.package.ttf-parser] +opt-level = 3 + +[profile.dev.package.fontdb] +opt-level = 3 + +[profile.dev.package.svgtypes] +opt-level = 3 From 9bb392bafe263db0aa48d2094c208fef7be3d5e2 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:25:40 -0700 Subject: [PATCH 12/59] build: optimize the salsa/indexmap query substrate in dev builds Every model any test compiles goes through salsa's query engine and its indexmap-backed dependency edges, so unlike the render pin -- which is concentrated in 16 tests -- this one is spread thinly across the entire suite. Measured on the engine's two test binaries, three runs each: lib-test CPU 57.0s -> 46.5s and integration CPU 64.8s -> 54.0s, about -17% on both. That is the number a 4-core CI runner collects, since both binaries are throughput-bound rather than floor-bound there. The cost is one rebuild of these three crates and everything above them: 64.8s under `taskset -c 0-3`, i.e. paid once per CI cache generation and never again. It does NOT recur on the edit-compile-test loop, because a dependency's artifacts are keyed by its own fingerprint and an edit to a workspace crate cannot invalidate them. Checked rather than assumed: a real content edit to `src/simlin-engine/src/vm.rs` followed by `cargo test -p simlin-engine --no-run` averages 10.8s with the pin and 12.0s without it -- the same number, within noise, and certainly not worse. That asymmetry is why pinning a DEPENDENCY and pinning a WORKSPACE crate are different trades, and why only the former is done here. `hashbrown` here is the standalone crate behind indexmap and salsa, not the copy vendored into std; `std::collections::HashMap` is untouched. --- Cargo.toml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 3595cd6f4..e3afa124b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,13 @@ strip = false # `perf`, with the un-inlined `u8::max` inside it alone at ~35%). Pinning # the stack takes those 16 tests from 2.52s to 0.33s and drops the engine's # lib-test binary below its previous one-test parallel floor. +# - salsa/hashbrown/indexmap: the incremental compilation substrate. Every +# model any test compiles goes through salsa's query engine and its +# indexmap-backed dependency edges, so this is the one pin that is spread +# across the whole suite rather than concentrated in a few tests: it cuts +# ~17% of BOTH engine test binaries' CPU. Note this `hashbrown` is the +# standalone crate behind indexmap and salsa, not the copy vendored into +# std, so `std::collections::HashMap` is unaffected. # # The asymmetry that makes these worth it: a dependency is not recompiled by an # edit to a workspace crate, so a pin costs one rebuild of that dependency (and @@ -83,3 +90,12 @@ opt-level = 3 [profile.dev.package.svgtypes] opt-level = 3 + +[profile.dev.package.salsa] +opt-level = 3 + +[profile.dev.package.hashbrown] +opt-level = 3 + +[profile.dev.package.indexmap] +opt-level = 3 From e74d4d69487594a6d796104cc1e4d4354b7a63f4 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:26:18 -0700 Subject: [PATCH 13/59] engine: pin which slots carry information across a simulation step Between one Euler step and the next, exactly three classes of slot carry a value forward rather than being rewritten by the Flows or Stocks phase: 1. The `IMPLICIT_VAR_COUNT` implicit globals. `run_initials` pre-fills DT/INITIAL_TIME/FINAL_TIME across EVERY chunk of the slab once, after which `run_to` advances only TIME. A run-initialization invariant, not a per-step one. 2. Stocks, written into `next` by the Stocks phase and reaching the next step's `curr` through the chunk ring. 3. Standalone lookup-only table holders (#606) -- excluded from every runlist AND from the saved output, their data reached through `base_gf` into `graphical_functions` and never through the slot. Storage no consumer can observe. `Vm::poison_next_chunk_for_test` fills `next` past the implicit prefix with a sentinel at the top of every Euler step, so a slot that carries forward silently surfaces as the sentinel in the saved results. The test compares the slots reachable through `Results::offsets`, which is exactly the set a consumer can name. Preserving the prefix rather than poisoning it and ignoring it afterwards is the whole point of the fixture. `Context::build_stock_update_expr` emits `stock + (inflows - outflows) * Expr::Dt` and `Expr::Dt` lowers to a `LoadGlobalVar { off: DT_OFF }` read of `curr[DT_OFF]`, so poisoning `dt` corrupts every stock in the model. A whole-chunk poison reports widespread staleness across 28 corpus models that is really one slot, which is precisely the false signal this fixture has to avoid producing. Class 3 was found by running the fixture, not by reading: with the prefix preserved, no model diverges on a stock and exactly one does on two unnamed slots -- the two `` holders in `lookups_simlin/test_lookups.xmile`. The invariant is load-bearing for any change that stops carrying a chunk's contents forward: swapping the chunk indices instead of copying, hoisting run-invariant work out of the per-step program, or partially evaluating a step. Each must carry all three classes explicitly, and none of them has a test of its own that would notice. --- src/simlin-engine/src/vm.rs | 29 ++++++ .../tests/integration/simulate.rs | 91 +++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/src/simlin-engine/src/vm.rs b/src/simlin-engine/src/vm.rs index 1eb97bef3..bcc0cd5a0 100644 --- a/src/simlin-engine/src/vm.rs +++ b/src/simlin-engine/src/vm.rs @@ -365,6 +365,9 @@ pub struct Vm { // returns the fallback during the initial timestep even when // RK stages advance TIME away from INITIAL_TIME. prev_values_valid: bool, + // Test-only: fill the `next` chunk with a sentinel at the top of every + // Euler step. See `poison_next_chunk_for_test`. + poison_next: bool, // Conveyor support (docs/design/conveyors.md §9.3). Empty for every // non-conveyor model, and all conveyor logic is guarded on a non-empty // plan list -- so an ordinary simulation runs with zero overhead and @@ -783,6 +786,12 @@ pub(crate) fn increment_indices(indices: &mut [u16], dims: &[u16]) { } } +/// Sentinel written into the `next` chunk by `poison_next_chunk_for_test`. A +/// distinctive finite value rather than NaN, so a slot that carries forward is +/// distinguishable from a model's own NaN. +#[doc(hidden)] +pub const POISON_SENTINEL: f64 = -1.234567e123; + impl Vm { pub fn new(sim: CompiledSimulation) -> Result { if sim.specs.stop < sim.specs.start { @@ -847,6 +856,7 @@ impl Vm { stock_offsets, rk_scratch, prev_values_valid: false, + poison_next: false, conveyor_plans: Vec::new(), conveyors: Vec::new(), conveyor_last_unit: i64::MIN, @@ -905,6 +915,20 @@ impl Vm { crate::queue_compile::CouplingTable::build(&self.conveyor_plans, &self.queue_plans); } + /// Test-support: fill the `next` chunk PAST the implicit-global prefix with + /// a sentinel at the top of every Euler step, before the Flows phase runs. + /// + /// Exposes which slots carry information across a step: anything not + /// rewritten by the Flows or Stocks phase surfaces as the sentinel in the + /// saved results. The prefix is deliberately preserved -- `Expr::Dt` lowers + /// to a `curr[DT_OFF]` read inside every stock update, so poisoning it + /// corrupts every stock and hides the property under test. See + /// `only_documented_classes_carry_across_a_step`. + #[doc(hidden)] // test-support: used by tests/integration/simulate.rs + pub fn poison_next_chunk_for_test(&mut self) { + self.poison_next = true; + } + pub fn run_to_end(&mut self) -> Result<()> { let end = self.specs.stop; self.run_to(end) @@ -994,12 +1018,17 @@ impl Vm { }}; } + let poison_next = self.poison_next; + match self.specs.method { Method::Euler => loop { let (curr, next) = borrow_two(&mut data, n_slots, self.curr_chunk, self.next_chunk); if curr[TIME_OFF] > end { break; } + if poison_next { + next[IMPLICIT_VAR_COUNT..].fill(POISON_SENTINEL); + } if self.conveyor_plans.is_empty() && self.queue_plans.is_empty() { Self::eval_step(&self.sliced_sim, &mut state, root_idx, curr, next); diff --git a/src/simlin-engine/tests/integration/simulate.rs b/src/simlin-engine/tests/integration/simulate.rs index 684fe3701..521e12248 100644 --- a/src/simlin-engine/tests/integration/simulate.rs +++ b/src/simlin-engine/tests/integration/simulate.rs @@ -6408,3 +6408,94 @@ $192-192-192,0,Times New Roman|12||0-0-0|0-0-0|0-0-255|-1--1--1|-1--1--1|72,72,1 } } } + +// -- What carries across a step ------------------------------------------- +// +// Between one Euler step and the next, the ONLY slots that carry information +// forward are: +// +// 1. The `IMPLICIT_VAR_COUNT` implicit globals. `run_initials` pre-fills +// `DT_OFF`/`INITIAL_TIME_OFF`/`FINAL_TIME_OFF` across EVERY chunk of the +// slab once (`vm.rs`, the `curr[DT_OFF] = dt` block), after which `run_to` +// advances only `TIME`. They are a run-initialization invariant, not a +// per-step one. +// 2. Stocks, which the Stocks phase writes into `next` and which reach the +// following step's `curr` through the chunk ring. +// 3. Standalone lookup-only table holders (#606). These are excluded from +// every runlist AND from the saved output, and a `LOOKUP` reaches their +// data through `base_gf` into `graphical_functions`, never through the +// slot -- so the slot is storage no consumer can observe. +// +// Everything else is rewritten by the Flows or Stocks phase before it is read. +// +// This test pins that by filling `next` -- PAST the implicit prefix -- with a +// sentinel at the top of every Euler step, so any slot that silently carries a +// value forward surfaces as the sentinel in the saved results. It compares the +// slots reachable through `Results::offsets`, which is precisely the set a +// consumer can name. +// +// Why the prefix must be preserved rather than poisoned and then ignored: +// `Context::build_stock_update_expr` emits `stock + (inflows - outflows) * +// Expr::Dt`, and `Expr::Dt` lowers to a `LoadGlobalVar { off: DT_OFF }` read of +// `curr[DT_OFF]`. Poisoning `dt` therefore corrupts every stock in the model, +// which looks like widespread staleness and is really one slot. +// +// The invariant is load-bearing for any change that stops carrying a chunk's +// contents forward -- swapping the chunk indices instead of copying, hoisting +// run-invariant work out of the step, or partially evaluating a step. Such a +// change must carry classes 1-3 explicitly. +// +// Scope: Euler, which is what the corpus exercises. An RK model runs unpoisoned +// and passes trivially. +fn assert_poisoned_next_matches(xmile_path: &str) { + let f = File::open(xmile_path).unwrap(); + let mut f = BufReader::new(f); + let Ok(datamodel_project) = xmile::project_from_reader(&mut f) else { + return; // not a loadable model; the corpus tests already gate that + }; + let compiled = compile_vm(&datamodel_project); + + let mut clean = Vm::new(compiled.clone()).unwrap(); + clean.run_to_end().unwrap(); + let clean = clean.into_results(); + + let mut poisoned = Vm::new(compiled).unwrap(); + poisoned.poison_next_chunk_for_test(); + poisoned.run_to_end().unwrap(); + let poisoned = poisoned.into_results(); + + assert_eq!( + clean.step_size, poisoned.step_size, + "{xmile_path}: step_size" + ); + assert_eq!( + clean.step_count, poisoned.step_count, + "{xmile_path}: step_count" + ); + + let mut named: Vec<(usize, &str)> = clean + .offsets + .iter() + .map(|(k, v)| (*v, k.as_str())) + .collect(); + named.sort(); + for (step, (a, b)) in clean.iter().zip(poisoned.iter()).enumerate() { + for (slot, name) in &named { + let (x, y) = (a[*slot], b[*slot]); + assert!( + x == y || (x.is_nan() && y.is_nan()), + "{xmile_path}: step {step} slot {slot} ({name}) changed when the \ + `next` chunk was poisoned: clean {x} vs poisoned {y}. That slot \ + carried a value across a step without being rewritten, which is \ + outside the three classes documented above." + ); + } + } +} + +#[test] +fn only_documented_classes_carry_across_a_step() { + for path in TEST_MODELS.iter() { + assert_poisoned_next_matches(&format!("../../{path}")); + } +} From bdf605c545da3258134bd80bfdcf0e482be9c498 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:33:13 -0700 Subject: [PATCH 14/59] test: run the metasd macro expansion tier one model per test The expansion tier was three overlapping bulk tests: `metasd_expansion_tier` over the light models, an `#[ignore]`d `metasd_expansion_tier_heavy` over the rest, and `metasd_expansion_tier_full` over all 17. `_full` and the light subset both ran by default, so every light model was compiled and diagnosed twice per suite run; `_heavy`'s own doc comment already made the argument for why -- "`_full` is a strict superset, so running both would buy nothing" -- and it applied just as well to the light one. It is now one `#[test]` per corpus model. The reason is the rule in docs/dev/rust.md: a binary's parallel wall is `max(longest test, total/threads)`, so a serial loop over a corpus sets a floor no number of cores can get under. `metasd_expansion_tier_full` was the second-longest test in the whole integration harness at 3.20s solo. The longest of the 17 per-model tests is now 0.892s (scirev8), and the module's total drops from 3.77s to 2.64s -- the difference being the duplicate light pass. A failure also now names the model in the test name rather than only inside an accumulated list. Coverage goes UP, not down: `_heavy`'s five models were only ever run on demand and now run by default. They fit easily -- scirev8 0.892s, scirev7 0.766s, free6 0.258s, beer-game 0.250s, covid19 0.081s -- which is also why the `heavy` field is gone rather than reworded. Its per-entry annotations ("~3.6s compile" for scirev8) were stale by 4x after the compile work of the last few rounds, and nothing read the field once both filters went away; a stale measurement in a comment is worse than no comment. The one way a per-model split can silently under-cover is a CORPUS entry added without a matching test, so `corpus_is_exactly_the_17_macro_using_metasd_files` now asserts the generated test-name set equals the corpus name set in both directions (and that the names are unique). Checked by mutation: deleting one name from the `expansion_tier_tests!` list reds that guard. --- .../tests/integration/metasd_macros.rs | 153 +++++++++++------- 1 file changed, 94 insertions(+), 59 deletions(-) diff --git a/src/simlin-engine/tests/integration/metasd_macros.rs b/src/simlin-engine/tests/integration/metasd_macros.rs index 492391aaf..699a09e30 100644 --- a/src/simlin-engine/tests/integration/metasd_macros.rs +++ b/src/simlin-engine/tests/integration/metasd_macros.rs @@ -95,19 +95,19 @@ enum SimTier { /// `TEST_SDEVERYWHERE_MODELS` style (a small struct rather than parallel /// commented sections so the reason travels with the path). struct CorpusModel { + /// The name of this model's generated expansion-tier `#[test]`, which is + /// also how `corpus_entry` looks the entry up. Derived from the path + /// (directory under `test/metasd/` plus file stem, snake_cased) so it + /// stays legible in a failure report. + name: &'static str, /// Path relative to `src/simlin-engine/` (the `../../test/...` prefix). path: &'static str, - /// `true` => the expansion tier for this model is `#[ignore]`d into - /// `metasd_expansion_tier_heavy` (it is a large real-world model whose - /// compile exceeds the per-test time budget; see `docs/dev/rust.md`). - /// `false` => it runs in the fast default `metasd_expansion_tier`. - heavy: bool, sim: SimTier, } /// The full corpus: every macro-using `.mdl` under `test/metasd/` (the -/// exact 17-file list, 14 directories). Each entry's `sim` reason and -/// `heavy` flag is the *measured, verified* status as of Phase 7 +/// exact 17-file list, 14 directories). Each entry's `sim` reason is the +/// *measured, verified* status as of Phase 7 /// (2026-05-15). The expansion tier asserts NONE of these -- all 17 -- has a /// macro-attributable diagnostic. (Historical note: `thyroid-2008-d.mdl` was /// once excluded for a #554-class false-positive `delayn -> delayn` @@ -118,8 +118,8 @@ struct CorpusModel { const CORPUS: &[CorpusModel] = &[ // -- 12 single-file directories -- CorpusModel { + name: "bathtub_statistics_integration3", path: "../../test/metasd/bathtub-statistics/integration3.mdl", - heavy: false, // Macros trend2/init/pink_noise all expand (correct MacroSpecs). sim: SimTier::Skip( "no reference output checked in; also unrelated blocker: \ @@ -128,8 +128,8 @@ const CORPUS: &[CorpusModel] = &[ ), }, CorpusModel { + name: "beer_game_realbeer4_sterman13", path: "../../test/metasd/beer-game/RealBeer4-Sterman13.mdl", - heavy: true, // ~1.2s compile sim: SimTier::Skip( "no reference output checked in; also unrelated blocker: \ RANDOM NORMAL (UnknownBuiltin), a model-logic peak->peak \ @@ -138,8 +138,8 @@ const CORPUS: &[CorpusModel] = &[ ), }, CorpusModel { + name: "covid19_us_homer_covid19us_v8", path: "../../test/metasd/covid19-us-homer/homer v8/Covid19US v8.mdl", - heavy: true, // ~0.17s but large; grouped with the opt-in corpus sim: SimTier::Skip( "no reference output checked in; also unrelated blocker: the \ *_data variables are unresolved GET DIRECT/GET XLS DATA refs \ @@ -148,24 +148,24 @@ const CORPUS: &[CorpusModel] = &[ ), }, CorpusModel { + name: "critical_slowing_critical_slowing", path: "../../test/metasd/critical-slowing/critical-slowing.mdl", - heavy: false, sim: SimTier::Skip( "no reference output checked in; also unrelated blocker: \ pink_noise body uses RANDOM NORMAL (UnknownBuiltin)", ), }, CorpusModel { + name: "early_warnings_catastrophe_catastropewarning2", path: "../../test/metasd/early-warnings-catastrophe/catastropeWarning2.mdl", - heavy: false, sim: SimTier::Skip( "no reference output checked in; also unrelated blocker: \ pink_noise body uses RANDOM NORMAL (UnknownBuiltin)", ), }, CorpusModel { + name: "free_free_6", path: "../../test/metasd/FREE/FREE6/FREE6-original/free 6.mdl", - heavy: true, // ~1.2s compile // A sibling `all_data2.vdf` EXISTS, but `free 6.mdl` has heavy // unrelated MDL-parse / dimension blockers, so it is NOT // simulation-tier-eligible (the `init` macro itself expands). @@ -178,8 +178,8 @@ const CORPUS: &[CorpusModel] = &[ ), }, CorpusModel { + name: "industrial_dynamics_idch15d", path: "../../test/metasd/industrial-dynamics/IDch15/IDch15d.mdl", - heavy: false, sim: SimTier::Skip( "no reference output checked in; also unrelated blocker: \ main-model UnrecognizedToken / UnknownBuiltin (the `clip` \ @@ -187,8 +187,8 @@ const CORPUS: &[CorpusModel] = &[ ), }, CorpusModel { + name: "interpolating_arrays_interpolatingarrays", path: "../../test/metasd/interpolating-arrays/InterpolatingArrays.mdl", - heavy: false, sim: SimTier::Skip( "no reference output checked in; also unrelated blocker: \ main-model ExtraToken / UnrecognizedToken / CantSubscriptScalar \ @@ -196,16 +196,16 @@ const CORPUS: &[CorpusModel] = &[ ), }, CorpusModel { + name: "pink_noise_pinknoise2010", path: "../../test/metasd/pink-noise/PinkNoise2010.mdl", - heavy: false, sim: SimTier::Skip( "no reference output checked in; also unrelated blocker: \ pink_noise body uses RANDOM NORMAL (UnknownBuiltin)", ), }, CorpusModel { + name: "theil_statistics_theil_2011", path: "../../test/metasd/theil-statistics/Theil_2011.mdl", - heavy: false, // Theil_2011 COMPILES with ZERO errors (the THEIL multi-output // macro materializes + simulates -- pinned end-to-end by // simulate.rs::corpus_theil_multi_output_materializes_and_simulates). @@ -219,8 +219,8 @@ const CORPUS: &[CorpusModel] = &[ ), }, CorpusModel { + name: "thyroid_dynamics_thyroid_2008_d", path: "../../test/metasd/thyroid-dynamics/thyroid-2008-d.mdl", - heavy: false, // The #554-class false-positive `delayn -> delayn` macro-registry // recursion is FIXED (the #554 follow-up extended the shared // renamed-builtin self-edge suppression to the stdlib-module-backed @@ -238,8 +238,8 @@ const CORPUS: &[CorpusModel] = &[ ), }, CorpusModel { + name: "wonderland_wonderland3", path: "../../test/metasd/wonderland/Wonderland3.mdl", - heavy: false, sim: SimTier::Skip( "no reference output checked in; also unrelated blocker: \ main-model UnknownBuiltin / UnknownDependency (the `p_exp` / \ @@ -248,8 +248,8 @@ const CORPUS: &[CorpusModel] = &[ }, // -- scientific-revolution: two macro-using files -- CorpusModel { + name: "scientific_revolution_scirev7", path: "../../test/metasd/scientific-revolution/scirev7.mdl", - heavy: true, // ~2.5s compile sim: SimTier::Skip( "no reference output checked in; also unrelated blocker: \ main-model UnknownBuiltin / UnknownDependency / Generic (the \ @@ -257,8 +257,8 @@ const CORPUS: &[CorpusModel] = &[ ), }, CorpusModel { + name: "scientific_revolution_scirev8", path: "../../test/metasd/scientific-revolution/scirev8.mdl", - heavy: true, // ~3.6s compile (over the 5s soft ceiling combined) sim: SimTier::Skip( "no reference output checked in; also unrelated blocker: \ main-model UnknownBuiltin / UnknownDependency / Generic (the \ @@ -270,8 +270,8 @@ const CORPUS: &[CorpusModel] = &[ // but every groupon model has heavy unrelated MDL-parse blockers, so // none is simulation-tier-eligible (the `report` macro expands). -- CorpusModel { + name: "social_network_valuation_groupon_1", path: "../../test/metasd/social-network-valuation/groupon 1.mdl", - heavy: false, sim: SimTier::Skip( "unrelated blocker: data_* variables are EmptyEquation / \ UnrecognizedToken (unresolved external data) despite sibling \ @@ -279,8 +279,8 @@ const CORPUS: &[CorpusModel] = &[ ), }, CorpusModel { + name: "social_network_valuation_groupon_2", path: "../../test/metasd/social-network-valuation/groupon 2.mdl", - heavy: false, sim: SimTier::Skip( "unrelated blocker: data_* variables are EmptyEquation / \ UnrecognizedToken (unresolved external data) despite sibling \ @@ -288,8 +288,8 @@ const CORPUS: &[CorpusModel] = &[ ), }, CorpusModel { + name: "social_network_valuation_groupon_3", path: "../../test/metasd/social-network-valuation/groupon 3.mdl", - heavy: false, sim: SimTier::Skip( "unrelated blocker: data_* variables are EmptyEquation / \ UnrecognizedToken (unresolved external data) despite sibling \ @@ -453,44 +453,57 @@ fn run_expansion_tier(entries: impl Iterator) { } } -/// macros.AC6.4 (expansion tier, fast subset). The light macro-using -/// metasd models compile via the salsa path with NO macro-attributable -/// diagnostic. Runs by default (each model compiles in well under the -/// per-test budget); the heavy real-world models are in the `#[ignore]`d -/// `metasd_expansion_tier_heavy` opt-in below (`docs/dev/rust.md` -/// test-time-budget rules). Together they cover all 14 macro-using metasd -/// directories / all 17 macro-using files. -#[test] -fn metasd_expansion_tier() { - run_expansion_tier(CORPUS.iter().filter(|m| !m.heavy)); +/// The `CORPUS` entry a generated expansion-tier test is about. +fn corpus_entry(name: &str) -> &'static CorpusModel { + CORPUS + .iter() + .find(|m| m.name == name) + .unwrap_or_else(|| panic!("no CORPUS entry named {name}")) } -/// macros.AC6.4 (expansion tier, the heavy real-world models). Same -/// assertion as `metasd_expansion_tier` for the large models whose -/// compile used to exceed the per-test time budget. +/// macros.AC6.4 (expansion tier): one `#[test]` per corpus model, asserting +/// that model compiles via the salsa path with NO macro-attributable +/// diagnostic. All 17 files / 14 directories run by default. /// -/// Still `#[ignore]`d, but no longer for time: `metasd_expansion_tier_full` -/// now runs by default and is a strict superset of this, so running both in -/// the default suite would buy nothing. Kept as the focused subset to reach for -/// when the full tier fails and the light models are not the culprit. -// Run with: cargo test -p simlin-engine --test integration -- --ignored metasd_expansion_tier_heavy -#[test] -#[ignore] -fn metasd_expansion_tier_heavy() { - run_expansion_tier(CORPUS.iter().filter(|m| m.heavy)); +/// One test per model rather than one loop over all of them, for the reason +/// `docs/dev/rust.md` gives: a binary's parallel wall is +/// `max(longest test, total/threads)`, so a serial loop over a corpus sets a +/// floor no number of cores can get under. It also puts the failing model in +/// the test NAME instead of only in an accumulated list. +/// +/// `EXPANSION_TEST_NAMES` is what `corpus_is_exactly_the_17_macro_using_metasd_files` +/// checks the generated set against, so a `CORPUS` entry added without a test +/// here -- the one way this list can silently under-cover -- fails loudly. +macro_rules! expansion_tier_tests { + ($($name:ident),* $(,)?) => { + static EXPANSION_TEST_NAMES: &[&str] = &[$(stringify!($name)),*]; + $( + #[test] + fn $name() { + run_expansion_tier(std::iter::once(corpus_entry(stringify!($name)))); + } + )* + }; } -/// The full expansion tier over ALL 17 macro-using files in one run -/// (light + heavy), the AC6.4 "all 14 macro-using metasd models pass the -/// expansion tier" check. -/// -/// Runs by default. The "sum of compiles ~10s" that put it over the per-test -/// budget is now under three seconds on a debug build, and this is the -/// assertion the acceptance criterion is actually about -- the light-subset -/// `metasd_expansion_tier` was the compromise, not the goal. -#[test] -fn metasd_expansion_tier_full() { - run_expansion_tier(CORPUS.iter()); +expansion_tier_tests! { + bathtub_statistics_integration3, + beer_game_realbeer4_sterman13, + covid19_us_homer_covid19us_v8, + critical_slowing_critical_slowing, + early_warnings_catastrophe_catastropewarning2, + free_free_6, + industrial_dynamics_idch15d, + interpolating_arrays_interpolatingarrays, + pink_noise_pinknoise2010, + theil_statistics_theil_2011, + thyroid_dynamics_thyroid_2008_d, + wonderland_wonderland3, + scientific_revolution_scirev7, + scientific_revolution_scirev8, + social_network_valuation_groupon_1, + social_network_valuation_groupon_2, + social_network_valuation_groupon_3, } /// Positive regression guard (inverted premise -- the bug is FIXED): @@ -503,8 +516,9 @@ fn metasd_expansion_tier_full() { /// follow-up (`module_functions::is_renamed_stdlib_module_builtin`) /// suppresses that false self-edge, so thyroid is now in the asserted /// expansion tier; this test additionally pins thyroid *specifically* (so a -/// regression of the follow-up is caught here with a focused message, not -/// only in the bulk `metasd_expansion_tier`). It deliberately does NOT +/// regression of the follow-up is caught here with a focused message rather +/// than only as a generic macro-attributable-diagnostic failure in +/// `thyroid_dynamics_thyroid_2008_d`). It deliberately does NOT /// assert `compiled_ok`: the macro handling is correct, but the body's /// `DELAY N(...,Order)` with the order a macro *port* still hits the /// orthogonal, pre-existing stdlib "order must be a compile-time constant" @@ -800,6 +814,27 @@ fn corpus_is_exactly_the_17_macro_using_metasd_files() { got {dirs:?}" ); + // Every corpus entry has a generated expansion-tier test, and every + // generated test names a real entry. Without this, adding a model to + // CORPUS without adding it to `expansion_tier_tests!` would silently + // leave it unasserted -- the exact under-coverage the per-model split + // could otherwise introduce, and invisible in a green run. + let corpus_names: std::collections::BTreeSet<&str> = CORPUS.iter().map(|m| m.name).collect(); + assert_eq!( + corpus_names.len(), + CORPUS.len(), + "corpus entry names must be unique; a duplicate would make two \ + generated tests assert the same model and leave another unasserted" + ); + let test_names: std::collections::BTreeSet<&str> = + EXPANSION_TEST_NAMES.iter().copied().collect(); + assert_eq!( + corpus_names, test_names, + "every CORPUS entry must have a generated expansion-tier test and \ + vice versa; add the model's `name` to the `expansion_tier_tests!` \ + list (or remove the stale entry from it)" + ); + for m in CORPUS { let p = std::path::Path::new(m.path); assert!(p.exists(), "corpus model missing on disk: {}", m.path); From a32766df1d2f562de0a0953e9e24f5db7e789d11 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:36:45 -0700 Subject: [PATCH 15/59] test: stop re-simulating the protobuf round-trip in the corpus harness `simulate_path_with_excluding` compiled and ran every corpus model three times on the VM -- the original, the protobuf round-trip, and the XMILE round-trip -- plus once through the wasm backend. The protobuf leg already asserts `datamodel_project == datamodel_project2` before recompiling, so what the recompile-and-compare added was the question "is compilation a function of the datamodel?", asked once per corpus model. That is a real property but a different one, and it is owned by `db::fragment_determinism_tests`, which asserts it far more directly: byte-identical compiled output from independent fresh databases, twelve repeats, on fixtures chosen to expose the specific HashMap orderings that can break it. The XMILE leg keeps simulating, and the comment now says why: it asserts no datamodel equality (the reader legitimately normalizes), so simulating the re-read project is the only thing pinning its behaviour. Measured on the 58 `simulate::corpus::` tests -- the ones that actually go through this helper -- 0.965s to 0.796s solo, about -17%. Across the whole `simulate::` module it is -0.4s, since that module is dominated by the C-LEARN tests, which do not use this path. A small win, reported at its size. --- .../tests/integration/simulate.rs | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/simlin-engine/tests/integration/simulate.rs b/src/simlin-engine/tests/integration/simulate.rs index 684fe3701..7eed13c54 100644 --- a/src/simlin-engine/tests/integration/simulate.rs +++ b/src/simlin-engine/tests/integration/simulate.rs @@ -1188,8 +1188,18 @@ fn simulate_path_with_excluding(xmile_path: &str, compile: CompileFn, excluded: let expected = load_expected_results(xmile_path).unwrap(); ensure_results_excluding(&expected, &results, excluded); - // serialize our project through protobufs and ensure we don't see problems - let results_proto = { + // Protobuf round-trip: the decoded project must equal the original. That + // equality is the whole claim -- re-compiling and re-simulating the decoded + // copy would be asking whether compilation is a function of the datamodel, + // which is a different property, owned by `db::fragment_determinism_tests` + // and asserted there far more directly (byte-identical output from + // independent fresh databases). Here it would only re-derive, once per + // corpus model, an answer the `assert_eq!` already gives. + // + // The XMILE round-trip below deliberately still simulates: it asserts NO + // datamodel equality (the reader legitimately normalizes), so simulating + // the re-read project is the only thing that pins its behaviour. + { use simlin_engine::prost::Message; let pb_project_inner = serialize(&datamodel_project).unwrap(); @@ -1199,12 +1209,7 @@ fn simulate_path_with_excluding(xmile_path: &str, compile: CompileFn, excluded: let datamodel_project2 = deserialize(project_io::Project::decode(&*buf).unwrap()); assert_eq!(datamodel_project, datamodel_project2); - let compiled_sim = compile(&datamodel_project2); - let mut vm = Vm::new(compiled_sim).unwrap(); - vm.run_to_end().unwrap(); - vm.into_results() - }; - ensure_results_excluding(&expected, &results_proto, excluded); + } // serialize our project back to XMILE let serialized_xmile = xmile::project_to_xmile(&datamodel_project).unwrap(); From 89261c3257c2ea8ddeb455db44c73995adcb1c95 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:50:53 -0700 Subject: [PATCH 16/59] doc: separate the measurement channels and their noise floors The ~4% figure this file records is a WALL-CLOCK/CYCLES floor, and it has been getting applied to instruction counts, where the measured sd across independent builds is 0.026% -- three orders of magnitude apart. Retired instructions are a property of the program; cycles are a property of the machine executing it, and only the second is subject to the binary-layout lottery the figure describes. Conflating them is expensive in both directions. It makes a real instruction-level win look unmeasurable and get abandoned, and it sends anyone who wants an instruction-count result into a multi-build A/B that one build pair would have settled. The new section states the three channels and what each answers, the measured per-channel floors, and the rule that a cycles claim must clear the SAME session's null control rather than any floor recorded here -- machine conditions vary hour to hour, and taking a historical figure for the current one is how noise becomes a reported result. The null control is the evidence for all of it: the identical binary run as both sides of an interleaved A/B reports -0.003% on instructions and -1.540% on cycles, i.e. the cycles channel manufactures a 1.5% "win" out of nothing at load average 4-9. Two general techniques ride along because they are what actually decides these questions. Prefer a structural check to a statistical one where the change admits it: a change confined to an `#[inline(never)]` function with an unchanged signature should leave its callers byte-identical, which is a binary answer rather than a sample and directly detects the leak-into-`eval_bytecode` failure mode this file has recorded more than once. And decide the falsification signatures before measuring, so the eventual number is a result rather than a reading. The closing convention is the common cause of the whole problem: a recorded verdict should name the channel its number came from. A bare "only ~1.5%" invites the next reader to compare it against whatever floor they have in mind. The "methodology consequence" note under the #712 B2 result is rewritten to point here and to say which claim its ~4% bounds, rather than giving one threshold for every channel. --- docs/design/engine-performance.md | 94 +++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/docs/design/engine-performance.md b/docs/design/engine-performance.md index 15d8c5e25..7a071f868 100644 --- a/docs/design/engine-performance.md +++ b/docs/design/engine-performance.md @@ -23,6 +23,91 @@ set of larger proposals grounded in the measured data. unless noted. Profile builds add `CARGO_PROFILE_RELEASE_DEBUG=1 CARGO_PROFILE_RELEASE_STRIP=false`. +### Measuring a change + +Three channels, each answering a different question. **None substitutes for +another**, and a change is not established until the question you are actually +asking has been answered by the channel that can answer it. + +| channel | question it answers | tool | +|---|---|---| +| exact instruction attribution | *did the intended work disappear?* | `valgrind --tool=callgrind` | +| retired instructions / branches | *how much work disappeared?* | `perf stat` | +| cycles / wall clock | *did it get faster?* | `perf stat`, interleaved A/B | + +**Callgrind is deterministic** and immune to both binary layout and machine +load. It is the right first measurement for any change with a mechanism: it +says whether the work you meant to remove is gone, per function and per source +line, with no statistics. A change whose per-call cost is unchanged did not +fire, whatever the end-to-end counters say. + +**Retired instructions and branches are properties of the program**; cycles are +a property of the machine executing it. That distinction sets the noise floors, +and they are three orders of magnitude apart. Measured on the C-LEARN run +across six independent build+run pairs of identical source: + +| channel | sd across builds | a 2.7% effect is | +|---|---|---| +| instructions | **0.026%** | ~104 sigma | +| branches | **0.028%** | ~96 sigma | +| cycles, quiet machine | 1.65% | 1.7 sigma | +| cycles, machine under load | 9.9%–11% | 0.24 sigma | + +So a few-percent effect is resolved by one build pair on the instruction +channel and is **not** resolvable on the cycles channel without a deliberate +protocol. Reaching for multi-build A/Bs to establish an instruction-count +reduction wastes hours the instruction channel settles in one pair; quoting a +cycles delta from one pair asserts something the measurement cannot support. + +**Every cycles claim needs a null control from the same session.** Run the +identical binary as both sides of the A/B, interleaved, alongside the real +comparison. The apparent delta it produces is that session's floor. A measured +example, taken at load average 4–9: + +``` +identical binary, both sides, 5 interleaved rounds, medians: + instructions -0.003% + branches -0.004% + cycles -1.540% <- a "win" from nothing +``` + +A cycles delta that does not clearly exceed the session's own null delta is +**unresolved, and must be reported as unresolved rather than as a small win**. +Use the session's null, never a floor recorded here or anywhere else: machine +conditions vary hour to hour, and taking a historical figure for the current +one is what turns noise into a reported result. + +**Contention is a reason to wait, not to average harder.** Resolving 3% at the +quiet-machine sd of 1.65% needs about 5 builds per side; at a contended 9.9% it +needs about 175. The second is not a measurement plan. Check the load average +before starting, pin with `taskset`, interleave A/B/A/B so drift is shared, take +medians, and reject outliers explicitly rather than letting them widen the +spread. + +**Prefer a structural check to a statistical one where the change admits it.** +When a change is confined to a function that is `#[inline(never)]` and keeps its +signature, the callers' machine code should be *byte-identical*; disassemble +both binaries and diff the caller modulo addresses. That is a binary answer +rather than a sample, and it directly detects the failure mode that has bitten +this file's eval-loop work repeatedly: a change leaking into `eval_bytecode` and +perturbing the register allocation of a very large function. Treat a single +differing instruction as a hard stop and explain it before quoting any number. + +**Decide what would falsify the change before measuring it.** Write down the +predicted delta per channel, and the signatures that would mean it did not work: +end-to-end instructions falling while the callgrind per-call cost is unchanged +means something other than the intended mechanism moved; instructions falling +while the branch count holds means a branchy inner loop was not actually +replaced. Stating these in advance is what makes the eventual number a result +instead of a reading. + +**State which channel a recorded number came from.** A verdict written as "only +~1.5%" invites the next reader to compare it against whatever floor they happen +to have in mind, and the floors differ by three orders of magnitude between +channels. Write "1.5% of retired instructions" or "1.5% of cycles"; a +percentage with no channel attached is how a cycles floor ends up being applied +to an instruction measurement. + ## Measured baseline (before this work) | Phase | Wall (per iter) | Allocations | Dominant costs | @@ -325,10 +410,11 @@ parity, zero-alloc all hold -- but did not clear the keep bar: - Branch-misses fell 8.4%, so a mispredict-bound core (the round-1 Ryzen) might see a real win -- that is the retry condition recorded on GH #712. -Methodology consequence for future rounds: for effects under ~3%, either -compare layout-stable counters (instructions/branch-misses via `perf stat`) -or A/B multiple independent builds per side; a single worktree build pair is -only conclusive for effects that exceed ~4%. +Methodology consequence for future rounds: the ~4% figure above bounds a +WALL-CLOCK/CYCLES claim from a single build pair, and nothing else. Retired +instructions and branches have an sd of ~0.026% across builds, so the same +effect is resolved there by one pair; see "Measuring a change" above for the +per-channel floors and the null-control rule. ### R4. `RuntimeView` allocation + `flat_offset` (~20% of post-win run) From eef3b31c3fffdc48e63026eb9f8d89f8a41d974d Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:51:50 -0700 Subject: [PATCH 17/59] engine: make compile_implicit_var_fragment a salsa query `compile_implicit_var_fragment` was a plain function, on the reasoning that "the parent variable's parse result already provides salsa caching". That is true of the PARSE and of nothing else: the lowering (`lower_implicit_var` -> `parse_var` -> `lower_variable`) and the per-phase codegen ran on every call, and both production call sites call it once per helper per assembly. So every SMOOTH/DELAY/TREND/PREVIOUS/INIT helper in a model was recompiled from scratch whenever `assemble_module` re-ran -- which any equation edit causes. On C-LEARN that is 651 helper compiles per cold assembly (~12% of a cold compile) and ~28% of the cost of a WARM single-equation edit, by far the largest share of a recompile that should have touched one variable and its consumers. Keyed on the helper's own canonical name. That is the only identity a helper has -- it exists solely inside its parent's parse -- and it is the key `model_implicit_var_info` files it under, so `model_implicit_var_by_name` resolves the metadata inside the query instead of the caller passing a borrowed `&ImplicitVarMeta` no salsa key could carry. `ImplicitVarMeta::name` already carries the argument for a name over a position, and bounds the one case where a name resolves to a different helper than the metadata meant -- a case that fails to compile regardless. The runlist gate reads a new `implicit_var_runlist_membership` rather than the whole `ModelDepGraphResult` the callers used to pass in, for the reason `compile_var_fragment` reads `var_runlist_membership`: a three-bit projection backdates when this helper's membership is unchanged, where depending on the whole result would re-execute every helper's fragment whenever any variable's dependencies moved -- reintroducing the coarseness this change removes. The two keyed projections share one `membership_in` body so they cannot answer the same question differently. Measured on C-LEARN (40 single-equation edits + 5 no-op recompiles, interleaved, three rounds): 20.49G -> 14.33G retired instructions, -30% of the whole warm workload and -32% per edit. Wall-clock median for one edit falls from ~40 ms to ~5-9 ms in the uncontended rounds. Cold compile and the compiled artifact are unchanged (5215 slots, 58291 opcodes). `implicit_and_ltm_fragment_cache_granularity` was a characterization pin on the old behaviour and is restated rather than deleted: it asserted that every helper recompiles on an edit to a variable none of them reads, and now asserts that none does. Its new complement -- editing the variable a helper DOES read -- turned out to recompile only ONE of the fixture's two helpers, which is correct and finer than expected: `builtins_visitor` passes a bare `Var` argument through by name and synthesizes a helper only for a computed one, so `SMTH1(src, 2)` wires `src` into the module instance and captures only the literal `2`. The granularity is per helper, not per parent. --- src/simlin-engine/src/db/assemble.rs | 6 +- src/simlin-engine/src/db/dep_graph.rs | 38 ++++++++- src/simlin-engine/src/db/diagnostic.rs | 16 ++-- .../src/db/fragment_char_tests.rs | 78 ++++++++++++------- src/simlin-engine/src/db/fragment_compile.rs | 68 ++++++++++++---- .../src/db/fragment_determinism_tests.rs | 26 +++---- 6 files changed, 162 insertions(+), 70 deletions(-) diff --git a/src/simlin-engine/src/db/assemble.rs b/src/simlin-engine/src/db/assemble.rs index 269fa54f5..8dfda0e95 100644 --- a/src/simlin-engine/src/db/assemble.rs +++ b/src/simlin-engine/src/db/assemble.rs @@ -1448,11 +1448,11 @@ pub fn assemble_module<'db>( } } - for (name, meta) in implicit_info.iter() { + for name in implicit_info.keys() { if let Some(result) = - compile_implicit_var_fragment(db, meta, model, project, dep_graph, module_input_names) + compile_implicit_var_fragment(db, model, project, name.clone(), module_inputs) { - all_fragments.insert(name.clone(), result); + all_fragments.insert(name.clone(), result.clone()); } } diff --git a/src/simlin-engine/src/db/dep_graph.rs b/src/simlin-engine/src/db/dep_graph.rs index 7a3a90356..a46b2571e 100644 --- a/src/simlin-engine/src/db/dep_graph.rs +++ b/src/simlin-engine/src/db/dep_graph.rs @@ -1733,11 +1733,41 @@ pub fn var_runlist_membership<'db>( module_inputs: ModuleInputSet<'db>, ) -> RunlistMembership { let dep_graph = model_dependency_graph(db, model, project, module_inputs); - let name = canonicalize(var.ident(db)).into_owned(); + membership_in(dep_graph, &canonicalize(var.ident(db))) +} + +/// The same projection for a variable that has no `SourceVariable` handle: an +/// implicit SMOOTH/DELAY/TREND/PREVIOUS helper, which exists only inside its +/// parent's parse and is filed in the runlists under its canonical synthesized +/// name. +/// +/// Keyed on that name because it is the only identity such a helper has -- the +/// same key `model_implicit_var_by_name` uses. `compile_implicit_var_fragment` +/// reads this instead of the whole `ModelDepGraphResult` for the identical +/// reason the explicit twin does: a helper's fragment must not re-execute +/// because some unrelated variable's dependencies moved. +#[salsa::tracked(returns(clone))] +pub fn implicit_var_runlist_membership<'db>( + db: &'db dyn Db, + model: SourceModel, + project: SourceProject, + name: String, + module_inputs: ModuleInputSet<'db>, +) -> RunlistMembership { + let dep_graph = model_dependency_graph(db, model, project, module_inputs); + membership_in(dep_graph, &name) +} + +/// The projection itself, stated once so the two keyed entry points above +/// cannot answer the same question differently. +fn membership_in(dep_graph: &ModelDepGraphResult, name: &str) -> RunlistMembership { + // The runlists are ordered `Vec`s (the topological emission order), so each + // of these is the same linear scan `Vec::contains` performed before; going + // through `iter().any` only lets the key stay a `&str`. RunlistMembership { - initials: dep_graph.runlist_initials.contains(&name), - flows: dep_graph.runlist_flows.contains(&name), - stocks: dep_graph.runlist_stocks.contains(&name), + initials: dep_graph.runlist_initials.iter().any(|n| n == name), + flows: dep_graph.runlist_flows.iter().any(|n| n == name), + stocks: dep_graph.runlist_stocks.iter().any(|n| n == name), } } diff --git a/src/simlin-engine/src/db/diagnostic.rs b/src/simlin-engine/src/db/diagnostic.rs index ed86e5a9f..809c2dfea 100644 --- a/src/simlin-engine/src/db/diagnostic.rs +++ b/src/simlin-engine/src/db/diagnostic.rs @@ -220,12 +220,16 @@ pub fn model_all_diagnostics(db: &dyn Db, model: SourceModel, project: SourcePro // nothing. { let implicit_info = crate::db::model_implicit_var_info(db, model, project); - let dep_graph = crate::db::model_dependency_graph(db, model, project, empty_inputs); - let mut sorted_implicit: Vec<_> = implicit_info.iter().collect(); - sorted_implicit.sort_unstable_by_key(|(name, _)| name.as_str()); - for (_name, meta) in sorted_implicit { - let _ = - crate::db::compile_implicit_var_fragment(db, meta, model, project, dep_graph, &[]); + let mut sorted_implicit: Vec<&String> = implicit_info.keys().collect(); + sorted_implicit.sort_unstable_by_key(|name| name.as_str()); + for name in sorted_implicit { + let _ = crate::db::compile_implicit_var_fragment( + db, + model, + project, + name.clone(), + empty_inputs, + ); } } diff --git a/src/simlin-engine/src/db/fragment_char_tests.rs b/src/simlin-engine/src/db/fragment_char_tests.rs index 26b45f447..abde88465 100644 --- a/src/simlin-engine/src/db/fragment_char_tests.rs +++ b/src/simlin-engine/src/db/fragment_char_tests.rs @@ -551,15 +551,10 @@ fn collect_model_fragments( let mut implicit_names: Vec<&String> = implicit_info.keys().collect(); implicit_names.sort(); for name in implicit_names { - if let Some(result) = compile_implicit_var_fragment( - db, - &implicit_info[name], - model, - project, - dep_graph, - &owned_inputs, - ) { - push(&mut out, FragmentKind::Implicit, &result); + if let Some(result) = + compile_implicit_var_fragment(db, model, project, name.clone(), inputs) + { + push(&mut out, FragmentKind::Implicit, result); } } @@ -2479,16 +2474,24 @@ fn equation_only_edit_recompiles_only_the_edited_fragment() { /// The cache granularity of the OTHER two fragment compilers, measured rather /// than assumed. /// -/// `compile_implicit_var_fragment` is not a salsa query at all -- it is a plain -/// function called from the tracked `assemble_module` -- so every implicit -/// (SMOOTH/DELAY/TREND/PREVIOUS/INIT) helper recompiles whenever assembly -/// re-runs, which an equation edit to ANY variable in the model causes. -/// `compile_ltm_var_fragment` IS tracked, per `(from, to)` link. +/// `compile_implicit_var_fragment` is a salsa query keyed on the helper's own +/// canonical name, so an implicit (SMOOTH/DELAY/TREND/PREVIOUS/INIT) helper +/// recompiles only when something it reads changes -- NOT merely because +/// assembly re-ran, which an equation edit to any variable in the model +/// causes. `compile_ltm_var_fragment` is likewise tracked, per `(from, to)` +/// link. +/// +/// The implicit assertion below is the whole reason the query is keyed the way +/// it is. While it was a plain function every helper in the model recompiled on +/// every assembly: this fixture recompiled both of `smoothed`'s helpers when +/// `unrelated` was edited, and on C-LEARN it was 651 helper compiles per cold +/// assembly and ~28% of the cost of a warm single-equation edit. A change that +/// reverts the query to a plain function reds here on the count rather than +/// merely running slower. /// -/// Pinned because stage 3 of GH #964 routes all three emitters through one -/// implementation: if that implementation is a salsa query, these numbers -/// should drop, and if it is a plain function, the explicit path could -/// silently acquire the implicit path's granularity instead. +/// Pinned also because stage 3 of GH #964 routes all three emitters through one +/// implementation: the explicit path must not silently acquire the implicit +/// path's granularity, or vice versa. #[test] fn implicit_and_ltm_fragment_cache_granularity() { use salsa::Setter; @@ -2517,7 +2520,7 @@ fn implicit_and_ltm_fragment_cache_granularity() { // Edit a variable the SMTH1 helper does not read. let edited = project_with("3", "2"); - let (_state3, execs) = resync_and_assemble(&mut db, &edited, Some(&state2)); + let (state3, execs) = resync_and_assemble(&mut db, &edited, Some(&state2)); assert_eq!( explicit_execs(&execs), vec!["unrelated"], @@ -2530,13 +2533,36 @@ fn implicit_and_ltm_fragment_cache_granularity() { .collect(); assert_eq!( implicit, - vec![ - "smoothed#$\u{205A}smoothed\u{205A}0\u{205A}arg1", - "smoothed#$\u{205A}smoothed\u{205A}0\u{205A}smth1" - ], - "every implicit helper of the model recompiles on an edit to a variable \ - none of them reads: `compile_implicit_var_fragment` has no cache entry \ - of its own, so its granularity is `assemble_module`'s" + Vec::<&str>::new(), + "no implicit helper recompiles on an edit to a variable none of them \ + reads: `compile_implicit_var_fragment` has its own cache entry per \ + helper, so its granularity is the helper's, not `assemble_module`'s" + ); + + // The complement, so the assertion above cannot pass by the query having + // become unreachable: editing a variable a helper DOES read must still + // recompile it. + // + // Only ONE of the two helpers reads `src`, and which one is a property of + // `builtins_visitor`'s synthesis rather than of this cache: an argument + // that is already a bare `Var` is passed through by name and gets no helper + // at all, so `SMTH1(src, 2)` synthesizes `⁚arg1` for the literal `2` and + // wires `src` straight into the `⁚smth1` module instance. The granularity + // is therefore per HELPER, not per parent variable -- editing `src` leaves + // the constant-capture helper's fragment cached. + let src_edited = project_with("5", "2"); + let (_state4, src_execs) = resync_and_assemble(&mut db, &src_edited, Some(&state3)); + let implicit_after_src: Vec<&str> = src_execs + .iter() + .filter(|(kind, _)| *kind == FragmentExecKind::Implicit) + .map(|(_, name)| name.as_str()) + .collect(); + assert_eq!( + implicit_after_src, + vec!["smoothed#$\u{205A}smoothed\u{205A}0\u{205A}smth1"], + "editing `src` must still recompile the helper that reads it (a query \ + that never re-executed would be a cache bug, not a cache win), and \ + must NOT recompile `\u{205A}arg1`, which captures the literal `2`" ); // The LTM link fragments, on the same shape of edit. diff --git a/src/simlin-engine/src/db/fragment_compile.rs b/src/simlin-engine/src/db/fragment_compile.rs index 194d47515..07905e5b7 100644 --- a/src/simlin-engine/src/db/fragment_compile.rs +++ b/src/simlin-engine/src/db/fragment_compile.rs @@ -514,19 +514,47 @@ fn lower_implicit_var<'db>( Some((implicit_name, lowered)) } -/// Compile a single implicit variable (generated by SMOOTH/DELAY/TREND builtins) -/// to symbolic bytecodes. Not a tracked function -- the parent variable's -/// parse result already provides salsa caching. -pub(crate) fn compile_implicit_var_fragment( - db: &dyn Db, - meta: &ImplicitVarMeta, +/// Compile a single implicit variable (generated by SMOOTH/DELAY/TREND +/// builtins) to symbolic bytecodes. +/// +/// **Salsa-tracked, keyed on the helper's own canonical name.** It used not to +/// be, on the reasoning that "the parent variable's parse result already +/// provides salsa caching" -- which is true of the PARSE and of nothing else. +/// The lowering (`lower_implicit_var` -> `variable::parse_var` -> +/// `lower_variable`) and the per-phase codegen ran on every assembly, so a +/// model's helpers were recompiled from scratch each time `assemble_module` +/// re-ran. On C-LEARN that is 651 calls costing ~12% of a cold compile, and +/// ~28% of the cost of a WARM single-equation edit -- by far the largest share +/// of a recompile that should have touched one variable and its consumers. +/// +/// The name is the only identity a helper has (it exists solely inside its +/// parent's parse), and it is the key `model_implicit_var_info` files it +/// under, so `model_implicit_var_by_name` resolves the metadata inside the +/// query rather than the caller passing a borrowed `&ImplicitVarMeta` that no +/// salsa key could carry. `ImplicitVarMeta::name`'s own rustdoc explains why a +/// name and not a position, and bounds the one case where a name resolves to a +/// different helper than the metadata meant -- a case that already fails to +/// compile. +/// +/// The runlist gate reads `implicit_var_runlist_membership` rather than the +/// whole `ModelDepGraphResult` the caller used to pass in, for the same reason +/// `compile_var_fragment` reads `var_runlist_membership`: a three-bit +/// projection backdates when this helper's membership is unchanged, where the +/// whole result re-executes every helper's fragment whenever any variable's +/// dependencies move. +#[salsa::tracked(returns(ref))] +pub(crate) fn compile_implicit_var_fragment<'db>( + db: &'db dyn Db, model: SourceModel, project: SourceProject, - dep_graph: &ModelDepGraphResult, - module_input_names: &[String], + implicit_var_name: String, + module_inputs: ModuleInputSet<'db>, ) -> Option { use crate::compiler::symbolic::CompiledVarFragment; + let meta = &model_implicit_var_by_name(db, model, project, implicit_var_name.clone())?; + let module_input_names = module_inputs.names(db); + // Recorded at body entry (before the helper is even resolved), keyed by the // parent variable and the helper's own name -- the identity this compiler is // called with. Recording after `lower_implicit_var` would silently omit @@ -544,10 +572,17 @@ pub(crate) fn compile_implicit_var_fragment( // the per-phase compile returns (the helper is absent from this parse / // equation errors). let module_ident_context = - model_module_ident_context(db, model, project, module_input_names.to_vec()); + model_module_ident_context(db, model, project, module_input_names.clone()); let (implicit_name, _lowered) = lower_implicit_var(db, meta, model, project, module_ident_context)?; let var_ident_str = canonicalize(&implicit_name).into_owned(); + let membership = crate::db::dep_graph::implicit_var_runlist_membership( + db, + model, + project, + var_ident_str, + module_inputs, + ); // Runlist-gated phase selection (unchanged output behavior): the // Initial phase is compiled only for implicit vars in @@ -611,22 +646,21 @@ pub(crate) fn compile_implicit_var_fragment( bytecodes }; - let initial_bytecodes = if dep_graph.runlist_initials.contains(&var_ident_str) { + let initial_bytecodes = if membership.initials { phase(true) } else { None }; - let flow_bytecodes = if !meta.is_stock && dep_graph.runlist_flows.contains(&var_ident_str) { + let flow_bytecodes = if !meta.is_stock && membership.flows { + phase(false) + } else { + None + }; + let stock_bytecodes = if (meta.is_stock || meta.is_module) && membership.stocks { phase(false) } else { None }; - let stock_bytecodes = - if (meta.is_stock || meta.is_module) && dep_graph.runlist_stocks.contains(&var_ident_str) { - phase(false) - } else { - None - }; Some(VarFragmentResult { fragment: CompiledVarFragment { diff --git a/src/simlin-engine/src/db/fragment_determinism_tests.rs b/src/simlin-engine/src/db/fragment_determinism_tests.rs index 473398cb6..43c086912 100644 --- a/src/simlin-engine/src/db/fragment_determinism_tests.rs +++ b/src/simlin-engine/src/db/fragment_determinism_tests.rs @@ -27,8 +27,8 @@ use crate::datamodel; use crate::db::{ ModuleInputSet, SimlinDb, assemble_simulation, collect_all_diagnostics, compile_implicit_var_fragment, compile_project_incremental, compile_var_fragment, - model_dependency_graph, model_implicit_var_info, model_module_ident_context, - parse_source_variable_with_module_context, sync_from_datamodel, + model_implicit_var_info, model_module_ident_context, parse_source_variable_with_module_context, + sync_from_datamodel, }; use crate::test_common::TestProject; use crate::testutils::{sim_specs_with_units, x_aux, x_model, x_module, x_project}; @@ -831,7 +831,6 @@ fn check_helpers_resolve_to_their_own_names() { "the fixture's sub-model must be instantiated WITH a bound input, or the \ two parse contexts coincide and this test proves nothing" ); - let dep_graph = model_dependency_graph(&db, sub, project, inputs); let info = model_implicit_var_info(&db, sub, project); assert!( @@ -839,15 +838,15 @@ fn check_helpers_resolve_to_their_own_names() { "the fixture must synthesize more than one helper in `sub`, or a \ mis-resolution has nowhere to land; got {info:?}" ); - for (name, meta) in info.iter() { - let fragment = - compile_implicit_var_fragment(&db, meta, sub, project, dep_graph, inputs.names(&db)) - .unwrap_or_else(|| { - panic!( - "implicit helper `{name}` failed to lower under its own \ + for name in info.keys() { + let fragment = compile_implicit_var_fragment(&db, sub, project, name.clone(), inputs) + .as_ref() + .unwrap_or_else(|| { + panic!( + "implicit helper `{name}` failed to lower under its own \ instance's module-input set (GH #1002)" - ) - }); + ) + }); assert_eq!( &fragment.fragment.ident, name, "the fragment compiled for helper `{name}` is actually \ @@ -1074,14 +1073,13 @@ fn an_implicit_helper_declines_when_the_contexts_synthesize_different_sets() { helper lists, or it does not exercise anything the order fix left open" ); - let dep_graph = model_dependency_graph(&db, sub, project, inputs); let info = model_implicit_var_info(&db, sub, project); assert!( !info.is_empty(), "the fixture must derive some helpers, or the loop below is vacuous" ); let mut declined = 0usize; - for (name, meta) in info.iter() { + for name in info.keys() { // `None` is the correct answer here: this parse holds no helper of that // name. What must never happen is `Some` carrying a DIFFERENT name -- // that is the one thing `find_in`'s name check does guarantee. It does @@ -1089,7 +1087,7 @@ fn an_implicit_helper_declines_when_the_contexts_synthesize_different_sets() { // context-stable, which // `a_cross_context_helper_name_collision_is_confined_to_a_failing_compile` // builds and bounds. - match compile_implicit_var_fragment(&db, meta, sub, project, dep_graph, inputs.names(&db)) { + match compile_implicit_var_fragment(&db, sub, project, name.clone(), inputs) { Some(fragment) => assert_eq!( &fragment.fragment.ident, name, "the fragment compiled for helper `{name}` is filed under `{}`; \ From ada43393bf9ce70a88dffe4713b675ed86710ceb Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:52:39 -0700 Subject: [PATCH 18/59] doc: correct the stated reason for the lazy-If NO-GO (#711) The verdict is UNCHANGED. Only its stated reason is corrected; this is not a reopening of #711. The recorded reason was that the ~1.5% instruction share is "below the ~4% layout-noise measurement floor". That applies a cycles/binary-layout floor to an instruction-count measurement. The instruction channel's sd across independent builds is ~0.026%, so ~1.5% there is roughly 58 sigma -- comfortably measurable. The number was never the problem. Two reasons survive and are sufficient on their own. The return is small against the highest design cost of the three candidates: forward-jump opcodes touching codegen, `max_stack_depth` join validation, the peephole/fusion jump maps, the symbolic layer and wasmgen parity. And the cheap part of the win has since been taken without any of that machinery -- the `SetCond;If[;AssignCurr]` fusion removes 12.0% of executed dispatches against the projected 15.9% -- so what is left here is the residual, not the headline. Worth correcting rather than leaving inert because a wrong measurement premise is not caught by review: reviewers check the reasoning against the stated premise, not the premise against the world. "Below the measurement floor" reads as a fact and ends the enquiry, and this one had already carried a verdict. The same correction is posted on the issue, since that is where someone picks the work up and a doc-only fix would leave the trap in the more-read place. --- docs/design/engine-performance.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/design/engine-performance.md b/docs/design/engine-performance.md index 7a071f868..761846055 100644 --- a/docs/design/engine-performance.md +++ b/docs/design/engine-performance.md @@ -513,9 +513,19 @@ a data verdict in round 3 (2026-06-04): stack-effect branch-span reconstruction over the fused stream) measured C-LEARN at exactly **30,524 executed dispatches/step**, of which lazy-If would skip 4,859 (**15.9%**) — but 93% of the skipped opcodes are cheap - scalar loads/binops, so the *instruction* share is only **~1.5%** (~35k of - ~2.4M instr/step): below the ~4% layout-noise measurement floor, at the - highest complexity of the three candidates. WORLD3: 3.25% dispatch share. + scalar loads/binops, so the share of RETIRED INSTRUCTIONS is only **~1.5%** + (~35k of ~2.4M instr/step). That is measurable (the instruction channel's sd + is ~0.026%; see "Measuring a change"), so the verdict does not rest on it + being unresolvable — it rests on ~1.5% of instructions being a small return + for the highest design cost of the three candidates. WORLD3: 3.25% dispatch + share. + The cheap part of the win has since been taken WITHOUT that machinery: + fusing `SetCond;If[;AssignCurr]` into conditional-select opcodes removes + **12.0% of executed dispatches** against this item's projected 15.9%, resting + on the pair being adjacent by construction (`compiler::codegen`'s `Expr::If` + arm is the sole producer of both and emits them together; executed counts are + exactly equal at 1,874,169 each). What remains here is the residual after + that fusion, against the full forward-jump cost. Notably **69.8% of the skippable dispatches sit behind constant conditions** (1,300 of 1,679 flow `If` sites take the same branch for the whole run) — a compile-time / #712-family observation, not a runtime-jump From 44ff66937835f54851d80ba0a5ae40ab9c1eab77 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 07:58:29 -0700 Subject: [PATCH 19/59] build: cache prettier results in the format check `pnpm js-needs-format` runs on every commit, on pipeline B, which is the pre-commit hook's critical path. It was ~6s of that pipeline (and ~11s on a busier run) purely to re-decide, from scratch, that 379 unchanged files are still formatted. The suspicion that it was scanning generated output does not hold, and it is worth writing down so nobody re-checks it: of the files `find` emits, the existing filter already drops `lib/`, `lib.browser/` and `lib.module/`, and of the 379 that survive, ZERO are under `node_modules`, `build`, `dist` or `coverage` (pnpm's per-package `node_modules` are symlinks, which `find` does not follow). They are 379 real sources, 9 of them hand-maintained `.d.ts`. The cost is prettier itself at roughly 10ms per file. So cache it. Standalone the step goes 3.8s -> 0.9s warm (-76%, three runs). What this does NOT do, measured, is move the hook's wall clock: a same-session A/B of the whole hook is 29-30s with and without the cache, because inside pipeline B the binding costs are `pnpm build` (~15s) and tsc+test (~8s), and the format check was never the constraint. Landed because it is a one-line change that strictly removes work, not because a developer will feel it. `--cache-strategy content` rather than the default `metadata`: it keys on a content hash instead of mtime+size, so a checkout that rewrites mtimes cannot produce a false "clean". Both invalidation axes were checked by mutation rather than assumed -- unformatting a file with a warm cache still reports it, and narrowing `printWidth` to 40 reports 366 files, identical to an uncached run. (The first attempt at that second check appended a duplicate `printWidth` key, which YAML ignored, and briefly looked like a cache bug; the real mutation edits the existing line.) Also replaces `egrep` with `grep -E`, which drops an "egrep is obsolescent" warning that printed on every hook run. --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 547406ed1..f70fbb00c 100644 --- a/package.json +++ b/package.json @@ -19,9 +19,9 @@ "yaml": "^2.4.2" }, "scripts": { - "js-needs-format": "find src -name '*.ts' -o -name '*.tsx' | egrep -v '/(lib(\\.(browser|module))?)/' | xargs prettier -l", + "js-needs-format": "find src -name '*.ts' -o -name '*.tsx' | grep -E -v '/(lib(\\.(browser|module))?)/' | xargs prettier -l --cache --cache-strategy content", "rust-needs-format": "cargo fmt -- --check", - "js-format": "find src -name '*.ts' -o -name '*.tsx' | egrep -v '/(lib(\\.(browser|module))?)/' | xargs prettier --write", + "js-format": "find src -name '*.ts' -o -name '*.tsx' | grep -E -v '/(lib(\\.(browser|module))?)/' | xargs prettier --write --cache --cache-strategy content", "rust-format": "cargo fmt", "format": "cargo fmt && pnpm js-format", "precommit": "pnpm js-needs-format && pnpm rust-needs-format && pnpm lint", From 457fb25615a9f848b959c85b72d1bbc23b90483c Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:03:11 -0700 Subject: [PATCH 20/59] engine: memoize the cycle gate's symbolic fragment probe `var_phase_symbolic_fragment_prod` is the engine's own per-variable lowering plus codegen -- the same work `compile_var_fragment` does, under the no-module-input wiring -- and it was a plain function called once per recurrence-SCC member per phase by the cycle gate's element-order probe. Instrumented on C-LEARN, the probe called it **135 times per cold compile for 57 distinct `(variable, phase)` keys**. The 2.4x duplication is structural, not incidental: `refine_scc_to_element_verdict`'s dt arm verifies the init element graph as a precondition, and `resolve_recurrence_sccs` is then run again for the init phase, so every init-phase member fragment is built twice. The probe is ~16% of a cold C-LEARN compile, and all of it recurred on every recompile of an unchanged model. The body is now a `#[salsa::tracked]` query keyed on `(model, project, var_name, phase)` -- the arguments it already varied over -- behind an unchanged wrapper that clones the memo out, so every call site keeps the ownership it had. `SccPhase` gains `Hash` to serve as a key. The `#[cfg(test)]` `UnsourceableVarsGuard` short-circuit stays OUTSIDE the memo, which is the one thing that could have gone quietly wrong here. Inside the tracked body its verdict would be cached against a key the guard is not part of, so a guard toggled between two calls on one `db` would be ignored by the second -- and in the direction that makes the AC3.2 loud-safe regression test (`unsourceable_in_scc_node_falls_back_to_circular_no_panic`) pass for the wrong reason. Short-circuiting in the wrapper keeps the override exactly as immediate as it was. Measured on C-LEARN, interleaved A/B/A over four rounds at load average 6.9-12.2 (so retired instructions, not cycles: 18.769G -> 16.293G over nine compiles = **-275.1M instructions per compile, -14.3%**). The repeated baseline arm spread 0.21% across all eight of its runs, against an effect ~60x that. Wall clock is not resolvable at this load and is not quoted. The compiled artifact is unchanged: 5215 slots, 58291 opcodes (31525 flow + 1477 stock + 25289 initial), same literal, GF, temp, dimension, view, name and module counts. --- src/simlin-engine/src/db/assemble.rs | 42 +++++++++++++++++++++++++-- src/simlin-engine/src/db/dep_graph.rs | 2 +- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/simlin-engine/src/db/assemble.rs b/src/simlin-engine/src/db/assemble.rs index 8dfda0e95..e96851498 100644 --- a/src/simlin-engine/src/db/assemble.rs +++ b/src/simlin-engine/src/db/assemble.rs @@ -904,8 +904,6 @@ pub(crate) fn var_phase_symbolic_fragment_prod( var_name: &str, phase: SccPhase, ) -> Option { - use crate::db::var_fragment::{LoweredVarFragment, lower_var_fragment}; - // `#[cfg(test)]` only: an active `UnsourceableVarsGuard` forces this // node to take the loud-safe `None` arm, so the AC3.2 regression test // can exercise the genuinely-unsourceable in-SCC path through the @@ -915,11 +913,51 @@ pub(crate) fn var_phase_symbolic_fragment_prod( // trigger). It returns the SAME `None` a real no-`SourceVariable` // node returns, so the test observes the real loud-safe behavior, not // a shim. No effect in non-test builds. + // + // It sits OUTSIDE the memo deliberately. Inside the tracked body its + // verdict would be cached against a key the guard is not part of, so a + // guard toggled between two calls on one `db` would be ignored by the + // second -- silently, and in the direction that makes the AC3.2 test pass + // for the wrong reason. Short-circuiting here keeps the override exactly + // as immediate as it was when this whole function was a plain call. #[cfg(test)] if crate::db::dep_graph::var_is_forced_unsourceable(var_name) { return None; } + var_phase_symbolic_fragment_memo(db, model, project, var_name.to_string(), phase).clone() +} + +/// The memoized body of [`var_phase_symbolic_fragment_prod`]. +/// +/// Salsa-tracked because this is the engine's own per-variable lowering plus +/// codegen -- the same work `compile_var_fragment` does, under the +/// no-module-input wiring -- run once per SCC member per phase by the cycle +/// gate's element-order probe, and it was a plain function. Instrumented on +/// C-LEARN the probe called it **135 times per cold compile for 57 distinct +/// `(variable, phase)` keys**: the dt refinement verifies BOTH phases as a +/// precondition and the init refinement then re-derives the init order, so a +/// 2.4x duplication was structural rather than incidental. It is ~16% of a +/// cold compile, and the whole of it recurs on every recompile of the same +/// unchanged model. +/// +/// The key is `(model, project, var_name, phase)` -- the arguments the body +/// already varied over. `var_name` is a `String` rather than a `&str` because +/// a salsa key must be owned; the wrapper above does that one allocation on +/// the caller's behalf and clones the memo out, which is what keeps every +/// existing call site's ownership unchanged. Both are trivial next to the +/// lowering they replace. +#[salsa::tracked(returns(ref))] +fn var_phase_symbolic_fragment_memo( + db: &dyn Db, + model: SourceModel, + project: SourceProject, + var_name: String, + phase: SccPhase, +) -> Option { + use crate::db::var_fragment::{LoweredVarFragment, lower_var_fragment}; + + let var_name = var_name.as_str(); let source_vars = model.variables(db); // No `SourceVariable` (a synthetic INIT/PREVIOUS/SMOOTH/macro-expansion // helper, `$\u{205A}` prefix, absent from `model.variables`): before diff --git a/src/simlin-engine/src/db/dep_graph.rs b/src/simlin-engine/src/db/dep_graph.rs index a46b2571e..6e4a76677 100644 --- a/src/simlin-engine/src/db/dep_graph.rs +++ b/src/simlin-engine/src/db/dep_graph.rs @@ -1628,7 +1628,7 @@ mod dep_graph_tests; /// /// Derives the same trait set as `ModelDepGraphResult` (it is reachable /// from a salsa return value, so it must participate in salsa equality). -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum SccPhase { Dt, Initial, From 0fa26211182898d79156511e8c13e0cdf545dfe6 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:04:40 -0700 Subject: [PATCH 21/59] engine: omit LTM arms provably equal to PREVIOUS(target) For a per-element (`Ast::Arrayed`) link-score target, one arm is shaped per target element whether or not that element's equation reads the link's source. An arm with no live source reference has every occurrence frozen by the ceteris-paribus wrap, so it recomputes the value that produced `PREVIOUS(target)` and its guarded numerator is identically zero -- yet it was still printed, parsed, lowered and executed every timestep. On C-LEARN those arms are 7.63 MB of 9.63 MB of generated arm text (GH #977). `partial_is_provably_previous_target` decides when such an arm may be dropped, and `build_arrayed_link_score_equation` drops it by leaving the slot ABSENT from the element map; `compiler::expand_arrayed_with_hoisting` already lowers an absent slot to a single `AssignCurr(off, Const(0.0))`. Absence is deliberately a different channel from an arm whose generated text is empty, so a generator bug stays distinguishable from an intended zero slot. Omission is gated on `apply_default_to_missing == false`: under EXCEPT semantics a missing slot takes the DEFAULT equation rather than zero. The predicate is POSITIVE -- outside every `PREVIOUS`/`INIT` subtree the arm holds only literals, operators, keywords and pure builtins. The obvious negative criterion, "the wrap froze every occurrence of the link's source" (`WrapOutcome::live_ref == None`), is UNSOUND, and that is the most important thing here for a reviewer, because the cheap version looks obviously correct: it says nothing about what else the arm reads. #977 measured it as changing 187 result slots across 35 link-score variables on C-LEARN, 151 of them by >= 1.0, worst case 8,086.97 -> 0. The wrap does not freeze everything that varies -- a live `time()` survives it, and a raw-vs-canonical element-spelling mismatch can leave the source itself unwrapped -- so the negative criterion is also unstable under fixes to either. The positive test asks about the emitted tree instead of the wrap's bookkeeping, so it stays correct whether or not those are fixed. The pure-builtin allowlist is deliberately broader than the minimum this corpus needs, and what admits a builtin is a PROPERTY rather than membership in a list: a builtin may join iff it is deterministic in its arguments and reads neither the clock, nor state, nor a table. Extending it is then a rule to apply rather than a taste call. `lookup` stays out even though a graphical function is a compile-time constant, because #977 measured that relaxation as buying exactly zero additional arms. The structure follows #977's standing constraints: every match arm returns a named `Reach` verdict, the builtin-argument walk is a verdict-returning fold so `=> {}` is a type error, and the matches carry no catch-all so a new `Expr0` variant is a compile error rather than a silent `Established`. The predicate lives in a `#[path]`-mounted sibling, `ltm_augment_zero_slot.rs`, following the six siblings already split off `ltm_augment.rs` for the per-file line cap -- adding it inline put that file at 6,059 lines against a 6,000-line threshold. Splitting it out also repaired a doc-comment detachment the inline version introduced: `shaped_guard_form_text`'s rustdoc, which describes its three freeze conventions and its `gf_table_ref` parameter, had been left documenting the interposed enum instead of the function. Measured on C-LEARN v77 with LTM in discovery mode: flow opcodes 1,208,106 -> 976,581 (-19.2%) total opcodes 1,238,728 -> 1,007,203 (-18.7%) The control that shows the predicate does not fire where it must not: WORLD3-03 with LTM is UNCHANGED -- 23,165 flow opcodes both ways, with the full opcode histogram, all 23 fused-binop counts and the 1,202-slot result geometry identical. `clearn_ltm_var_count_guardrail` stays green, so the C-LEARN variable count and slot width are unmoved as well. Value-neutrality was measured over the WHOLE result slab rather than over base slots, since comparing only `offsets`' base slots is blind to exactly the per-element arms at issue: 251 steps x 30,123 slots = 7,560,873 slot-steps, of which 51,358 differ (0.68%) in exactly one bit-pattern pair, `-0.0 -> +0.0`. Numerically zero differ. That probe is not checked in, and a value-level LTM gate at any scale is still missing -- see the note on goldens below. The one bit-pattern pair has a disclosed cosmetic surface. Rust's `Display` preserves the sign of zero (`format!("{}", -0.0f64)` is `-0`, verified), and `simlin-cli` prints result values with `{}` at src/simlin-cli/src/main.rs:337, :398 and :407 -- so a column that printed `-0` now prints `0`. Nothing in production branches on the sign of zero: `vm.rs:4160` pins `eval_op2(Eq, 0.0, -0.0) == 1.0`, `vm.rs:4129` pins `!is_truthy(-0.0)`, and `float.rs:201` pins `approx_eq(0.0, -0.0)`. Peak live bytes during `compile_project_incremental`, via the counting allocator in `examples/clearn_profile` -- stable to +/-0.2 MiB across runs, where peak RSS on this box varies by 10% and cannot carry a percentage: 5a82634a (merge base) 438.8 MiB + 4c68ef33 + 9a9934ec 440.7 MiB (+1.9 MiB: the memoization and the shared equation ASTs cost essentially nothing in memory) + this commit 353.2 MiB (-19.9% here, -19.5% cumulative) Allocation count across the same three points: 50.80M -> 49.55M -> 41.37M. Four pre-existing tests asserted on arms that are now correctly absent. Each was repointed at the property it guards rather than at materialization: * `test_arrayed_link_score_stock_to_flow_per_element_partials` (ltm-503-cross-element-agg.AC1.3) guards a generator giving up and emitting a literal "0". Its assertion could no longer tell that from a deliberate omission, so it now derives the present-slot set, asserts it is exactly the live-source arms, and requires every present arm to be non-empty and free of the `((0) - ` give-up form. * `a_colliding_index_name_is_resolved_against_the_axis_it_indexes` and `an_index_naming_the_axis_own_element_stays_a_static_selector` (GH #986). Repointing these onto slot absence would have silently gutted them: under the #986 defect the index is rewritten to a static element selector, which the wrap freezes as an other-dep, so the arm is provably `PREVIOUS(target)` and is omitted just the same. The control test demonstrates that directly -- its correctly-qualified `q[slot.s1]` arm was omitted too. After the change no emitted arm on that fixture carried the index at all. Both boston equations therefore gain `+ 0 * pop[nyc]`, the idiom the neighbouring `0 * ctr` already uses: it gives the arm a live reference to the link's source, which is exactly the materialization condition, at zero numeric weight. The documented double-lag residual series reproduces bit-identically, so every original assertion -- `PREVIOUS(ctr, ctr)`, the `bucket.ctr` negative, and the two-variant series equality -- stands unchanged. * `test_disjoint_dim_arrayed_target_per_source_element_link_scores` described its `[a,y]` slot as "the trivial-zero guard form", which is the omitted class exactly. Its runtime claim is untouched, so the structural half now asserts absence and the VM assertion is TIGHTENED from `abs < 1e-6` to exactly zero. `[b,y]` on the same variable must stay materialized, so "every slot but `[a,x]` vanished" cannot pass. The characterization goldens do not move, and that is a coverage hole rather than good news: of the 16 fixtures exactly one carries an `Ast::Arrayed` link score and it is an EXCEPT-default target, hence pinned to `Materialize`, so the omission had no characterization coverage at all. A new `char_arrayed_target_no_default_slot_scores` fixture covers it at `apply_default=false`. That is structural; the value-level gate #977 asks for does not exist yet. --- src/simlin-engine/CLAUDE.md | 2 +- .../arrayed_target_no_default_slot_scores.txt | 8 + src/simlin-engine/src/db/ltm_char_tests.rs | 49 ++++++ src/simlin-engine/src/db/ltm_tests.rs | 49 +++++- src/simlin-engine/src/ltm_augment.rs | 121 ++++++++++--- src/simlin-engine/src/ltm_augment_tests.rs | 85 +++++++-- .../src/ltm_augment_zero_slot.rs | 162 ++++++++++++++++++ .../tests/integration/simulate_ltm.rs | 53 ++++-- 8 files changed, 471 insertions(+), 58 deletions(-) create mode 100644 src/simlin-engine/src/db/ltm_char_golden/arrayed_target_no_default_slot_scores.txt create mode 100644 src/simlin-engine/src/ltm_augment_zero_slot.rs diff --git a/src/simlin-engine/CLAUDE.md b/src/simlin-engine/CLAUDE.md index ac67bbba6..bc3d90eff 100644 --- a/src/simlin-engine/CLAUDE.md +++ b/src/simlin-engine/CLAUDE.md @@ -184,7 +184,7 @@ Unit checking is **opt-in by declaring units**: a model that declares units on N - `tests.rs` - integration-style tests spanning all submodules; preserved as a single file to avoid splitting shared test fixtures. - **`src/ltm_finding.rs`** - Strongest-path loop discovery algorithm (Eberlein & Schoenberg 2020). Post-processes simulation results containing link score synthetic variables to find the most important loops at each timestep. `discover_loops_with_graph(results, causal_graph, stocks, ltm_vars, dims, expansion, sub_model_output_ports, budget)` is the primary entry point: `ltm_vars` and `dims` enable A2A link score expansion (per-element edges from `parse_link_offsets`); when empty, all link scores are treated as scalar. A Bare A2A link score is dimensioned over the TARGET's dims, so `expand_a2a_link_offsets` subscripts the TARGET node per element but PROJECTS the SOURCE node onto `from`'s OWN declared dims -- bare for a scalar feeder (`scale → growth`, GH #790), the same-element diagonal for an equal-dim feeder, the broadcast/partial-collapse form for a lower-dim feeder, and the positionally-mapped diagonal for a `State→Region` pair (GH #527) -- by reusing the element graph's own `db::expand_same_element` rule, so the discovery search graph's node names match `model_element_causal_edges` node-for-node (GH #754; before this, subscripting BOTH endpoints with the score's full tuple minted phantom from-nodes like `scale[a]`/`boost[r,a]`/`x[s]` that named no real node, so every loop through such a feeder dangled and was silently undiscoverable). The from-var declared dims + dimension-mapping context ride in a `LinkExpansionContext` (`declared_dims` + `dim_ctx`) that `analyze_model` builds via the public `analysis::build_link_expansion_context` (the SAME `variable_dimensions` / `project_dimensions_context` queries the element graph reads); the db-less `discover_loops(&Results, &Project)` convenience path passes `LinkExpansionContext::default()` (no A2A expansion runs there). Element-mapped (non-positional) pairs never reach the projection: `db::ltm::link_score_dimensions` declines to retarget them (no Bare A2A score; the GH #758 loud skip fires instead) under the GH #756 positional-only gate. `SearchGraph` provides DFS-based strongest-path traversal from stock nodes. The post-simulation score recompute (path → `FoundLoop`) applies the **per-exit-port pathway selection** (`recompute_module_input_edge_series`, GH #698): a loop edge `x → m` into a module is recomputed by max-abs-selecting over the sub-model's `m·$⁚ltm⁚path⁚{entry}⁚{idx}` pathway scores that end at the exit port the loop reads (recovered from the next link `m → y`), instead of reading the module *composite* (which max-abs-selects across ALL ports and so can pick a wrong-signed port for a multi-output module -- flipping the loop polarity vs exhaustive). The pathway indices come from the module's sub-graph via the same `enumerate_pathways_to_outputs_with_truncation` over the same sorted output-port set the sub-model emitted against, so they match index-for-index (the discovery-mode equivalent of exhaustive's `compute_module_link_overrides`). That port set is NOT re-derived parent-scoped (which would shift indices when another project model reads an extra output port -- GH #698 / PR #705 r3353097150): `discover_loops_with_graph` takes a `SubModelOutputPorts` map (sub-model canonical name -> emitted sorted port set) that `analyze_model` builds from the SAME emission decision via `analysis::build_sub_model_output_ports` -> `db::ltm::sub_model_output_ports` (identity-by-construction); the db-less `discover_loops(&Results, &Project)` convenience path reconstructs the same project-wide-union + stdlib-`output` semantics in `project_sub_model_output_ports`. Falls back to the base composite offset for single-output / pathless / indeterminate-port / sub-model-absent-from-map edges. Because the discovery graph is element-level, the recompute `strip_subscript`s `link.from`/`link.to`/`next.from`/`next.to` before its name matches (arrayed loop nodes carry `[elem]`; mirrors the exhaustive twin's stripping -- PR #705 r3353758167); that stripping is LIVE rather than latent since GH #716 closed: a scalar module output feeding an arrayed reader used to emit ONE scalar constant-0 link score, which dropped the loop, and is now scored per target element by `db::ltm::link_scores::try_implicit_scalar_to_arrayed_link_scores` (which also owns the per-element module INSTANCES a per-element expansion mints, whose partials were previously `scalarize`d onto element 0's arm), so an arrayed loop through a multi-output module is discoverable end-to-end -- `analysis::tests::analyze_model_arrayed_module_loop_is_discovered_per_element`. Returns a `DiscoveryResult` whose `FoundLoop`s carry per-timestep link/loop/pathway scores, ranked **competitive-first** (`rank_and_filter`): loops sharing their cycle partition with at least one other discovered loop come first, ordered by mean partition-relative importance; loops trivially ALONE in their partition (relative score exactly +/-1 by construction -- zero information, e.g. C-LEARN's isolated gas-uptake decay loops) sort after all competing loops and are dropped first under the `MAX_LOOPS` cap. Each `FoundLoop` carries a result-scoped dense `partition` index into `DiscoveryResult::partitions` (`DiscoveredPartition { stocks, loop_count }`, first-appearance order; NOT stable across runs/edits -- key on the stock set for durable identity), threaded through `analysis::ModelAnalysis::partitions` / `LoopSummary::partition`, the FFI `SimlinDiscoveredPartition` / `SimlinDiscoveredLoop.partition`, and pysimlin `Analysis.partitions` / `Loop.partition`. Also hosts the link-set synthetic-node collapse: `trim_synthetic_aggs_from_loop_links` collapses `$⁚ltm⁚agg⁚{n}` nodes out of a single loop's link *cycle*, and the public `collapse_synthetic_links(Vec)` generalizes that to ALL synthetic/macro/module-internal nodes (`ltm::is_synthetic_node_name`) over an arbitrary link *set* -- each chain `X -> $⁚…internal… -> Y` collapses to one composite edge `X -> Y` with product polarity and the per-timestep max-magnitude path score (the composite link score, LTM ref 6.3/6.4); a purely-internal cycle/source is dropped. `CollapsibleLink { from, to, polarity, score: Option> }` is the abstract shape, so structural-only callers (`score = None`) and LTM-run callers share one impl. libsimlin's `simlin_analyze_get_links(.., include_internal=false)` collapses the `get_links()` set through this; `include_internal=true` returns the raw graph. The `#[cfg(test)]` tests live in the sibling `src/ltm_finding_tests.rs` (mounted via `#[cfg(test)] #[path]`, split out for the per-file line cap). - **`src/ltm_agg.rs`** - Aggregate-node enumeration for LTM. `enumerate_agg_nodes` (salsa-tracked) walks every variable's `Expr2` AST left-to-right depth-first and identifies each maximal array-reducer subexpression (`SUM`/`MEAN`/`MIN`/`MAX`/`STDDEV`/`RANK`/`SIZE`); AST-identical subexpressions (keyed by canonical printed equation text) dedupe to one `AggNode`. Two kinds: **synthetic** (`is_synthetic == true`, the reducer is a sub-expression of a larger equation -- a `$⁚ltm⁚agg⁚{n}` aux is minted) and **variable-backed** (`is_synthetic == false`, the reducer is the entire dt-equation of a scalar/A2A variable like `total_population = SUM(pop[*])` -- the variable itself is the agg; EXCEPT a whole-RHS reducer whose shape the variable-backed machinery cannot express -- a MAPPED iterated axis (GH #534: the name-based link-score path cannot remap, so its `Wildcard` partial would silently stub to 0) or NON-ALIGNED result dims (GH #764 / shape-expressiveness T4: a broadcast over extra owner dims `out[D1,D3] = SUM(matrix[D1,*])` or permuted axes `out[D2,D1] = SUM(cube[D1,D2,*])`, where a per-`(row, slot)` slot cannot name a complete owner element) -- which mints a synthetic agg instead (`variable_backed_shape_is_expressible`, the ONE minting condition), riding the two-half emitters + the GH #528 projection). Each `AggNode` carries a `sources: Vec` -- one entry per source variable, SORTED by canonical name and deduped (salsa cache-equality + emission-order determinism; T2 of the shape-expressiveness design), each with its OWN `read_slice: Vec` (one `AxisRead ∈ {Pinned(elem), Iterated{dim, source_dim}, Reduced{subset}}` per THAT source's axes -- which rows of it the reducer actually reads; a scalar feeder like `scale` in `SUM(pop[*] * scale)` carries an empty slice; `Iterated` carries the (target, source) canonical dim pair, equal for the literal case; `Reduced.subset` is `None` for the full extent or the proper-subdimension element subset for a `SUM(arr[*:Sub])` StarRange, GH #766, decided per axis by `classify_axis_access` -- the single per-axis classifier of the shape-expressiveness design) -- and a `result_dims` (the `Iterated` axes' TARGET dims, datamodel-cased -- empty for a whole-extent or pinned-slice reduce). Under the I1 acceptance (`accept_source_slices`, T5 of the shape-expressiveness design / GH #767) every arrayed CO-SOURCE (`Reduced`-bearing slice) carries the identical *canonical* slice (`AggNode::canonical_read_slice` -- the first `Reduced`-bearing source slice, falling back to the first non-empty for the degenerate no-co-source agg), while an ITERATED-DIM PROJECTION FEEDER -- a source whose slice is all-`Iterated` over exactly the canonical slice's iterated target dims, in order, unmapped (`frac[D1]` in `SUM(matrix[D1,*] * frac[D1])`, per-result-slot constant) -- is accepted with ITS OWN slice (`AggNode::source_is_projection_feeder` is the discriminator); per-source consumers (`emit_agg_routed_edges`, `emit_source_to_agg_link_scores`) read `AggNode::source_read_slice(from)`, and name-keyed consumers use `AggNode::reads_var`. A feeder's link-score half is the per-`(row, slot)` CHANGED-LAST equation (`ltm_augment::generate_iterated_feeder_to_agg_equation`, emitted via `link_scores::iterated_feeder_row_scores` from both the synthetic source-half and `try_cross_dimensional_link_scores`' variable-backed feeder branch): the reducer text pinned to the slot with only the feeder frozen -- the arrayed generalization of the GH #737 scalar-feeder convention, exactly complementary per slot to the co-source rows' changed-first numerators for a bilinear body; the co-source rows' changed-first body partial pins the mismatched-arity feeder dep BY DIM NAME (`pin_body_to_row`'s GH #767 extension) so `PREVIOUS(frac[d1·r])` is held frozen at the row instead of bailing to the delta-ratio fallback. `compute_read_slice` decides hoistability: a whole-extent reduce (`SUM(pop[*])` ⇒ all-`Reduced`), a sliced one (`SUM(pop[NYC,*])` ⇒ `[Pinned(nyc), Reduced]`, `SUM(matrix[D1,*])` over an A2A-`D1` body ⇒ `[Iterated(d1,d1), Reduced]` → an arrayed agg over `D1`), a mixed one (`SUM(matrix3d[D1,NYC,*])` over an A2A-`D1` body ⇒ `[Iterated(d1,d1), Pinned(nyc), Reduced]`), or a positionally-MAPPED sliced one (GH #534: `SUM(matrix[State,*])` over `matrix[Region,D2]` with a positional `State→Region` mapping ⇒ `[Iterated{state, region}, Reduced]`, `result_dims = [State]` -- the agg is arrayed over the TARGET's iterated dim and the three `Iterated`-axis consumers remap each source row to the slot of its positionally-corresponding target element via `iterated_axis_slot_elements`, the preimage inversion of `mapped_element_correspondence`, so the positional-only/element-map gate is inherited) is hoisted; the carve-outs are (a) a reducer over a *dynamic index* (`SUM(pop[idx,*])`, `idx` non-literal ⇒ not statically describable ⇒ not hoisted, reference stays on the conservative path -- `db::ltm_ir` reclassifies it as `DynamicIndex`), (b) an ELEMENT-mapped sliced reducer the correspondence declines (execution resolves positionally and ignores the map, GH #756; a POSITIONAL mapping is accepted in either declaration direction since GH #757) ⇒ `compute_read_slice` returns `None`, conservative -- (c) a multi-source slice combination outside the I1 acceptance -- co-sources with differing slices, one variable read with two different slices (I3b), or a no-`Reduced` source that is not the pure iterated projection (a Pinned-axis mix like `SUM(matrix[D1,*] * w[D1, c1])`, a dim-subset/permuted feeder, or any mapped Iterated axis in a feeder combination) -- `combined_read_slice`/`accept_source_slices` return `None`, (d) a StarRange naming a NON-subdimension of its axis (a mid-edit inconsistency; declined rather than silently widened to the full extent), and (e) `RANK` (GH #771: array-valued, so `reducer_is_hoistable` requires `reducer_collapses_to_scalar` and RANK references stay `Direct` -- a bare arg classifies `Bare`, scored by the GH #742 arrayed-capture path; loops through the rank ORDERING are a documented residual). A multi-source reducer whose arrayed args *agree* (`SUM(a[*] + b[*])`, `a`, `b` over the same dim) mints one agg with one `AggSource` per variable, each carrying the shared canonical slice. `AggNodesResult` exposes `aggs` (first-encounter order), `agg_for_key` (by canonical reducer text), and `aggs_in_var` (which aggs occur in a variable's equation, so the element-graph reroute can ask "which agg of `to` reads `from`?"). A variable-backed reduce with a NON-TRIVIAL statically-describable slice is gated by `variable_backed_reduce_agg` (GH #752, generalized by GH #765 / T3 of the shape-expressiveness design) -- the SAME gate `model_element_causal_edges`' dispatch, `build_element_level_loops`' per-circuit routing, and `try_cross_dimensional_link_scores`' row derivation consume, so edges, loop routing, and scores always cover the identical read rows (all three derive from `read_slice_rows`, invariant I4). Accepted: an ALIGNED partial reduce (`row_sum[D1] = SUM(matrix[D1,*])`, `result_dims` equal to the variable's own dims in order -- Pinned-mixed `outf[D1] = MEAN(cube[D1,x,*])` and subset `out[D1] = MEAN(matrix[D1,*:Sub])` slices included: the divisor is the true read count and unread rows get neither edges nor scores) gets read-slice element edges straight onto the variable's element nodes and per-circuit scalar loops whose both-subscripted `matrix[d1,d2]→row_sum[d1]` links resolve the per-`(row, slot)` scores; a scalar-result Pinned/subset slice on a SCALAR owner (`total = SUM(pop[nyc,*])`, `total = SUM(arr[*:Sub])`) routes the read rows into the bare `to` node with matching per-read-row scores; and the ARRAYED-owner scalar-result Pinned/subset BROADCAST slice (`share[Region] = SUM(pop[nyc,*])`, no `Iterated` axis -- GH #777) fans each read row across the FULL target element set (`emit_agg_routed_edges`' broadcast arm emits `pop[nyc,d2] → share[e]` for every `e`; `emit_broadcast_reduce_link_scores` emits the matching per-(read-row, full-target-element) scalar scores `pop[nyc,d2]→share[e]`, the section-3 `PerElement` rule applied to a variable-backed reducer owner), with loop circuits routed to the per-circuit scalar path via `is_broadcast_reduce_edge` -- the read rows are independent of `to`'s dims, so the RELATED-dim (`share[Region]`) and DISJOINT-dim (`share[D9]`) spellings emit identically. Declined: a pure full-extent variable-backed agg keeps the normal reference walker's edges (the true reads for that shape, inert skip). The GH #764 broadcast/permuted result shapes never reach this gate since T4 -- they mint synthetic aggs at enumeration -- so the gate's Iterated-arm alignment check is defense-in-depth. `scalar_feeder_of_variable_backed_agg` (GH #790) is the scalar sibling of `source_is_projection_feeder`: it composes `variable_backed_reduce_agg` with an empty-slice + genuine-`Reduced`-canonical check to recognize a SCALAR FEEDER of a whole-RHS variable-backed reduce (`scale` in `growth[D1] = SUM(matrix[D1,*] * scale)`), so `try_scalar_to_arrayed_link_scores` can route it to the single Bare A2A changed-last feeder score instead of the uncompilable per-target-element partials. Two predicates here answer what LOOKS like one question -- "is this reference inside a reducer?" -- and their answers are inverted on exactly `SIZE` and `RANK`; the inversion is the DEFINITION of the difference, not a disagreement (GH #982, assessed and left as two predicates). `reducer_collapses_to_scalar` is about the reducer's RESULT TYPE (does the subtree fit in a scalar slot?) and is read by the two freeze/capture gates plus the GH #779 bare-reducer-feeder decline: `SIZE` is a count so it collapses, `RANK` is array-valued so it does not. `builtin_routes_through_agg` is about LTM ROUTING (did `enumerate_agg_nodes` mint a node for this call?) and sets `db::ltm_ir::OccurrenceSite::in_reducer`: `SIZE` is `Constant` and is never hoisted, `RANK` gets an array-valued agg. Both read the ONE `reducer_kind_from_name` table, and `builtin_routes_through_agg` is the disjunction of the enumerator's own two minting branches (`reducer_is_hoistable` and `array_valued_rank_arg`) rather than a restatement of them. The `#[cfg(test)]` `REDUCER_DECISION_TABLE` pins all three derived predicates row by row over every arm of the kind table -- including the agreement gate's name-keyed twin of the routing predicate -- so no cell can move silently. `is_synthetic_agg_name` / `synthetic_agg_name` are the `$⁚ltm⁚agg⁚{n}` name helpers. `classify_axis_access` resolves a bare-identifier index through the shared `dimensions::resolve_axis_index_name` (element-first, GH #986), so it and `ltm_augment_post_transform::pin_dimension_name_indices` cannot disagree about which row a colliding name selects. The `#[cfg(test)]` tests live in the sibling `src/ltm_agg_tests.rs` (split out for the per-file line cap). Each node also carries `reducer: BuiltinFn` -- the reducer call the enumerator classified when it decided the hoist, of which `equation_text` is the printed rendering (GH #983). It is what makes the link-score and polarity emitters parse-free: `ltm_augment::classify_reducer_in_builtin` reads the kind/name/body off it and `ltm::CausalGraph::source_to_agg_polarity` analyses it directly, where both used to print `equation_text`, re-parse it, re-lower it against a freshly built scope and re-derive the classification -- per (agg, source) pair, with both fallible steps returning early and silently zeroing the agg's loop score. It is stored in `Expr2::strip_loc_and_bounds` form, which removes two of the three ways this field could make the salsa-cached `AggNodesResult` compare unequal to an identical rebuild: `Loc` (load-bearing -- two AST-identical occurrences differ in byte offsets, so raw storage would make the dedup winner observable and would stop `enumerate_agg_nodes` backdating across an offset-only edit) and `ArrayBounds` (a guard -- inert today, since `reconstruct_model_variables` lowers against an empty model scope and allocates no bound). Neither reader looks at either. It cannot remove the third -- dropping a `nan` literal would change what the equation means -- so that one is closed at the ROOT instead: `Expr2::Const` holds an `ast::Literal`, compared by BIT PATTERN, so a model whose hoisted reducer contains a `nan` literal backdates like any other (GH #987/#981; with a bare `f64` it never could, since `NaN != NaN`). Pinned by `a_nan_literal_in_a_reducer_does_not_defeat_agg_backdating`. The stored builtin is read only for SYNTHETIC aggs (both readers filter to those); the variable-backed arm's copy is unread today, kept so `AggNode` has one shape. `AggNode`/`AggNodesResult` derive `Eq` (reflexivity is a compile-checked property now that the literal is not a bare `f64`) and `Debug` only under `debug-derive`. -- **`src/ltm_augment.rs`** - Equation generators for LTM synthetic variables: `generate_link_score_equation_for_link` (ceteris-paribus link scores; takes `RefShape` and source dimension elements to drive per-shape PREVIOUS wrapping), `generate_loop_score_variables` (emits one `loop_score` per loop as a dimension-shaped `datamodel::Equation`: `Scalar` for scalar loops, `ApplyToAll` for dimensioned loops whose links resolve through Bare A2A names, and per-slot `Equation::Arrayed` for dimensioned loops backed by per-element circuits via `Loop.slot_links` -- GH #653; relative loop scores are computed post-simulation in `ltm_post.rs`), `build_partial_equation_shaped` (the `#[cfg(test)]` TEXT entry point for the ceteris-paribus wrap; arrayed-per-element-equation (`Ast::Arrayed`) targets get one partial per element assembled into an `Equation::Arrayed`). **Production never parses a target equation**: `wrap_changed_first_ast` takes an `Expr0` lowered straight from the target's `Expr2` by `patch::expr2_to_expr0` (which is what `expr2_to_string` prints, so the former print->reparse was a parse of our own output), and every per-occurrence decision -- access shape, the GH #526 other-dep verdict, the literal-element index guard, and the `PerElement` row pinning -- is a lookup into the `db::ltm_ir` occurrence IR by the structural child-index path the wrap tracks, which equals the occurrence's `SiteId` BY CONSTRUCTION now that both walk the same tree. Every `PREVIOUS` the wrap SYNTHESIZES goes through `freeze_at_previous`, which chooses the call's first-DT initial value from the position being frozen: a VALUE position keeps the unary spelling (desugared to `0`, the XMILE-documented default, and unobservable behind the guard form's `TIME = INITIAL_TIME` arm), while a SUBSCRIPT INDEX names its own un-lagged operand -- `PREVIOUS(idx, idx)` -- because `0` is out of range for every 1-based dimension, so the frozen read yielded NaN at t=0 and `make_temp_arg`'s capture helper served that NaN as the score's FIRST LIVE step (GH #975). Only the two walkers that descend into indices need it (`wrap_non_matching_in_previous` via `wrap_index_non_matching_in_previous`, and `wrap_matching_in_previous`); `wrap_live_shaped_in_previous` and `freeze_pinned_body` document that they never do. A subscripted source reference the IR did NOT record -- reachable under a `LOOKUP` TABLE argument, which the walker skips as static data ("not a causal edge", which is right about ATTRIBUTION) -- still has to COMPILE, so the pin-only descent discharges it by NAME (`pin_dimension_name_indices`: an index naming one of the TARGET's iterated dimensions becomes the source element this target element reads on that axis, the same structural substitution `pin_bare_source_ref` performs for a bare `Var`). That is a lowering-completeness rule, not a second classifier: it consults no occurrence, infers no shape, and never makes the reference live-selectable -- it asks the SHARED row derivation `per_element_row_for_target` (hence `DimensionsContext::mapped_element_correspondence`) which element an axis reads, so the identity axis and a positionally-MAPPED one (`effect[State, old]` over an `effect[Region, Age]` source, either declaration direction) are one arm and it accepts exactly the mapped pairs `ltm_agg::classify_axis_access` accepts. An index the source's axis DECLARES as an element is resolved BEFORE that dimension-name reading, and that precedence is not the pin's own: it is the shared `dimensions::resolve_axis_index_name`, which `ltm_agg::classify_axis_access` reads too (GH #986 closed the divergence -- the classifier had the opposite order, so a mapped collision described the axis as `Iterated` over a dimension the compiler never iterates there). Element-first is what `compiler::subscript`'s `normalize_subscripts3` does ("First check if it's a named dimension element (takes priority)"), and the simulation is the authority: the two readings collide when a dimension declares an element whose name is also a dimension name (`Bucket = [old, region]` beside a `Region` dimension), and a describer that breaks the tie the other way names rows the simulation never reads (`a_colliding_index_name_reads_the_axis_element_in_the_simulation` is the numeric oracle; the XMILE spec's footnote [9] settles the adjacent VARIABLE-vs-element pair outright and sections 2.1/3.7.1 argue this pair from the namespace rule -- `resolve_axis_index_name`'s rustdoc says which is which). Everything that is NOT a bare identifier is left verbatim, needing no pin: a numeric literal, arithmetic over literals, an `@N` position (which `compiler::context` resolves to a concrete element offset in scalar context -- spelling it out here would be a second implementation of position syntax), and a compound expression selecting the element at RUNTIME. That last one used to be a conditional REFUSAL, and deleting it is GH #984: the wrap now freezes a `LOOKUP` table argument's index reads itself (`freeze_lookup_table_indices`), so a runtime index arrives here already lagged and the rule keeps it. That freeze WIDENS its own descent's dep set with the index idents, and without that it would not fire at all -- `variable::classify_dependencies`' `BuiltinContents::LookupTable` arm records the table's ident and never walks the table expression, so an index variable referenced only there is not a dependency and the wrap's `other_deps` freeze could never reach it. The widened set is scoped to that argument's indices, and the element / dimension-name guards run before the dep check, so it cannot make a selector wrap. (Leaving the dep set itself alone is deliberate: an index dropped from a variable's dependencies is a runlist-ordering question, not an LTM one.) What the SHARED derivation declines -- an unmapped or element-mapped pair (GH #756), a transposition, a dimension the target does not iterate -- is declined LOUDLY (`WrapOutcome::missing_occurrence` -> warned skip) rather than emitted with its dimension-name subscript intact, which would not resolve in a scalar fragment. The verdict space (`Pinned`, `Keep`, and the one loud `Unspellable`) is ENUMERATED cell by cell in `ltm_augment_pin_tests.rs`'s three verdict enumerations rather than sampled. There is ONE access-shape classifier family, on `Expr2`; the Expr0 mirror is test-support only and lives in `ltm_augment_wrap_test_support.rs` (kept because three wrap unit tests -- unparseable text, empty text, and the Fig. 2 Q4 `SUM(w[from]) + from` shape the engine REJECTS as a model -- cannot be db-backed fixtures), with `ltm_classifier_agreement_tests.rs` proving it matches the IR field for field (`SiteId` path, `shape`, `axes`, `in_reducer`) corpus-wide, `link_score_var_name` (synthetic name helper: Bare gets the canonical `{from}\u{2192}{to}` form, FixedIndex prepends `[elem]` to from; the obsolete per-shape `\u{205A}wildcard`/`\u{205A}dynamic` Wildcard/DynamicIndex suffixes were retired -- those shapes now collapse onto the Bare name, since *every statically-describable* inlined reducer (whole-extent or sliced) is hoisted into a `$⁚ltm⁚agg⁚{n}` node and only a `DynamicIndex` reference -- `arr[i+1]`, a range, or the not-hoistable dynamic-index reducer carve-out `SUM(pop[idx,*])` -- a whole-RHS variable-backed reducer's `Wildcard` argument, or a de-hoisted array-valued reducer's `Wildcard` arg (`RANK(pop[*], 1)`, GH #771) reach this function), `quote_ident` (identifier quoting for equations). Array support: `classify_reducer` (walks target Expr2 AST to identify reducing builtins -- Linear for SUM/MEAN, Nonlinear for MIN/MAX/STDDEV/RANK, Constant for SIZE -- a thin reader of `ltm_agg::reducer_kind`; it also hands back the reducer's array-argument AST as `ClassifiedReducer::body`, lowered from `Expr2` rather than printed, so the body-aware row partials never re-parse it), `generate_element_to_scalar_equation` (per-element link score equations for arrayed-to-scalar edges, used by both the variable-backed-reducer path and the `source[d] → $⁚ltm⁚agg⁚{n}` half) which dispatches on `ReducerKind` -- `generate_linear_partial` (SUM/MEAN algebraic shortcut), `generate_nonlinear_partial` (MIN/MAX nested binary calls; STDDEV the unrolled population-variance `sqrt` ceteris-paribus partial -- divisor `N`, matching `vm.rs::Opcode::ArrayStddev`, with the mean string-inlined; RANK the documented delta-ratio stand-in pinned by `test_generate_rank_keeps_delta_ratio` -- an order statistic, non-differentiable and unreachable via a real model RHS), `generate_scalar_to_element_equation` (per-element link score for the `$⁚ltm⁚agg⁚{n} → target[e]` half; takes a `source_ref_override: Option<&str>` so a multi-slot arrayed agg's `Δsource` denominator carries the projected `agg[]` subscript instead of the bare agg name, which wouldn't compile as a scalar), `substitute_reducers_in_expr0` (textually replaces a recognized reducer subexpression in an `Expr0` with its agg name, for the `$⁚ltm⁚agg⁚{n} → target` link score), `resolve_link_score_name_for_loop` (picks the Bare-or-FixedIndex link-score name a loop-score reference should target). Module link score formulas (black-box delta-ratio and composite-ref) are inlined directly into `module_link_score_equation` in `db.rs` (called by the per-shape `link_score_equation_text_shaped`). `subscript_idents_at_element` pins a target's arrayed deps for a per-element scalar partial, and it pins each one over the dimensions THAT DEP declares rather than over the target's element tuple (GH #974): a bare arrayed reference in an apply-to-all body reads its own axes' coordinates matched by dimension NAME, so a subset-dims dep (`w[Age]` under `growth[Region,Age]`) got an over-arity subscript whose fragment failed to compile, and a REORDERED one (`w[Age,Region]`) got a subscript that compiled and silently read the transposed element. The projection is `post_transform::dep_element_pins`/`dep_row_for_target`, reused by `pin_bare_source_ref` for a bare reference to the LIVE SOURCE (which is why a positionally-MAPPED bare source reference resolves through `mapped_element_correspondence` instead of being left bare and frozen into an uncompilable multi-slot `PREVIOUS`). The partial-equation builders (`build_partial_equation_shaped`/`_with_live_ref`, `subscript_idents_at_element`) and every link-score equation generator return `Result<_, PartialEquationError>`: a parse failure (genuine `Err`, or an empty `Ok(None)` equation) has no AST to PREVIOUS-wrap, so emitting the unwrapped input would silently produce a non-ceteris-paribus "partial" identical to the full equation (link score magnitude constant |Δz/Δz| = 1) -- a hidden attribution error that compiles cleanly (GH #311). The db-bearing callers (`link_score_equation_text_shaped` and the `src/db/ltm/link_scores.rs` emitters) convert the error into a `Warning` (`emit_ltm_partial_equation_warning`, naming the variable + offending equation text) and skip the variable -- distinct from `model_ltm_fragment_diagnostics`, which only catches *compile* failures. The failure is effectively unreachable in production (the text is always a `print_eqn` re-print; an empty equation is rejected as an `EmptyEquation` Error upstream), so this is defense-in-depth. The `#[cfg(test)]` tests live in the sibling `src/ltm_augment_tests.rs` (split out for the per-file line cap). Six more siblings are `#[path]`-mounted into `ltm_augment` purely for that cap, so every caller still names their items `crate::ltm_augment::*`: **`ltm_augment_partial_error.rs`** (the `PartialEquationError`/`PartialEquationErrorKind` loud-failure vocabulary plus the `contains_rank_like_builtin` walk the `RankLikePartial` class is decided by), **`ltm_augment_occurrence.rs`** (the wrap's read side of the occurrence IR -- `SlotOccurrences` groups a target's stream by slot ONCE and is the only way to obtain an `OccurrenceLookup`, so the borrow forces callers to hoist it out of their per-element loop), **`ltm_augment_post_transform.rs`** (the concrete-form lowerings: the agg-name substitution, and the `PerElement` row pinning the wrap calls AS IT DESCENDS -- the wrap is the only place that knows both the occurrence and whether it is about to FREEZE the reference, which is what picks the bare row for the live occurrence over the qualified row for every other one), **`ltm_augment_with_lookup.rs`** (the GH #910 implicit-WITH-LOOKUP rules), **`ltm_augment_wrap_test_support.rs`** (the `#[cfg(test)]` occurrence reconstruction + the Expr0 classifier mirror described above), and **`ltm_augment_freeze.rs`** (the GH #975 first-DT initial value of every synthesized `PREVIOUS`). +- **`src/ltm_augment.rs`** - Equation generators for LTM synthetic variables: `generate_link_score_equation_for_link` (ceteris-paribus link scores; takes `RefShape` and source dimension elements to drive per-shape PREVIOUS wrapping), `generate_loop_score_variables` (emits one `loop_score` per loop as a dimension-shaped `datamodel::Equation`: `Scalar` for scalar loops, `ApplyToAll` for dimensioned loops whose links resolve through Bare A2A names, and per-slot `Equation::Arrayed` for dimensioned loops backed by per-element circuits via `Loop.slot_links` -- GH #653; relative loop scores are computed post-simulation in `ltm_post.rs`), `build_partial_equation_shaped` (the `#[cfg(test)]` TEXT entry point for the ceteris-paribus wrap; arrayed-per-element-equation (`Ast::Arrayed`) targets get one partial per element assembled into an `Equation::Arrayed`). **Production never parses a target equation**: `wrap_changed_first_ast` takes an `Expr0` lowered straight from the target's `Expr2` by `patch::expr2_to_expr0` (which is what `expr2_to_string` prints, so the former print->reparse was a parse of our own output), and every per-occurrence decision -- access shape, the GH #526 other-dep verdict, the literal-element index guard, and the `PerElement` row pinning -- is a lookup into the `db::ltm_ir` occurrence IR by the structural child-index path the wrap tracks, which equals the occurrence's `SiteId` BY CONSTRUCTION now that both walk the same tree. Every `PREVIOUS` the wrap SYNTHESIZES goes through `freeze_at_previous`, which chooses the call's first-DT initial value from the position being frozen: a VALUE position keeps the unary spelling (desugared to `0`, the XMILE-documented default, and unobservable behind the guard form's `TIME = INITIAL_TIME` arm), while a SUBSCRIPT INDEX names its own un-lagged operand -- `PREVIOUS(idx, idx)` -- because `0` is out of range for every 1-based dimension, so the frozen read yielded NaN at t=0 and `make_temp_arg`'s capture helper served that NaN as the score's FIRST LIVE step (GH #975). Only the two walkers that descend into indices need it (`wrap_non_matching_in_previous` via `wrap_index_non_matching_in_previous`, and `wrap_matching_in_previous`); `wrap_live_shaped_in_previous` and `freeze_pinned_body` document that they never do. A subscripted source reference the IR did NOT record -- reachable under a `LOOKUP` TABLE argument, which the walker skips as static data ("not a causal edge", which is right about ATTRIBUTION) -- still has to COMPILE, so the pin-only descent discharges it by NAME (`pin_dimension_name_indices`: an index naming one of the TARGET's iterated dimensions becomes the source element this target element reads on that axis, the same structural substitution `pin_bare_source_ref` performs for a bare `Var`). That is a lowering-completeness rule, not a second classifier: it consults no occurrence, infers no shape, and never makes the reference live-selectable -- it asks the SHARED row derivation `per_element_row_for_target` (hence `DimensionsContext::mapped_element_correspondence`) which element an axis reads, so the identity axis and a positionally-MAPPED one (`effect[State, old]` over an `effect[Region, Age]` source, either declaration direction) are one arm and it accepts exactly the mapped pairs `ltm_agg::classify_axis_access` accepts. An index the source's axis DECLARES as an element is resolved BEFORE that dimension-name reading, and that precedence is not the pin's own: it is the shared `dimensions::resolve_axis_index_name`, which `ltm_agg::classify_axis_access` reads too (GH #986 closed the divergence -- the classifier had the opposite order, so a mapped collision described the axis as `Iterated` over a dimension the compiler never iterates there). Element-first is what `compiler::subscript`'s `normalize_subscripts3` does ("First check if it's a named dimension element (takes priority)"), and the simulation is the authority: the two readings collide when a dimension declares an element whose name is also a dimension name (`Bucket = [old, region]` beside a `Region` dimension), and a describer that breaks the tie the other way names rows the simulation never reads (`a_colliding_index_name_reads_the_axis_element_in_the_simulation` is the numeric oracle; the XMILE spec's footnote [9] settles the adjacent VARIABLE-vs-element pair outright and sections 2.1/3.7.1 argue this pair from the namespace rule -- `resolve_axis_index_name`'s rustdoc says which is which). Everything that is NOT a bare identifier is left verbatim, needing no pin: a numeric literal, arithmetic over literals, an `@N` position (which `compiler::context` resolves to a concrete element offset in scalar context -- spelling it out here would be a second implementation of position syntax), and a compound expression selecting the element at RUNTIME. That last one used to be a conditional REFUSAL, and deleting it is GH #984: the wrap now freezes a `LOOKUP` table argument's index reads itself (`freeze_lookup_table_indices`), so a runtime index arrives here already lagged and the rule keeps it. That freeze WIDENS its own descent's dep set with the index idents, and without that it would not fire at all -- `variable::classify_dependencies`' `BuiltinContents::LookupTable` arm records the table's ident and never walks the table expression, so an index variable referenced only there is not a dependency and the wrap's `other_deps` freeze could never reach it. The widened set is scoped to that argument's indices, and the element / dimension-name guards run before the dep check, so it cannot make a selector wrap. (Leaving the dep set itself alone is deliberate: an index dropped from a variable's dependencies is a runlist-ordering question, not an LTM one.) What the SHARED derivation declines -- an unmapped or element-mapped pair (GH #756), a transposition, a dimension the target does not iterate -- is declined LOUDLY (`WrapOutcome::missing_occurrence` -> warned skip) rather than emitted with its dimension-name subscript intact, which would not resolve in a scalar fragment. The verdict space (`Pinned`, `Keep`, and the one loud `Unspellable`) is ENUMERATED cell by cell in `ltm_augment_pin_tests.rs`'s three verdict enumerations rather than sampled. There is ONE access-shape classifier family, on `Expr2`; the Expr0 mirror is test-support only and lives in `ltm_augment_wrap_test_support.rs` (kept because three wrap unit tests -- unparseable text, empty text, and the Fig. 2 Q4 `SUM(w[from]) + from` shape the engine REJECTS as a model -- cannot be db-backed fixtures), with `ltm_classifier_agreement_tests.rs` proving it matches the IR field for field (`SiteId` path, `shape`, `axes`, `in_reducer`) corpus-wide, `link_score_var_name` (synthetic name helper: Bare gets the canonical `{from}\u{2192}{to}` form, FixedIndex prepends `[elem]` to from; the obsolete per-shape `\u{205A}wildcard`/`\u{205A}dynamic` Wildcard/DynamicIndex suffixes were retired -- those shapes now collapse onto the Bare name, since *every statically-describable* inlined reducer (whole-extent or sliced) is hoisted into a `$⁚ltm⁚agg⁚{n}` node and only a `DynamicIndex` reference -- `arr[i+1]`, a range, or the not-hoistable dynamic-index reducer carve-out `SUM(pop[idx,*])` -- a whole-RHS variable-backed reducer's `Wildcard` argument, or a de-hoisted array-valued reducer's `Wildcard` arg (`RANK(pop[*], 1)`, GH #771) reach this function), `quote_ident` (identifier quoting for equations). Array support: `classify_reducer` (walks target Expr2 AST to identify reducing builtins -- Linear for SUM/MEAN, Nonlinear for MIN/MAX/STDDEV/RANK, Constant for SIZE -- a thin reader of `ltm_agg::reducer_kind`; it also hands back the reducer's array-argument AST as `ClassifiedReducer::body`, lowered from `Expr2` rather than printed, so the body-aware row partials never re-parse it), `generate_element_to_scalar_equation` (per-element link score equations for arrayed-to-scalar edges, used by both the variable-backed-reducer path and the `source[d] → $⁚ltm⁚agg⁚{n}` half) which dispatches on `ReducerKind` -- `generate_linear_partial` (SUM/MEAN algebraic shortcut), `generate_nonlinear_partial` (MIN/MAX nested binary calls; STDDEV the unrolled population-variance `sqrt` ceteris-paribus partial -- divisor `N`, matching `vm.rs::Opcode::ArrayStddev`, with the mean string-inlined; RANK the documented delta-ratio stand-in pinned by `test_generate_rank_keeps_delta_ratio` -- an order statistic, non-differentiable and unreachable via a real model RHS), `generate_scalar_to_element_equation` (per-element link score for the `$⁚ltm⁚agg⁚{n} → target[e]` half; takes a `source_ref_override: Option<&str>` so a multi-slot arrayed agg's `Δsource` denominator carries the projected `agg[]` subscript instead of the bare agg name, which wouldn't compile as a scalar), `substitute_reducers_in_expr0` (textually replaces a recognized reducer subexpression in an `Expr0` with its agg name, for the `$⁚ltm⁚agg⁚{n} → target` link score), `resolve_link_score_name_for_loop` (picks the Bare-or-FixedIndex link-score name a loop-score reference should target). Module link score formulas (black-box delta-ratio and composite-ref) are inlined directly into `module_link_score_equation` in `db.rs` (called by the per-shape `link_score_equation_text_shaped`). `subscript_idents_at_element` pins a target's arrayed deps for a per-element scalar partial, and it pins each one over the dimensions THAT DEP declares rather than over the target's element tuple (GH #974): a bare arrayed reference in an apply-to-all body reads its own axes' coordinates matched by dimension NAME, so a subset-dims dep (`w[Age]` under `growth[Region,Age]`) got an over-arity subscript whose fragment failed to compile, and a REORDERED one (`w[Age,Region]`) got a subscript that compiled and silently read the transposed element. The projection is `post_transform::dep_element_pins`/`dep_row_for_target`, reused by `pin_bare_source_ref` for a bare reference to the LIVE SOURCE (which is why a positionally-MAPPED bare source reference resolves through `mapped_element_correspondence` instead of being left bare and frozen into an uncompilable multi-slot `PREVIOUS`). The partial-equation builders (`build_partial_equation_shaped`/`_with_live_ref`, `subscript_idents_at_element`) and every link-score equation generator return `Result<_, PartialEquationError>`: a parse failure (genuine `Err`, or an empty `Ok(None)` equation) has no AST to PREVIOUS-wrap, so emitting the unwrapped input would silently produce a non-ceteris-paribus "partial" identical to the full equation (link score magnitude constant |Δz/Δz| = 1) -- a hidden attribution error that compiles cleanly (GH #311). The db-bearing callers (`link_score_equation_text_shaped` and the `src/db/ltm/link_scores.rs` emitters) convert the error into a `Warning` (`emit_ltm_partial_equation_warning`, naming the variable + offending equation text) and skip the variable -- distinct from `model_ltm_fragment_diagnostics`, which only catches *compile* failures. The failure is effectively unreachable in production (the text is always a `print_eqn` re-print; an empty equation is rejected as an `EmptyEquation` Error upstream), so this is defense-in-depth. The `#[cfg(test)]` tests live in the sibling `src/ltm_augment_tests.rs` (split out for the per-file line cap). Nine more siblings are `#[path]`-mounted into `ltm_augment` purely for that cap, so every caller still names their items `crate::ltm_augment::*`: **`ltm_augment_partial_error.rs`** (the `PartialEquationError`/`PartialEquationErrorKind` loud-failure vocabulary plus the `contains_rank_like_builtin` walk the `RankLikePartial` class is decided by), **`ltm_augment_occurrence.rs`** (the wrap's read side of the occurrence IR -- `SlotOccurrences` groups a target's stream by slot ONCE and is the only way to obtain an `OccurrenceLookup`, so the borrow forces callers to hoist it out of their per-element loop), **`ltm_augment_post_transform.rs`** (the concrete-form lowerings: the agg-name substitution, and the `PerElement` row pinning the wrap calls AS IT DESCENDS -- the wrap is the only place that knows both the occurrence and whether it is about to FREEZE the reference, which is what picks the bare row for the live occurrence over the qualified row for every other one), **`ltm_augment_with_lookup.rs`** (the GH #910 implicit-WITH-LOOKUP rules), **`ltm_augment_wrap_test_support.rs`** (the `#[cfg(test)]` occurrence reconstruction + the Expr0 classifier mirror described above), **`ltm_augment_freeze.rs`** (the GH #975 first-DT initial value of every synthesized `PREVIOUS`), **`ltm_augment_index.rs`** (the wrap's subscript-INDEX pass -- the one position where "is this a causal reference?" has a different answer than everywhere else, since a bare identifier between the brackets may be an element selector, a dimension name the apply-to-all expansion resolves per element, or a genuine variable read), **`ltm_augment_array_freeze.rs`** (materializing a frozen ARRAY SLICE as its own synthetic variable -- GH #995 option B, since `PREVIOUS(arr[pin, *])` has no inline spelling codegen accepts), and **`ltm_augment_zero_slot.rs`** (`ZeroSlotPolicy` + `partial_is_provably_previous_target`, the GH #977 decision of when a per-element arm's transformed partial is provably `PREVIOUS(target)` and the slot may be OMITTED rather than materialized -- a POSITIVE test over the emitted tree, deliberately not the unsound "the link's source stayed frozen", and gated on `apply_default_to_missing == false` because an EXCEPT-default target's absent slot takes the default rather than zero). - **`src/ltm_augment_with_lookup.rs`** - The implicit-WITH-LOOKUP rules for LTM (GH #910), re-exported from `ltm_augment` so every `crate::ltm_augment::*` path is unchanged. A `v = WITH LOOKUP(input, table)` variable is lowered by the compiler to `LOOKUP(v, input)` (`compiler::apply_implicit_with_lookup`), so a link-score partial that RE-EVALUATES such a target's equation is in gf-INPUT units while the guard form's `PREVIOUS(target)` anchor is in gf-OUTPUT units. `is_implicit_with_lookup` carries the coverage doc: every partial is either a **full re-evaluation** (class 1 -- must be wrapped in the gf application) or a **delta-ratio stand-in** (class 2 -- the RANK arm and the nested-arithmetic arm, already in output units, must NEVER be wrapped or a gf output is fed back through the gf). `WithLookupSlotRefs` resolves the target's table reference ONCE per target (`NoGf` / `Shared` / `PerElement`), so a per-element-gf target costs one row-major `SubscriptIterator` walk rather than one per element; an arrayed target's shared table is pinned as `to[1,...]` because a bare arrayed reference resolves each iterated element's own table offset, which the VM reads as NaN past `table_count`. `compose_with_lookup_polarity` (in `src/ltm/polarity.rs`) is the polarity twin, mirroring `apply_implicit_with_lookup`'s placeholder-Positive and zero-point-table rules. The polarity tests live in `src/ltm/with_lookup_tests.rs`, a child module of `ltm::tests`. - **`src/ltm_post.rs`** - Post-simulation relative loop *and link* score computation. `compute_rel_loop_scores(results, loop_partitions)` normalizes each loop's `loop_score` series against the sum of absolute scores within its cycle partition, using SAFEDIV-0 semantics (zero denominator -> zero result). Called after simulation rather than emitted as synthetic equations to avoid O(P^2) equation-text growth on models with dense partitions. `loop_partitions` is an `IndexMap` (re-exported `engine::indexmap`). `compute_rel_loop_scores*` walk its **emission** order rather than re-sorting the loop ids: the partition-sum denominator accumulates `|loop_score|` in that order, and emission order keeps the IEEE-754 (non-associative) sum bit-for-bit identical to the pre-#461 compile-time emitter (GH #468). Emission order is itself deterministic across salsa cache invalidations / processes because `assign_loop_ids` orders loops by a content-derived key (`ltm::graph::loop_id_sort_key`), so it never flaps even though `IndexMap`'s `PartialEq` (salsa cache equality) is order-insensitive. `compute_rel_link_scores(links, step_count)` is the link-level analogue (GH #652): raw link scores divide by the change in the *target*, so they are not comparable across targets and ranking by raw magnitude surfaces numerically-degenerate links (near-constant targets blow up the score). It groups the input `RelLinkInput { to, score }` links by their `to` target and normalizes each link's score by the per-target, per-timestep sum of `|score|` over all that target's *scored* inputs -- a **signed** value in `[-1, 1]` (sign kept like `compute_rel_loop_scores`), with the same `denom_summand` NaN-exclusion / Inf-retention / SAFEDIV-0 semantics. Denominator scope is the scored inputs only: complete in discovery mode (every causal edge scored) but covering just the in-loop subset in exhaustive mode (documented caveat on the fn). libsimlin's `analyze_links_core` calls it over the final (post-synthetic-collapse) link set so the per-target denominator matches the links the caller receives. - **`src/ltm_dominance.rs`** - Dominant-period selection over LTM loop importance series (GH #998): `FeedbackLoop` (a loop + its signed partition-relative importance series), `DominantPeriod`, the coarse 3-way display `LoopPolarity`, and `calculate_dominant_periods` -- the per-cycle-partition Praxis-style selection, parameterized by the caller-declared `PartitionSurface` (`PartitionBearing`: each partition-`None` loop is its own solo dominance group, mirroring discovery's `NormGroup::Solo`; `NoMetadata`: the flat legacy group, used only by layout's persisted-loop-metadata fallback). Two consumer families: `analysis::analyze_model` (the discovery surface, reaching FFI/pysimlin/MCP/TS via `ModelAnalysis::dominant_loops_by_period`) and the layout pipeline (`layout::detect_ltm_loops` + `layout::metadata::ComputedMetadata`). These types historically lived in `layout::metadata`; they were moved out so the LTM/analysis surface does not depend on a layout submodule -- layout consuming LTM is the right dependency direction, the reverse was not. diff --git a/src/simlin-engine/src/db/ltm_char_golden/arrayed_target_no_default_slot_scores.txt b/src/simlin-engine/src/db/ltm_char_golden/arrayed_target_no_default_slot_scores.txt new file mode 100644 index 000000000..adaabdbfa --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/arrayed_target_no_default_slot_scores.txt @@ -0,0 +1,8 @@ +$⁚ltm⁚link_score⁚pop[boston]→mp dims=[Region] +arrayed[Region] (apply_default=false): + boston => if (TIME = INITIAL_TIME) then 0 else if ((mp - PREVIOUS(mp)) = 0) OR ((pop[region·boston] - PREVIOUS(pop[region·boston])) = 0) then 0 else SAFEDIV((((pop[boston] - PREVIOUS(pop[region·nyc])) * 0.01) - PREVIOUS(mp)), ABS((mp - PREVIOUS(mp))), 0) * SIGN((pop[region·boston] - PREVIOUS(pop[region·boston]))) + nyc => if (TIME = INITIAL_TIME) then 0 else if ((mp - PREVIOUS(mp)) = 0) OR ((pop[region·boston] - PREVIOUS(pop[region·boston])) = 0) then 0 else SAFEDIV((((PREVIOUS(pop[region·nyc]) - pop[boston]) * 0.01) - PREVIOUS(mp)), ABS((mp - PREVIOUS(mp))), 0) * SIGN((pop[region·boston] - PREVIOUS(pop[region·boston]))) +$⁚ltm⁚link_score⁚pop[nyc]→mp dims=[Region] +arrayed[Region] (apply_default=false): + boston => if (TIME = INITIAL_TIME) then 0 else if ((mp - PREVIOUS(mp)) = 0) OR ((pop[region·nyc] - PREVIOUS(pop[region·nyc])) = 0) then 0 else SAFEDIV((((PREVIOUS(pop[region·boston]) - pop[nyc]) * 0.01) - PREVIOUS(mp)), ABS((mp - PREVIOUS(mp))), 0) * SIGN((pop[region·nyc] - PREVIOUS(pop[region·nyc]))) + nyc => if (TIME = INITIAL_TIME) then 0 else if ((mp - PREVIOUS(mp)) = 0) OR ((pop[region·nyc] - PREVIOUS(pop[region·nyc])) = 0) then 0 else SAFEDIV((((pop[nyc] - PREVIOUS(pop[region·boston])) * 0.01) - PREVIOUS(mp)), ABS((mp - PREVIOUS(mp))), 0) * SIGN((pop[region·nyc] - PREVIOUS(pop[region·nyc]))) diff --git a/src/simlin-engine/src/db/ltm_char_tests.rs b/src/simlin-engine/src/db/ltm_char_tests.rs index 7485afef5..6610454de 100644 --- a/src/simlin-engine/src/db/ltm_char_tests.rs +++ b/src/simlin-engine/src/db/ltm_char_tests.rs @@ -1665,6 +1665,55 @@ fn char_arrayed_target_slot_scores() { ); } +// --------------------------------------------------------------------------- +// Model E2: the same `Ast::Arrayed` target WITHOUT an EXCEPT default -- the GH +// #977 omission path. +// +// Model E's `mp` carries an EXCEPT default, and a target with one is pinned to +// `ZeroSlotPolicy::Materialize`: an absent slot there takes the DEFAULT +// equation, not zero, so no arm may be dropped. Model E is the only arrayed +// target in this file, which is why the omission reached ZERO characterization +// coverage when it landed -- this fixture is that coverage. +// +// `mp[la]` reads no `pop` at all, so for either `pop[e] -> mp` edge every +// occurrence in the `la` arm is frozen by the ceteris-paribus wrap and the arm +// is provably `PREVIOUS(mp)`. It is omitted from the element map, which +// `compiler::expand_arrayed_with_hoisting` lowers to a single constant-zero +// assign. What the golden shows is the slot being ABSENT -- deliberately +// distinct from an arm that is present holding a `"0"` partial, which is what a +// generator that gave up would emit. +// --------------------------------------------------------------------------- + +fn arrayed_target_no_default_model() -> datamodel::Project { + let mut p = TestProject::new("arrayed_target_no_default_char") + .named_dimension("Region", &["nyc", "boston", "la"]) + .aux("drift", "1", None) + .array_aux("pop[Region]", "10"); + // `array_with_ranges` builds an `Equation::Arrayed` with no default, so + // `apply_default_to_missing` is false and omission is sound. + p = p.array_with_ranges( + "mp[Region]", + vec![ + ("nyc", "(pop[nyc] - pop[boston]) * 0.01"), + ("boston", "(pop[boston] - pop[nyc]) * 0.01"), + ("la", "drift * 0.01"), + ], + ); + p.array_stock("stock[Region]", "0", &["mpflow"], &[], None) + .array_flow("mpflow[Region]", "mp", None) + .build_datamodel() +} + +#[test] +fn char_arrayed_target_no_default_slot_scores() { + assert_char_fixture( + "arrayed_target_no_default_slot_scores", + arrayed_target_no_default_model(), + "link_score\u{205A}pop", + FragmentExpectation::AllCompile, + ); +} + // --------------------------------------------------------------------------- // Model F (Track A3 stage 2, review finding 2): the GH #517 whole-reducer // freeze over an INDEX-NESTED live-source occurrence (Fig. 2 Q4). diff --git a/src/simlin-engine/src/db/ltm_tests.rs b/src/simlin-engine/src/db/ltm_tests.rs index 92904a2ef..87689b549 100644 --- a/src/simlin-engine/src/db/ltm_tests.rs +++ b/src/simlin-engine/src/db/ltm_tests.rs @@ -1041,14 +1041,32 @@ fn collect_agg_petals_groups_single_agg_circuits() { // which still compiles and reads a different slot than the anchor did. // --------------------------------------------------------------------------- -/// `share[boston]` reads `q`/`gtab` at the runtime index `ctr`, and has no -/// causal dependence on `pop[nyc]` whatsoever. +/// `share[boston]` reads `q`/`gtab` at the runtime index `ctr`, and its only +/// dependence on `pop[nyc]` is the zero-coefficient term `0 * pop[nyc]`. /// /// `declare_bucket` adds a dimension **no equation references**, whose first /// element is named `ctr` -- the same canonical name as the model variable. It /// changes nothing about the simulation; before the fix it changed the emitted /// link score. /// +/// **The `0 * pop[nyc]` term is load-bearing for the TEST, not for the model**, +/// and it is the same idiom the neighbouring `0 * ctr` already uses. Its job is +/// to give the `boston` arm a live reference to the link's SOURCE, which is what +/// makes the arm materialize at all: since GH #977 a slot whose transformed +/// partial is provably `PREVIOUS(target)` is omitted from the `Arrayed` element +/// map and lowered to a constant zero, and without this term every occurrence in +/// this arm is frozen, so the arm this test reads would not exist. Its +/// coefficient is zero, so it changes no value the test asserts on: the residual +/// series below is bit-identical with and without it. +/// +/// That materialization matters because "the slot is absent under both variants" +/// would NOT be an adequate stand-in for the assertions below. Both readings of +/// `ctr` -- frozen (`PREVIOUS(ctr, ctr)`, correct) and qualified onto the +/// unrelated dimension (`bucket·ctr`, the defect) -- leave the arm provably +/// `PREVIOUS(target)`, so an omission-based assertion passes on the defect too. +/// The sibling control below demonstrates exactly that: a genuinely static +/// `q[slot·s1]` selector produces an omitted arm as well. +/// /// `indexed_name` only varies the subscripted variable's NAME. Both iterations /// exercise the SAME path -- an ordinary arrayed variable subscripted directly -- /// and that is deliberate, because it is the only path the fix reaches. @@ -1091,7 +1109,10 @@ fn colliding_index_name_model(declare_bucket: bool, second_name: bool) -> datamo "share[Region]", vec![ ("nyc", "pop[nyc] * 0.01"), - ("boston", &format!("{indexed}[ctr] * 0.002 + 0 * ctr")), + ( + "boston", + &format!("{indexed}[ctr] * 0.002 + 0 * ctr + 0 * pop[nyc]"), + ), ("la", "pop[la] * 0.03"), ], ) @@ -1128,6 +1149,10 @@ fn colliding_index_boston_arm(project: &datamodel::Project) -> (String, usize) { .iter() .find(|(e, _)| e == "boston") .map(|(_, arm)| arm.text.clone()) + // A missing arm here means the GH #977 omission claimed the + // slot, which would gut every assertion downstream rather than + // fail it -- see `colliding_index_name_model`'s note on the + // zero-coefficient term that keeps this arm materialized. .unwrap_or_else(|| panic!("no boston arm in {:?}", elements)), other => panic!("expected an arrayed score, got {other:?}"), }) @@ -1193,9 +1218,10 @@ fn a_colliding_index_name_is_resolved_against_the_axis_it_indexes() { /// The simulated `boston` slot of the `pop[nyc] -> share` link score. /// /// NOTE what this deliberately does NOT assert: that the series is ZERO. -/// `share[boston]` has no causal dependence on `pop[nyc]`, so a fully -/// ceteris-paribus partial would be identically zero -- and it is not; it runs -/// -1.06 / +0.73 / -1.03 / +0.82 on this fixture. That residual is a SEPARATE +/// `share[boston]`'s only dependence on `pop[nyc]` carries a zero coefficient, +/// so a fully ceteris-paribus partial would be identically zero -- and it is +/// not; it runs -1.06 / +0.73 / -1.03 / +0.82 on this fixture, bit-identically +/// with and without that term. That residual is a SEPARATE /// defect from the one above and predates this branch: an index frozen inside an /// already-frozen head is DOUBLE-lagged (the partial reads `q` at `t-1` indexed /// by `ctr` at `t-2`, where the anchor `PREVIOUS(share)` used `ctr` at `t-1`). @@ -1249,6 +1275,15 @@ fn an_index_naming_the_axis_own_element_stays_a_static_selector() { // The control that keeps the fix from being "freeze every bare index": // `s1` IS an element of `gtab`'s own `Slot` axis, so it is a selector and // must stay unwrapped (and qualified onto its own dimension). + // + // `0 * pop[nyc]` plays the same role it does in `colliding_index_name_model` + // and for the same reason: a `boston` arm holding only frozen reads is + // provably `PREVIOUS(target)` and GH #977 omits it, and an omitted arm has + // no text to inspect. This control is also the direct evidence that + // omission cannot substitute for the assertions here -- WITHOUT the term + // this correctly-qualified static selector produces an omitted arm, exactly + // as the frozen runtime index does, so "absent under both variants" cannot + // tell a selector from a freeze. let project = TestProject::new("axis_element_index") .named_dimension("Region", &["nyc", "boston", "la"]) .named_dimension("Slot", &["s1", "s2"]) @@ -1260,7 +1295,7 @@ fn an_index_naming_the_axis_own_element_stays_a_static_selector() { "share[Region]", vec![ ("nyc", "pop[nyc] * 0.01"), - ("boston", "q[s1] * 0.002"), + ("boston", "q[s1] * 0.002 + 0 * pop[nyc]"), ("la", "pop[la] * 0.03"), ], ) diff --git a/src/simlin-engine/src/ltm_augment.rs b/src/simlin-engine/src/ltm_augment.rs index 6e3bf2357..ef0ae3df5 100644 --- a/src/simlin-engine/src/ltm_augment.rs +++ b/src/simlin-engine/src/ltm_augment.rs @@ -267,6 +267,15 @@ mod array_freeze; pub(crate) use array_freeze::{ArrayFreezeHelper, FREEZE_HELPER_PREFIX, materialize_array_freezes}; +/// Deciding when a per-element link-score arm is provably `PREVIOUS(target)` +/// and may therefore be OMITTED rather than materialized (GH #977), in its own +/// file only to keep this one under the project line-count lint. +#[path = "ltm_augment_zero_slot.rs"] +mod zero_slot; + +pub(crate) use zero_slot::ZeroSlotPolicy; +use zero_slot::partial_is_provably_previous_target; + /// Append child index `i` to `path`, yielding the child node's structural path. /// The wrap's recursion mirrors `db::ltm_ir::walk_all_in_expr`'s child-index /// construction exactly, so the path at any node equals that occurrence's @@ -1626,6 +1635,12 @@ fn wrap_live_shaped_in_previous( /// reference is a *layout* reference (`classify_dependencies` records it /// in `referenced_tables`, not `all`), so it adds no causal edge. `None` /// leaves the partial unwrapped (an ordinary target). +/// +/// `zero_slot_policy` decides what happens when the changed-first wrap froze +/// EVERY occurrence of the source ([`WrapOutcome::live_ref`] is `None`), so +/// the partial is the fully-frozen target and the guard form it would build +/// evaluates to ~0. Under [`ZeroSlotPolicy::OmitStructuralZero`] the arm is +/// dropped (`Ok(None)`) instead of materialized; see that variant's docs. #[allow(clippy::too_many_arguments)] // threads the link-score generation context fn shaped_guard_form_text( target_expr: &Expr0, @@ -1639,8 +1654,9 @@ fn shaped_guard_form_text( target_ref: &str, gf_table_ref: Option<&str>, occ: &OccurrenceLookup<'_>, + zero_slot_policy: ZeroSlotPolicy, freeze_helpers: &mut Vec, -) -> Result { +) -> Result, PartialEquationError> { let gf_wrap = |partial: String| -> String { match gf_table_ref { Some(table_ref) => format!("LOOKUP({table_ref}, {partial})"), @@ -1704,6 +1720,31 @@ fn shaped_guard_form_text( let mut first_leg_helpers = Vec::new(); let changed_first = materialize(changed_first, &mut first_leg_helpers); if !out.other_dep_mismatch && !contains_unfreezable_previous(&changed_first) { + // GH #977: the wrap produced a partial that reads nothing which can have + // changed since the previous step, so it recomputes `PREVIOUS(target)` + // and the guard form's numerator is identically zero. Drop the arm + // rather than print, parse, lower and execute a full equation to arrive + // at the constant an absent slot already lowers to. + // + // The test runs on the MATERIALIZED partial, after the array-freeze + // rewrite, so it judges the tree that would actually be emitted rather + // than the one before helper substitution. + // + // The check sits INSIDE the changed-first success block, not before it: + // an arm that also trips the doom checks must keep falling through to + // the changed-last leg, which rejects it with `Err(UnfreezablePartial)` + // and so declares the whole edge unscoreable (the #758/#780 contract, + // which drops dependent loop scores). Omitting it earlier would quietly + // keep that edge scoreable and change which loops get dropped. + // + // `first_leg_helpers` is deliberately NOT appended: the arm that would + // have referenced those freeze helpers is gone, so appending them would + // mint variables no equation reads. + if zero_slot_policy == ZeroSlotPolicy::OmitStructuralZero + && partial_is_provably_previous_target(&changed_first) + { + return Ok(None); + } let source_ref = source_ref_for_guard( from, shape, @@ -1712,11 +1753,11 @@ fn shaped_guard_form_text( source_dim_elements, ); freeze_helpers.append(&mut first_leg_helpers); - return Ok(link_score_guard_form( + return Ok(Some(link_score_guard_form( &gf_wrap(print_eqn(&changed_first)), target_ref, &source_ref, - )); + ))); } // Changed-last fallback: freeze only the live source, starting from the @@ -1783,11 +1824,11 @@ fn shaped_guard_form_text( // so it needs the same implicit WITH-LOOKUP application the target's // own compiled value gets (GH #910). let numerator = format!("({target_ref} - ({}))", gf_wrap(print_eqn(&changed_last))); - Ok(link_score_guard_form_with_numerator( + Ok(Some(link_score_guard_form_with_numerator( &numerator, target_ref, &source_ref, - )) + ))) } /// Wrap every reference to `target` in `PREVIOUS()` -- the *inverse* of @@ -3512,11 +3553,21 @@ fn build_arrayed_link_score_equation( // below are in range by the LTM front door, which refuses a target needing // more slots than `db::ltm_ir::MAX_SITE_CHILDREN` can tell apart -- so this // model would have emitted no link score to reach here. + // GH #977: a slot whose partial holds no live source reference scores a + // structural zero, and an omitted slot already lowers to a single + // constant-zero assign -- but ONLY when a missing slot means zero. Under + // EXCEPT semantics it means "apply the default equation", so those targets + // keep every arm. This is the only place the target's flag is in scope. + let zero_slot_policy = if apply_default_to_missing { + ZeroSlotPolicy::Materialize + } else { + ZeroSlotPolicy::OmitStructuralZero + }; let slot_equation = |expr: &crate::ast::Expr2, gf_table_ref: Option<&str>, slot: u16, freeze_helpers: &mut Vec| - -> Result { + -> Result, PartialEquationError> { let elem_eqn = crate::patch::expr2_to_expr0(expr); // Per-element dependency set: walk *only this slot's* expression // (the union over all elements -- what `identifier_set` on the @@ -3524,15 +3575,16 @@ fn build_arrayed_link_score_equation( // from this slot). Pass the target's dimensions so literal // element-name subscripts of the *target*'s dims are filtered out; // strip the *source*'s dim/element names afterward (see above). - let deps_e: HashSet> = crate::variable::classify_dependencies( + let classified = crate::variable::classify_dependencies( &crate::ast::Ast::Scalar(expr.clone()), target_ast_dims, None, - ) - .all - .into_iter() - .filter(|d| !source_dim_token_set.contains(d.as_str())) - .collect(); + ); + let deps_e: HashSet> = classified + .all + .into_iter() + .filter(|d| !source_dim_token_set.contains(d.as_str())) + .collect(); let occ = slot_occurrences.for_slot(slot); shaped_guard_form_text( &elem_eqn, @@ -3546,6 +3598,7 @@ fn build_arrayed_link_score_equation( target_ref, gf_table_ref, &occ, + zero_slot_policy, freeze_helpers, ) }; @@ -3562,15 +3615,26 @@ fn build_arrayed_link_score_equation( per_elem.iter().collect(); sorted_slots.sort_by(|a, b| a.0.cmp(b.0)); + // A slot the policy omitted is simply absent from `elements`; nothing is + // pushed for it. That is deliberately NOT the same channel as an arm whose + // generated text is EMPTY, which is still pushed and dropped later by + // `LtmEquation::to_flow_ast` -- keeping the two distinct is what lets an + // empty generated arm stay a symptom of a generator bug rather than a + // second, silent way to zero a slot. let mut elements: Vec<(String, String)> = Vec::with_capacity(sorted_slots.len()); for (slot, (elem, expr)) in sorted_slots.iter().enumerate() { let gf_table_ref = slot_refs.for_element(elem); - elements.push(( - elem.as_str().to_string(), - slot_equation(expr, gf_table_ref.as_deref(), slot as u16, freeze_helpers)?, - )); + if let Some(text) = + slot_equation(expr, gf_table_ref.as_deref(), slot as u16, freeze_helpers)? + { + elements.push((elem.as_str().to_string(), text)); + } } + // The default arm follows the same policy. When the policy is + // `OmitStructuralZero` the target's `apply_default_to_missing` is false, so + // `expand_arrayed_with_hoisting` never consults a default anyway; when it + // is `Materialize` the arm is always built and the flatten is a no-op. let default_gf_table_ref = slot_refs.for_default(); let default_slot = default_expr .map(|expr| { @@ -3581,7 +3645,8 @@ fn build_arrayed_link_score_equation( freeze_helpers, ) }) - .transpose()?; + .transpose()? + .flatten(); Ok(LtmEquation::arrayed( target_dim_names, @@ -3763,7 +3828,7 @@ fn generate_auxiliary_to_auxiliary_equation( // occurrence stream. let slot_occurrences = SlotOccurrences::new(to_occurrences); let occ = slot_occurrences.for_slot(0); - let text = shaped_guard_form_text( + let Some(text) = shaped_guard_form_text( &to_equation, &deps, from, @@ -3777,8 +3842,16 @@ fn generate_auxiliary_to_auxiliary_equation( // (GH #910); `None` for an ordinary aux. with_lookup_table_ref(to_var).as_deref(), &occ, + // This builds a whole variable's equation, not one slot of an arrayed + // one, so a structural zero still has to be materialized: there is no + // slot to leave absent, and dropping the variable would change the + // emitted score set. + ZeroSlotPolicy::Materialize, freeze_helpers, - )?; + )? + else { + unreachable!("ZeroSlotPolicy::Materialize never omits an arm") + }; Ok(link_score_equation_for_target(text, to_var)) } @@ -4078,7 +4151,7 @@ fn generate_stock_to_flow_equation( // changed-last fallback for an unfreezable changed-first partial). let slot_occurrences = SlotOccurrences::new(to_occurrences); let occ = slot_occurrences.for_slot(0); - let text = shaped_guard_form_text( + let Some(text) = shaped_guard_form_text( &flow_equation, &deps, stock, @@ -4091,8 +4164,14 @@ fn generate_stock_to_flow_equation( // A flow can be an implicit WITH-LOOKUP variable too (GH #910). with_lookup_table_ref(flow_var).as_deref(), &occ, + // A whole variable's equation -- see the twin call in + // `generate_link_score_equation_for_link`. + ZeroSlotPolicy::Materialize, freeze_helpers, - )?; + )? + else { + unreachable!("ZeroSlotPolicy::Materialize never omits an arm") + }; Ok(link_score_equation_for_target(text, flow_var)) } diff --git a/src/simlin-engine/src/ltm_augment_tests.rs b/src/simlin-engine/src/ltm_augment_tests.rs index a263563f0..d076e9108 100644 --- a/src/simlin-engine/src/ltm_augment_tests.rs +++ b/src/simlin-engine/src/ltm_augment_tests.rs @@ -3264,6 +3264,18 @@ fn arrayed_slot<'a>(equation: &'a crate::db::LtmEquation, element: &str) -> &'a } } +/// The elements an `Equation::Arrayed` score actually carries an arm for, in +/// emission order. A slot the GH #977 predicate omitted is simply absent, which +/// is the marker that keeps an intended zero distinguishable from a generator +/// that gave up and emitted a `"0"` arm. +fn arrayed_slot_names(equation: &crate::db::LtmEquation) -> Vec { + use crate::db::LtmEquation; + match equation { + LtmEquation::Arrayed { elements, .. } => elements.iter().map(|(e, _)| e.clone()).collect(), + other => panic!("expected LtmEquation::Arrayed, got: {other:?}"), + } +} + fn region_dm_dimension() -> crate::datamodel::Dimension { crate::datamodel::Dimension::named( "Region".to_string(), @@ -3784,11 +3796,35 @@ fn test_arrayed_link_score_population_to_migration_pressure_fixed_boston() { ); } +/// ltm-503-cross-element-agg.AC1.3 (unit-level): a stock-to-flow link score into +/// a per-element-equation arrayed flow must never report a slot by GIVING UP -- +/// emitting a literal `"0"` partial where it could not build a real one. +/// +/// The instrument changed with GH #977 and the guarded property did not. A slot +/// whose transformed partial is provably `PREVIOUS(target)` is now OMITTED from +/// the element map (`compiler::expand_arrayed_with_hoisting` lowers an absent +/// slot to one constant-zero assign), so "every slot references the flow's +/// equation contents" can no longer be asked of `boston` and `la` -- those arms +/// are gone by design. Asking it anyway would pin materialization, not +/// non-degradation. +/// +/// So this asserts the two things that still distinguish the failure from the +/// intent, over the DERIVED slot set rather than over named slots: +/// +/// * exactly the slots with a live source reference are present -- `nyc` here, +/// since the FixedIndex(nyc) shape matches only that arm -- and the rest are +/// ABSENT, which is the distinct omission marker #977 requires. An arm that is +/// present but empty, or present holding `"0"`, is a generator bug and stays +/// distinguishable from an intended zero slot precisely because the intended +/// one is not in the map at all. +/// * every PRESENT arm is a real partial: non-empty, and never the `((0) - ...)` +/// give-up form. +/// +/// What this does NOT guard: that the omitted slots are numerically zero. That +/// is the predicate's own claim and is gated by the whole-slab differential and +/// the char goldens, not here. #[test] fn test_arrayed_link_score_stock_to_flow_per_element_partials() { - // ltm-503-cross-element-agg.AC1.3 (unit-level): a stock-to-flow link - // score into a per-element-equation arrayed flow yields per-element - // partials referencing the flow's actual equation contents. let dims = vec![crate::datamodel::Dimension::named( "Region".to_string(), vec!["NYC".to_string(), "Boston".to_string(), "LA".to_string()], @@ -3829,23 +3865,32 @@ fn test_arrayed_link_score_stock_to_flow_per_element_partials() { ) .unwrap(); + let present = arrayed_slot_names(&equation); + // Derived from the flow's own element list and the link's shape: `nyc` is + // the only arm referencing `population[nyc]`, so it is the only arm with a + // live source and the only one that may survive. + assert_eq!( + present, + vec!["nyc".to_string()], + "exactly the live-source arms may be present; got {present:?} from {equation:?}" + ); + let nyc_slot = arrayed_slot(&equation, "nyc"); - let boston_slot = arrayed_slot(&equation, "boston"); - // The NYC slot keeps population[nyc] live (shape match); the other - // slots freeze their population refs but still reference - // `population` -- never a bare `(0)` partial. assert!( nyc_slot.contains("population[nyc] * 0.03"), "nyc slot partial should keep population[nyc] live; got: {nyc_slot}" ); - assert!( - boston_slot.contains("population"), - "boston slot should reference population; got: {boston_slot}" - ); - assert!( - !nyc_slot.contains("((0) -") && !boston_slot.contains("((0) -"), - "no slot may use a '0' partial; nyc={nyc_slot} boston={boston_slot}" - ); + for name in &present { + let slot = arrayed_slot(&equation, name); + assert!( + !slot.trim().is_empty(), + "a present slot must carry a real partial, never empty text; slot {name}" + ); + assert!( + !slot.contains("((0) -"), + "no present slot may use a '0' partial; {name}={slot}" + ); + } } #[test] @@ -5016,8 +5061,14 @@ fn sgft( target_ref, gf_table_ref, &occ, + // These tests exercise the wrap and guard-form construction for a whole + // variable's equation, which is what `Materialize` models; the slot + // omission is covered at the `model_ltm_variables` level in + // `db/ltm_tests.rs`. + ZeroSlotPolicy::Materialize, &mut Vec::new(), ) + .map(|text| text.expect("ZeroSlotPolicy::Materialize never omits an arm")) } /// Finding 1 (loud-degradation, not silent-zero): a live-source subscript node @@ -5089,6 +5140,10 @@ fn wrap_missing_live_source_occurrence_is_loud_not_silent_freeze() { "combined", None, &occ, + // The desync must be loud under EITHER policy; `OmitStructuralZero` is + // the interesting one, since it is the policy that has a non-Err way to + // decline an arm and so is the one that could swallow this. + ZeroSlotPolicy::OmitStructuralZero, &mut Vec::new(), ); assert!( diff --git a/src/simlin-engine/src/ltm_augment_zero_slot.rs b/src/simlin-engine/src/ltm_augment_zero_slot.rs new file mode 100644 index 000000000..b5acfc857 --- /dev/null +++ b/src/simlin-engine/src/ltm_augment_zero_slot.rs @@ -0,0 +1,162 @@ +// Copyright 2026 The Simlin Authors. All rights reserved. +// Use of this source code is governed by the Apache License, +// Version 2.0, that can be found in the LICENSE file. + +//! Deciding when a per-element link-score arm may be OMITTED rather than +//! materialized (GH #977), in its own file only to keep `ltm_augment.rs` under +//! the project line-count lint. Mounted into `ltm_augment`, so callers keep +//! naming these items `crate::ltm_augment::*`. + +use crate::ast::Expr0; +use crate::builtins::UntypedBuiltinFn; + +/// Whether the caller's result is a whole VARIABLE's equation or one slot of an +/// `Ast::Arrayed` one -- which is the only thing that decides whether a +/// structurally-zero partial may be dropped instead of built. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum ZeroSlotPolicy { + /// Always build the guard form, even when the partial is the fully-frozen + /// target. Required wherever the result is a whole VARIABLE's equation + /// rather than one slot of an `Ast::Arrayed` one: dropping it there would + /// delete the variable, changing the emitted score set and the layout. + Materialize, + /// Drop the arm when the transformed partial is PROVABLY `PREVIOUS(target)` + /// ([`partial_is_provably_previous_target`], GH #977). The omitted slot is + /// then absent from the `Arrayed` element map, and + /// `compiler::expand_arrayed_with_hoisting` lowers an absent slot to a + /// single `AssignCurr(off, Const(0.0))` -- one opcode in place of a full + /// guard form that recomputes the same zero the long way. + /// + /// Sound ONLY when the target's `apply_default_to_missing` is FALSE. Under + /// EXCEPT semantics an absent slot picks up the DEFAULT equation instead of + /// zero, so an omitted arm would silently take the default's value. + /// [`super::build_arrayed_link_score_equation`] enforces that, being the only + /// caller that knows the target's flag. + /// + /// This is a BIT-EXACT transformation, and that is the whole point of the + /// positive predicate. The tempting negative test -- "the link's source + /// stayed frozen" -- says nothing about what else the arm reads, and + /// collapsing on it changes 187 C-LEARN result slots across 35 link-score + /// variables (151 by >= 1.0, worst 8,086.97 -> 0), because the wrap does not + /// freeze everything that varies: a live `time()` remains, and a + /// raw-vs-canonical element-spelling mismatch can leave the source itself + /// unwrapped. Those are tracked as #1016 and the wrap defects in #977; this + /// predicate is correct whether or not they are fixed, because it asks about + /// the emitted tree rather than about the wrap's bookkeeping. + OmitStructuralZero, +} + +/// Whether a walk established that a subtree cannot vary between the previous +/// and current step. +/// +/// A named verdict rather than a `bool` because the failure mode this predicate +/// exists to prevent is a match arm that inspects a node and then neglects to +/// decide (GH #977: a prior negative-criterion collapse was withdrawn after +/// seven adversarial review rounds, each finding a different node whose +/// "trivial" arm was not actually zero). Every arm below returns one of these, +/// the argument walk is a verdict-returning fold rather than a unit-returning +/// callback, and the matches carry no catch-all -- so a new `Expr0` variant is a +/// compile error rather than a silent `Established`. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Reach { + /// Every leaf reachable here is a literal or sits inside a frozen subtree. + Established, + /// Something reachable here can differ between steps -- or the walk could + /// not prove otherwise, which is the same answer. + NotEstablished, +} + +impl Reach { + /// `Established` iff both halves are. The fold's combining step, named so + /// the walk never open-codes the conjunction. + fn and(self, other: Reach) -> Reach { + match (self, other) { + (Reach::Established, Reach::Established) => Reach::Established, + (Reach::NotEstablished, _) | (_, Reach::NotEstablished) => Reach::NotEstablished, + } + } +} + +/// How a builtin call bears on the walk. The classification is by NAME, so it +/// cannot be exhaustive over a type -- which is exactly why the unrecognized +/// case is a named variant decided at the match below rather than a fall-through. +#[derive(Clone, Copy, PartialEq, Eq)] +enum BuiltinReach { + /// The call's contents are read at the PREVIOUS step, so the subtree is + /// frozen whatever it contains and the walk must NOT descend into it. + FrozenSubtree, + /// Deterministic in its arguments and independent of the step, so the + /// verdict is the fold over its arguments. + PureInArgs, + /// Reads the clock, a table, or something otherwise unrecognized. Either + /// way the walk cannot establish invariance. + Varying, +} + +/// Classify a builtin by name for [`partial_is_provably_previous_target`]. +/// +/// `lookup` is deliberately `Varying` even though a graphical function is a +/// compile-time constant: it would only matter for an arm whose lookup index is +/// itself invariant, and GH #977 measured that relaxation as buying **exactly +/// zero** additional arms on C-LEARN (those arms hit a live `time()` inside the +/// lookup's own index immediately afterwards). Conservative and free. +fn classify_builtin_reach(name: &str) -> BuiltinReach { + // Lowercased at parse time, but classify case-insensitively so a future + // caller with raw source spelling cannot silently fall into `Varying`. + let lowered = name.to_ascii_lowercase(); + match lowered.as_str() { + "previous" | "init" => BuiltinReach::FrozenSubtree, + "abs" | "arccos" | "arcsin" | "arctan" | "cos" | "exp" | "inf" | "int" | "ln" | "log10" + | "max" | "min" | "pi" | "safediv" | "sign" | "sin" | "sqrt" | "tan" => { + BuiltinReach::PureInArgs + } + // Everything else -- `time`, `dt`, `initial_time`, `final_time`, `step`, + // `ramp`, `pulse`, `lookup`, the stateful macros, and any builtin added + // after this was written -- cannot be established as invariant here. + _ => BuiltinReach::Varying, + } +} + +/// Is `partial` provably equal to `PREVIOUS(target)` -- i.e. does it recompute +/// the target from inputs that cannot have changed since the previous step? +/// +/// This is the soundness condition for dropping an `Ast::Arrayed` link-score arm +/// (GH #977). The arm's numerator is `partial - PREVIOUS(target)`; when the +/// partial reads nothing that varies, it reproduces the value that PRODUCED +/// `PREVIOUS(target)`, the numerator is identically zero, and an absent slot's +/// `AssignCurr(off, Const(0.0))` computes the same thing for one opcode. +/// +/// The test is POSITIVE -- "everything reachable outside a frozen subtree is a +/// literal" -- rather than the negative "the link's source stayed frozen". The +/// negative form asks a different question, one that says nothing about the rest +/// of the arm; see [`ZeroSlotPolicy::OmitStructuralZero`] for what that costs. +/// +/// A `Var` or `Subscript` reached outside a frozen subtree is a live read and +/// ends the walk, which is why subscript INDICES are never descended into: the +/// whole reference is already `NotEstablished`, so `IndexExpr0` needs no arm +/// here and a new index variant cannot change any verdict. +pub(super) fn partial_is_provably_previous_target(partial: &Expr0) -> bool { + reach_of(partial) == Reach::Established +} + +fn reach_of(expr: &Expr0) -> Reach { + match expr { + Expr0::Const(..) => Reach::Established, + // A live read of model state: the value it yields this step is exactly + // what the wrap was supposed to freeze and did not. + Expr0::Var(..) => Reach::NotEstablished, + Expr0::Subscript(..) => Reach::NotEstablished, + Expr0::Op1(_, inner, _) => reach_of(inner), + Expr0::Op2(_, lhs, rhs, _) => reach_of(lhs).and(reach_of(rhs)), + Expr0::If(cond, then, other, _) => reach_of(cond).and(reach_of(then)).and(reach_of(other)), + Expr0::App(UntypedBuiltinFn(name, args), _) => match classify_builtin_reach(name) { + // Do NOT descend: the contents are read at the previous step, so + // whatever they reference is frozen by construction. + BuiltinReach::FrozenSubtree => Reach::Established, + BuiltinReach::PureInArgs => args + .iter() + .fold(Reach::Established, |acc, arg| acc.and(reach_of(arg))), + BuiltinReach::Varying => Reach::NotEstablished, + }, + } +} diff --git a/src/simlin-engine/tests/integration/simulate_ltm.rs b/src/simlin-engine/tests/integration/simulate_ltm.rs index 7a5cf6b67..b26831808 100644 --- a/src/simlin-engine/tests/integration/simulate_ltm.rs +++ b/src/simlin-engine/tests/integration/simulate_ltm.rs @@ -7266,9 +7266,20 @@ fn build_disjoint_dim_unscoreable_model(name: &str) -> simlin_engine::datamodel: /// `Equation::Arrayed` over `target`'s dims (`["D1","D2"]`); the `[a,x]` slot /// of the `source[m]→target` var holds `source[m]` live (its partial differs /// from `PREVIOUS`-evaluated) and the `[a,y]` slot (references `source[n]`, -/// not `m`) is the trivial-zero guard form; and running the VM, the +/// not `m`) scores a structural zero; and running the VM, the /// `source[m]→target` link score is non-zero at the `[a,x]` slot at some step -/// >= 2 and ~0 at `[a,y]` at every step >= 2. +/// >= 2 and zero at `[a,y]` at every step >= 2. +/// +/// The `[a,y]` slot's INSTRUMENT moved with GH #977 and its claim did not. It +/// used to be a materialized guard form whose ratio evaluated to a trivial +/// zero; that partial is provably `PREVIOUS(target)`, so the slot is now +/// OMITTED from the element map and `compiler::expand_arrayed_with_hoisting` +/// lowers it to a single constant-zero assign. The VM assertion below is +/// therefore tightened from "~0" to exactly zero -- what the omission promises, +/// and the check that would catch it dropping a slot that was not a structural +/// zero. `[b,y]` stays materialized on the same variable, which is what keeps +/// this from degenerating into "every non-`[a,x]` slot vanishes": its equation +/// multiplies a frozen `source[n]` by a LIVE `source[m]`. #[test] fn test_disjoint_dim_arrayed_target_per_source_element_link_scores() { let project = build_disjoint_dim_arrayed_target_model("disjoint_dim_arrayed"); @@ -7327,20 +7338,30 @@ fn test_disjoint_dim_arrayed_target_per_source_element_link_scores() { .unwrap_or_else(|| panic!("slot {elem:?} not found in {elements:?}")) }; let ax = slot("a,x"); - let ay = slot("a,y"); assert!( ax.contains("source[m]"), "the [a,x] slot of source[m]→target should reference source[m] live; got: {ax}" ); - // The [a,y] slot's partial: every source reference is `source[n]`, - // which for the `source[m]` link score is "other content" and gets - // PREVIOUS-frozen, so the partial equals PREVIOUS(target[a,y]) and - // the guarded ratio is the trivial-zero form. (We don't pin the - // exact text -- the VM check below is the substantive one -- but it - // must not hold `source[m]` live.) + // The [a,y] slot's every source reference is `source[n]`, which for + // the `source[m]` link score is "other content" and gets + // PREVIOUS-frozen -- so the partial IS `PREVIOUS(target[a,y])` and + // the slot is omitted rather than materialized (GH #977). Absence + // is the distinct omission marker; an arm present holding a `"0"` + // partial would be a generator giving up, and the two must stay + // distinguishable. + assert!( + !elements.iter().any(|(e, _)| e == "a,y"), + "the [a,y] slot scores a structural zero and must be OMITTED, \ + not materialized; got slots {:?}", + elements.iter().map(|(e, _)| e.as_str()).collect::>() + ); + // The counterweight: `[b,y]` multiplies a frozen `source[n]` by a + // LIVE `source[m]`, so it must survive. Without this, "every slot + // but [a,x] disappeared" would pass. + let by = slot("b,y"); assert!( - !ax.contains("source[n]") || ay.contains("PREVIOUS(source[n]"), - "sanity: [a,y] slot freezes source[n] for the source[m] link score; got: {ay}" + by.contains("PREVIOUS(source[d3\u{B7}n]"), + "[b,y] freezes source[n] for the source[m] link score; got: {by}" ); } other => panic!("expected Equation::Arrayed for source[m]→target, got {other:?}"), @@ -7373,9 +7394,13 @@ fn test_disjoint_dim_arrayed_target_per_source_element_link_scores() { if ax_val.abs() > 1e-9 && ax_val.is_finite() { saw_ax_nonzero = true; } - assert!( - ay_val.abs() < 1e-6, - "step {step}: source[m]→target [a,y] slot should be ~0 (it references source[n], not m); got {ay_val}" + // Exactly zero, not merely small: the omitted slot lowers to a single + // `AssignCurr(off, Const(0.0))`, so any nonzero here means the omission + // claimed a slot that was not a structural zero. + assert_eq!( + ay_val, 0.0, + "step {step}: source[m]→target [a,y] slot is an omitted structural \ + zero (it references source[n], not m); got {ay_val}" ); checked += 1; } From ea342b2c01eab91120f641057dbfaac63cefec58 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:06:32 -0700 Subject: [PATCH 22/59] engine: fold constant operands into LoadPrev and Apply `Opcode::LoadPrev` pops its PREVIOUS() fallback off the arithmetic stack, so codegen emits a `LoadConstant` immediately before every one: on C-LEARN 171,120 of 177,106 LoadPrev sites (96.6%) are preceded by a literal-0 load that exists only to be popped. `LoadPrevConst { off, lit }` reads the fallback from the literal table instead, folding the pair 2->1. Neither existing window can reach it -- the 3- and 2-window combiners are `Op2`/`BinOpAssign`, and `LoadPrev` is neither a combiner nor a leaf load, since it pops. `ApplyTerConst { func, lit }` does the same for a 3-arity builtin whose trailing argument is a literal. This is NOT the operand padding removed in 2d34ba48: that commit established the arity is a property of the builtin and stopped emitting pads, but a 3-arity builtin's third operand is a real value the builtin reads -- for SAFEDIV it is the divide-by-zero result, which the LTM guard form supplies as a literal at 21,040 sites on C-LEARN. That load survives the arity fix and is still worth folding. The `arity() == 3` guard is load-bearing rather than a narrowing: for a 1- or 2-arity builtin the preceding `LoadConstant` is one of the operands the builtin actually reads, so folding it as a "third operand" would consume a real argument and leave the stack short. A test pins that. Both are late-fusion forms, created only by `ByteCode::fuse_three_address` on the Vm's private execution copy. They never enter the symbolic layer and never reach wasmgen, which lowers the pre-fusion bytecode. Both absorbed instructions are guarded against jump targets like every other fusion in the pass, and both fused forms LOWER the peak stack depth (an operand that no longer transits the stack), which is the safe direction for `resolve_bytecode`'s fixed-stack proof -- that proof is computed on the pre-fusion stream, so fusion must never raise the peak. Measured against e74d4d69, retired instructions per run and post-fusion flow opcodes (instructions have an across-build sd of 0.026%, so one build pair resolves an effect this size): C-LEARN +LTM 920,966 -> 728,352 opcodes -14.15% WORLD3 +LTM 17,415 -> 14,041 opcodes -16.29% C-LEARN -LTM -0.41% Non-LTM barely moves because that program has few PREVIOUS sites; this is an LTM-shaped win. Behaviour-preserving: 5487 engine lib tests and 632 integration tests pass, including `clearn_residual_exactness`, `oracle_clearn`, and `vdf_parity`. --- src/simlin-engine/src/bytecode.rs | 147 +++++++++++++++++++++++++++++- src/simlin-engine/src/vm.rs | 16 ++++ 2 files changed, 162 insertions(+), 1 deletion(-) diff --git a/src/simlin-engine/src/bytecode.rs b/src/simlin-engine/src/bytecode.rs index f2f5d5213..6ade16843 100644 --- a/src/simlin-engine/src/bytecode.rs +++ b/src/simlin-engine/src/bytecode.rs @@ -725,6 +725,26 @@ pub(crate) enum Opcode { LoadPrev { off: VariableOffset, }, + /// Fused `LoadConstant lit; LoadPrev off`. + /// + /// `LoadPrev` pops its `PREVIOUS()` fallback off the arithmetic stack, so + /// codegen emits a `LoadConstant` immediately before every one. Reading the + /// fallback from the literal table instead folds the pair into one dispatch. + LoadPrevConst { + off: VariableOffset, + lit: LiteralId, + }, + /// Fused `LoadConstant lit; Apply` for a 3-arity builtin whose trailing + /// argument is a literal. + /// + /// This is NOT the operand padding `Apply` once carried: the arity is a + /// property of the builtin and no pads are emitted. A 3-arity builtin's + /// third operand is a value it reads -- for `SAFEDIV` it is the + /// divide-by-zero result -- so the load survives and is worth folding. + ApplyTerConst { + func: BuiltinId, + lit: LiteralId, + }, /// Load the initial (t=0) value of a variable from the initial-value buffer. /// Pushes `initial_values[module_off + off]` onto the stack. LoadInitial { @@ -1451,6 +1471,12 @@ impl Opcode { // LoadPrev pops the caller-provided fallback, then pushes // either the fallback (at t=INITIAL_TIME) or prev_values[off]. Opcode::LoadPrev { .. } => (1, 1), + // The fused `LoadConstant; LoadPrev` pair: the fallback comes from + // the literal table, so nothing is popped. + Opcode::LoadPrevConst { .. } => (0, 1), + // The fused `LoadConstant; Apply` pair for a 3-arity builtin: two + // operands still come off the stack, the third from the literals. + Opcode::ApplyTerConst { .. } => (2, 1), // Legacy subscript: PushSubscriptIndex pops an index from the // arithmetic stack and appends it to a separate subscript_index @@ -1635,6 +1661,8 @@ impl Opcode { Opcode::LoadVar { .. } => "LoadVar", Opcode::LoadGlobalVar { .. } => "LoadGlobalVar", Opcode::LoadPrev { .. } => "LoadPrev", + Opcode::LoadPrevConst { .. } => "LoadPrevConst", + Opcode::ApplyTerConst { .. } => "ApplyTerConst", Opcode::LoadInitial { .. } => "LoadInitial", Opcode::PushSubscriptIndex { .. } => "PushSubscriptIndex", Opcode::LoadSubscript { .. } => "LoadSubscript", @@ -2317,7 +2345,22 @@ impl ByteCode { Opcode::AssignStackConstCurr { dst, b: *b, op } } }) - .or_else(|| push2.map(|op| Opcode::BinStackConst { r: *b, op })), + .or_else(|| push2.map(|op| Opcode::BinStackConst { r: *b, op })) + // A trailing constant that is not a binop operand: the + // `PREVIOUS()` fallback `LoadPrev` pops, or the third + // operand of a 3-arity builtin. + .or_else(|| match &self.code[i + 1] { + Opcode::LoadPrev { off } => { + Some(Opcode::LoadPrevConst { off: *off, lit: *b }) + } + Opcode::Apply { func } if func.arity() == 3 => { + Some(Opcode::ApplyTerConst { + func: *func, + lit: *b, + }) + } + _ => None, + }), // `(lhs on stack) op global`. No global stack-leaf-assign // opcode exists (the global ops are pushing-only), so a // `BinOpAssign` combiner (push2 == None) is left unfused: the @@ -4262,6 +4305,108 @@ mod tests { assert_eq!(bc.max_stack_depth().unwrap(), 0); } + // === Trailing-constant fusion (PREVIOUS fallback, 3-arity builtin) === + // + // Neither pattern is a binop, so no existing window can reach either: the + // 3- and 2-window combiners are `Op2`/`BinOpAssign`, and `LoadPrev`/`Apply` + // are neither. Both absorbed instructions are guarded against jump targets + // like every other fusion in the pass. + + #[test] + fn test_fuse_previous_fallback_into_load() { + // `PREVIOUS(v)` compiles to `LoadConstant ; LoadPrev v`. + let mut bc = ByteCode { + literals: vec![0.0], + code: vec![Opcode::LoadConstant { id: 0 }, Opcode::LoadPrev { off: 7 }], + }; + let before = bc.max_stack_depth().unwrap(); + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 1); + assert!(matches!( + bc.code[0], + Opcode::LoadPrevConst { off: 7, lit: 0 } + )); + // The fused form pushes the same single value the pair did. + assert_eq!(bc.max_stack_depth().unwrap(), before); + } + + #[test] + fn test_fuse_previous_fallback_blocked_by_jump_target() { + // A jump landing on the `LoadPrev` means the pair is not a unit. + let mut bc = ByteCode { + literals: vec![0.0], + code: vec![ + Opcode::LoadConstant { id: 0 }, + Opcode::LoadPrev { off: 7 }, + Opcode::NextIterOrJump { jump_back: -1 }, + ], + }; + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 3); + assert!(matches!(bc.code[0], Opcode::LoadConstant { .. })); + assert!(matches!(bc.code[1], Opcode::LoadPrev { .. })); + } + + #[test] + fn test_fuse_third_operand_of_three_arity_builtin() { + // `SAFEDIV(a, b, 0)` -- the trailing literal IS the divide-by-zero + // result, an operand `apply` reads, not padding. + let mut bc = ByteCode { + literals: vec![0.0], + code: vec![ + Opcode::LoadVar { off: 1 }, + Opcode::LoadVar { off: 2 }, + Opcode::LoadConstant { id: 0 }, + Opcode::Apply { + func: BuiltinId::SafeDiv, + }, + ], + }; + let before = bc.max_stack_depth().unwrap(); + bc.fuse_three_address(); + assert!( + bc.code.iter().any(|op| matches!( + op, + Opcode::ApplyTerConst { + func: BuiltinId::SafeDiv, + lit: 0 + } + )), + "got {:?}", + bc.code.iter().map(|o| o.name()).collect::>() + ); + // The third operand no longer transits the stack, so the peak DROPS. + // Never rising is what keeps `resolve_bytecode`'s fixed-stack proof -- + // computed on the pre-fusion stream -- valid for what the Vm executes. + assert!(bc.max_stack_depth().unwrap() <= before); + } + + #[test] + fn test_trailing_constant_not_fused_into_lower_arity_builtin() { + // The guard is `arity() == 3`, and it is load-bearing: for a 1- or + // 2-arity builtin the preceding `LoadConstant` is one of the operands + // the builtin actually reads, so folding it as a "third operand" would + // consume a real argument and leave the stack short. + for func in [BuiltinId::Abs, BuiltinId::Max] { + let mut bc = ByteCode { + literals: vec![3.0], + code: vec![ + Opcode::LoadVar { off: 1 }, + Opcode::LoadConstant { id: 0 }, + Opcode::Apply { func }, + ], + }; + bc.fuse_three_address(); + assert!( + !bc.code + .iter() + .any(|op| matches!(op, Opcode::ApplyTerConst { .. })), + "{func:?} (arity {}) must not take the ApplyTerConst form", + func.arity() + ); + } + } + // === 3-address fusion with GLOBAL operands and two-constant operands === // // Globals (TIME/DT/...) load via `LoadGlobalVar`; the fusion now folds them diff --git a/src/simlin-engine/src/vm.rs b/src/simlin-engine/src/vm.rs index bcc0cd5a0..024293461 100644 --- a/src/simlin-engine/src/vm.rs +++ b/src/simlin-engine/src/vm.rs @@ -2041,6 +2041,22 @@ impl Vm { // sole mechanism -- it replaces the old TIME == INITIAL_TIME // check, which broke when RK stages advanced TIME to trial // points before prev_values was initialized. + Opcode::LoadPrevConst { off, lit } => { + let value = if use_prev_fallback { + bytecode.literals[*lit as usize] + } else { + prev_values[module_off + *off as usize] + }; + stack.push(value); + } + Opcode::ApplyTerConst { func, lit } => { + let time = curr[TIME_OFF]; + let dt = curr[DT_OFF]; + let c = bytecode.literals[*lit as usize]; + let b = stack.pop(); + let a = stack.pop(); + stack.push(apply(*func, time, dt, a, b, c)); + } Opcode::LoadPrev { off } => { let fallback = stack.pop(); let value = if use_prev_fallback { From c07afbff3ac69543a2e779efdbfbde3709f4ab02 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:16:31 -0700 Subject: [PATCH 23/59] engine: FxHash the topological-sort probe maps `topo_sort_str` and its `build_scc_grouping` helper kept four probe-only collections on `std`'s default SipHash: the allowed-name set, the visited set, and the two resolved-SCC lookup maps. The sort runs once per phase per model per module-input set and probes them once per dependency edge, which on C-LEARN is 136,116 hashes per compile -- the largest single SipHash site left in the compile path. All four are probed by key and never iterated for output (`scc_members`' values are pre-sorted `Vec`s, and `root_shifted`-style map-to-map copies do not depend on order), so the hasher is invisible to the result. The runlists this produces are byte-stable for the reason they already were: the visit order is a pre-sorted `names` list and each dependency set is a `BTreeSet`. FxHash's fixed seed additionally makes these maps' iteration order reproducible across processes, which is the direction GH #595 wants; the `IdentMap` alias's rustdoc carries the constraint this obeys -- the keys are variable names out of a model file, supplied by the party paying for the compile. Measured on C-LEARN (retired instructions, interleaved A/B, three rounds at load average 3.7-10.3): 16.283G -> 15.980G over nine compiles = -33.7M instructions per compile, -1.8%. The compiled artifact is unchanged. --- src/simlin-engine/src/db/dep_graph.rs | 50 +++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/simlin-engine/src/db/dep_graph.rs b/src/simlin-engine/src/db/dep_graph.rs index 6e4a76677..f01e72f8d 100644 --- a/src/simlin-engine/src/db/dep_graph.rs +++ b/src/simlin-engine/src/db/dep_graph.rs @@ -2339,38 +2339,38 @@ pub(crate) fn model_dependency_graph_impl( // `Dt`-phase aux SCC's members carry the SAME recurrence in their init // equations AND an `Initial`-phase SCC obviously recurs, so BOTH // phases are grouped for the initials runlist. - let build_scc_grouping = |only_dt: bool| -> (HashMap<&str, usize>, HashMap>) { - let mut scc_of: HashMap<&str, usize> = HashMap::new(); - let mut scc_members: HashMap> = HashMap::new(); - for (idx, scc) in resolved_sccs.iter().enumerate() { - if only_dt && scc.phase != SccPhase::Dt { - continue; - } - // `scc.members` is a BTreeSet, so this member list is sorted - // and byte-stable. - let members: Vec<&str> = scc.members.iter().map(|m| m.as_str()).collect(); - for m in &members { - scc_of.insert(*m, idx); + let build_scc_grouping = + |only_dt: bool| -> (FxHashMap<&str, usize>, FxHashMap>) { + let mut scc_of: FxHashMap<&str, usize> = FxHashMap::default(); + let mut scc_members: FxHashMap> = FxHashMap::default(); + for (idx, scc) in resolved_sccs.iter().enumerate() { + if only_dt && scc.phase != SccPhase::Dt { + continue; + } + // `scc.members` is a BTreeSet, so this member list is sorted + // and byte-stable. + let members: Vec<&str> = scc.members.iter().map(|m| m.as_str()).collect(); + for m in &members { + scc_of.insert(*m, idx); + } + scc_members.insert(idx, members); } - scc_members.insert(idx, members); - } - (scc_of, scc_members) - }; + (scc_of, scc_members) + }; let (flows_scc_of, flows_scc_members) = build_scc_grouping(true); let (init_scc_of, init_scc_members) = build_scc_grouping(false); let topo_sort_str = |names: Vec<&String>, deps: &HashMap, BTreeSet>>, - scc_of: &HashMap<&str, usize>, - scc_members: &HashMap>| + scc_of: &FxHashMap<&str, usize>, + scc_members: &FxHashMap>| -> Vec { - use std::collections::HashSet; // Build the allowed set: only variables in the filtered input list // should appear in the output. Dependencies are used solely for // ordering, not for expanding the set. - let allowed: HashSet<&str> = names.iter().map(|n| n.as_str()).collect(); + let allowed: FxHashSet<&str> = names.iter().map(|n| n.as_str()).collect(); let mut result: Vec = Vec::new(); - let mut used: HashSet = HashSet::new(); + let mut used: FxHashSet = FxHashSet::default(); // `deps` is now interned-keyed, but this sort still works in `&str` // space: probes go through `Borrow` and each dep-set iteration @@ -2379,11 +2379,11 @@ pub(crate) fn model_dependency_graph_impl( // `names`, same `BTreeSet` dep order). fn add( deps: &HashMap, BTreeSet>>, - allowed: &HashSet<&str>, - scc_of: &HashMap<&str, usize>, - scc_members: &HashMap>, + allowed: &FxHashSet<&str>, + scc_of: &FxHashMap<&str, usize>, + scc_members: &FxHashMap>, result: &mut Vec, - used: &mut HashSet, + used: &mut FxHashSet, name: &str, ) { if used.contains(name) { From eec0c81f3fd9ccf212b4e55450f0614ca72830bf Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:16:33 -0700 Subject: [PATCH 24/59] engine: fuse the PREVIOUS delta into a single opcode The LTM link-score guard computes `v - PREVIOUS(v)` four times per link -- twice for the target and twice for the source -- and on C-LEARN its 80,952 evaluations cover only 4,166 distinct deltas. Each is four dispatches (`LoadVar; LoadConstant; LoadPrev; Op2 Sub`), which no existing window matches: their combiner is an `Op2`/`BinOpAssign` over two leaf LOADS, and `LoadPrev` is not a leaf load -- it pops. `SubVarPrev { l, r, lit }` folds that 4-window to one dispatch, and `BinStackPrev { r, lit, op }` folds the 3-window `LoadConstant; LoadPrev; Op2` where the lhs is already on the stack. `SubVarPrev` keeps its operator in the variant tag (only Sub occurs) so the payload stays 3xu16 = 6 bytes and `size_of::()` stays at 8 -- the same trick `AssignSubVarVarCurr` uses. A test pins that any other operator falls through rather than being silently encoded as a subtraction. Both match the ORIGINAL windows rather than a `LoadPrevConst`-rewritten stream: `fuse_three_address` is a single greedy left-to-right pass, so when the window at i is tested, position i+1 has not been rewritten. They are therefore independent of `LoadPrevConst` and compose with it in either order -- the longer windows claim the sites they cover, and `LoadPrevConst` mops up the rest. Score a helper-variable proposal against the POST-fusion stream. Hoisting a repeated subexpression into a shared aux replaces each use with a `LoadVar` -- one dispatch, exactly what a fused opcode costs -- so the hoist is worth zero wherever a superinstruction can match the pattern, while still paying for a store and a slot. Scored against the pre-fusion stream the same hoist looks like a 3-to-1 win. Measured here: hoisting the delta is worth nothing next to `SubVarPrev`; hoisting `ABS`/`SIGN` of it is a net LOSS on WORLD3 (375 uses over 163 distinct); and hoisting `TIME = INITIAL_TIME` is worth nothing at all, because the pass already folds it to one `BinGlobalGlobal`. Measured on top of the previous commit, retired instructions per run and post-fusion flow opcodes: C-LEARN +LTM 728,352 -> 513,054 opcodes -16.32% WORLD3 +LTM 14,041 -> 9,896 opcodes -20.74% Cumulative over both commits, against e74d4d69: C-LEARN +LTM 920,966 -> 513,054 opcodes and -28.3% retired instructions; WORLD3 +LTM 17,415 -> 9,896 and -33.6%. Behaviour-preserving: the engine lib and integration suites pass, including `clearn_residual_exactness`, `oracle_clearn`, and `vdf_parity`. --- src/simlin-engine/src/bytecode.rs | 306 +++++++++++++++++++++++------- src/simlin-engine/src/vm.rs | 20 ++ 2 files changed, 256 insertions(+), 70 deletions(-) diff --git a/src/simlin-engine/src/bytecode.rs b/src/simlin-engine/src/bytecode.rs index 6ade16843..5979c237c 100644 --- a/src/simlin-engine/src/bytecode.rs +++ b/src/simlin-engine/src/bytecode.rs @@ -734,6 +734,24 @@ pub(crate) enum Opcode { off: VariableOffset, lit: LiteralId, }, + /// Fused `LoadVar l; LoadConstant lit; LoadPrev r; Op2 Sub` -- the delta + /// `v - PREVIOUS(v)`, four dispatches in one. + /// + /// The operator lives in the variant tag (only `Sub` occurs) so the payload + /// stays 3xu16 = 6 bytes and `size_of::()` stays at 8, the same + /// trick the `Assign{Add,Sub,Mul,Div}VarVar*` family uses. + SubVarPrev { + l: VariableOffset, + r: VariableOffset, + lit: LiteralId, + }, + /// Fused `LoadConstant lit; LoadPrev r; Op2 op` with the lhs already on the + /// arithmetic stack. + BinStackPrev { + r: VariableOffset, + lit: LiteralId, + op: Op2, + }, /// Fused `LoadConstant lit; Apply` for a 3-arity builtin whose trailing /// argument is a literal. /// @@ -1474,6 +1492,8 @@ impl Opcode { // The fused `LoadConstant; LoadPrev` pair: the fallback comes from // the literal table, so nothing is popped. Opcode::LoadPrevConst { .. } => (0, 1), + Opcode::SubVarPrev { .. } => (0, 1), + Opcode::BinStackPrev { .. } => (1, 1), // The fused `LoadConstant; Apply` pair for a 3-arity builtin: two // operands still come off the stack, the third from the literals. Opcode::ApplyTerConst { .. } => (2, 1), @@ -1662,6 +1682,8 @@ impl Opcode { Opcode::LoadGlobalVar { .. } => "LoadGlobalVar", Opcode::LoadPrev { .. } => "LoadPrev", Opcode::LoadPrevConst { .. } => "LoadPrevConst", + Opcode::SubVarPrev { .. } => "SubVarPrev", + Opcode::BinStackPrev { .. } => "BinStackPrev", Opcode::ApplyTerConst { .. } => "ApplyTerConst", Opcode::LoadInitial { .. } => "LoadInitial", Opcode::PushSubscriptIndex { .. } => "PushSubscriptIndex", @@ -2232,78 +2254,123 @@ impl ByteCode { // op (3->1) rather than `Bin*` (3->1 pushing) + a separate store. // Only {Add,Sub,Mul,Div} have dedicated leaf-assign opcodes; any // other operator falls through and keeps the existing form. + // 4-window: `LoadVar l; LoadConstant lit; LoadPrev r; Op2 Sub` -- + // the `v - PREVIOUS(v)` delta. Matched on the ORIGINAL stream: this + // is a single greedy left-to-right pass, so when the window at `i` + // is tested nothing at `i+1` has been rewritten yet. That is why + // this does not depend on `LoadPrevConst` having run first, and why + // the two are independent of each other. + let four = i + 3 < self.code.len() + && !jump_targets[i + 1] + && !jump_targets[i + 2] + && !jump_targets[i + 3]; + if four + && let ( + Opcode::LoadVar { off: l }, + Opcode::LoadConstant { id: lit }, + Opcode::LoadPrev { off: r }, + Opcode::Op2 { op: Op2::Sub }, + ) = ( + &self.code[i], + &self.code[i + 1], + &self.code[i + 2], + &self.code[i + 3], + ) + { + optimized.push(Opcode::SubVarPrev { + l: *l, + r: *r, + lit: *lit, + }); + pc_map.push(new_pc); // old i + pc_map.push(new_pc); // old i+1 + pc_map.push(new_pc); // old i+2 + pc_map.push(new_pc); // old i+3 + i += 4; + continue; + } + let three = i + 2 < self.code.len() && !jump_targets[i + 1] && !jump_targets[i + 2]; - let fused3 = if three { - // Decode the combiner once into two mutually-exclusive options: - // `assign3 = (op, dst, is_next)` for a leaf-assign, or - // `push3 = op` for a pushing Op2. - let (assign3, push3) = match &self.code[i + 2] { - Opcode::BinOpAssignCurr { op, off } => (Some((*op, *off, false)), None), - Opcode::BinOpAssignNext { op, off } => (Some((*op, *off, true)), None), - Opcode::Op2 { op } => (None, Some(*op)), - _ => (None, None), - }; - match (&self.code[i], &self.code[i + 1]) { - // Leaf assignment `dst = a op b` -> one fused op (3->1). - (Opcode::LoadVar { off: l }, Opcode::LoadVar { off: r }) => assign3 - .and_then(|(op, dst, n)| fused_leaf_var_var(op, n, *l, *r, dst)) - .or_else(|| push3.map(|op| Opcode::BinVarVar { l: *l, r: *r, op })), - (Opcode::LoadVar { off: l }, Opcode::LoadConstant { id: r }) => assign3 - .and_then(|(op, dst, n)| fused_leaf_var_const(op, n, *l, *r, dst)) - .or_else(|| push3.map(|op| Opcode::BinVarConst { l: *l, r: *r, op })), - (Opcode::LoadConstant { id: l }, Opcode::LoadVar { off: r }) => assign3 - .and_then(|(op, dst, n)| fused_leaf_const_var(op, n, *l, *r, dst)) - .or_else(|| push3.map(|op| Opcode::BinConstVar { l: *l, r: *r, op })), - // Two constant leaves: no leaf-assign form, so this only fuses - // a pushing `Op2`. Computes `literals[l] op literals[r]` at run - // time (NOT compile-time folding -- the operands are two - // distinct interned literals). - (Opcode::LoadConstant { id: l }, Opcode::LoadConstant { id: r }) => { - push3.map(|op| Opcode::BinConstConst { l: *l, r: *r, op }) - } - // Global-operand leaf pairs. A global has no dedicated - // leaf-assign opcode, so these fuse only a pushing `Op2`; a - // `BinOpAssign` combiner (push3 == None) falls through to the - // 2-window, which folds the rhs+store and leaves the global - // load as a standalone push. `l_global`/`r_global` index - // `curr[g]` (absolute), the var operand `curr[module_off + v]`. - (Opcode::LoadGlobalVar { off: l }, Opcode::LoadVar { off: r }) => { - push3.map(|op| Opcode::BinGlobalVar { - l_global: *l, - r: *r, - op, - }) - } - (Opcode::LoadVar { off: l }, Opcode::LoadGlobalVar { off: r }) => { - push3.map(|op| Opcode::BinVarGlobal { - l: *l, - r_global: *r, - op, - }) + let fused3 = + if three { + // Decode the combiner once into two mutually-exclusive options: + // `assign3 = (op, dst, is_next)` for a leaf-assign, or + // `push3 = op` for a pushing Op2. + let (assign3, push3) = match &self.code[i + 2] { + Opcode::BinOpAssignCurr { op, off } => (Some((*op, *off, false)), None), + Opcode::BinOpAssignNext { op, off } => (Some((*op, *off, true)), None), + Opcode::Op2 { op } => (None, Some(*op)), + _ => (None, None), + }; + match (&self.code[i], &self.code[i + 1]) { + // Leaf assignment `dst = a op b` -> one fused op (3->1). + (Opcode::LoadVar { off: l }, Opcode::LoadVar { off: r }) => assign3 + .and_then(|(op, dst, n)| fused_leaf_var_var(op, n, *l, *r, dst)) + .or_else(|| push3.map(|op| Opcode::BinVarVar { l: *l, r: *r, op })), + (Opcode::LoadVar { off: l }, Opcode::LoadConstant { id: r }) => assign3 + .and_then(|(op, dst, n)| fused_leaf_var_const(op, n, *l, *r, dst)) + .or_else(|| push3.map(|op| Opcode::BinVarConst { l: *l, r: *r, op })), + (Opcode::LoadConstant { id: l }, Opcode::LoadVar { off: r }) => assign3 + .and_then(|(op, dst, n)| fused_leaf_const_var(op, n, *l, *r, dst)) + .or_else(|| push3.map(|op| Opcode::BinConstVar { l: *l, r: *r, op })), + // Two constant leaves: no leaf-assign form, so this only fuses + // a pushing `Op2`. Computes `literals[l] op literals[r]` at run + // time (NOT compile-time folding -- the operands are two + // distinct interned literals). + // `LoadConstant lit; LoadPrev r; Op2` with the lhs already + // on the stack. Same reason as the 4-window above: matched + // against the original `LoadPrev`, not a rewritten form. + (Opcode::LoadConstant { id: lit }, Opcode::LoadPrev { off: r }) => push3 + .map(|op| Opcode::BinStackPrev { + r: *r, + lit: *lit, + op, + }), + (Opcode::LoadConstant { id: l }, Opcode::LoadConstant { id: r }) => { + push3.map(|op| Opcode::BinConstConst { l: *l, r: *r, op }) + } + // Global-operand leaf pairs. A global has no dedicated + // leaf-assign opcode, so these fuse only a pushing `Op2`; a + // `BinOpAssign` combiner (push3 == None) falls through to the + // 2-window, which folds the rhs+store and leaves the global + // load as a standalone push. `l_global`/`r_global` index + // `curr[g]` (absolute), the var operand `curr[module_off + v]`. + (Opcode::LoadGlobalVar { off: l }, Opcode::LoadVar { off: r }) => push3 + .map(|op| Opcode::BinGlobalVar { + l_global: *l, + r: *r, + op, + }), + (Opcode::LoadVar { off: l }, Opcode::LoadGlobalVar { off: r }) => push3 + .map(|op| Opcode::BinVarGlobal { + l: *l, + r_global: *r, + op, + }), + (Opcode::LoadGlobalVar { off: l }, Opcode::LoadConstant { id: r }) => push3 + .map(|op| Opcode::BinGlobalConst { + l_global: *l, + r: *r, + op, + }), + (Opcode::LoadConstant { id: l }, Opcode::LoadGlobalVar { off: r }) => push3 + .map(|op| Opcode::BinConstGlobal { + l: *l, + r_global: *r, + op, + }), + (Opcode::LoadGlobalVar { off: l }, Opcode::LoadGlobalVar { off: r }) => { + push3.map(|op| Opcode::BinGlobalGlobal { + l_global: *l, + r_global: *r, + op, + }) + } + _ => None, } - (Opcode::LoadGlobalVar { off: l }, Opcode::LoadConstant { id: r }) => push3 - .map(|op| Opcode::BinGlobalConst { - l_global: *l, - r: *r, - op, - }), - (Opcode::LoadConstant { id: l }, Opcode::LoadGlobalVar { off: r }) => push3 - .map(|op| Opcode::BinConstGlobal { - l: *l, - r_global: *r, - op, - }), - (Opcode::LoadGlobalVar { off: l }, Opcode::LoadGlobalVar { off: r }) => push3 - .map(|op| Opcode::BinGlobalGlobal { - l_global: *l, - r_global: *r, - op, - }), - _ => None, - } - } else { - None - }; + } else { + None + }; if let Some(op) = fused3 { optimized.push(op); pc_map.push(new_pc); // old i @@ -4407,6 +4474,105 @@ mod tests { } } + // === PREVIOUS-delta fusion (SubVarPrev / BinStackPrev) === + + #[test] + fn test_fuse_previous_delta_four_window() { + // `v - PREVIOUS(v)`, the shape the LTM link-score guard emits four + // times per link. + let mut bc = ByteCode { + literals: vec![0.0], + code: vec![ + Opcode::LoadVar { off: 5 }, + Opcode::LoadConstant { id: 0 }, + Opcode::LoadPrev { off: 5 }, + Opcode::Op2 { op: Op2::Sub }, + ], + }; + let before = bc.max_stack_depth().unwrap(); + bc.fuse_three_address(); + assert_eq!(bc.code.len(), 1); + assert!(matches!( + bc.code[0], + Opcode::SubVarPrev { l: 5, r: 5, lit: 0 } + )); + assert!(bc.max_stack_depth().unwrap() <= before); + } + + #[test] + fn test_previous_delta_only_fuses_subtraction() { + // The operator is in the variant tag, so only `Sub` has a fused form. + // Any other operator must fall through to the shorter windows rather + // than be silently encoded as a subtraction. + let mut bc = ByteCode { + literals: vec![0.0], + code: vec![ + Opcode::LoadVar { off: 5 }, + Opcode::LoadConstant { id: 0 }, + Opcode::LoadPrev { off: 5 }, + Opcode::Op2 { op: Op2::Add }, + ], + }; + bc.fuse_three_address(); + assert!( + !bc.code + .iter() + .any(|op| matches!(op, Opcode::SubVarPrev { .. })) + ); + } + + #[test] + fn test_fuse_previous_delta_blocked_by_jump_target() { + // A jump into the middle of the window means it is not a unit. + let mut bc = ByteCode { + literals: vec![0.0], + code: vec![ + Opcode::LoadVar { off: 5 }, + Opcode::LoadConstant { id: 0 }, + Opcode::LoadPrev { off: 5 }, + Opcode::Op2 { op: Op2::Sub }, + Opcode::NextIterOrJump { jump_back: -2 }, + ], + }; + bc.fuse_three_address(); + assert!( + !bc.code + .iter() + .any(|op| matches!(op, Opcode::SubVarPrev { .. })) + ); + } + + #[test] + fn test_fuse_previous_as_binop_rhs_on_stack() { + // lhs already on the stack: `; LoadConstant; LoadPrev; Op2`. + // Div is used because it is non-commutative, so a swapped operand + // encoding fails loudly rather than silently. + let mut bc = ByteCode { + literals: vec![0.0], + code: vec![ + Opcode::LoadVar { off: 1 }, + Opcode::LoadVar { off: 2 }, + Opcode::Op2 { op: Op2::Mul }, + Opcode::LoadConstant { id: 0 }, + Opcode::LoadPrev { off: 9 }, + Opcode::Op2 { op: Op2::Div }, + ], + }; + bc.fuse_three_address(); + assert!( + bc.code.iter().any(|op| matches!( + op, + Opcode::BinStackPrev { + r: 9, + lit: 0, + op: Op2::Div + } + )), + "got {:?}", + bc.code.iter().map(|o| o.name()).collect::>() + ); + } + // === 3-address fusion with GLOBAL operands and two-constant operands === // // Globals (TIME/DT/...) load via `LoadGlobalVar`; the fusion now folds them diff --git a/src/simlin-engine/src/vm.rs b/src/simlin-engine/src/vm.rs index 024293461..53fd51f7e 100644 --- a/src/simlin-engine/src/vm.rs +++ b/src/simlin-engine/src/vm.rs @@ -2041,6 +2041,26 @@ impl Vm { // sole mechanism -- it replaces the old TIME == INITIAL_TIME // check, which broke when RK stages advanced TIME to trial // points before prev_values was initialized. + Opcode::SubVarPrev { l, r, lit } => { + let lhs = curr[module_off + *l as usize]; + let rhs = if use_prev_fallback { + bytecode.literals[*lit as usize] + } else { + prev_values[module_off + *r as usize] + }; + // Through `eval_op2` so the fused form is bit-identical to + // the sequence by construction, not by inspection. + stack.push(eval_op2(Op2::Sub, lhs, rhs)); + } + Opcode::BinStackPrev { r, lit, op } => { + let rhs = if use_prev_fallback { + bytecode.literals[*lit as usize] + } else { + prev_values[module_off + *r as usize] + }; + let lhs = stack.pop(); + stack.push(eval_op2(*op, lhs, rhs)); + } Opcode::LoadPrevConst { off, lit } => { let value = if use_prev_fallback { bytecode.literals[*lit as usize] From 2e96af605bddbde9883663668abe1b8424d556fd Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:24:23 -0700 Subject: [PATCH 25/59] engine: answer the engine's own separators without the case tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `canonicalize`'s fused fast-path scan decodes every non-ASCII byte and asks `changes_when_lowercased` whether lowercasing would change it. The engine writes a non-ASCII character into identifiers itself -- the module-hierarchy separator `·` is in every `submodel·var` ident, and LTM's synthetic names carry `⁚` and `→` -- so that question was reaching the Unicode case tables 342,138 times per C-LEARN compile to re-derive that a middle dot is not an uppercase letter. Short-circuiting those three characters is sound only because the case tables agree, so `engine_separators_are_lowercase_invariant` asks the tables rather than restating the answer: it checks `char::to_lowercase` yields exactly the same single character for each, and reds if a separator is ever added to the list that lowercasing does change. Any character not listed still takes the general path, so this narrows the work without narrowing the domain. Measured on C-LEARN (retired instructions, interleaved A/B, three rounds at load average 5.6-9.8): 15.994G -> 15.590G over nine compiles = -44.9M instructions per compile, -2.4%. The compiled artifact is unchanged. --- src/simlin-engine/src/common.rs | 58 +++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/simlin-engine/src/common.rs b/src/simlin-engine/src/common.rs index 51188949b..70d769b5d 100644 --- a/src/simlin-engine/src/common.rs +++ b/src/simlin-engine/src/common.rs @@ -915,12 +915,37 @@ pub(crate) type IdentMap = std::collections::HashMap bool { + if is_engine_separator(c) { + return false; + } let mut lower = c.to_lowercase(); lower.next() != Some(c) || lower.next().is_some() } +/// The non-ASCII characters the engine writes into identifiers itself: the +/// module-hierarchy separator, and the two LTM synthetic-name separators. +/// +/// Listed here only as a fast path for [`changes_when_lowercased`]; membership +/// carries no meaning beyond "the case tables say this character is unchanged +/// by lowercasing, and it is common enough in our identifiers to be worth not +/// asking them". +#[inline] +fn is_engine_separator(c: char) -> bool { + matches!(c, '\u{00B7}' | '\u{205A}' | '\u{2192}') +} + /// Per-byte "this byte alone cannot make a name non-canonical" table, the /// fast path's whole decision. /// @@ -1593,6 +1618,39 @@ mod canonicalize_invariant_tests { } } + /// `changes_when_lowercased` short-circuits the characters the engine + /// mints into identifiers itself. The shortcut is only sound because the + /// Unicode case tables agree, so ask them here rather than asserting it: + /// this is the test that reds if a future separator is added to + /// `is_engine_separator` that lowercasing DOES change. + /// + /// Checked against the general path (`c.to_lowercase()`) rather than + /// against a hardcoded `false`, which would restate the shortcut instead + /// of verifying it. + #[test] + fn engine_separators_are_lowercase_invariant() { + for c in ['\u{00B7}', '\u{205A}', '\u{2192}'] { + assert!( + is_engine_separator(c), + "{c:?} must be on the fast path for this test to be checking it" + ); + let mut lower = c.to_lowercase(); + assert_eq!( + lower.next(), + Some(c), + "{c:?} lowercases to something else; the fast path in \ + changes_when_lowercased is unsound for it" + ); + assert_eq!( + lower.next(), + None, + "{c:?} lowercases to more than one character; the fast path in \ + changes_when_lowercased is unsound for it" + ); + assert!(!changes_when_lowercased(c)); + } + } + /// Hand-written cases for the interactions the fused ASCII rewrite has to /// get right, each of which composes two steps whose order matters. #[test] From b7ca9f84099258936203ab2181ef13398668c3d7 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:31:00 -0700 Subject: [PATCH 26/59] doc: record compile round 3 and the parallel fan-out constraints Three things this file could not tell a reader before. The compile-side proposals C1/C2/C3 were written against a pre-salsa profile and each is now answered, two of them differently than proposed: C2 is moot (`reconstruct_variable` is salsa-cached and off the ordinary compile path entirely), and C3's interning half is already done while its ASCII-fast-path half reaches only the 4.6% of `canonicalize` calls that allocate. Left as written, they would send the next reader at a correctness-critical function guarded by the GH #559 idempotence proptests to chase a twentieth of the cost that call elimination reached without touching it. C4 records the parallel fan-out: designed, prototyped and measured (2.23x achieved parallelism but only 1.34x wall, against an Amdahl ceiling of 1.44x over the then-serial 68%), and deliberately not implemented. It carries the structural constraint that decides its shape -- `salsa::Database` is `Send` but not `Sync`, so the fan-out cannot live inside the query graph at all -- and the two hazards that were found by running the suite rather than by thinking: a prewarm placed ahead of the module-cycle gate reopens GH #806's process abort, and an ungated prewarm regresses the fully-cached recompile 2.5-4x. Both are silent. Determinism, the hazard that was expected, is recorded as measured-absent so nobody spends the round re-establishing it. C5 records why the top allocation site was left alone: `NameId` assignment order is part of the compiled artifact. The round-3 section states its two findings as standing constraints rather than as a list of fixes -- a per-variable helper needs a key of its own, and a projection is what keeps a per-variable query per-variable -- because both are cheap to violate and neither is visible in a diff that violates them. --- docs/design/engine-performance.md | 163 +++++++++++++++++++++++++++++- 1 file changed, 160 insertions(+), 3 deletions(-) diff --git a/docs/design/engine-performance.md b/docs/design/engine-performance.md index 15d8c5e25..daa6afbbc 100644 --- a/docs/design/engine-performance.md +++ b/docs/design/engine-performance.md @@ -1,7 +1,8 @@ # Engine performance: profile and optimization opportunities -Status: analysis + two rounds of wins landed. Round 1 2026-05-19; round 2 -(constant folding + linear-run fast paths, below) 2026-06-03. +Status: analysis + three rounds of wins landed. Round 1 2026-05-19; round 2 +(constant folding + linear-run fast paths) 2026-06-03; compile round 3 (the +salsa pipeline's own redundancy) 2026-08-10. This documents an empirical CPU/memory profile of **compiling and simulating the C-LEARN hero model** (the largest model we have: ~53k MDL lines / 1.4 MB, 934 @@ -356,6 +357,17 @@ changes**. The following are second-order and worth it only if compile latency remains a UX problem after the build levers (it matters for the salsa *incremental* edit loop more than cold compile). +### C1. Arena-allocate the transient parse AST — NOT the dominant allocator + +Re-measured after compile round 3: the parser is no longer where the +allocations are. Per cold C-LEARN compile, `Expr0::clone` accounts for 212,184 +allocations (3.4% of compile instructions) and the `Expr0`/`Expr2`/`Expr3` drop +glue for ~7% — so an arena is worth ~10% for a large, medium-risk change, and +the top allocation site is not the parser at all but `Compiler::intern_name` +(320,650 allocations per compile, ~10% of all 3.24M; see C5). The original +figure below (3.86M transient allocations) predates the salsa pipeline and no +longer describes the code. + ### C1. Arena-allocate the transient parse AST The equation parser builds `Expr0` with `Box` children + `Vec` args — 3.86M+ @@ -370,6 +382,15 @@ only if profiling after B still shows the parser as a hotspot. - Effort: large (thread an arena through the parser; verify nothing cached retains an arena reference). Risk: medium. +### C2. Halve `reconstruct_variable` — MOOT + +`reconstruct_variable` is now the salsa-cached `reconstruct_model_variables`, +and every caller is on the LTM / analysis / patch path; it does not appear in +an ordinary compile profile at all. The 2x duplication that WAS real, and is +fixed, was a different function: `variable_dimensions` demanded the per-variable +parse under an empty `ModuleIdentContext`, a cache key nothing else used, so +every variable was parsed twice per compile. + ### C2. Halve `reconstruct_variable` (6.4% of compile) `reconstruct_variable` rebuilds a full `datamodel::Variable` (ident/equation/ @@ -382,6 +403,31 @@ avoid ~half the full reconstructions (and their clones). - Effort: medium. Risk: low–medium (changes the `collect_module_idents` input type; behavior must stay identical). +### C3. `canonicalize` — the lever is call elimination, not a faster slow path + +`canonicalize` is still the largest non-allocator cost of a cold compile, but +neither half of the proposal below is the way to reduce it, and the reason is +worth keeping because the profile invites the wrong conclusion. + +**(b) interning is done.** `Ident` is a 64-shard-interned `Arc` +(`common.rs`): `Clone` is a refcount bump and `PartialEq` is pointer equality. + +**(a) an ASCII fast path exists and already carries almost all traffic.** +`is_canonical_needing_no_trim` is a single-pass byte-table scan returning +`Cow::Borrowed`, measured at ~90 instructions per call at the hottest site. +Only **4.6% of calls allocate** (99,322 of 2,146,745 per compile), so making +the slow path cheaper cannot reach the other 95.4% — while rewriting it puts +the GH #559 idempotence proptests, which guard the Unicode arm (titlecase, +U+00A0, quoted sections, backslash unescaping), at risk for that 4.6%. + +**What works is not calling it.** Half of all calls came from one predicate +re-canonicalizing names that are canonical by construction; deleting that inner +call, which touches `canonicalize` not at all, measured −5.0% of a cold compile. +The residual worth having is narrower still: `changes_when_lowercased` is asked +about the separators the engine itself mints (`·` in every `submodel·var` +ident), which is answered from a three-character list rather than the Unicode +case tables. + ### C3. `canonicalize` ASCII fast-path + ident interning 6.1M `to_lowercase` calls; ~4.6M are the `canonicalize` slow path (Vensim names @@ -395,6 +441,109 @@ re-derivation. (b) is broader but touches many call sites. - Effort: (a) small/careful, (b) medium–large. Risk: (a) medium (correctness- critical function), (b) medium. +### Compile round 3 (2026-08-10): the salsa pipeline's own redundancy + +Cold C-LEARN compile 2.119G -> 1.602G retired instructions, **−24.4%**, and a +warm single-equation edit **−47%** (median wall 38 ms -> 4.3 ms). Every change +is artifact-identical: 5215 slots, 58291 opcodes (31525 flow + 1477 stock + +25289 initial), same literal / GF / temp / dimension / view / name / module +counts. Measured as retired instructions throughout, because the machine was +contended and the cycles channel cannot resolve effects this size there. + +What the round found, stated as the standing shape of the problem rather than +as five fixes: **the cold compile's redundancy was in the salsa layer's own +keying, not in the compiler.** Four of the five were a query being asked a +question it had already answered, under a key that did not say so. + +| what | mechanism | share of cold compile | +|---|---|---| +| `is_dimension_name` | re-canonicalized every declared dimension name per call | −5.0% | +| `variable_dimensions` | demanded the parse under an empty `ModuleIdentContext` -> every variable parsed twice | −3.5% | +| `compile_implicit_var_fragment` | not tracked: every SMOOTH/DELAY/TREND helper recompiled per assembly | −12% cold, **−28% of a warm edit** | +| `var_phase_symbolic_fragment_prod` | not tracked: cycle gate built 135 fragments per compile for 57 distinct keys | −14.3% | +| topo-sort probe maps, `changes_when_lowercased` | SipHash and Unicode tables on the engine's own idents | −1.8%, −2.4% | + +Two constraints follow, and both are cheap to violate: + +- **A per-variable helper needs a per-variable key.** The two biggest wins were + functions whose comment said salsa already cached them, because their *parse* + was cached. Lowering and codegen are the expensive half and were not. When + adding a per-variable compiler, the question is not "is something upstream + memoized" but "does this function have a key of its own". +- **A projection is what keeps a per-variable query per-variable.** Both new + queries read a three-bit `RunlistMembership` rather than the whole + `ModelDepGraphResult`; taking the whole result would re-execute every + fragment whenever any variable's dependencies moved, silently restoring the + coarseness the key was introduced to remove. + +### C4. Parallel fan-out of per-variable fragment compilation — designed and measured, NOT implemented + +The compile is **exactly serial** (`task-clock` / `elapsed` = 1.000 over two +independent measurements). A prototype fan-out was built and measured on +C-LEARN before round 3 landed; it is not in the tree, and these are the facts +whoever implements it needs so they are not rediscovered. + +**Achievable, and bounded well below the core count.** Staged prewarm (parse + +dependency memos, then per-variable fragments) reached **2.23x achieved +parallelism but only 1.34x wall speedup** (132.7 -> 99.3 ms), at +12% retired +instructions. The ceiling is a property of the query decomposition: at the time +of measurement `model_dependency_graph` (35.5% of compile) was one query per +`(model, input-set)` and could not be split by variable, `compile_implicit_var_fragment` +(12.2%) had no key to prewarm, and symbolic->concrete resolution (~20%) is +inherently sequential. Amdahl over that ~68% serial floor predicts 1.44x; the +measurement was 1.34x. Round 3 has since moved the middle two rows into keyed +queries, so the floor is lower and the ceiling correspondingly higher — but it +is still a decomposition question, not a thread-count one. + +**The fan-out cannot live inside the salsa query graph.** `salsa::Database` is +`Send` but **not `Sync`**, so `&dyn Db` cannot cross a rayon boundary and +neither `assemble_module` nor `assemble_simulation` can fan out from within. +It has to run from `compile_project_incremental`, which holds a concrete +`&SimlinDb`. `Storage: Clone` clones the shared `Arc` and mints a +fresh per-thread `ZalsaLocal`, so each worker takes its own **moved** handle +(`SimlinDb` is `Send`, not `Sync` — a handle may be given to a thread, never +shared with one). Every handle must drop before the next `db.sync`: `zalsa_mut` +cancels and blocks on outstanding handles, so a leaked one deadlocks the next +edit. + +**Two hazards found by measurement, not by reasoning.** Both are silent. + +1. **The prewarm must run AFTER the module-cycle gate, never before.** + `compile_var_fragment` demands the recursive `model_module_map`, which salsa + turns into a dependency-graph cycle panic — a process abort under + `panic = abort` (GH #806). A prewarm placed ahead of + `assemble_simulation`'s `project_module_graph(..).cycle_error_from(..)` + check reopened exactly that hole: the lib suite went from its baseline to + two extra failures, both module-cycle regression tests, and repeating the + gate ahead of the prewarm restored the baseline exactly. +2. **The fan-out must be gated on cold-ness.** Unconditionally prewarming + regressed the fully-cached recompile from 0.85–1.32 ms to 3.29–3.42 ms — a + 2.5–4x regression on the path that matters most for interactive editing — + because it builds a work list over every variable and spins up workers to + re-verify memos that are already valid. + +Determinism is **not** a hazard here, and that is a measured result rather than +an assumption: the 12-repeat byte-identical determinism suites +(`fragment_determinism_tests`, `diagnostic_determinism_tests`) pass with the +prewarm active. Salsa's accumulator drain is a dependency DFS, not an execution +order. + +### C5. `Compiler::intern_name` — the top allocation site, blocked on artifact identity + +320,650 allocations per cold C-LEARN compile, ~10% of all 3.24M, from two +independent causes: `intern_name` calls `name.to_string()` twice per new name +(once for `names`, once for the `name_ids` key), and `Compiler::new` re-interns +every project dimension and element name for each of ~1,600 per-variable +fragments. + +The second is the real cost and cannot be hoisted naively: `NameId` assignment +order is baked into the compiled artifact (`base_gf`, `DimId`, and every +`name_id` operand), and the ids are assigned per fragment from 0 and merged by +`FragmentMerger`. Sharing a project-global prefix changes those ids. Any +attempt here must either preserve the assignment exactly or accept an artifact +change and re-baseline the goldens deliberately — which is why round 3 stopped +short of it rather than taking a ~2-3%. + ## Suggested ordering 1. ~~**Build levers A (opt=3 native) + B (mimalloc native)**~~ — DONE. Measured @@ -415,7 +564,15 @@ re-derivation. (b) is broader but touches many call sites. cache is the next idea there — and see the round-2 negative result before attempting it). 5. **R3 superinstructions** — incremental dispatch wins, low risk. -6. **C2 / C3** — only if incremental-compile latency still bites after A+B. +6. ~~**C2 / C3**~~ — answered, and not as proposed: C2 is moot (the function + is salsa-cached and off the ordinary compile path) and C3's two halves are + already done or the wrong lever. The compile round 3 section above records + what the profile actually pointed at, and what it cost. +7. **C4 (parallel fan-out)** — the largest remaining compile lever and the only + one that needs a design rather than a fix. Read its two hazards before + starting; both are silent, and one is a process abort. +8. **C5 (`Compiler::intern_name`)** — the top allocation site, blocked on + `NameId` assignment order being part of the compiled artifact. Larger run-side swings identified during round 2 — all three were taken to a data verdict in round 3 (2026-06-04): From 26b65be0e1f508ec678ba67b83950ae8e975b115 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:35:53 -0700 Subject: [PATCH 27/59] engine: add a value-level LTM gate at two scales MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing asserted what an LTM link score is WORTH. `clearn_residual_exactness` never enables LTM; `clearn_ltm_var_count_guardrail` pins the emitted variable count and the slot width, and neither moves when an arm's value is rewritten. The characterization goldens are text, so they catch an arm whose spelling changes and say nothing about an arm whose spelling is right and whose value is not. A change that rewrote 149 C-LEARN LTM slots to zero passed every named C-LEARN gate (GH #977). Two halves, split by runtime rather than by coverage. `db::ltm_value_gate_tests` is the sub-second default-suite half: a three-arm `Ast::Arrayed` target with no EXCEPT default, so `OmitStructuralZero` is live and each arm's fate is decided independently, and each arm is one of the ways an arm-level change goes wrong. nyc pop[nyc] * 0.01 + TIME * 0.002 live TIME, no live source for the other links -- must NOT be omitted boston alt[a1] * 0.02 source reached through a bare element subscript of a DISJOINT dimension -- must NOT be omitted la base * 0.03 genuinely source-free and invariant -- must be EXACTLY +0.0 `alt` is wired back through `pop_total = SUM(pop[*])` so the edge sits on a real loop; the first draft omitted that and emitted no `alt[a1]→growth` score at all, which would have failed on a missing variable rather than on a wrong value. The whole LTM slab is pinned as a golden, but a golden alone would not do this job: per the root CLAUDE.md, a golden that pins an artifact is blind to that artifact being stably absent, and a careless `UPDATE_LTM_VALUE_GOLDEN=1` re-capture would bless a zeroed slot. So each mechanism also carries a named assertion that does not read the golden. The structural-zero row asserts EXACT equality rather than a tolerance, because a near-zero residual passing a tolerance is precisely the signal that the arm was not provably `PREVIOUS(target)`. The `boston` row's rustdoc is explicit about what it does NOT establish: it pins the access SHAPE in which #977's 322 unwrapped-bare-variable arms arise, not the raw-vs-canonical mismatch itself. `alt[a1]` is that link's own source, so the occurrence match and the emitted tree agree about it. No fixture reproduces the mismatch yet, and claiming one would be worse than having none. `simulate_ltm::clearn_ltm_slot_maxima_digest` is the C-LEARN half, `#[ignore]`d purely for runtime (~3.5 s release, but it needs a release build; the debug build is far past the 3-minute cap in docs/dev/rust.md). It is a digest rather than a slab because 30k slots x 251 steps is 60 MB of golden nobody would read: `nonzero_slots` (which a silent zeroing moves DOWN), `finite_slots` (so a regression to NaN cannot hide behind an unchanged non-zero count), and an order-independent sum of per-slot maximum magnitudes quantized to nine significant digits -- fine enough to catch any real zeroing, coarse enough not to red on last-bit drift, which is what turns a pin into something people re-capture without reading. Its teeth were measured, not assumed. Same binary, three runs differing only in `ltm_augment_zero_slot`: predicate as shipped (1369, 7000, 10_248_673_492_482_445_132_733_301) omission disabled (Materialize) identical in all three numbers predicate forced true (1287, 7000, 10_248_673_492_258_319_975_585_940) The second run is the point: this digest is the reproducible, checked-in form of the whole-slab differential that established 0fa26211's value-neutrality on C-LEARN, which until now existed only as a throwaway probe nobody could re-run. The third run is what makes the second meaningful -- 82 slots carrying real scores go to zero and the digest reds, so "unchanged when the omission is disabled" is not merely a digest that cannot see the omission. --- src/simlin-engine/src/db.rs | 2 + .../src/db/ltm_value_gate_tests.rs | 294 ++++++++++++++++++ .../src/db/ltm_value_golden/value_gate.txt | 18 ++ .../tests/integration/simulate_ltm.rs | 131 ++++++++ 4 files changed, 445 insertions(+) create mode 100644 src/simlin-engine/src/db/ltm_value_gate_tests.rs create mode 100644 src/simlin-engine/src/db/ltm_value_golden/value_gate.txt diff --git a/src/simlin-engine/src/db.rs b/src/simlin-engine/src/db.rs index f57c3e5ca..83fb94093 100644 --- a/src/simlin-engine/src/db.rs +++ b/src/simlin-engine/src/db.rs @@ -1436,6 +1436,8 @@ mod ltm_rank_decline_tests; #[cfg(test)] mod ltm_unified_tests; #[cfg(test)] +mod ltm_value_gate_tests; +#[cfg(test)] mod module_cycle_tests; #[cfg(test)] mod module_wiring_tests; diff --git a/src/simlin-engine/src/db/ltm_value_gate_tests.rs b/src/simlin-engine/src/db/ltm_value_gate_tests.rs new file mode 100644 index 000000000..c1c17b9a9 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_value_gate_tests.rs @@ -0,0 +1,294 @@ +// Copyright 2026 The Simlin Authors. All rights reserved. +// Use of this source code is governed by the Apache License, +// Version 2.0, that can be found in the LICENSE file. + +//! The value-level LTM gate: what every LTM synthetic variable's slots are +//! WORTH, step by step, on a fixture built to reproduce the ways an arm-level +//! change silently zeroes a score. +//! +//! Nothing else covers this. `clearn_residual_exactness` never enables LTM at +//! all; `clearn_ltm_var_count_guardrail` pins the emitted variable count and the +//! slot width, and neither of those moves when an arm's VALUE is rewritten. A +//! change that rewrote 149 C-LEARN LTM slots to zero passed every named C-LEARN +//! gate (GH #977). The characterization goldens are text, so they catch an arm +//! whose spelling changes and say nothing about an arm whose spelling is right +//! and whose value is not. +//! +//! Two halves, and the split is about run time rather than about coverage: this +//! file is the sub-second default-suite half, and +//! `simulate_ltm::clearn_ltm_slot_maxima_digest` is the `#[ignore]`d C-LEARN +//! half (~25 s release, well past the debug-build 3-minute cap in +//! `docs/dev/rust.md`). +//! +//! **A golden alone would not do this job**, and the reason is the standing +//! constraint in the root `CLAUDE.md`: a golden that pins an artifact is blind +//! to that artifact being stably absent, and a careless re-capture blesses a +//! vanished value. So every mechanism below carries a NAMED assertion that does +//! not depend on the golden's contents, and the golden's job is to catch +//! everything nobody thought to name. + +use super::*; +use crate::datamodel; +use crate::test_common::TestProject; + +/// The three ways a per-element link-score arm can be wrong about whether it is +/// a structural zero, in one model. +/// +/// The target `growth[Region]` is a per-element (`Ast::Arrayed`) flow with no +/// EXCEPT default, so `ZeroSlotPolicy::OmitStructuralZero` is live for it and +/// each arm's fate is decided independently: +/// +/// * `nyc` reads the link source `pop[nyc]` AND carries `TIME`. For the +/// `pop[nyc]` link this arm is live on both counts; for the OTHER links it is +/// the load-bearing row -- every occurrence of their source is frozen, and +/// the arm must still be materialized because `TIME` advances. This is the +/// mechanism that makes the naive "the source stayed frozen" collapse unsound +/// (5,035 of C-LEARN's 9,514 no-live-source arms are blocked solely by a live +/// `time()`; GH #1016). If a future relaxation drops it, this arm goes to zero +/// and the assertion below reds. +/// * `boston` reads `alt[a1]` -- a source in a dimension DISJOINT from the +/// target's, subscripted by a bare element name. This is the ACCESS SHAPE in +/// which GH #977's 322 unwrapped-bare-variable arms arise (raw +/// `[developing_b_countries]` against canonical +/// `[aggregated_regions.developing_b_countries]`), and what this row pins is +/// that such an arm is scored LIVE rather than claimed as a structural zero. +/// +/// Be precise about what that is NOT: `alt[a1]` is the link's own source +/// here, so the occurrence match and the emitted tree agree about it, and +/// this fixture does not exhibit the raw-vs-canonical MISMATCH itself -- the +/// state where the shape match records no live reference while the wrap +/// leaves the source unwrapped. Sizing that defect is separate work; until it +/// is characterized there is no fixture that reproduces it, and claiming one +/// here would be the more expensive error than having none. +/// * `la` reads only the constant `base`. For every link into `growth` its +/// partial is provably `PREVIOUS(growth)`, so the slot is genuinely omitted +/// and must be EXACTLY `+0.0` -- the promise the omission makes. +/// +/// `alt` is deliberately wired back through `pop_total = SUM(pop[*])` so that +/// `alt -> growth -> pop -> pop_total -> alt` is a genuine feedback loop. +/// Without that the exhaustive path emits no `alt[a1] -> growth` score at all +/// and every assertion below would fail on a missing variable rather than on a +/// wrong value -- which is how the first draft of this fixture was caught. +fn ltm_value_gate_project() -> datamodel::Project { + TestProject::new("ltm_value_gate") + .with_sim_time(0.0, 5.0, 1.0) + .named_dimension("Region", &["nyc", "boston", "la"]) + .named_dimension("Alt", &["a1", "a2"]) + .aux("base", "2", None) + .aux("pop_total", "SUM(pop[*])", None) + .array_aux("alt[Alt]", "pop_total * 0.05 + TIME * 0.1") + .array_flow_with_ranges( + "growth[Region]", + vec![ + ("nyc", "pop[nyc] * 0.01 + TIME * 0.002"), + ("boston", "alt[a1] * 0.02"), + ("la", "base * 0.03"), + ], + ) + .array_stock("pop[Region]", "10", &["growth"], &[], None) + .build_datamodel() +} + +/// One LTM synthetic variable's per-slot, per-step series. +struct LtmSlotSeries { + name: String, + /// Slot index within the variable (0 for a scalar). + slot: usize, + values: Vec, +} + +/// Simulate `project` with LTM on and return every LTM synthetic variable's +/// slots, name-sorted then slot-ordered. +/// +/// Widths come from each variable's own `dimensions` via the project's +/// dimension context rather than from a hand-written table, so a variable that +/// changes shape is read at its real width instead of being silently truncated. +fn ltm_slot_series(project: &datamodel::Project) -> Vec { + let mut db = SimlinDb::default(); + let sync = sync_from_datamodel(&db, project); + use salsa::Setter; + sync.project.set_ltm_enabled(&mut db).to(true); + // Re-sync so every downstream query sees the flag (mirrors the other + // db-level LTM fixtures in this crate). + let sync = sync_from_datamodel(&db, project); + sync.project.set_ltm_enabled(&mut db).to(true); + + let ltm = crate::db::model_ltm_variables(&db, sync.models["main"].source, sync.project); + let dim_ctx = crate::db::project_dimensions_context(&db, sync.project); + + let compiled = crate::db::compile_project_incremental(&db, sync.project, "main") + .expect("the value-gate fixture must compile with LTM enabled"); + let offsets = compiled.offsets.clone(); + let mut vm = crate::vm::Vm::new(compiled).expect("vm"); + vm.run_to_end().expect("run"); + let results = vm.into_results(); + + let mut out: Vec = Vec::new(); + let mut vars: Vec<&crate::db::LtmSyntheticVar> = ltm.vars.iter().collect(); + vars.sort_by(|a, b| a.name.cmp(&b.name)); + for var in vars { + let Some(&base) = offsets.get(&crate::common::Ident::new(&var.name)) else { + // A variable with no layout slot is a real defect, but it is + // `model_ltm_fragment_diagnostics`' to report; this gate is about + // values, so record it loudly rather than skipping it silently. + panic!("LTM variable {} has no result offset", var.name); + }; + let width: usize = var + .dimensions + .iter() + .map(|d| { + let canonical = crate::common::CanonicalDimensionName::from_raw(d); + dim_ctx.get(&canonical).map(|dim| dim.len()).unwrap_or(1) + }) + .product::() + .max(1); + for slot in 0..width { + let off = base + slot; + out.push(LtmSlotSeries { + name: var.name.clone(), + slot, + values: (0..results.step_count) + .map(|s| results.data[s * results.step_size + off]) + .collect(), + }); + } + } + out +} + +/// Render the slab as a stable text table. `{:.12e}` keeps the sign of zero +/// (`-0.000000000000e0`), which matters here: an omitted slot is `+0.0` and a +/// materialized trivial arm ending in `* SIGN(dx)` can be `-0.0`, and the two +/// must stay distinguishable in the pin. +fn render_slab(series: &[LtmSlotSeries]) -> String { + let mut out = String::new(); + for s in series { + out.push_str(&format!("{}[{}]", s.name, s.slot)); + for v in &s.values { + out.push_str(&format!(" {:.12e}", v)); + } + out.push('\n'); + } + out +} + +fn assert_value_golden(name: &str, actual: &str) { + let path = format!( + "{}/src/db/ltm_value_golden/{name}.txt", + env!("CARGO_MANIFEST_DIR") + ); + if std::env::var("UPDATE_LTM_VALUE_GOLDEN").is_ok() { + let dir = format!("{}/src/db/ltm_value_golden", env!("CARGO_MANIFEST_DIR")); + std::fs::create_dir_all(&dir).expect("create golden dir"); + std::fs::write(&path, actual).expect("write golden"); + return; + } + let expected = std::fs::read_to_string(&path).unwrap_or_else(|e| { + panic!("missing golden {path}: {e}; run once with UPDATE_LTM_VALUE_GOLDEN=1 to capture") + }); + if actual != expected { + eprintln!("\n===== LTM VALUE GOLDEN MISMATCH ({name}): actual below ====="); + eprintln!("{actual}"); + eprintln!("===== end actual (expected in {path}) =====\n"); + } + assert_eq!(actual, &expected, "LTM value golden mismatch for {name}"); +} + +/// Find one slot's series by variable name substring + slot index. +fn slot<'a>(series: &'a [LtmSlotSeries], name_contains: &str, slot: usize) -> &'a [f64] { + let hits: Vec<&LtmSlotSeries> = series + .iter() + .filter(|s| s.name.contains(name_contains) && s.slot == slot) + .collect(); + assert_eq!( + hits.len(), + 1, + "expected exactly one slot matching {name_contains:?}[{slot}]; got {:?}", + series + .iter() + .map(|s| format!("{}[{}]", s.name, s.slot)) + .collect::>() + ); + &hits[0].values +} + +#[test] +fn ltm_slot_values_are_pinned_on_the_value_gate_fixture() { + let series = ltm_slot_series(<m_value_gate_project()); + assert!( + !series.is_empty(), + "the fixture emitted no LTM slots at all, so this gate would pass vacuously" + ); + assert_value_golden("value_gate", &render_slab(&series)); +} + +/// Mechanism 1: an arm with NO live source reference but a live `TIME` must be +/// materialized and must carry a non-zero value. +/// +/// The `alt[a1] -> growth` link's `nyc` slot is that arm: `pop[nyc]` and `base` +/// are frozen for this link, `alt[a1]` does not appear in the `nyc` equation at +/// all, and what remains live is `TIME * 0.002`. Under the negative "the +/// source stayed frozen" criterion this slot would be dropped to zero; under +/// the positive predicate `TIME` is `BuiltinReach::Varying`, so the arm stays. +/// +/// This assertion does not depend on the golden, which is the point: a careless +/// `UPDATE_LTM_VALUE_GOLDEN=1` re-capture would bless the zeroed slot, and this +/// would still red. +#[test] +fn a_time_bearing_arm_with_no_live_source_is_not_zeroed() { + let series = ltm_slot_series(<m_value_gate_project()); + // Region declaration order: nyc=0, boston=1, la=2. + let nyc = slot(&series, "link_score\u{205A}alt[a1]\u{2192}growth", 0); + assert!( + nyc.iter().any(|v| v.abs() > 1e-12 && v.is_finite()), + "the TIME-bearing `nyc` arm was zeroed: an arm whose only live content \ + is a time-dependent builtin is NOT a structural zero; got {nyc:?}" + ); +} + +/// Mechanism 2: an arm whose source is reached through a bare element name of a +/// DISJOINT dimension must be materialized and non-zero. +/// +/// +/// The `alt[a1] -> growth` link's `boston` slot is that arm -- `growth[boston] +/// = alt[a1] * 0.02`, the source subscripted by a raw element spelling, in a +/// dimension disjoint from the target's. It is the access shape GH #977's 322 +/// unwrapped-bare-variable arms live in, and the guard is that an arm reached +/// this way is scored rather than omitted. See the fixture's rustdoc for what +/// this deliberately does not claim: it does not reproduce the raw-vs-canonical +/// mismatch, only the shape it occurs in. +#[test] +fn a_disjoint_dim_element_subscript_arm_is_not_zeroed() { + let series = ltm_slot_series(<m_value_gate_project()); + let boston = slot(&series, "link_score\u{205A}alt[a1]\u{2192}growth", 1); + assert!( + boston.iter().any(|v| v.abs() > 1e-12 && v.is_finite()), + "the `boston` arm, which reads its source through a bare element \ + subscript of a disjoint dimension, was zeroed; got {boston:?}" + ); +} + +/// Mechanism 3, the other direction: a genuinely structural-zero arm must be +/// EXACTLY zero, at every step. +/// +/// `growth[la] = base * 0.03` reads no link source and nothing that varies, so +/// every link into `growth` omits that slot and +/// `compiler::expand_arrayed_with_hoisting` lowers it to one +/// `AssignCurr(off, Const(0.0))`. Asserting exact equality rather than a +/// tolerance is what makes this catch the omission claiming a slot it should +/// not have: a near-zero residual would pass a tolerance and is precisely the +/// signal that the arm was NOT provably `PREVIOUS(target)`. +#[test] +fn a_structural_zero_arm_is_exactly_zero() { + let series = ltm_slot_series(<m_value_gate_project()); + for source in ["alt[a1]\u{2192}growth", "pop[nyc]\u{2192}growth"] { + let la = slot(&series, &format!("link_score\u{205A}{source}"), 2); + for (step, v) in la.iter().enumerate() { + assert_eq!( + *v, 0.0, + "{source} `la` slot must be an exact structural zero at every \ + step; step {step} was {v}" + ); + } + } +} diff --git a/src/simlin-engine/src/db/ltm_value_golden/value_gate.txt b/src/simlin-engine/src/db/ltm_value_golden/value_gate.txt new file mode 100644 index 000000000..92c1228e8 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_value_golden/value_gate.txt @@ -0,0 +1,18 @@ +$⁚ltm⁚link_score⁚alt[a1]→growth[0] 0.000000000000e0 6.666666666667e-1 6.600660066007e-1 6.535306996046e-1 6.470600986184e-1 6.406535629885e-1 +$⁚ltm⁚link_score⁚alt[a1]→growth[1] 0.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 +$⁚ltm⁚link_score⁚alt[a1]→growth[2] 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 +$⁚ltm⁚link_score⁚growth→pop[0] 0.000000000000e0 0.000000000000e0 1.000000000000e0 9.999999999998e-1 1.000000000000e0 9.999999999999e-1 +$⁚ltm⁚link_score⁚growth→pop[1] 0.000000000000e0 0.000000000000e0 9.999999999997e-1 9.999999999999e-1 1.000000000000e0 1.000000000000e0 +$⁚ltm⁚link_score⁚growth→pop[2] 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 +$⁚ltm⁚link_score⁚pop[boston]→pop_total[0] 0.000000000000e0 1.578947368421e-1 1.649162354629e-1 1.715697797158e-1 1.778798792994e-1 1.838689119179e-1 +$⁚ltm⁚link_score⁚pop[la]→pop_total[0] 0.000000000000e0 3.157894736842e-1 3.073927967621e-1 2.993785051921e-1 2.917210936525e-1 2.843972770067e-1 +$⁚ltm⁚link_score⁚pop[nyc]→growth[0] 0.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 +$⁚ltm⁚link_score⁚pop[nyc]→growth[1] 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 +$⁚ltm⁚link_score⁚pop[nyc]→growth[2] 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 +$⁚ltm⁚link_score⁚pop[nyc]→pop_total[0] 0.000000000000e0 5.263157894737e-1 5.276909677750e-1 5.290517150920e-1 5.303990270480e-1 5.317338110755e-1 +$⁚ltm⁚link_score⁚pop_total→alt[a1][0] 0.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 +$⁚ltm⁚link_score⁚pop_total→alt[a2][0] 0.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 1.000000000000e0 +$⁚ltm⁚loop_score⁚r1[0] 0.000000000000e0 0.000000000000e0 1.649162354628e-1 1.715697797158e-1 1.778798792994e-1 1.838689119179e-1 +$⁚ltm⁚loop_score⁚r2[0] 0.000000000000e0 0.000000000000e0 1.000000000000e0 9.999999999998e-1 1.000000000000e0 9.999999999999e-1 +$⁚ltm⁚loop_score⁚r2[1] 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 +$⁚ltm⁚loop_score⁚r2[2] 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 0.000000000000e0 diff --git a/src/simlin-engine/tests/integration/simulate_ltm.rs b/src/simlin-engine/tests/integration/simulate_ltm.rs index b26831808..5875b1098 100644 --- a/src/simlin-engine/tests/integration/simulate_ltm.rs +++ b/src/simlin-engine/tests/integration/simulate_ltm.rs @@ -11185,3 +11185,134 @@ fn test_whole_rhs_mapped_reducer_routes_through_synthetic_agg() { } } } + +/// The C-LEARN half of the value-level LTM gate: a digest over every LTM slot's +/// per-step maximum magnitude. +/// +/// The sub-second half is `db::ltm_value_gate_tests`, which pins exact values on +/// a three-arm fixture built around the known ways an arm-level change zeroes a +/// score. It cannot show that the same change leaves 7,000 real variables alone, +/// and C-LEARN is the only model in the repo at that scale. Hence this: same +/// property, real model, `#[ignore]`d purely for runtime (~25 s release, against +/// the 3-minute debug-build cap in `docs/dev/rust.md`). +/// +/// The digest is deliberately NOT a checked-in series slab -- 30k slots x 251 +/// steps is 60 MB of golden nobody would read. It is three numbers that move +/// under exactly the failure this gate exists for: +/// +/// * `nonzero_slots` -- how many LTM slots are ever non-zero. Rewriting live +/// arms to zero moves this DOWN, which is the GH #977 failure (a change that +/// zeroed 149 C-LEARN LTM slots passed every named C-LEARN gate); wrongly +/// materializing structural zeros as small residuals moves it UP. +/// * `finite_slots` -- how many are finite throughout, so a regression that +/// replaces values with NaN cannot hide behind an unchanged non-zero count. +/// * `magnitude_digest` -- an order-independent sum over each slot's maximum +/// magnitude, quantized to 1e-9 relative. Two slots swapping values keeps the +/// first two numbers and moves this. +/// +/// Quantizing is what makes the pin usable rather than a per-run coin flip: raw +/// f64 maxima carry last-bit noise across allocator and layout changes, and a +/// digest that reds on that is a digest people learn to re-capture without +/// reading. A real zeroing moves it far outside the quantum. +/// +/// **"It passes" and "it constrains the code" are different claims, so both +/// were measured.** Three runs of this digest, same binary, differing only in +/// `ltm_augment_zero_slot`: +/// +/// * predicate as shipped -- `(1369, 7000, 10_248_673_492_482_445_132_733_301)` +/// * `ZeroSlotPolicy::Materialize` forced everywhere, i.e. GH #977's omission +/// disabled -- **identical in all three numbers**. That is this gate's other +/// job: it is the reproducible, checked-in form of the whole-slab differential +/// that established the omission's value-neutrality on C-LEARN, which +/// previously existed only as a throwaway probe nobody could re-run. +/// * `partial_is_provably_previous_target` forced to `true`, so every arm is +/// omitted whether or not it is a structural zero -- `(1287, 7000, +/// 10_248_673_492_258_319_975_585_940)`. 82 slots that carry real scores go to +/// zero and the digest reds. +/// +/// The third run is what makes the second meaningful. Without it, "unchanged +/// when the omission is disabled" would be equally consistent with a digest that +/// cannot see the omission at all. +/// +/// Run with: +/// cargo test -p simlin-engine --release --test integration -- --ignored \ +/// clearn_ltm_slot_maxima_digest +#[test] +#[ignore] +fn clearn_ltm_slot_maxima_digest() { + use simlin_engine::open_vensim; + + let mdl_path = "../../test/xmutil_test_models/C-LEARN v77 for Vensim.mdl"; + let contents = std::fs::read_to_string(mdl_path) + .unwrap_or_else(|e| panic!("failed to read {mdl_path}: {e}")); + let project = + open_vensim(&contents).unwrap_or_else(|e| panic!("failed to parse {mdl_path}: {e}")); + + let compiled = compile_ltm_discovery_incremental(&project); + let mut vm = Vm::new(compiled).expect("vm"); + vm.run_to_end() + .expect("C-LEARN must simulate with LTM enabled"); + let results = vm.into_results(); + + // Which result slots belong to LTM, taken from the run's own offset map + // rather than from a name list, so a renamed synthetic prefix fails loudly + // here instead of quietly shrinking the gate's scope. + let ltm_offsets: Vec = results + .offsets + .iter() + .filter(|(name, _)| name.as_str().starts_with("$\u{205A}ltm\u{205A}")) + .map(|(_, &off)| off) + .collect(); + assert!( + !ltm_offsets.is_empty(), + "no LTM slots found in the results; the gate would pass vacuously" + ); + + let mut nonzero_slots = 0usize; + let mut finite_slots = 0usize; + let mut magnitude_digest: i128 = 0; + for &off in <m_offsets { + let mut max_mag = 0.0f64; + let mut all_finite = true; + let mut ever_nonzero = false; + for step in 0..results.step_count { + let v = results.data[step * results.step_size + off]; + if !v.is_finite() { + all_finite = false; + continue; + } + if v != 0.0 { + ever_nonzero = true; + } + if v.abs() > max_mag { + max_mag = v.abs(); + } + } + if ever_nonzero { + nonzero_slots += 1; + } + if all_finite { + finite_slots += 1; + } + // Nine significant digits: far finer than any real zeroing, far coarser + // than last-bit drift. + magnitude_digest += (max_mag * 1e9).round() as i128; + } + + assert_eq!( + (nonzero_slots, finite_slots, magnitude_digest), + ( + CLEARN_LTM_NONZERO_SLOTS, + CLEARN_LTM_FINITE_SLOTS, + CLEARN_LTM_MAGNITUDE_DIGEST + ), + "C-LEARN's LTM slot values moved. A DROP in nonzero_slots is the \ + silent-zeroing regression this gate exists for; re-derive before \ + re-pinning, and say in the commit which arms changed and why" + ); +} + +/// Pinned by `clearn_ltm_slot_maxima_digest`; see its rustdoc before changing. +const CLEARN_LTM_NONZERO_SLOTS: usize = 1369; +const CLEARN_LTM_FINITE_SLOTS: usize = 7000; +const CLEARN_LTM_MAGNITUDE_DIGEST: i128 = 10_248_673_492_482_445_132_733_301; From 32351241d8c3819e5c13f62d9b4498f5eb084481 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:38:13 -0700 Subject: [PATCH 28/59] build: skip wasm-opt in the pre-commit hook's TypeScript build On an engine change that alters emitted code, `src/engine/build.sh` takes 188.7s, and `wasm-opt -O3` is ~170s of it (90.6s + 76.2s for the two blobs). The hook needs an artifact that builds and passes the TypeScript tests, not a size-optimized one: the engine suite runs 1.26s/1.29s against the unoptimized blob and 1.40s/1.42s against the optimized one -- no measurable cost. With this, a whole-workspace `pnpm build` after an engine change is 11.1s. This mirrors `.github/workflows/ci.yaml`, which already sets DISABLE_WASM_OPT=1 in both of its build steps, one of them with the same reasoning written out ("release-quality WASM is irrelevant for a smoke test"). The variable belongs on THIS call site and not in `src/engine/build.sh` or `package.json`, which is the obvious way to do it and is wrong. Six callers run `pnpm build`: the two CI steps (already opted out), this hook, and `scripts/deploy-web.sh`, `scripts/deploy-web-staged.sh` and the release workflow -- which must keep the optimized artifact, because the browser bundle is download-size-dominated, the same reason `.cargo/config.toml` forces opt-level=z on wasm32. Flipping the default would route through all of them and ship a 24% larger bundle (5.00MB -> 6.20MB) to every user. What this gives up, stated plainly (GH #1019): `build.sh` runs wasm-opt IN PLACE over `core/.wasm`, the file the TypeScript tests load, so a developer's machine was until now the only place the optimized bundle was ever executed under test -- neither automated lane runs it, and only `ts-release.yml` installs binaryen at all. After this, nothing does until an npm publish. That coverage was accidental (it depended on every developer running the hook) and sat in the wrong place, but it was real; #1019 tracks converting it into a deliberate CI lane that builds with wasm-opt and runs the engine suite against the optimized blob. Verified by running the exact edited command after a codegen-altering change (a new `#[no_mangle]` export in libsimlin): both blobs report "Skipping wasm-opt". That indirection matters here -- an earlier probe that appended an unused `pub const` recompiled Rust but produced a byte-identical wasm, so build.sh's `cmp` guard skipped wasm-opt on its own and measured a build that never did the expensive part. The hook this commit edits is also not the one that runs on this commit: git resolves it through the main checkout's copy. --- scripts/pre-commit | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/pre-commit b/scripts/pre-commit index d0f47ce92..3be0a19a4 100755 --- a/scripts/pre-commit +++ b/scripts/pre-commit @@ -204,8 +204,24 @@ PIPELINE_PIDS+=($PID_A) pnpm -r --parallel run lint # 3. Build WASM and TS packages (needed for .d.ts files) + # + # DISABLE_WASM_OPT=1 skips the wasm-opt -O3 pass, mirroring + # .github/workflows/ci.yaml's two build steps. On an engine change that + # alters emitted code it is ~170s of the ~190s this step takes, and the + # hook needs an artifact that builds and passes the TypeScript tests, not + # a size-optimized one: the engine suite runs 1.26s against the + # unoptimized blob and 1.40s against the optimized one. See GH #1019 for + # the coverage this gives up. + # + # The variable belongs HERE and not in src/engine/build.sh or + # package.json. Six callers run `pnpm build`; the four that are not CI + # (scripts/deploy-web.sh, scripts/deploy-web-staged.sh, and the release + # workflows) must keep the optimized artifact, because the browser bundle + # is download-size-dominated -- the same reason .cargo/config.toml forces + # opt-level=z on wasm32. Flipping the default would route through all of + # them and ship a 24% larger bundle (5.00MB -> 6.20MB) to every user. echo "[ts] Building..." - pnpm build + DISABLE_WASM_OPT=1 pnpm build # 4. Type check and tests in parallel echo "[ts] Running type check and tests..." From 99bf33d779e25b0b2f2f8440ad93626f9b8f3b07 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:42:05 -0700 Subject: [PATCH 29/59] engine: pin that fusion never raises a program's peak stack depth `compiler::symbolic::resolve_bytecode` proves the compiled stream fits `STACK_CAPACITY`, and `vm::Stack` uses unchecked access on the strength of that proof. But the proof is computed on the PRE-fusion stream while the Vm executes the fused one, so `ByteCode::fuse_three_address` carries a standing obligation: a fused opcode's `stack_effect` must account for every operand the sequence it replaces consumed, and a program's peak depth may fall but never rise. Nothing already in the suite covers that. The deepest stack any corpus model reaches is 8-12 against a `STACK_CAPACITY` of 64, so a wrong stack effect has more than 5x of headroom to hide in: it would not overflow, the arithmetic would still be correct, and every saved value would match. A passing suite and matching results fingerprints are strong evidence for other failure modes and weak evidence for this one -- comparing the two depths is what detects it. The `Err` arm of `max_stack_depth` covers the other half: an underflow means an opcode's declared effect is wrong rather than the program. `CompiledSimulation::fusion_depth_audit` reports both depths per (module, phase) alongside the opcode counts; it lives in `vm_profile.rs`, the diagnostics-only sibling that already exposes bytecode shape without leaking the private `Opcode` type. The test sweeps the curated corpus (`TEST_MODELS`), both executed phases, every module -- initials are excluded because `Vm::new` leaves them unfused. It covers the shapes the corpus curates rather than every file on disk, which keeps its cost proportional to a list someone maintains deliberately: 0.20s on a debug build, against the 2s per-test target in docs/dev/rust.md. --- src/simlin-engine/src/vm_profile.rs | 52 ++++++++++++++++++ .../tests/integration/simulate.rs | 54 +++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/src/simlin-engine/src/vm_profile.rs b/src/simlin-engine/src/vm_profile.rs index 465dd7f45..9a0e7aa4a 100644 --- a/src/simlin-engine/src/vm_profile.rs +++ b/src/simlin-engine/src/vm_profile.rs @@ -83,6 +83,58 @@ impl CompiledSimulation { } } +/// One `(module, phase)` program's peak arithmetic-stack depth either side of +/// `ByteCode::fuse_three_address`. `Err` means an opcode's declared +/// [`Opcode::stack_effect`] underflowed the stack, i.e. the metadata is wrong. +pub struct FusionDepthCheck { + pub module: String, + pub phase: &'static str, + pub pre_depth: Result, + pub post_depth: Result, + pub pre_opcodes: usize, + pub post_opcodes: usize, +} + +impl CompiledSimulation { + /// Peak stack depth before and after fusion, for every module and executed + /// phase. + /// + /// **Standing constraint on `fuse_three_address`: fusion must never RAISE a + /// program's peak stack depth.** `compiler::symbolic::resolve_bytecode` + /// proves the compiled stream fits `STACK_CAPACITY`, and `vm::Stack` uses + /// unchecked access on the strength of that proof -- but the proof is + /// computed on the PRE-fusion stream, while the Vm executes the fused one. + /// A fused opcode whose `stack_effect` understates its pops would leave the + /// Vm running a program the proof does not cover. + /// + /// Neither the hero models nor a results fingerprint covers this. The + /// deepest stack any corpus model reaches is ~12 against a `STACK_CAPACITY` + /// of 64, so a wrong stack effect has more than 5x of headroom to hide in: + /// it would not overflow, the arithmetic would still be correct, and every + /// value would match. Only comparing the two depths detects it. + pub fn fusion_depth_audit(&self) -> Vec { + let mut out = Vec::new(); + for (key, module) in self.modules.iter() { + for (phase, bc) in [ + ("flows", module.compiled_flows.as_ref()), + ("stocks", module.compiled_stocks.as_ref()), + ] { + let mut fused = bc.clone(); + fused.fuse_three_address(); + out.push(FusionDepthCheck { + module: key.0.as_str().to_string(), + phase, + pre_depth: bc.max_stack_depth(), + post_depth: fused.max_stack_depth(), + pre_opcodes: bc.code.len(), + post_opcodes: fused.code.len(), + }); + } + } + out + } +} + /// Aggregate composition of a compiled simulation's bytecode and side tables. /// Produced by [`CompiledSimulation::bytecode_profile`]. `histogram` maps each /// opcode variant name to its occurrence count across all modules and phases. diff --git a/src/simlin-engine/tests/integration/simulate.rs b/src/simlin-engine/tests/integration/simulate.rs index 521e12248..2891780b5 100644 --- a/src/simlin-engine/tests/integration/simulate.rs +++ b/src/simlin-engine/tests/integration/simulate.rs @@ -6493,6 +6493,60 @@ fn assert_poisoned_next_matches(xmile_path: &str) { } } +// -- Fusion must never raise a program's peak stack depth ------------------ +// +// `compiler::symbolic::resolve_bytecode` proves the compiled stream fits +// `STACK_CAPACITY`, and `vm::Stack` uses unchecked access on the strength of +// that proof -- but the proof is computed on the PRE-fusion stream, while the +// Vm executes the fused one. So `fuse_three_address` carries a standing +// obligation: a fused opcode's `stack_effect` must account for every operand +// the sequence it replaces consumed, and the peak may fall but never rise. +// +// Neither the hero models nor a results fingerprint covers this, which is why +// it gets its own test. The deepest stack any corpus model reaches is 8-12 +// against a `STACK_CAPACITY` of 64, so a wrong stack effect has >5x of headroom +// to hide in: it would not overflow, the arithmetic would still be right, and +// every saved value would match. Comparing the two depths is what detects it. +// A stack-effect that underflows shows up as the `Err` arm, which means the +// metadata is wrong rather than the program. +// +// Scope: every corpus model, both executed phases, all modules. Initials are +// excluded because `Vm::new` leaves them unfused. +#[test] +fn fusion_never_raises_peak_stack_depth() { + let mut checked = 0usize; + for path in TEST_MODELS.iter() { + let path = format!("../../{path}"); + let Ok(f) = File::open(&path) else { continue }; + let mut f = BufReader::new(f); + let Ok(datamodel_project) = xmile::project_from_reader(&mut f) else { + continue; + }; + for check in compile_vm(&datamodel_project).fusion_depth_audit() { + let (module, phase) = (&check.module, check.phase); + let pre = check + .pre_depth + .unwrap_or_else(|e| panic!("{path}: {module}/{phase}: pre-fusion {e}")); + let post = check + .post_depth + .unwrap_or_else(|e| panic!("{path}: {module}/{phase}: post-fusion {e}")); + assert!( + post <= pre, + "{path}: {module}/{phase}: fusion RAISED peak stack depth {pre} -> {post} \ + ({} -> {} opcodes). `resolve_bytecode`'s capacity proof is computed on the \ + pre-fusion stream, so it no longer covers what the Vm executes.", + check.pre_opcodes, + check.post_opcodes, + ); + checked += 1; + } + } + assert!( + checked > 100, + "expected a real corpus sweep, checked {checked}" + ); +} + #[test] fn only_documented_classes_carry_across_a_step() { for path in TEST_MODELS.iter() { From 116b855ce8254a22e8bdc8670bca53440ed30306 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:45:29 -0700 Subject: [PATCH 30/59] ci: run wasm-opt and test the optimized bundle on engine changes `src/engine/build.sh` runs `wasm-opt -O3` IN PLACE over `core/*.wasm`, the file the TypeScript tests load, so "did wasm-opt run" and "was the optimized bundle executed under test" are one question -- and until now nothing automated answered yes. Both of ci.yaml's build steps set DISABLE_WASM_OPT=1, `scripts/pre-commit` now does too, and `ts-release.yml` is the only workflow that installs binaryen at all, by which point the artifact is being published. The coverage that existed was a side effect of developers running the hook on machines that happened to have binaryen. This makes it deliberate: a path-filtered job on the sources that can change emitted WASM, which builds with wasm-opt on and runs the TypeScript suite against the result. It closes GH #1019 and is what makes the hook's DISABLE_WASM_OPT change a coverage improvement rather than a trade -- before it, nothing automated ran the pass before an npm publish; after it, an automated gate runs it on exactly the PRs that can break it. Its own workflow file rather than a job in ci.yaml because GitHub applies `paths` per workflow, not per job; filtering inside ci.yaml would mean taking a third-party paths-filter action. Two things the job does beyond the obvious, both because a silent pass here would be worse than no job at all: `build.sh` SKIPS wasm-opt and exits 0 when binaryen is missing, so a broken install would quietly turn this into a slower duplicate of ci.yaml's frontend job. The "Assert the blobs are actually optimized" step compares each blob against the `.raw` copy build.sh stages and fails if they are byte-identical. Checked by mutation in all three states: optimized (passes, reporting both sizes), blob copied from .raw (fails naming wasm-opt), .raw missing (fails naming build.sh). The header says what a failure MEANS. The same TypeScript tests run in ci.yaml against an unoptimized blob, so green there and red here isolates the difference to binaryen's -O3 pass -- a miscompilation, an unsupported feature, or a version incompatibility -- not a defect in the TypeScript under test, with the two commands to reproduce the pair locally. Also notes that it must not be made a required status check as-is: a path-filtered workflow reports nothing on a PR that touches none of its paths, and a required check that never reports blocks the PR forever. Cost: ~170s of wasm-opt (90s + 76s across the two blobs) plus the wasm cargo build, on engine PRs only, in parallel with jobs already minutes long. Fixes #1019 --- Also corrects a stale number in scripts/verify-deploy-build.sh, which said DISABLE_WASM_OPT "bumps it to ~12MB". The unoptimized opt-level=z blob is 7.9MB; 12.7MB is what opt-level=1 produces, which is not a configuration anything uses. Restates what that check does and does not gate, so nobody mistakes it for a wasm-opt gate -- it deliberately passes either way. --- .github/workflows/wasm-opt.yml | 135 +++++++++++++++++++++++++++++++++ scripts/verify-deploy-build.sh | 7 +- 2 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/wasm-opt.yml diff --git a/.github/workflows/wasm-opt.yml b/.github/workflows/wasm-opt.yml new file mode 100644 index 000000000..38b1a47f7 --- /dev/null +++ b/.github/workflows/wasm-opt.yml @@ -0,0 +1,135 @@ +--- +# The ONLY automated job that runs `wasm-opt` and then executes the resulting +# blob under test. +# +# `src/engine/build.sh` runs `wasm-opt -O3` IN PLACE over `core/*.wasm`, which +# is the file the TypeScript tests load -- so "did wasm-opt run" and "was the +# optimized bundle executed under test" are the same question. Everywhere else +# deliberately answers no: both of `ci.yaml`'s build steps set +# DISABLE_WASM_OPT=1 (the pass is multi-minute and a PR gate does not need +# release-quality WASM), and `scripts/pre-commit` does the same. Only +# `ts-release.yml` installs binaryen at all, and by then the artifact is being +# published. This job is what keeps that gap from reaching a release. See +# GH #1019. +# +# It is path-filtered to the sources that can change the emitted WASM, which is +# why it lives in its own workflow rather than as a job in `ci.yaml`: GitHub +# applies `paths` per workflow, not per job, and filtering inside `ci.yaml` +# would mean adding a third-party paths-filter action. +# +# Do NOT make this a required status check in branch protection as-is. A +# path-filtered workflow reports nothing at all on a PR that touches none of +# these paths, and a required check that never reports blocks the PR forever. +# If it needs to be required, the standard workaround is an always-triggered +# companion job that succeeds trivially when the filter does not match. +# +# IF THIS JOB FAILS AND `ci.yaml` PASSED, the difference is wasm-opt. The same +# TypeScript tests run in `ci.yaml` against an UNOPTIMIZED blob; if they pass +# there and fail here, the engine's WASM did not survive binaryen's -O3 pass -- +# a miscompilation, an unsupported feature, or a binaryen version +# incompatibility -- not a defect in the TypeScript under test. Reproduce +# locally with `bash src/engine/build.sh && pnpm -C src/engine test` (note the +# absence of DISABLE_WASM_OPT), and compare against +# `DISABLE_WASM_OPT=1 bash src/engine/build.sh && pnpm -C src/engine test`. +name: WASM optimized-bundle check + +"on": + push: + branches: + - main + paths: + - 'src/simlin-engine/**' + - 'src/libsimlin/**' + - 'src/engine/**' + - 'Cargo.lock' + - 'Cargo.toml' + - '.cargo/config.toml' + - '.github/workflows/wasm-opt.yml' + pull_request: + branches: + - main + paths: + - 'src/simlin-engine/**' + - 'src/libsimlin/**' + - 'src/engine/**' + - 'Cargo.lock' + - 'Cargo.toml' + - '.cargo/config.toml' + - '.github/workflows/wasm-opt.yml' + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + +jobs: + wasm-opt: + name: Build with wasm-opt and run the engine tests against it + runs-on: ubuntu-latest + # wasm-opt -O3 is ~170s on our two blobs (90s + 76s), on top of the wasm + # cargo build. Generous cap so a cold cargo cache does not trip it. + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Rust toolchain + run: rustup show + + # Same recipe as ts-release.yml, the only other workflow that needs it. + - name: Install wasm-opt + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq binaryen + + - name: Install pnpm + uses: pnpm/action-setup@v4 + + - name: Install node + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'pnpm' + + - name: Cache cargo registry and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: cargo-wasmopt-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + cargo-wasmopt- + + - name: Install pnpm dependencies + run: pnpm install + + # No DISABLE_WASM_OPT here -- that omission is the entire point of this + # workflow, so do not "fix" it to match ci.yaml. + - name: Build with wasm-opt enabled + run: pnpm build + + # Fail loudly if the pass silently did not run: build.sh skips wasm-opt + # when binaryen is absent, printing "Skipping wasm-opt" and exiting 0, so + # without this check a broken install would turn this job into an + # expensive duplicate of ci.yaml's frontend job. + - name: Assert the blobs are actually optimized + run: | + set -euo pipefail + for f in src/engine/core/libsimlin.wasm src/engine/core/libsimlin-browser.wasm; do + if [ ! -f "$f.raw" ]; then + echo "ERROR: $f.raw missing -- src/engine/build.sh did not stage this blob" >&2 + exit 1 + fi + if cmp -s "$f" "$f.raw"; then + echo "ERROR: $f is byte-identical to the pre-wasm-opt output, so" >&2 + echo " wasm-opt did not run. Is binaryen installed, and is" >&2 + echo " DISABLE_WASM_OPT unset? This job exists to run it." >&2 + exit 1 + fi + printf '%s: optimized (%s -> %s bytes)\n' "$f" "$(wc -c < "$f.raw")" "$(wc -c < "$f")" + done + + - name: Run the TypeScript tests against the optimized bundle + run: pnpm test diff --git a/scripts/verify-deploy-build.sh b/scripts/verify-deploy-build.sh index 50fc48b2c..3b0c61d5a 100755 --- a/scripts/verify-deploy-build.sh +++ b/scripts/verify-deploy-build.sh @@ -158,8 +158,11 @@ fi # and its model-preview pipeline calls simlin_project_render_png; a # slim WASM here would 500 every preview render. A missing or empty # WASM means the Rust+WASM step was skipped or failed silently. -# ~1MB minimum is well under any real build (release WASM is ~6MB; -# DISABLE_WASM_OPT bumps it to ~12MB). +# ~1MB minimum is well under any real build (wasm-opt'd release WASM is +# ~6.5MB; DISABLE_WASM_OPT leaves the raw opt-level=z output at ~7.9MB). +# This check deliberately passes either way -- it gates "the WASM step +# ran and produced the full artifact", not "wasm-opt ran"; that is +# .github/workflows/wasm-opt.yml's job. if [ ! -f src/engine/core/libsimlin.wasm ]; then fail "src/engine/core/libsimlin.wasm missing (engine WASM build skipped?)" else From d3074a38a437949789869106a3f6236c22553d14 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:46:13 -0700 Subject: [PATCH 31/59] engine: name what enforces emit_apply's arity discipline Removing `Apply`'s operand padding took a safety property with it. The pads were freshly-zeroed values in `b`/`c`, so an arm that read past its arity read zeros; without them it reads whatever locals an earlier `Apply` left behind. `BuiltinId::arity()`'s exhaustive match forces a new builtin to DECLARE an arity, but nothing forces its `emit_apply` arm to stay within it -- and the natural way to add a builtin is to copy an adjacent arm, so copying a 3-arity arm for a 1-arity one is the shape of the mistake. The guarantee now lives in the `apply_*` tests in `lower_tests.rs`, which execute every builtin against the VM. That is a fine place for it, but nothing said so where someone copying an arm would look. Say it there. All 24 builtins are within their arity today and all 13 `apply_*` tests pass; this adds no behaviour. --- src/simlin-engine/src/wasmgen/lower.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/simlin-engine/src/wasmgen/lower.rs b/src/simlin-engine/src/wasmgen/lower.rs index 0c504d37b..208ae4ee6 100644 --- a/src/simlin-engine/src/wasmgen/lower.rs +++ b/src/simlin-engine/src/wasmgen/lower.rs @@ -2476,6 +2476,17 @@ fn emit_apply(func: BuiltinId, ctx: &EmitCtx, f: &mut Function) { // positions this builtin does not read keep whatever a previous `Apply` // left in them and are never read back: each `match` arm below touches only // the locals its own arity covers. + // + // OBLIGATION when adding a builtin: an arm must read only the locals its + // arity covers. `BuiltinId::arity()`'s exhaustive match forces a new + // builtin to DECLARE an arity; nothing forces its arm here to respect it, + // and the natural way to add one is to copy an adjacent arm -- so copying a + // 3-arity arm for a 1-arity builtin reads two stale locals left by an + // earlier `Apply`. Operand padding used to make that safe by accident + // (`b`/`c` were always freshly-zeroed pads); with the padding gone the + // guarantee moved from the data into the `apply_*` tests in + // `lower_tests.rs`, which execute every builtin against the VM. They are + // the enforcement -- extend them when adding one. let arity = func.arity(); if arity >= 3 { f.instruction(&Ins::LocalSet(c)); From 69d56f9162ca4c989bc0f6f8d139ee4e6a699eb9 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:47:42 -0700 Subject: [PATCH 32/59] build: resolve the cargo target dir instead of assuming it `src/engine/build.sh` staged the wasm artifact from a hardcoded `../../target/wasm32-unknown-unknown/release/simlin.wasm`, and `scripts/pysimlin-tests.sh` staleness-checked a hardcoded `$REPO_ROOT/target/release/libsimlin.a`. Neither path is where cargo puts things when `CARGO_TARGET_DIR`, `--target-dir`, or a cargo config's `build.target-dir` is set. The two failures are not equally loud, which is why both are worth fixing. The wasm one is a `cp: cannot stat` in the middle of the pre-commit hook's TypeScript stage, which reads as a broken wasm build rather than as a path mismatch; it has cost two agents a debugging cycle. The pysimlin one is silent: the `-nt` staleness test on a nonexistent file simply never fires, so the CFFI extension is not rebuilt against a changed static library and the tests run against a stale binding. `scripts/cargo-target-dir.sh` asks `cargo metadata` rather than reconstructing cargo's rules, since that is the only thing that accounts for every way the directory can be set. It is one script rather than a snippet copied into each caller: a hand-maintained second resolution drifts exactly where the real one is non-trivial. `jq` is already a dependency of `scripts/release-*.sh` and `scripts/ci/await-reviews.sh`. --- scripts/cargo-target-dir.sh | 23 +++++++++++++++++++++++ scripts/pysimlin-tests.sh | 7 ++++++- src/engine/build.sh | 8 +++++++- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100755 scripts/cargo-target-dir.sh diff --git a/scripts/cargo-target-dir.sh b/scripts/cargo-target-dir.sh new file mode 100755 index 000000000..c374bc968 --- /dev/null +++ b/scripts/cargo-target-dir.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Print the absolute path of this workspace's cargo target directory. +# +# Scripts that stage a built artifact need to know where cargo actually put it, +# and that is NOT always `/target`: `CARGO_TARGET_DIR`, `--target-dir`, and +# `build.target-dir` in any applicable cargo config all move it. Hardcoding the +# default turns a moved target directory into a `cp: cannot stat` at the staging +# step, which reads as a broken build rather than as a path mismatch -- and has +# cost more than one debugging session. +# +# `cargo metadata` is the only thing that accounts for every way the directory +# can be set, so this asks cargo rather than reconstructing its rules. Keep this +# the single copy: a second, hand-maintained resolution drifts exactly where the +# real one is non-trivial. +# +# Usage: TARGET_DIR="$(scripts/cargo-target-dir.sh)" +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null 2>&1 && pwd)" + +cargo metadata --format-version 1 --no-deps \ + --manifest-path "$REPO_ROOT/Cargo.toml" \ + | jq -r '.target_directory' diff --git a/scripts/pysimlin-tests.sh b/scripts/pysimlin-tests.sh index 4eec646e7..95ac73cae 100755 --- a/scripts/pysimlin-tests.sh +++ b/scripts/pysimlin-tests.sh @@ -12,12 +12,17 @@ fi echo "Building libsimlin (release)..." cargo build --release --manifest-path src/libsimlin/Cargo.toml +CARGO_TARGET_DIR_RESOLVED="$("$REPO_ROOT/scripts/cargo-target-dir.sh")" cd src/pysimlin # Only rebuild the CFFI extension if the static library, header, or build # script is newer than the .so (or the .so doesn't exist yet). -LIBSIMLIN_A="$REPO_ROOT/target/release/libsimlin.a" +# Resolved rather than assumed -- see scripts/cargo-target-dir.sh. A stale path +# here is quieter than the wasm one: the staleness check below simply never +# fires, so the CFFI extension is silently not rebuilt against a changed +# library. +LIBSIMLIN_A="$CARGO_TARGET_DIR_RESOLVED/release/libsimlin.a" SIMLIN_H="$REPO_ROOT/src/libsimlin/simlin.h" CFFI_SO=$(find simlin -maxdepth 1 -name '_clib*.so' -print -quit 2>/dev/null || true) if [ -z "$CFFI_SO" ] || [ "$LIBSIMLIN_A" -nt "$CFFI_SO" ] || [ "$SIMLIN_H" -nt "$CFFI_SO" ] || [ simlin/_ffi_build.py -nt "$CFFI_SO" ]; then diff --git a/src/engine/build.sh b/src/engine/build.sh index cb24e9c15..f2be5a1cc 100755 --- a/src/engine/build.sh +++ b/src/engine/build.sh @@ -22,7 +22,13 @@ mkdir -p core # so we stage into core/ immediately after each build. # # The xmutil feature is always off here (C++ dependency, not wasm-buildable). -WASM_SRC="../../target/wasm32-unknown-unknown/release/simlin.wasm" +# +# The target directory is RESOLVED, not assumed: `CARGO_TARGET_DIR` and a cargo +# config's `build.target-dir` both move it, and a hardcoded `../../target` turns +# that into a `cp: cannot stat` below -- which reads as a broken wasm build +# rather than as a path mismatch. +TARGET_DIR="$("$DIR/../../scripts/cargo-target-dir.sh")" +WASM_SRC="$TARGET_DIR/wasm32-unknown-unknown/release/simlin.wasm" build_wasm() { local out_name="$1" From b43ec32bfcdd53317cfeb469e0712f6b3256cd42 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:48:52 -0700 Subject: [PATCH 33/59] doc: record what a warm single-equation edit costs (C6) Interactive edit latency had never been measured on a real model -- the `salsa_incremental` bench covers a synthetic 200-variable chain -- so C6 records it, together with the two ways the measurement misleads, because both were got wrong on the way to the numbers and either one misreports the result by an order of magnitude. An "equation edit" is not one workload: appending a term can change the dependency STRUCTURE rather than the text, and turning a bare `INITIAL(x)` into an expression containing one is exactly that. C-LEARN has 177 `INITIAL(` equations, so a probe that edits each variable once measures the structural cost for every one of them; pre-seeding so later edits only change digits moves the reported p90 by a factor of twelve. And consumer count does not predict cost -- the slowest variable has 3 references and a fast one has 47 -- which is recorded because it is the obvious hypothesis and it is false. The result: the median edit was already ~2.8 ms and is unchanged; the TAIL was the problem and is gone (p90 36.0 -> 3.0 ms, max 73.9 -> 6.7 ms, 11.03G -> 5.27G retired instructions). A cheap edit is now 40.6M instructions whose profile is almost entirely salsa's own verification plus re-lexing the one edited equation. What still costs a near-full recompile is a dependency-structure edit, at 1.798G instructions, and the decomposition names the mechanism rather than the symptom: 911 of ~955 explicit fragments and all 651 implicit helpers recompile because `model_module_ident_context` is an interned handle whose id changes when the helper set grows, and a new key cannot backdate. That bound is already pinned by `implicit_helper_add_is_tight_but_module_helper_add_is_not`. So the next interactive lever is GH #372's context-stable naming, not the dependency graph and not the fragment compilers -- which is worth stating, because the profile points at those two and they are the symptom. --- docs/design/engine-performance.md | 69 +++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/docs/design/engine-performance.md b/docs/design/engine-performance.md index daa6afbbc..4f9de5335 100644 --- a/docs/design/engine-performance.md +++ b/docs/design/engine-performance.md @@ -528,6 +528,71 @@ an assumption: the 12-repeat byte-identical determinism suites prewarm active. Salsa's accumulator drain is a dependency DFS, not an execution order. +### C6. Warm-edit latency: what a single-equation edit costs, and what still does not scale + +Interactive edit latency, not cold compile, is what a modeller experiences, and +it is measured with an out-of-tree probe that drives `SimlinDb::sync` + +`compile_project_incremental` over real edits to a real model. Two facts about +the measurement itself come first, because both were got wrong on the way to +the numbers and either one silently misreports the result by an order of +magnitude. + +**An "equation edit" is not one workload.** Appending a term to an equation can +change the DEPENDENCY STRUCTURE rather than just the text -- turning a bare +`INITIAL(x)` into an expression containing an `INITIAL(x)` is the case that bit +here, and C-LEARN has 177 `INITIAL(` equations. A probe that edits each variable +once measures the structural cost for every one of them. Pre-applying one edit +so that later edits only change digits is what separates the two, and it moves +the reported p90 by a factor of twelve. + +**Consumer count does not predict cost.** The obvious explanation for an +expensive edit -- a constant read by many variables, each recompiling under the +one-hop rule -- is false and was measured false: the slowest variable +(`2x CO2 forcing`) has 3 references in the model and a fast one (`c uptake`) has +47. Do not spend a day on fan-out. + +**Structure-preserving single-equation edit, C-LEARN** (40 edits, paired over +the same variables, before = the first two round-3 commits, after = all six): + +| | before | after | +|---|---:|---:| +| median | 2.8 ms | 2.8 ms | +| **p90** | **36.0 ms** | **3.0 ms** | +| max | 73.9 ms | **6.7 ms** | +| retired instructions | 11.03G | **5.27G** | + +The median was already fine; **the tail was the problem and the tail is gone.** +That tail was the per-assembly recompile of every implicit helper and the cycle +gate's un-memoized fragment probe -- the two changes keyed in round 3. A cheap +edit now costs 40.6M instructions and its profile is almost entirely salsa's +own `maybe_changed_after` verification plus the lexer re-reading the one edited +equation, which is what proportional looks like. + +**What still costs a full recompile: an edit that changes the dependency +structure.** Measured at 1.798G instructions -- 85% of a cold compile -- and it +decomposes as: + +| | calls | Ir | share | +|---|---:|---:|---:| +| `compile_var_fragment` | **911** of ~955 | 402M | 22% | +| `model_dependency_graph` | 1 | 565M | 31% | +| ...of which `resolve_recurrence_sccs` | 2 | 245M | 14% | +| `compile_implicit_var_fragment` | **651** (all) | 233M | 13% | + +Nearly every fragment in the model recompiles, which the per-variable keys +should have prevented. The reason is already written down one level away, on +`model_implicit_var_by_name`: a structural edit can change the model's implicit +helper set, `model_module_ident_context` is an INTERNED handle whose id changes +when that set grows, and a new key cannot backdate at all -- so every variable's +parse is re-keyed and every fragment behind it recompiles. The bound is pinned +by `implicit_helper_add_is_tight_but_module_helper_add_is_not`, which asserts +exactly this asymmetry. + +So the next lever for interactive latency is **not** the dependency graph and +not the fragment compilers: it is the granularity of the module-ident context's +interning, which is GH #372's context-stable naming. Anything else attacks the +22% and 13% rows while leaving the mechanism that produced them in place. + ### C5. `Compiler::intern_name` — the top allocation site, blocked on artifact identity 320,650 allocations per cold C-LEARN compile, ~10% of all 3.24M, from two @@ -573,6 +638,10 @@ short of it rather than taking a ~2-3%. starting; both are silent, and one is a process abort. 8. **C5 (`Compiler::intern_name`)** — the top allocation site, blocked on `NameId` assignment order being part of the compiled artifact. +9. **C6's residual** — the remaining interactive lever, and it is GH #372's + context-stable helper naming rather than anything in the compile path: a + structural edit re-keys `model_module_ident_context`, and a new interned key + cannot backdate, so every fragment behind it recompiles. Larger run-side swings identified during round 2 — all three were taken to a data verdict in round 3 (2026-06-04): From 105def7fac9a3839d61880f005eb66e4d68f0d6f Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:49:16 -0700 Subject: [PATCH 34/59] engine: correct four comments claiming compilation is rayon-parallel `common.rs`'s interner sharding, `db/query.rs`'s shared `DimensionsContext`, and the counting allocators in `examples/clearn_profile.rs` and `examples/backend_bench.rs` each justified a thread-safety decision by saying compilation fans out across rayon threads. It does not: measured at 0.9996 CPUs utilized, and the engine's only rayon call site is `layout::generate_best_layout`'s best-of-k seed fan-out. The designs are all correct and unchanged -- only the stated reason was false, and each has a real one. The interner is a process-global reachable from any thread, so sharding bounds contention among the layout fan-out and any host driving several databases. The context's `Mutex` is what makes the memo `Sync` and therefore shareable at all, so one context serves every variable's compilation instead of one per variable. A `GlobalAlloc` must be `Sync` and serves every thread in the process, which is a property of the allocator position rather than of the workload. Each now says compilation is not parallel *today* rather than making a claim about whether it will be. --- src/simlin-engine/examples/backend_bench.rs | 4 +++- src/simlin-engine/examples/clearn_profile.rs | 7 +++++-- src/simlin-engine/src/common.rs | 10 ++++++++-- src/simlin-engine/src/db/query.rs | 10 ++++++---- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/simlin-engine/examples/backend_bench.rs b/src/simlin-engine/examples/backend_bench.rs index a887bf88b..b63f0e7bf 100644 --- a/src/simlin-engine/examples/backend_bench.rs +++ b/src/simlin-engine/examples/backend_bench.rs @@ -56,7 +56,9 @@ use wasm::validate; // ── Counting allocator ────────────────────────────────────────────────────── // // Mirrors `examples/clearn_profile.rs`: cumulative alloc calls/bytes plus live -// bytes and a high-water peak, all atomic (compile fans out across rayon). The +// bytes and a high-water peak, all atomic because a `GlobalAlloc` must be +// `Sync` and serves every thread in the process -- not because compilation is +// parallel, which it is not today. The // time pass leaves counting OFF so the per-allocation atomics don't distort // wall-clock; the memory pass turns it ON. The default `GlobalAlloc::realloc` // routes through alloc/dealloc, so realloc is counted without an override. diff --git a/src/simlin-engine/examples/clearn_profile.rs b/src/simlin-engine/examples/clearn_profile.rs index b8c65c6b4..a48914d06 100644 --- a/src/simlin-engine/examples/clearn_profile.rs +++ b/src/simlin-engine/examples/clearn_profile.rs @@ -44,8 +44,11 @@ use simlin_engine::{CompiledSimulation, Vm, open_vensim}; // --- Counting allocator ----------------------------------------------------- // // Tracks cumulative allocation calls/bytes plus live bytes and a high-water -// mark. compile_project_incremental can fan out across rayon threads, so all -// counters are atomic and the peak is maintained with a CAS loop. The default +// mark. A `GlobalAlloc` must be `Sync` and serves every thread in the process, +// so the counters are atomic and the peak is maintained with a CAS loop. That +// is a requirement of the allocator position, not of the workload: +// compile_project_incremental runs on one thread today (measured at 0.9996 CPUs +// utilized). The default // GlobalAlloc::realloc routes through our alloc/dealloc, so realloc is counted // without an explicit override. diff --git a/src/simlin-engine/src/common.rs b/src/simlin-engine/src/common.rs index 51188949b..be7511859 100644 --- a/src/simlin-engine/src/common.rs +++ b/src/simlin-engine/src/common.rs @@ -43,8 +43,14 @@ struct Interned { } /// Number of shards. A power of two so the shard index is a cheap mask of the -/// hash. Compilation fans out across rayon threads, so sharding keeps lock -/// contention low without a concurrent-map dependency. +/// hash. The interner is a process-global (`GLOBAL` below) reachable from any +/// thread, so sharding bounds lock contention without pulling in a +/// concurrent-map dependency. +/// +/// The concurrency it bounds is NOT compilation, which runs on one thread today +/// (measured at 0.9996 CPUs utilized). It is `layout::generate_best_layout`'s +/// best-of-k seed fan-out -- the engine's only rayon call site -- plus any host +/// driving several `SimlinDb`s at once. const INTERNER_SHARDS: usize = 64; /// One shard: a content-keyed map from string -> weak handle. A `Weak` diff --git a/src/simlin-engine/src/db/query.rs b/src/simlin-engine/src/db/query.rs index 72e2306d9..cfe72e638 100644 --- a/src/simlin-engine/src/db/query.rs +++ b/src/simlin-engine/src/db/query.rs @@ -136,10 +136,12 @@ pub fn project_datamodel_dims(db: &dyn Db, project: SourceProject) -> Vec Date: Mon, 10 Aug 2026 08:52:47 -0700 Subject: [PATCH 35/59] engine: name the two corpus lists apart `roundtrip.rs` and `simulate.rs` each held a private `static TEST_MODELS` with different contents -- 35 models and 58. Nothing stops two modules sharing a private name, and nothing warns when a sweep is pointed at the wrong one: it compiles, runs, and reports a clean pass over a corpus it never touched. That is not hypothetical. Verifying which slots carry information across a simulation step, a whole-corpus check run against the 35-model list reported zero divergences -- while the only model that diverges, the lookup-only table holders in `lookups_simlin/test_lookups.xmile`, exists solely in the 58-model one. The clean sweep was the tell: a probe built to find something, finding nothing, is a result to distrust before it is a result to report. The narrower list becomes `ROUNDTRIP_TEST_MODELS`, and each now carries a comment naming the other so the next reader sees both from either side. --- src/simlin-engine/tests/integration/roundtrip.rs | 12 ++++++++++-- src/simlin-engine/tests/integration/simulate.rs | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/simlin-engine/tests/integration/roundtrip.rs b/src/simlin-engine/tests/integration/roundtrip.rs index 8d678084a..fe7263215 100644 --- a/src/simlin-engine/tests/integration/roundtrip.rs +++ b/src/simlin-engine/tests/integration/roundtrip.rs @@ -10,7 +10,15 @@ use simlin_engine::db::{ }; use simlin_engine::xmile; -static TEST_MODELS: &[&str] = &[ +/// The models this file round-trips through XMILE serialization. +/// +/// A NARROWER list than `simulate.rs`'s same-shaped corpus, which its +/// `corpus_tests!` macro generates under the name `TEST_MODELS` from its own +/// (larger) set. The two are private to their own modules, so nothing stops +/// them sharing a name -- and a sweep run against the wrong one looks like a +/// clean pass over a corpus it never touched. Named apart so that cannot +/// happen silently. +static ROUNDTRIP_TEST_MODELS: &[&str] = &[ "test/test-models/samples/bpowers-hares_and_lynxes_modules/model.xmile", "test/test-models/tests/logicals/test_logicals.xmile", "test/test-models/samples/SIR/SIR.xmile", @@ -51,7 +59,7 @@ static TEST_MODELS: &[&str] = &[ #[test] fn roundtrips_model() { - for &path in TEST_MODELS { + for &path in ROUNDTRIP_TEST_MODELS { let file_path = format!("../../{path}"); eprintln!("model: {path}"); diff --git a/src/simlin-engine/tests/integration/simulate.rs b/src/simlin-engine/tests/integration/simulate.rs index 2891780b5..50747b1b6 100644 --- a/src/simlin-engine/tests/integration/simulate.rs +++ b/src/simlin-engine/tests/integration/simulate.rs @@ -58,6 +58,11 @@ macro_rules! corpus_tests { const OUTPUT_FILES: &[(&str, u8)] = &[("output.csv", b','), ("output.tab", b'\t')]; +// The simulation corpus. `array:` makes the macro emit the backing +// `static TEST_MODELS` as well as the per-model tests. This is the LARGER of +// the two corpus lists -- `roundtrip.rs` keeps its own, narrower one as +// `ROUNDTRIP_TEST_MODELS`; sweep against that one and you get a clean pass over +// a corpus you never touched. corpus_tests! { array: TEST_MODELS; module: corpus; From 7d9c60ca889a1ceeab4d278ece12fd93bcecfeac Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:56:00 -0700 Subject: [PATCH 36/59] engine: record why the LTM implicit-var parse is duplicated `model_ltm_implicit_var_info` parses every LTM equation to harvest the implicit helpers the PREVIOUS/INIT and stdlib-module visitor synthesizes, and `compile_ltm_equation_fragment` parses each of them again when it compiles the variable. On C-LEARN that is ~7,125 equations parsed to find 738 helpers, and roughly 6.7% of a compile spent parsing twice (GH #655 finding 3). It reads as an obvious cleanup and has now been re-derived three times. The answer is a measured trade: publishing the parses for the fragment compile to consume -- the only non-cycling shape, since that compile already reads this query -- moves allocations 41.37M to 38.33M (-7.3%) and peak live bytes during `compile_project_incremental` 353.2 to 435.4 MiB (+82.2 MiB), because it turns a transient parse into a permanent salsa memo held for every LTM variable rather than for the 738 whose helpers survive. That is substantially all of GH #977's peak reduction traded for a -7.3% allocation count, on a compile that is allocation-bound rather than peak-bound. Retaining only the helper-bearing parses is the same trade an order of magnitude smaller, not a better one: the pass must parse everything to discover which variables synthesize helpers, so it can choose only what to retain. The note names retention -- a smaller parsed representation, or memo eviction -- as what would change the answer, so a fourth reader can tell a promising direction from one already measured. --- src/simlin-engine/src/db/ltm/mod.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/simlin-engine/src/db/ltm/mod.rs b/src/simlin-engine/src/db/ltm/mod.rs index e7e2d32bd..2f8812232 100644 --- a/src/simlin-engine/src/db/ltm/mod.rs +++ b/src/simlin-engine/src/db/ltm/mod.rs @@ -843,6 +843,32 @@ pub struct LtmImplicitVarMeta { /// caching the results. Both `compute_layout` and `assemble_module` read /// this to allocate slots and compile fragments for those implicit vars /// within LTM equations. +/// +/// **The parse here is DELIBERATELY duplicated** with the one +/// `compile_ltm_equation_fragment` performs, and that is a measured space-time +/// trade rather than an oversight. It looks like pure waste: on C-LEARN this +/// parses ~7,125 equations to harvest 738 implicit helpers, and every one is +/// parsed again when its fragment is compiled -- about 6.7% of a compile +/// (GH #655 finding 3). +/// +/// Publishing these parses for the fragment compile to consume is the only +/// non-cycling shape, since the fragment compile already reads this query and +/// the reverse direction cycles. It was built and measured on C-LEARN: +/// **allocations 41.37M -> 38.33M (-7.3%), peak live bytes during +/// `compile_project_incremental` 353.2 -> 435.4 MiB (+82.2 MiB, +23.3%)**. The +/// retention is what costs -- a transient parse becomes a permanent salsa memo, +/// held for every LTM variable rather than for the 738 whose helpers survive -- +/// and it gives back substantially all of GH #977's peak reduction to buy a +/// -7.3% allocation count, on a compile that is allocation-bound rather than +/// peak-bound. +/// +/// Publishing only the helper-bearing parses scales both sides by the same +/// ~738/7,125: this pass must parse everything to discover WHICH variables +/// synthesize helpers and can only choose what to RETAIN, so it is the same +/// trade an order of magnitude smaller, not a better one. +/// +/// What would change the answer is the retention -- a smaller parsed +/// representation, or salsa memo eviction -- not the call graph. #[salsa::tracked(returns(ref))] pub fn model_ltm_implicit_var_info( db: &dyn Db, From da10cfb951ca7c6c165a6da1c6d75dc248b2df22 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 08:59:13 -0700 Subject: [PATCH 37/59] engine: make the graphical-function block scan exhaustive over opcodes This fixes no bug. Of the 61 `SymbolicOpcode` variants, exactly three carry a `base_gf` -- `Lookup`, `LookupArray`, `LookupDirect` -- and `gf_blocks_of_fragment` already listed all three. What changes is who enforces that: the obligation moves from a comment into the compiler. The hole it closes is a silent one. The scan reconstructs a fragment's GF block layout by matching opcodes for their `(base_gf, table_count)` runs and ended in `_ => continue`, so a lookup-family opcode it did not recognise was not an error. The block would simply stop being seen as referenced, collapse into a maximal un-referenced GAP, and the de-duplicated table layout would come out wrong with no diagnostic anywhere -- wrong numbers rather than a failure. A test cannot close it. `test_gf_block_scan_sees_lookup_direct_runs` exercises one lookup opcode, and it passes unchanged when a second is added and ignored; that is exactly the hazard the repo names, where a test pinning one arm of an N-way decision reads like a test pinning the decision. Only the compiler covers every arm, so the decision moves onto the type as `SymbolicOpcode::gf_run()` with an exhaustive match and no `_`. The 58 non-carrying variants are spelled out; that verbosity is the mechanism, not a cost of it. Same shape and same argument as `BuiltinId::arity()`: one table, read by every consumer, whose no-`_` match means a new variant cannot be added without deciding the question. `gf_run_reports_every_lookup_family_opcode` pins the answer for the three carriers and both non-carrier shapes. It is derived from the enum rather than sampled, but the compiler is what makes the set complete -- the test only fixes what each member reports. --- src/simlin-engine/src/compiler/symbolic.rs | 172 ++++++++++++++++++--- 1 file changed, 149 insertions(+), 23 deletions(-) diff --git a/src/simlin-engine/src/compiler/symbolic.rs b/src/simlin-engine/src/compiler/symbolic.rs index 0c88fb35e..bd1c55965 100644 --- a/src/simlin-engine/src/compiler/symbolic.rs +++ b/src/simlin-engine/src/compiler/symbolic.rs @@ -1823,6 +1823,109 @@ fn gf_block_key(tables: &[Vec<(f64, f64)>]) -> GfBlockKey { key } +impl SymbolicOpcode { + /// The graphical-function BLOCK this opcode references, as + /// `(base_gf, table_count)` -- i.e. the run `[base_gf, base_gf + table_count)` + /// in the fragment's own `graphical_functions`. + /// + /// This is the SINGLE place that decides whether an opcode carries a + /// graphical function, and the match is exhaustive with no `_` arm on + /// purpose: a new variant cannot be added without answering the question + /// here, which is a compile error rather than a silent omission. + /// + /// That matters because the consumer, `gf_blocks_of_fragment`, reconstructs + /// a fragment's GF block layout by scanning for these runs, and a lookup + /// opcode it does not recognise is not an error -- the block simply stops + /// being seen as referenced, collapses into a maximal un-referenced GAP, + /// and the de-duplicated table layout comes out wrong with no diagnostic + /// anywhere. Wrong numbers, not a failure. A test cannot close that hole + /// either: a fixture exercising one lookup opcode passes unchanged when a + /// second is added and ignored, which is the "a test that pins one arm of + /// an N-way decision reads exactly like a test that pins the decision" + /// hazard. Only the compiler covers every arm. + /// + /// The same reasoning, and the same shape, as `BuiltinId::arity`. + pub(crate) fn gf_run(&self) -> Option<(usize, usize)> { + match self { + SymbolicOpcode::Lookup { + base_gf, + table_count, + .. + } + | SymbolicOpcode::LookupDirect { + base_gf, + table_count, + .. + } + | SymbolicOpcode::LookupArray { + base_gf, + table_count, + .. + } => Some((*base_gf as usize, *table_count as usize)), + // Every remaining variant, spelled out rather than wildcarded -- + // that is what makes a new one a compile error here. + SymbolicOpcode::Op2 { .. } + | SymbolicOpcode::Not { .. } + | SymbolicOpcode::LoadConstant { .. } + | SymbolicOpcode::LoadVar { .. } + | SymbolicOpcode::SymLoadPrev { .. } + | SymbolicOpcode::SymLoadInitial { .. } + | SymbolicOpcode::LoadGlobalVar { .. } + | SymbolicOpcode::PushSubscriptIndex { .. } + | SymbolicOpcode::LoadSubscript { .. } + | SymbolicOpcode::SetCond { .. } + | SymbolicOpcode::If { .. } + | SymbolicOpcode::Ret + | SymbolicOpcode::LoadModuleInput { .. } + | SymbolicOpcode::EvalModule { .. } + | SymbolicOpcode::AssignCurr { .. } + | SymbolicOpcode::Apply { .. } + | SymbolicOpcode::AssignConstCurr { .. } + | SymbolicOpcode::BinOpAssignCurr { .. } + | SymbolicOpcode::BinOpAssignNext { .. } + | SymbolicOpcode::PushTempView { .. } + | SymbolicOpcode::PushStaticView { .. } + | SymbolicOpcode::PushVarViewDirect { .. } + | SymbolicOpcode::ViewSubscriptConst { .. } + | SymbolicOpcode::ViewSubscriptDynamic { .. } + | SymbolicOpcode::ViewRange { .. } + | SymbolicOpcode::ViewRangeDynamic { .. } + | SymbolicOpcode::ViewStarRange { .. } + | SymbolicOpcode::ViewWildcard { .. } + | SymbolicOpcode::ViewTranspose { .. } + | SymbolicOpcode::PopView { .. } + | SymbolicOpcode::DupView { .. } + | SymbolicOpcode::LoadTempConst { .. } + | SymbolicOpcode::LoadTempDynamic { .. } + | SymbolicOpcode::BeginIter { .. } + | SymbolicOpcode::LoadIterElement { .. } + | SymbolicOpcode::LoadIterTempElement { .. } + | SymbolicOpcode::LoadIterViewTop { .. } + | SymbolicOpcode::LoadIterViewAt { .. } + | SymbolicOpcode::StoreIterElement { .. } + | SymbolicOpcode::NextIterOrJump { .. } + | SymbolicOpcode::EndIter { .. } + | SymbolicOpcode::ArraySum { .. } + | SymbolicOpcode::ArrayMax { .. } + | SymbolicOpcode::ArrayMin { .. } + | SymbolicOpcode::ArrayMean { .. } + | SymbolicOpcode::ArrayStddev { .. } + | SymbolicOpcode::ArraySize { .. } + | SymbolicOpcode::VectorSelect { .. } + | SymbolicOpcode::VectorElmMap { .. } + | SymbolicOpcode::VectorSortOrder { .. } + | SymbolicOpcode::Rank { .. } + | SymbolicOpcode::AllocateAvailable { .. } + | SymbolicOpcode::AllocateByPriority { .. } + | SymbolicOpcode::BeginBroadcastIter { .. } + | SymbolicOpcode::LoadBroadcastElement { .. } + | SymbolicOpcode::StoreBroadcastElement { .. } + | SymbolicOpcode::NextBroadcastOrJump { .. } + | SymbolicOpcode::EndBroadcastIter { .. } => None, + } + } +} + /// Reconstruct the GF *block* layout of a single fragment as a list of /// `(start, len)` blocks covering `[0, gf_len)` exactly, sorted by `start` /// (#582). @@ -1860,29 +1963,8 @@ fn gf_blocks_of_fragment(frag: &PerVarBytecodes) -> Result, // Collect the distinct opcode runs. let mut runs: Vec<(usize, usize)> = Vec::new(); for op in &frag.symbolic.code { - let (base, count) = match op { - SymbolicOpcode::Lookup { - base_gf, - table_count, - .. - } - | SymbolicOpcode::LookupArray { - base_gf, - table_count, - .. - } - | SymbolicOpcode::LookupDirect { - base_gf, - table_count, - .. - } => (*base_gf as usize, *table_count as usize), - // OBLIGATION: every lookup-family opcode that carries a `base_gf` - // MUST be listed above. This arm is silent -- an unlisted one is - // skipped with no diagnostic, its block collapses into an - // un-referenced gap, and the de-duplicated table layout is wrong - // with no error anywhere. `test_gf_block_scan_sees_lookup_direct_runs` - // is the tripwire; extend it when adding a lookup opcode. - _ => continue, + let Some((base, count)) = op.gf_run() else { + continue; }; if count == 0 { continue; @@ -4560,6 +4642,50 @@ mod tests { } } + /// Every opcode that carries a `base_gf`, with the run it reports. + /// + /// Derived from the enum rather than sampled: `gf_run`'s match is + /// exhaustive with no `_`, so the compiler is what guarantees a new variant + /// answers the question, and this pins the answer for the three that do + /// carry one. A representative non-carrier of each shape (unit and struct) + /// is included so the `None` side is exercised too. + #[test] + fn gf_run_reports_every_lookup_family_opcode() { + let rows: Vec<(SymbolicOpcode, Option<(usize, usize)>)> = vec![ + ( + SymbolicOpcode::Lookup { + base_gf: 3, + table_count: 2, + mode: LookupMode::Interpolate, + }, + Some((3, 2)), + ), + ( + SymbolicOpcode::LookupDirect { + base_gf: 5, + table_count: 4, + elem: 1, + mode: LookupMode::Interpolate, + }, + Some((5, 4)), + ), + ( + SymbolicOpcode::LookupArray { + base_gf: 7, + table_count: 6, + mode: LookupMode::Interpolate, + write_temp_id: 0, + }, + Some((7, 6)), + ), + (SymbolicOpcode::Ret, None), + (SymbolicOpcode::SetCond {}, None), + ]; + for (op, want) in rows { + assert_eq!(op.gf_run(), want, "gf_run of {op:?}"); + } + } + /// Two SEPARATE single-table GF blocks in one fragment, each read by a /// `LookupDirect`, merged with a fragment holding only the second table's /// content. From 2ac7ac53a97d7b9fa5eee033d9113823215694ef Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 09:25:25 -0700 Subject: [PATCH 38/59] doc: size a fast path by the work it replaces, and record why one did not pay Two corrections to the measurement guidance, both from applying it. The structural check was stated as requiring the caller's machine code to be "byte-identical". That is unachievable whenever the change adds code: the text section shifts, and absolute branch targets and rip-relative displacements move with it even where nothing about the caller changed. The checkable claim is instruction-sequence-identical modulo relocation -- normalise the addresses, then require the same instruction count, mnemonics, operands and in-function branch offsets. Worth fixing before someone tries to apply the literal form, finds it fails on a correct change, and concludes the check is useless. The new sizing rule is the one that would have changed a decision. How many inputs are ELIGIBLE for a shortcut and how many BENEFIT from it are different questions, and only the second predicts the outcome: a shortcut has a fixed cost, so it wins only where the work it displaces exceeds that cost, which means a size threshold and a fallback paid on everything below it. Negative result #4 records the case. Replacing `lookup`'s binary search with an O(1) uniform-grid position was predicted at -2.7% / -3.1% instructions on the two hero models from a census showing ~100% of their tables eligible; it measured -0.63% and +0.59%. The guess costs ~50 instructions against ~12 per search probe, so it pays only above ~4 probes -- C-LEARN's 251-point tables win, WORLD3's 7-point tables lose -- and the prediction had costed the search removed without costing the guess replacing it. It is filed with the other three because it is the one to read before designing an experiment: in those the effect was merely small, whereas here the aggregate and the mechanism disagreed. C-LEARN alone reads as a win and a matching cycles figure could have been quoted; only the per-call channel showed WORLD3's `lookup` 7.7% worse underneath it, and only a pre-registered per-model prediction made the sign flip impossible to read as a smaller-than-hoped win. --- docs/design/engine-performance.md | 71 ++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 6 deletions(-) diff --git a/docs/design/engine-performance.md b/docs/design/engine-performance.md index 761846055..931241b67 100644 --- a/docs/design/engine-performance.md +++ b/docs/design/engine-performance.md @@ -86,12 +86,28 @@ spread. **Prefer a structural check to a statistical one where the change admits it.** When a change is confined to a function that is `#[inline(never)]` and keeps its -signature, the callers' machine code should be *byte-identical*; disassemble -both binaries and diff the caller modulo addresses. That is a binary answer -rather than a sample, and it directly detects the failure mode that has bitten -this file's eval-loop work repeatedly: a change leaking into `eval_bytecode` and -perturbing the register allocation of a very large function. Treat a single -differing instruction as a hard stop and explain it before quoting any number. +signature, its callers' machine code should be unchanged. Verify it by +disassembling both binaries and diffing the caller. + +The claim to check is *instruction-sequence-identical modulo relocation*, not +byte-identical: adding code anywhere shifts the text section, so absolute branch +targets and every rip-relative displacement move even when nothing about the +caller changed. Normalise those, then require the same instruction count, the +same mnemonics and operands, and the same in-function branch offsets. + +That is a binary answer rather than a sample, and it directly detects the +failure mode that has bitten this file's eval-loop work repeatedly: a change +leaking into `eval_bytecode` and perturbing the register allocation of a very +large function. Treat a single differing instruction as a hard stop and explain +it before quoting any number. + +**Size a fast path by the work it replaces, not by how often it applies.** How +many inputs are *eligible* for a shortcut and how many *benefit* from it are +different questions, and only the second predicts the outcome. A shortcut has +its own fixed cost, so it wins only where the work it displaces exceeds that +cost -- which usually means a size threshold, and a fallback that is now paid on +every input below it. Cost both sides before predicting, and gate on the +threshold rather than on eligibility. **Decide what would falsify the change before measuring it.** Write down the predicted delta per channel, and the signatures that would mean it did not work: @@ -410,6 +426,49 @@ parity, zero-alloc all hold -- but did not clear the keep bar: - Branch-misses fell 8.4%, so a mispredict-bound core (the round-1 Ryzen) might see a real win -- that is the retry condition recorded on GH #712. +**Negative result #4: the uniform-grid lookup index (implemented, not +landed).** Graphical-function x-axes are overwhelmingly uniform -- 86.6% of +corpus tables exactly, another 4.7% to within an ulp -- so `lookup`'s binary +search can be replaced by an O(1) position computed from the table's endpoints +and then verified, falling back to the search when the check fails. It is exact +on any sorted axis (the check `x[k-1] < index <= x[k]` identifies the same +position the search returns) and needs no stored metadata, so nothing is +threaded through the dispatch arm -- the property whose absence sank #602. + +Measured, against predictions registered before implementing: + +| | predicted | measured | +|---|---|---| +| C-LEARN instructions | -2.7% | **-0.63%** | +| WORLD3 instructions | -3.1% | **+0.59%** | +| C-LEARN `vm::lookup` Ir | -60% | **-34.8%** | +| WORLD3 `vm::lookup` Ir | -35% | **+7.7%** | + +The guess costs ~50 instructions (two divisions, a saturating float-to-int +cast, two bounds-checked loads for the check) against ~12 per search probe, so +it pays only above about four probes. C-LEARN's tables have a median of 251 +points -- an eight-probe search -- and win; WORLD3's median is 7, a three-probe +search, and lose. Gating on a 32-point minimum recovered C-LEARN and left +WORLD3 still 7.7% worse in `lookup`, because the restructured fallback is paid +by every table below the gate, which is most of the corpus. Forcing the helper +inline changed nothing (it was already inlined). + +**Why this one is worth reading before designing an experiment**: unlike the +three above, where the effect was merely small, here the aggregate and the +mechanism DISAGREED. End-to-end C-LEARN alone reads as a -0.63% win and a +plausible cycles figure could have been quoted to match it. Only the per-call +mechanism channel showed WORLD3's `lookup` getting 7.7% worse underneath that +aggregate, and only a pre-registered per-model prediction made the sign flip +impossible to read as "smaller than hoped". A single-model, single-channel +measurement ships this change. + +The standing lesson is the sizing rule under "Measuring a change": a census +established that ~100% of both hero models' tables were ELIGIBLE, which is not +the same as benefiting, and the prediction costed the search being removed +without costing the guess replacing it. The patch is recoverable from the +round's scratch artifacts (`p9_option_c.patch`) if a cheaper guess ever makes +the break-even worth revisiting. + Methodology consequence for future rounds: the ~4% figure above bounds a WALL-CLOCK/CYCLES claim from a single build pair, and nothing else. Retired instructions and branches have an sd of ~0.026% across builds, so the same From 3d488661d0846e458262d52e87200ffb261500c1 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 09:27:50 -0700 Subject: [PATCH 39/59] doc: record the superinstruction family and the LTM arm omission The R3 section described superinstructions as the portable lever still to be taken; the family is now implemented, so it records what exists, the two rules the work established (a fusion may live in the symbolic layer only if the fused opcode has a SymbolicOpcode form; score helper-variable ideas against the post-fusion stream), and the measured cost of a removed dispatch -- 25.9 instructions rather than the ~10 a bare dispatch costs, because fusion also deletes the operand work behind it. Sizing against the wrong one of those two misprices a proposal by 3.6x. It also records what the family cannot reach. The dispatches superinstructions remove best are the perfectly-predicted ones, so branches fell 6.3% while branch-misses fell only 2.6%. That bounds where #604's predictor-threshold hypothesis could still be tested. The ordering list gains the LTM link-score arms, whose residual is gated on the TIME semantics question in #1016 rather than on further engineering, with an explicit warning against the cheaper negative test that silently rewrites 187 result slots. --- docs/design/engine-performance.md | 75 ++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/docs/design/engine-performance.md b/docs/design/engine-performance.md index 0898cea9a..97a8c4695 100644 --- a/docs/design/engine-performance.md +++ b/docs/design/engine-performance.md @@ -1,8 +1,10 @@ # Engine performance: profile and optimization opportunities -Status: analysis + three rounds of wins landed. Round 1 2026-05-19; round 2 -(constant folding + linear-run fast paths) 2026-06-03; compile round 3 (the -salsa pipeline's own redundancy) 2026-08-10. +Status: analysis + four rounds of wins landed. Round 1 2026-05-19; round 2 +(constant folding + linear-run fast paths) 2026-06-03; round 3 2026-08-10 — +the salsa pipeline's own redundancy on the compile side, a superinstruction +family on the run side, and the LTM link-score arms that were being +materialized only to evaluate to zero. This documents an empirical CPU/memory profile of **compiling and simulating the C-LEARN hero model** (the largest model we have: ~53k MDL lines / 1.4 MB, 934 @@ -334,13 +336,49 @@ jump table (one indirect branch whose target is data-dependent → BTB-unfriendl Classic threaded dispatch (computed-goto / guaranteed tail calls) would spread the indirect branch across handlers for better prediction, but **stable Rust offers neither computed-goto nor guaranteed TCO** (the `become` keyword is unstable). -Portable options: - -- **More superinstructions** for the top opcode bigrams/trigrams (e.g. - `LoadVar; LoadVar; Op2`, `LoadConstant; Op2`). Each fused opcode removes a - dispatch; incremental and low-risk. This is the portable lever today. -- Revisit explicit tail-call dispatch if/when `become` stabilizes. -- R2 (register VM) reduces dispatch count more than any dispatch-mechanism change. +Superinstructions are the portable lever, and the family below is implemented. +Each removes a dispatch **and the operand work behind it**, which is why a +removed dispatch costs ~25.9 instructions rather than the ~10 a bare dispatch +costs — size proposals in this family against 25.9 or they read ~3x cheaper +than they are. Both figures were measured by injecting an empty `ProbeNop` +opcode at controlled rates and taking the instruction slope, validated by +bit-identical results at every rate and an exactly linear dispatch count. + +Landed, all created by `ByteCode::fuse_three_address` on the Vm's private +execution copy unless noted: + +| form | fuses | +|---|---| +| `SelectIf` / `SelectIfAssignCurr` | `SetCond; If[; AssignCurr]` | +| `AssignVarCurr` / `AssignInitialCurr` / `AssignModInputCurr` | a leaf load + its store | +| `BinStackModInput` / `AssignStackModInputCurr` | module inputs as a fusible leaf | +| `LoadPrevConst` | `LoadConstant; LoadPrev` (the `PREVIOUS` fallback) | +| `ApplyTerConst` | a 3-arity builtin's literal trailing operand | +| `SubVarPrev` / `BinStackPrev` | the `v - PREVIOUS(v)` delta, 4->1 and 3->1 | +| `LookupDirect` (codegen, so it reaches wasmgen) | a lookup's constant element offset | + +`SetCond; If` is safe to fuse because codegen is the sole producer of both and +emits them together, so the pair is adjacent by construction rather than by +luck. + +Two rules this family established. **A fusion may live in the symbolic layer +iff the fused opcode has a `SymbolicOpcode` form**, because `CompiledSimulation` +must stay the pure resolution of the cached symbolic fragments; the rest are +Vm-local and never reach wasmgen. And **score a helper-variable idea against +the post-fusion stream**: hoisting a repeated subexpression into a shared aux +replaces each use with a `LoadVar` — one dispatch, exactly what a fused opcode +costs — so the hoist is worth zero wherever a superinstruction can match the +pattern, while still paying for a store and a slot. + +What this family cannot reach: **mispredicts**. The dispatches superinstructions +remove best are the perfectly-predicted ones — `SetCond` always jumps to `If`'s +arm — so fusing them removes instructions and branches but not branch misses. +Measured: branches −6.3% against branch-misses −2.6%. The mispredict cost lives +in the genuinely-unpredictable dispatches, which is where #604's hypothesis +would have to be tested if anyone retries it. + +Remaining: revisit explicit tail-call dispatch if/when `become` stabilizes; a +register VM reduces dispatch count more than any dispatch-mechanism change. ### Round 2 wins (2026-06-03, measured on Apple M-series / Asahi) @@ -773,7 +811,11 @@ short of it rather than taking a ~2-3%. decompose path for shape-equal non-linear views (a per-loop access-plan cache is the next idea there — and see the round-2 negative result before attempting it). -5. **R3 superinstructions** — incremental dispatch wins, low risk. +5. ~~**R3 superinstructions**~~ — DONE; the family and its two rules are in the + R3 section above. Cumulative on the LTM-augmented run, which is where the + `PREVIOUS`-heavy forms pay most: post-fusion flow opcodes −44.3%, retired + instructions −28.3% on C-LEARN and −33.6% on WORLD3-03. An instruction/branch + win, not a predictor win. 6. ~~**C2 / C3**~~ — answered, and not as proposed: C2 is moot (the function is salsa-cached and off the ordinary compile path) and C3's two halves are already done or the wrong lever. The compile round 3 section above records @@ -787,6 +829,17 @@ short of it rather than taking a ~2-3%. context-stable helper naming rather than anything in the compile path: a structural edit re-keys `model_module_ident_context`, and a new interned key cannot backdate, so every fragment behind it recompiles. +10. **LTM link-score arms** — the dominant cost of an LTM-enabled run on an + arrayed model, and mostly a generation question rather than a VM one. An + arm whose ceteris-paribus partial is *provably* `PREVIOUS(target)` is + omitted and lowers to a single zero-store; on C-LEARN that is 4,335 arms + and −19.2% of the flow program. The residual is gated on a semantics + question, not on engineering: ~5,000 further arms are blocked solely by a + live `time()`, because TIME is excluded from the freeze (GH #1016), and + resolving that would roughly double the win. Do **not** substitute the + cheaper negative test ("the link's source stayed frozen") — it asks a + different question and silently rewrites 187 result slots. GH #977 carries + the decomposition and the standing constraints. Larger run-side swings identified during round 2 — all three were taken to a data verdict in round 3 (2026-06-04): From 929051e55e81694b9e66cb0afcde1b2e4cf60308 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 09:30:17 -0700 Subject: [PATCH 40/59] build: add an interleaved A/B harness for the profiling example Every perf claim in this round needed the same shape of measurement: alternate two binaries, take medians, and report a per-phase delta. Doing that by hand invites the two mistakes docs/design/engine-performance.md records -- running all of A then all of B, which compares two different machines, and quoting a cycles delta that a null control could have produced. The script warms both sides before measuring (whichever binary runs cold looks slower), alternates which side leads each round so a systematic first-vs-second effect cancels, and takes --perf for the layout-stable instruction counts that are the right channel for a few-percent effect. It refuses to inherit CLEARN_COUNT_ALLOCS, whose per-allocation atomics distort exactly the phase being timed. --- scripts/perf-ab.py | 153 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100755 scripts/perf-ab.py diff --git a/scripts/perf-ab.py b/scripts/perf-ab.py new file mode 100755 index 000000000..92d47ff7b --- /dev/null +++ b/scripts/perf-ab.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# Copyright 2026 The Simlin Authors. All rights reserved. +# Use of this source code is governed by the Apache License, +# Version 2.0, that can be found in the LICENSE file. + +"""Interleaved A/B timing for the `clearn_profile` engine harness. + +Why interleaved, and why medians: `docs/design/engine-performance.md` records +two rounds where a perf "win" turned out to be an artifact. Machine conditions +drift over minutes, so running all of A then all of B compares two different +machines; interleaving A,B,A,B... controls for that. Binary layout is a second, +independent lottery -- two builds of the *same* source can differ by several +percent -- which interleaving does NOT control, so treat a delta under ~4% as +unresolved unless you rebuild both sides and reproduce it. + +Both sides are warmed before the measured rounds because whichever binary runs +cold reliably looks slower. + +Usage: + + # build each side into its own target dir first, e.g. + # git worktree add ../simlin-base main + # CARGO_TARGET_DIR=/path/to/base-target cargo build --release \ + # -p simlin-engine --example clearn_profile --features file_io + scripts/perf-ab.py --a base-target/release/examples/clearn_profile \ + --b target/release/examples/clearn_profile \ + --rounds 7 --model test/metasd/WRLD3-03/wrld3-03.mdl --ltm + +`--perf` additionally reports retired-instruction counts via `perf stat`, which +are insensitive to machine load and to binary layout; when a wall-clock delta is +near the noise floor, the instruction delta is the number to trust. +""" + +from __future__ import annotations + +import argparse +import os +import re +import statistics +import subprocess +import sys + +# `phase()` in examples/clearn_profile.rs prints: +# " ms | allocs ..." +PHASE_RE = re.compile(r"^(\S.*?)\s{2,}([0-9.]+) ms \|") +# Trailing "compile x20: 12.34 ms/iter" / "run x200: 5.67 ms/iter" lines. +ITER_RE = re.compile(r"^(compile|run) x(\d+): ([0-9.]+) ms/iter") +PERF_INSNS_RE = re.compile(r"^\s*([0-9,]+)\s+instructions") + + +def run_once(binary: str, env: dict[str, str], use_perf: bool) -> dict[str, float]: + """One harness invocation; returns {phase name: milliseconds}.""" + cmd = [binary] + if use_perf: + cmd = ["perf", "stat", "-e", "instructions", "--"] + cmd + proc = subprocess.run( + cmd, env=env, capture_output=True, text=True, check=False + ) + if proc.returncode != 0: + sys.exit( + f"{binary} exited {proc.returncode}\n" + f"--- stdout ---\n{proc.stdout}\n--- stderr ---\n{proc.stderr}" + ) + + out: dict[str, float] = {} + for line in proc.stdout.splitlines(): + m = PHASE_RE.match(line) + if m: + out[m.group(1).strip()] = float(m.group(2)) + continue + m = ITER_RE.match(line) + if m: + out[f"{m.group(1)} x{m.group(2)}"] = float(m.group(3)) + # perf writes its summary to stderr. + for line in proc.stderr.splitlines(): + m = PERF_INSNS_RE.match(line) + if m: + out["instructions (M)"] = int(m.group(1).replace(",", "")) / 1e6 + if not out: + sys.exit(f"no timings parsed from {binary}; stdout was:\n{proc.stdout}") + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--a", required=True, help="baseline clearn_profile binary") + ap.add_argument("--b", required=True, help="candidate clearn_profile binary") + ap.add_argument("--rounds", type=int, default=7, help="measured rounds per side") + ap.add_argument("--warmup", type=int, default=2, help="discarded rounds per side") + ap.add_argument("--model", help="value for CLEARN_MODEL") + ap.add_argument("--ltm", action="store_true", help="set CLEARN_LTM=1") + ap.add_argument( + "--profile", + choices=["compile", "run", "both"], + help="CLEARN_PROFILE for the extra-iteration loops", + ) + ap.add_argument("--compile-iters", type=int, help="CLEARN_COMPILE_ITERS") + ap.add_argument("--run-iters", type=int, help="CLEARN_RUN_ITERS") + ap.add_argument( + "--perf", action="store_true", help="also report perf-stat instruction counts" + ) + args = ap.parse_args() + + env = dict(os.environ) + if args.model: + env["CLEARN_MODEL"] = args.model + if args.ltm: + env["CLEARN_LTM"] = "1" + if args.profile: + env["CLEARN_PROFILE"] = args.profile + if args.compile_iters is not None: + env["CLEARN_COMPILE_ITERS"] = str(args.compile_iters) + if args.run_iters is not None: + env["CLEARN_RUN_ITERS"] = str(args.run_iters) + # Allocation counting adds a pair of atomics to every allocation, which + # distorts exactly the phase this script is timing. + env.pop("CLEARN_COUNT_ALLOCS", None) + + for _ in range(args.warmup): + run_once(args.a, env, args.perf) + run_once(args.b, env, args.perf) + + samples: dict[str, dict[str, list[float]]] = {"a": {}, "b": {}} + for i in range(args.rounds): + # Alternate which side leads so a systematic first-vs-second-in-round + # effect (thermal, frequency ramp) cancels rather than always favouring + # one side. + order = [("a", args.a), ("b", args.b)] + if i % 2: + order.reverse() + for side, binary in order: + for phase, ms in run_once(binary, env, args.perf).items(): + samples[side].setdefault(phase, []).append(ms) + + phases = [p for p in samples["a"] if p in samples["b"]] + width = max((len(p) for p in phases), default=10) + print(f"\nrounds={args.rounds} (warmup {args.warmup}), medians") + print(f"{'phase':<{width}} {'A':>12} {'B':>12} {'delta':>9}") + for phase in phases: + a = statistics.median(samples["a"][phase]) + b = statistics.median(samples["b"][phase]) + delta = (b - a) / a * 100.0 if a else float("nan") + print(f"{phase:<{width}} {a:>12.2f} {b:>12.2f} {delta:>+8.1f}%") + print( + "\nA delta under ~4% on wall-clock is not resolved by one build pair " + "(binary-layout lottery); rebuild both sides and reproduce, or compare " + "instruction counts with --perf." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From a8fb7631e1d771c92e6cb14d9caebd5b49ea7dfb Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 09:52:25 -0700 Subject: [PATCH 41/59] engine: generate LookupDirect in the merge proptests, and read GF runs from gf_run The merge properties did not cover `LookupDirect`. Breaking its `base_gf` remap in `renumber_symbolic_opcode` -- the exact defect M1 exists to catch -- left all twelve proptests green. The cause was the GENERATOR, not the oracles. Neither `symbolic_merge_proptest`'s `frag_specs` nor `combined_fragment_proptest`'s builder ever constructed a `LookupDirect`; its only appearances were in `blank_resource_ids`, which rewrites an opcode it is handed and never creates one. So the oracles' arms were unreachable, and teaching them alone would have produced oracles that could see the opcode but never did -- still green under the mutation, and now looking covered. Both generators emit it, in the shape codegen does: `base_gf` at the block start, `table_count` spanning the block, and an interior `elem`. The two GF oracles now read `SymbolicOpcode::gf_run()` instead of re-listing the lookup family: `denote`'s GF arm, whose `_ => {}` made a merged `LookupDirect` denote no graphical functions and satisfy M1 vacuously, and the run-contiguity scan in `gf_dedup_preserves_runs_and_never_merges_distinct_content`, whose `_ => continue` skipped it. `gf_run`'s match is exhaustive with no `_`, so a future lookup variant is a compile error there rather than a silent skip in either oracle -- the same conversion already applied to `gf_blocks_of_fragment`, extended to the tests that check it. Verified by mutation rather than by inspection. With the remap broken, four properties now fail where twelve passed: `merged_ids_dereference_to_their_own_fragments_resources` (M1 itself), `forced_rich_fragments_exercise_every_resource_kind`, `forced_rich_phase_split_uses_non_zero_bases`, and `phase_split_assigns_the_same_ids_as_the_all_phases_merge`. Reverting the mutation returns all twelve to green. Two notes on scope. `gf_dedup_preserves_runs_...` stays green under this mutation and correctly so: it checks the `GfDedup` remap TABLE, not the rewrite of an opcode's `base_gf`, which is a different obligation. And `combined_fragment_proptest`'s temp-id scan is deliberately NOT changed. `LookupArray` appears in it because it carries a `write_temp_id`; `LookupDirect` has none, so it does not belong there. That file has no GF-run oracle at all, so generating the opcode there buys coverage of its interleave obligations -- 1:1 opcode conservation, element order, temp non-sharing -- and not of M1. --- .../src/compiler/symbolic_merge_proptest.rs | 46 +++++++++---------- .../src/db/combined_fragment_proptest.rs | 11 +++++ 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/simlin-engine/src/compiler/symbolic_merge_proptest.rs b/src/simlin-engine/src/compiler/symbolic_merge_proptest.rs index 9d4d33036..6126b8506 100644 --- a/src/simlin-engine/src/compiler/symbolic_merge_proptest.rs +++ b/src/simlin-engine/src/compiler/symbolic_merge_proptest.rs @@ -311,6 +311,17 @@ fn build_fragment(spec: &FragSpec) -> PerVarBytecodes { table_count: 1, mode: LookupMode::Interpolate, }); + // The constant-element form, addressing the same block by + // its start plus an interior offset. It has to be + // GENERATED, not merely tolerated by the oracles: an opcode + // the generator never emits leaves the property unverified + // however the oracles are written. + code.push(SymbolicOpcode::LookupDirect { + base_gf: start as GraphicalFunctionId, + table_count: block.len as u16, + elem: (block.len - 1).min(u8::MAX as usize) as u8, + mode: LookupMode::Interpolate, + }); } } next_block += 1; @@ -667,18 +678,14 @@ fn denote(op: &SymbolicOpcode, tables: &ResourceTables<'_>) -> Result { - for k in 0..(*table_count as usize) { - let slot = *base_gf as usize + k; + // Every opcode carrying a GF run, taken from `SymbolicOpcode::gf_run` + // rather than re-listed here, so this oracle cannot fall behind the + // opcode set: that match is exhaustive with no `_`, which makes a new + // lookup variant a compile error there instead of a silent skip here. + op if op.gf_run().is_some() => { + let (base_gf, table_count) = op.gf_run().unwrap(); + for k in 0..table_count { + let slot = base_gf + k; if slot >= tables.graphical_functions.len() { return Err(format!( "GF run [{base_gf}, {base_gf}+{table_count}) is past its table of {}", @@ -1014,18 +1021,9 @@ proptest! { ); } for op in &frag.symbolic.code { - let (base, count) = match op { - SymbolicOpcode::Lookup { - base_gf, - table_count, - .. - } - | SymbolicOpcode::LookupArray { - base_gf, - table_count, - .. - } => (*base_gf as usize, *table_count as usize), - _ => continue, + // Same single source of truth as `denote`'s GF arm. + let Some((base, count)) = op.gf_run() else { + continue; }; for k in 0..count { prop_assert_eq!( diff --git a/src/simlin-engine/src/db/combined_fragment_proptest.rs b/src/simlin-engine/src/db/combined_fragment_proptest.rs index 1fe1b7476..e766b171e 100644 --- a/src/simlin-engine/src/db/combined_fragment_proptest.rs +++ b/src/simlin-engine/src/db/combined_fragment_proptest.rs @@ -170,6 +170,17 @@ fn build_member(spec: &MemberSpec) -> PerVarBytecodes { table_count: 1, mode: crate::bytecode::LookupMode::Interpolate, }); + // The constant-element lookup form belongs in the stream too. This + // file asserts no GF-run property -- its obligations are 1:1 opcode + // conservation, element order and temp non-sharing -- but those + // apply to every opcode, and an opcode the generator never emits is + // outside all of them. + code.push(SymbolicOpcode::LookupDirect { + base_gf: e as u8, + table_count: 1, + elem: 0, + mode: crate::bytecode::LookupMode::Interpolate, + }); } if e < module_decls.len() { code.push(SymbolicOpcode::EvalModule { From f27f4659a28a7a28e09d239e3eda44d8af159419 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 09:59:45 -0700 Subject: [PATCH 42/59] build: make the WASM staging cache record whether wasm-opt ran `build_wasm`'s cache key was the raw cargo output alone, but the block it guards does two things: stage the blob AND optimize it. So the key described the input and not the artifact. A `DISABLE_WASM_OPT=1` build -- which is what `scripts/pre-commit` runs -- staged an unoptimized blob and wrote a `.raw` equal to the cargo output; a later `pnpm build` on an unchanged tree then found `.raw` matching, took the early-out, and kept the unoptimized blob without ever running wasm-opt. Reproduced before fixing: `DISABLE_WASM_OPT=1 pnpm build` then a plain `pnpm build` leaves `core/libsimlin.wasm` at 7.94MB, byte-identical to its `.raw`, with no wasm-opt line in the output at all. Post-fix the same sequence re-optimizes to 6.51MB. The reported consequence -- that a local deploy would ship this -- does NOT hold, and the check matters more than the fix. Both `scripts/deploy-web.sh` and `scripts/deploy-web-staged.sh` run `pnpm clean` immediately before `pnpm build`, and `src/engine`'s clean is `rm -rf ./lib ./lib.browser ./core ...`. With `core/` gone the `[ ! -f ... ]` arm fires and wasm-opt runs; verified by deleting `core/` and rebuilding (6.51MB, both blobs optimized). `ts-release.yml` builds a fresh checkout, so it is safe for the same reason. No unoptimized WASM reaches users today. It is still worth fixing, because nothing about the cache made that safe -- two callers happen to clean first. Dropping `pnpm clean` from a deploy script (its `cargo clean` is expensive and someone will eventually try) or adding a third deploy path without one makes it live, silently, at ~24% bundle growth. It also means a bare local `pnpm build` after a commit yields an unoptimized bundle, which quietly misleads anyone measuring bundle size. The fix hoists the wasm-opt decision above the cache check and stamps the staged mode in `core/.mode`, so the key covers both inputs. The stamp is written LAST, so an interrupted build leaves none and the next run redoes the work rather than trusting a half-staged artifact. Verified the caching this exists for is intact: a same-mode repeat still early-outs in BOTH modes (no wasm-opt rerun, no re-copy), and only a mode flip or a changed cargo output rebuilds. `verify-deploy-build.sh` gains the check that would have caught this, opt-in via `REQUIRE_WASM_OPT=1` because CI's frontend job runs the same script after a deliberate `DISABLE_WASM_OPT=1` build and must keep passing; only `deploy-web-staged.sh` sets it. Mutation-tested in four states: unset (silent, CI's case), mode=raw (fails naming the bundle cost), `.mode` absent (fails naming build.sh), mode=opt (passes). The stamp stays local: `src/engine/package.json`'s `files` lists the two `.wasm` paths explicitly and `build-deploy-staging.mjs` copies the single named file, so neither `.mode` nor the pre-existing `.raw` reaches a package or a deploy. --- scripts/deploy-web-staged.sh | 4 +++- scripts/verify-deploy-build.sh | 27 ++++++++++++++++++++++ src/engine/build.sh | 41 +++++++++++++++++++++++++++------- 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/scripts/deploy-web-staged.sh b/scripts/deploy-web-staged.sh index 796c2e3f7..ac7b6db13 100755 --- a/scripts/deploy-web-staged.sh +++ b/scripts/deploy-web-staged.sh @@ -72,7 +72,9 @@ echo "==> Staging app build into public/ (pnpm --filter @simlin/app run deploy:a pnpm --filter @simlin/app run deploy:assemble echo "==> Verifying assembled build artifacts (scripts/verify-deploy-build.sh)" -bash "$REPO_ROOT/scripts/verify-deploy-build.sh" +# REQUIRE_WASM_OPT=1: this is a deploy, so the WASM must be wasm-opt'd. CI runs +# the same script after a deliberately unoptimized build and does not set it. +REQUIRE_WASM_OPT=1 bash "$REPO_ROOT/scripts/verify-deploy-build.sh" echo "==> Assembling self-contained server staging dir (scripts/build-deploy-staging.mjs)" node "$REPO_ROOT/scripts/build-deploy-staging.mjs" "$STAGING_DIR" "$REPO_ROOT/.app.prod.yaml" diff --git a/scripts/verify-deploy-build.sh b/scripts/verify-deploy-build.sh index 3b0c61d5a..3379c3739 100755 --- a/scripts/verify-deploy-build.sh +++ b/scripts/verify-deploy-build.sh @@ -176,6 +176,33 @@ else fi fi +# 7b. On a DEPLOY, the WASM must additionally be wasm-opt'd. Opt-in via +# REQUIRE_WASM_OPT=1 rather than always-on, because CI's frontend job runs +# this same script after a deliberate `DISABLE_WASM_OPT=1 pnpm build` -- +# its subject is the deploy ASSEMBLY, not the artifact's optimization. +# Only the deploy scripts set it. +# +# This exists because the failure it catches is silent and user-facing: an +# unoptimized browser bundle is ~24% larger (5.0MB -> 6.2MB) and nothing +# else on the deploy path would notice. It is the backstop for the +# src/engine/build.sh cache-key bug -- a pre-commit build staging an +# unoptimized blob that then satisfied the next optimizing build's cache +# check -- which is fixed at the source but is worth a tripwire here too, +# since the deploy is a local command with no CI gate. +if [ "1" = "${REQUIRE_WASM_OPT-0}" ]; then + for wasm in src/engine/core/libsimlin.wasm src/engine/core/libsimlin-browser.wasm; do + if [ ! -f "$wasm" ]; then + fail "$wasm missing (REQUIRE_WASM_OPT=1 but the engine WASM build did not run)" + elif [ ! -f "$wasm.mode" ]; then + fail "$wasm.mode missing -- src/engine/build.sh did not stage $wasm, or predates the mode stamp" + elif [ "opt" != "$(cat "$wasm.mode")" ]; then + fail "$wasm was built WITHOUT wasm-opt (mode: $(cat "$wasm.mode")). Deploying it would ship a ~24% larger bundle. Is wasm-opt installed, and is DISABLE_WASM_OPT unset?" + else + pass "$wasm is wasm-opt'd ($(wc -c < "$wasm") bytes)" + fi + done +fi + # 8. The compiled server bundle exists. GAE runs `node src/server/lib` # on the instance; an empty lib/ would crash-loop without a useful # error. diff --git a/src/engine/build.sh b/src/engine/build.sh index f2be5a1cc..df9424ce2 100755 --- a/src/engine/build.sh +++ b/src/engine/build.sh @@ -37,17 +37,38 @@ build_wasm() { # cargo build is idempotent and no-ops when nothing has changed. cargo build -p simlin --lib --release --target wasm32-unknown-unknown "$@" - # Copy WASM only if the raw cargo output changed (avoids re-running - # wasm-opt and invalidating downstream TypeScript builds when Rust source - # is unchanged). We compare against a stashed copy of the pre-optimization - # WASM because wasm-opt transforms core/$out_name in-place, making it - # differ from the raw cargo output even when nothing changed. - if [ ! -f "core/$out_name" ] || ! cmp -s "$WASM_SRC" "core/$out_name.raw"; then + # Whether this invocation will optimize. Decided BEFORE the cache check + # because it is part of the cache key -- see below. + local want_mode="opt" + if ! command -v wasm-opt &> /dev/null || [ "1" = "${DISABLE_WASM_OPT-0}" ]; then + want_mode="raw" + fi + local have_mode="" + [ -f "core/$out_name.mode" ] && have_mode="$(cat "core/$out_name.mode")" + + # Copy WASM only if the staged artifact is stale (avoids re-running wasm-opt + # and invalidating downstream TypeScript builds when Rust source is + # unchanged). Staleness has TWO inputs, and both are in the key: + # + # 1. the raw cargo output changed -- compared against a stashed copy of the + # pre-optimization WASM, because wasm-opt transforms core/$out_name + # in-place and it therefore differs from the cargo output even when + # nothing changed; and + # 2. the staged artifact was produced in the OTHER mode. + # + # Without (2) the key described the input but not the artifact, and a + # DISABLE_WASM_OPT=1 build (`scripts/pre-commit`) staged an unoptimized blob + # whose .raw then satisfied the next optimizing build's check -- so a + # subsequent `pnpm build` on an unchanged tree kept the unoptimized blob and + # never ran wasm-opt again. Both deploy scripts happen to `pnpm clean` first, + # which deletes core/ and hid this; nothing about the cache made it safe. + if [ ! -f "core/$out_name" ] \ + || [ "$have_mode" != "$want_mode" ] \ + || ! cmp -s "$WASM_SRC" "core/$out_name.raw"; then cp "$WASM_SRC" "core/$out_name" cp "$WASM_SRC" "core/$out_name.raw" - # Optimize WASM if wasm-opt is available - if command -v wasm-opt &> /dev/null && [ "1" != "${DISABLE_WASM_OPT-0}" ]; then + if [ "$want_mode" = "opt" ]; then echo "Running wasm-opt on $out_name..." wasm-opt "core/$out_name" -o "core/$out_name-opt" -O3 \ --enable-mutable-globals \ @@ -58,6 +79,10 @@ build_wasm() { else echo "Skipping wasm-opt (not installed or disabled)" fi + + # Written LAST so an interrupted build leaves no stamp and the next run + # redoes the work rather than trusting a half-staged artifact. + printf '%s\n' "$want_mode" > "core/$out_name.mode" fi } From 37f441fbedd05195b189f9798f80f45a0b383e0e Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 10:03:20 -0700 Subject: [PATCH 43/59] engine: require lag alignment before omitting an LTM arm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GH #977 omission drops a per-element link-score arm whose ceteris-paribus partial is provably `PREVIOUS(target)`. The predicate treated every `PREVIOUS` subtree as established and stopped there, which is not sufficient: the partial reproduces `target(t-1)` only if every read in it is lagged by EXACTLY one step, and two shapes look entirely frozen while being misaligned. An ORIGINAL `PREVIOUS(z)` from the target's own equation is left untouched by `wrap_non_matching_in_previous`, deliberately and for good reasons -- re-wrapping would read two steps back and mint nested-PREVIOUS helper chains. So the partial reads `z(t-1)` where `PREVIOUS(target)` read `z(t-2)`, and the numerator is `0.02 * (z(t-1) - z(t-2))` rather than zero. Measured on a three-element fixture: the arm is worth 0.985, 0.985, 0.984, 0.984, 0.983 and was being rewritten to zero. A link score that near ±1 is the canonical single-input attribution and sets its loop's score, so this is the same failure the negative criterion was rejected for, reached by a different route. A synthesized `PREVIOUS` NESTED inside another is the second shape, and it needs no original `PREVIOUS` at all: the subscript-index freeze produces `PREVIOUS(q[PREVIOUS(ctr, ctr)])`, which reads `q` at `t-1` indexed at `t-2` where the anchor indexed at `t-1`. `db::ltm_tests::colliding_index_boston_series` already documented that residual as -1.06/+0.73/-1.03/+0.82 under the heading of an unadjudicated double-lag; those numbers are also the measurement showing such an arm is not a structural zero. The two checks do not subsume each other and are applied in different places for a structural reason. An original `PREVIOUS` must be found in the ORIGINAL element expression, because in the emitted tree it is literally the same node as a synthesized freeze and nothing distinguishes them. A nested `PREVIOUS` must be found in the PARTIAL, because the wrap is what introduces it. `BuiltinReach`'s former `FrozenSubtree` splits into `LagsOneStep` and `StepInvariant` so the distinction is carried by the type rather than by a comment: `INIT` really is step-invariant -- `INIT(x)` is the run's initial value, identical at `t` and `t-1` -- and stays established whatever it contains, including `INIT(PREVIOUS(z))`. Rejecting the cheaper repair matters here. "Require the partial to be `PREVIOUS`-free" omits nothing at all, since every frozen arm is built out of synthesized `PREVIOUS` calls; only an arm whose equation is pure literals would survive. Cost on C-LEARN: **zero**. Flow opcodes are 908,377 before and after, and `clearn_ltm_slot_maxima_digest` passes unchanged at its pinned `(1369, 7000, 10_248_673_492_482_445_132_733_301)` -- so the model contains no arm that is both omittable and misaligned, which is why the whole-slab differential and the digest were clean and why this buys soundness for free. Two new rows in `db::ltm_value_gate_tests`, one per shape, each with a named assertion outside the golden. They were mutation-tested against each clause separately: reverting only the original-`PREVIOUS` check reds `an_original_previous_arm_is_not_a_structural_zero` and leaves the nested row green, and reverting only the nested descent does the reverse. Neither row covers the other's clause, which is why there are two. The bit-exactness claim in `ltm_augment_zero_slot.rs` and in `docs/design/engine-performance.md` now states the lag-alignment requirement it rests on, rather than asserting the conclusion. --- docs/design/engine-performance.md | 11 ++ .../src/db/ltm_value_gate_tests.rs | 109 ++++++++++++++++++ src/simlin-engine/src/ltm_augment.rs | 2 +- .../src/ltm_augment_zero_slot.rs | 93 +++++++++++++-- 4 files changed, 203 insertions(+), 12 deletions(-) diff --git a/docs/design/engine-performance.md b/docs/design/engine-performance.md index 97a8c4695..461163635 100644 --- a/docs/design/engine-performance.md +++ b/docs/design/engine-performance.md @@ -841,6 +841,17 @@ short of it rather than taking a ~2-3%. different question and silently rewrites 187 result slots. GH #977 carries the decomposition and the standing constraints. + "Provably" carries a LAG-ALIGNMENT requirement that a walk stopping at the + first `PREVIOUS` will miss: the partial equals `PREVIOUS(target)` only if + every read is lagged by exactly one step. An ORIGINAL `PREVIOUS(z)` in the + target's equation (which the wrap deliberately leaves untouched, so the + partial reads `z(t-1)` where the anchor read `z(t-2)`) and a synthesized + `PREVIOUS` nested inside another (which the subscript-index freeze produces) + both look entirely frozen and are not aligned. Either one omits an arm worth + close to the canonical ±1 attribution. Both are rejected, each is pinned by + its own row in `db::ltm_value_gate_tests`, and rejecting them costs zero arms + on C-LEARN — the win above is measured with both checks in place. + Larger run-side swings identified during round 2 — all three were taken to a data verdict in round 3 (2026-06-04): diff --git a/src/simlin-engine/src/db/ltm_value_gate_tests.rs b/src/simlin-engine/src/db/ltm_value_gate_tests.rs index c1c17b9a9..f6cad4eaa 100644 --- a/src/simlin-engine/src/db/ltm_value_gate_tests.rs +++ b/src/simlin-engine/src/db/ltm_value_gate_tests.rs @@ -292,3 +292,112 @@ fn a_structural_zero_arm_is_exactly_zero() { } } } + +/// Mechanism 4: an arm whose lag is MISALIGNED with the anchor must not be +/// claimed as a structural zero, even though every leaf sits under a +/// `PREVIOUS`. +/// +/// The omission's soundness condition is `partial(t) == target(t-1)`, which +/// needs every read lagged by exactly ONE step. Two things break that while +/// leaving the emitted tree looking entirely frozen: +/// +/// * an ORIGINAL `PREVIOUS(z)` from the target's own equation, which +/// `wrap_non_matching_in_previous` deliberately leaves untouched -- so the +/// partial reads `z(t-1)` where `target(t-1)` read `z(t-2)`; +/// * a synthesized `PREVIOUS` nested inside another, which the subscript-index +/// freeze produces (`PREVIOUS(q[PREVIOUS(ctr, ctr)])` reads `q` at `t-1` +/// indexed at `t-2`). +/// +/// This fixture is the first. `growth[boston] = PREVIOUS(z) * 0.02 + pop[la] * +/// 0.001` has no live `pop[nyc]` reference, so the shape match records none and +/// every occurrence is frozen -- yet the arm is worth ~0.985, near the +/// canonical +/-1 single-input attribution. Omitting it rewrites a real score +/// to zero, which is exactly the failure GH #977 rejected the negative +/// criterion for, reached by a different route. +/// +/// `INIT` is deliberately NOT in this class and is not tested here as a +/// failure: `INIT(x)` is the run's initial value, identical at `t` and `t-1`, +/// so it aligns at every step. +fn lag_misalignment_project() -> datamodel::Project { + TestProject::new("lag_misalignment") + .with_sim_time(0.0, 5.0, 1.0) + .named_dimension("Region", &["nyc", "boston", "la"]) + .aux("z", "1 + TIME", None) + .array_flow_with_ranges( + "growth[Region]", + vec![ + ("nyc", "pop[nyc] * 0.01"), + ("boston", "PREVIOUS(z) * 0.02 + pop[la] * 0.001"), + ("la", "pop[la] * 0.03"), + ], + ) + .array_stock("pop[Region]", "10", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn an_original_previous_arm_is_not_a_structural_zero() { + let series = ltm_slot_series(&lag_misalignment_project()); + // Region declaration order: nyc=0, boston=1, la=2. + let boston = slot(&series, "link_score\u{205A}pop[nyc]\u{2192}growth", 1); + assert!( + boston.iter().any(|v| v.abs() > 0.5 && v.is_finite()), + "the `boston` arm carries an ORIGINAL PREVIOUS, so its partial reads \ + z(t-1) where the PREVIOUS(target) anchor read z(t-2); the arm is worth \ + ~0.985 and must not be omitted as a structural zero; got {boston:?}" + ); +} + +/// Mechanism 4, second clause: a synthesized `PREVIOUS` NESTED inside another. +/// +/// `growth[boston] = q[ctr] * 0.002` has no original `PREVIOUS` at all, so the +/// clause above cannot see it. The subscript-index freeze produces +/// `PREVIOUS(q[PREVIOUS(ctr, ctr)])` -- `q` read at `t-1` indexed by `ctr` at +/// `t-2`, where the `PREVIOUS(growth)` anchor indexed at `t-1`. Every leaf is +/// under a `PREVIOUS`, so a walk that stops at the first one calls this a +/// structural zero; it is not. +/// +/// The two clauses need separate rows because either one alone leaves the other +/// case omitted: reverting only the `contains_previous_call(original)` check +/// keeps this row green, and reverting only the nested-`PREVIOUS` descent keeps +/// the row above green. Both were measured that way. +/// +/// This is the shape `db::ltm_tests::colliding_index_boston_series` documents a +/// -1.06/+0.73/-1.03/+0.82 residual for, under the heading of an unadjudicated +/// double-lag. That residual is not only a semantics question: it is also the +/// measurement showing such an arm is not a structural zero, which is what this +/// row pins. +fn nested_freeze_project() -> datamodel::Project { + TestProject::new("nested_freeze") + .with_sim_time(0.0, 6.0, 1.0) + .named_dimension("Region", &["nyc", "boston", "la"]) + .named_dimension("Slot", &["s1", "s2"]) + .aux("tick", "1", None) + .stock("counter", "0", &["tick"], &[], None) + .aux("drive", "1 + counter", None) + // 1, 2, 1, 2, ... -- a genuine runtime index. + .aux("ctr", "1 + (INT(counter) MOD 2)", None) + .array_with_ranges("q[Slot]", vec![("s1", "1 * drive"), ("s2", "10 * drive")]) + .array_flow_with_ranges( + "growth[Region]", + vec![ + ("nyc", "pop[nyc] * 0.01"), + ("boston", "q[ctr] * 0.002"), + ("la", "pop[la] * 0.03"), + ], + ) + .array_stock("pop[Region]", "10", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn a_nested_freeze_arm_is_not_a_structural_zero() { + let series = ltm_slot_series(&nested_freeze_project()); + let boston = slot(&series, "link_score\u{205A}pop[nyc]\u{2192}growth", 1); + assert!( + boston.iter().any(|v| v.abs() > 0.5 && v.is_finite()), + "the `boston` arm freezes a runtime subscript INDEX inside an already \ + frozen head, so it reads two steps back and is not PREVIOUS(target); \ + it must not be omitted as a structural zero; got {boston:?}" + ); +} diff --git a/src/simlin-engine/src/ltm_augment.rs b/src/simlin-engine/src/ltm_augment.rs index ef0ae3df5..ecef398cd 100644 --- a/src/simlin-engine/src/ltm_augment.rs +++ b/src/simlin-engine/src/ltm_augment.rs @@ -1741,7 +1741,7 @@ fn shaped_guard_form_text( // have referenced those freeze helpers is gone, so appending them would // mint variables no equation reads. if zero_slot_policy == ZeroSlotPolicy::OmitStructuralZero - && partial_is_provably_previous_target(&changed_first) + && partial_is_provably_previous_target(target_expr, &changed_first) { return Ok(None); } diff --git a/src/simlin-engine/src/ltm_augment_zero_slot.rs b/src/simlin-engine/src/ltm_augment_zero_slot.rs index b5acfc857..7656abbf9 100644 --- a/src/simlin-engine/src/ltm_augment_zero_slot.rs +++ b/src/simlin-engine/src/ltm_augment_zero_slot.rs @@ -7,7 +7,7 @@ //! the project line-count lint. Mounted into `ltm_augment`, so callers keep //! naming these items `crate::ltm_augment::*`. -use crate::ast::Expr0; +use crate::ast::{Expr0, IndexExpr0}; use crate::builtins::UntypedBuiltinFn; /// Whether the caller's result is a whole VARIABLE's equation or one slot of an @@ -34,7 +34,19 @@ pub(crate) enum ZeroSlotPolicy { /// caller that knows the target's flag. /// /// This is a BIT-EXACT transformation, and that is the whole point of the - /// positive predicate. The tempting negative test -- "the link's source + /// positive predicate. Bit-exactness rests on a LAG-ALIGNMENT requirement + /// that is easy to state and easy to miss: the partial equals + /// `PREVIOUS(target)` only if every read in it is lagged by EXACTLY one + /// step. Two shapes look entirely frozen and are not aligned -- an ORIGINAL + /// `PREVIOUS(z)` from the target's own equation, which the wrap + /// deliberately leaves untouched (so the partial reads `z(t-1)` where the + /// anchor read `z(t-2)`), and a synthesized `PREVIOUS` nested inside + /// another, which the subscript-index freeze produces. Both are rejected by + /// [`partial_is_provably_previous_target`], and each is pinned by its own + /// row in `db::ltm_value_gate_tests`; skipping either omits an arm worth + /// close to the canonical +/-1 attribution. + /// + /// The tempting negative test -- "the link's source /// stayed frozen" -- says nothing about what else the arm reads, and /// collapsing on it changes 187 C-LEARN result slots across 35 link-score /// variables (151 by >= 1.0, worst 8,086.97 -> 0), because the wrap does not @@ -82,9 +94,16 @@ impl Reach { /// case is a named variant decided at the match below rather than a fall-through. #[derive(Clone, Copy, PartialEq, Eq)] enum BuiltinReach { - /// The call's contents are read at the PREVIOUS step, so the subtree is - /// frozen whatever it contains and the walk must NOT descend into it. - FrozenSubtree, + /// `PREVIOUS(..)`: its contents are read ONE step back. That is what the + /// wrap's synthesized freezes do, and it is what makes the partial + /// reproduce the target's previous value -- but only if the lag is exactly + /// one. A `PREVIOUS` nested inside this one reads two steps back, so the + /// walk MUST descend far enough to rule that out. + LagsOneStep, + /// `INIT(..)`: its value is the run's initial value, identical at every + /// step, so it is genuinely step-invariant whatever it contains and the + /// walk need not descend. + StepInvariant, /// Deterministic in its arguments and independent of the step, so the /// verdict is the fold over its arguments. PureInArgs, @@ -105,7 +124,8 @@ fn classify_builtin_reach(name: &str) -> BuiltinReach { // caller with raw source spelling cannot silently fall into `Varying`. let lowered = name.to_ascii_lowercase(); match lowered.as_str() { - "previous" | "init" => BuiltinReach::FrozenSubtree, + "previous" => BuiltinReach::LagsOneStep, + "init" => BuiltinReach::StepInvariant, "abs" | "arccos" | "arcsin" | "arctan" | "cos" | "exp" | "inf" | "int" | "ln" | "log10" | "max" | "min" | "pi" | "safediv" | "sign" | "sin" | "sqrt" | "tan" => { BuiltinReach::PureInArgs @@ -131,12 +151,54 @@ fn classify_builtin_reach(name: &str) -> BuiltinReach { /// negative form asks a different question, one that says nothing about the rest /// of the arm; see [`ZeroSlotPolicy::OmitStructuralZero`] for what that costs. /// +/// "Frozen" is not enough on its own: the partial reproduces `target(t-1)` only +/// if every read is lagged by EXACTLY one step, so this takes the ORIGINAL +/// element expression as well as the emitted partial. An original `PREVIOUS(z)` +/// has to be found in the original, because in the emitted tree it is the same +/// node as a synthesized freeze and nothing distinguishes them; a NESTED +/// `PREVIOUS` is found in the partial, because the wrap is what introduces it. +/// Neither check subsumes the other -- reverting either one alone leaves the +/// other case wrongly omitted, measured row by row in +/// `db::ltm_value_gate_tests`. +/// /// A `Var` or `Subscript` reached outside a frozen subtree is a live read and /// ends the walk, which is why subscript INDICES are never descended into: the /// whole reference is already `NotEstablished`, so `IndexExpr0` needs no arm /// here and a new index variant cannot change any verdict. -pub(super) fn partial_is_provably_previous_target(partial: &Expr0) -> bool { - reach_of(partial) == Reach::Established +pub(super) fn partial_is_provably_previous_target(original: &Expr0, partial: &Expr0) -> bool { + !contains_previous_call(original) && reach_of(partial) == Reach::Established +} + +/// Does `expr` call `PREVIOUS` outside every `INIT(..)` subtree? +/// +/// Asked of the ORIGINAL element equation, never of the partial, because in the +/// emitted tree an original `PREVIOUS(z)` and a synthesized freeze +/// `PREVIOUS(x)` are the same node and nothing distinguishes them. `INIT` +/// subtrees are skipped: `INIT(PREVIOUS(z))` is the run's initial value, a +/// constant, so it aligns at every step. +fn contains_previous_call(expr: &Expr0) -> bool { + match expr { + Expr0::Const(..) | Expr0::Var(..) => false, + Expr0::Subscript(_, indices, _) => indices.iter().any(|idx| match idx { + IndexExpr0::Expr(e) => contains_previous_call(e), + IndexExpr0::Range(l, r, _) => contains_previous_call(l) || contains_previous_call(r), + IndexExpr0::Wildcard(_) + | IndexExpr0::StarRange(_, _) + | IndexExpr0::DimPosition(_, _) => false, + }), + Expr0::Op1(_, inner, _) => contains_previous_call(inner), + Expr0::Op2(_, lhs, rhs, _) => contains_previous_call(lhs) || contains_previous_call(rhs), + Expr0::If(c, t, f, _) => { + contains_previous_call(c) || contains_previous_call(t) || contains_previous_call(f) + } + Expr0::App(UntypedBuiltinFn(name, args), _) => match classify_builtin_reach(name) { + BuiltinReach::LagsOneStep => true, + BuiltinReach::StepInvariant => false, + BuiltinReach::PureInArgs | BuiltinReach::Varying => { + args.iter().any(contains_previous_call) + } + }, + } } fn reach_of(expr: &Expr0) -> Reach { @@ -150,9 +212,18 @@ fn reach_of(expr: &Expr0) -> Reach { Expr0::Op2(_, lhs, rhs, _) => reach_of(lhs).and(reach_of(rhs)), Expr0::If(cond, then, other, _) => reach_of(cond).and(reach_of(then)).and(reach_of(other)), Expr0::App(UntypedBuiltinFn(name, args), _) => match classify_builtin_reach(name) { - // Do NOT descend: the contents are read at the previous step, so - // whatever they reference is frozen by construction. - BuiltinReach::FrozenSubtree => Reach::Established, + // Read one step back -- the lag the anchor expects -- but ONLY if + // nothing inside lags again. `PREVIOUS(q[PREVIOUS(ctr, ctr)])` reads + // `q` at `t-1` indexed by `ctr` at `t-2`, where the anchor indexed + // at `t-1`, so it is not `PREVIOUS(target)`. + BuiltinReach::LagsOneStep => { + if args.iter().any(contains_previous_call) { + Reach::NotEstablished + } else { + Reach::Established + } + } + BuiltinReach::StepInvariant => Reach::Established, BuiltinReach::PureInArgs => args .iter() .fold(Reach::Established, |acc, arg| acc.and(reach_of(arg))), From 1f7d1bad568fb97c365acc6e4d7f7f00068dd189 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 10:06:19 -0700 Subject: [PATCH 44/59] engine: gate variable_dimensions on the A2A equation having tokens `variable_dimensions` derives a variable's declared dimensions from its equation instead of demanding a parse, and its rustdoc claimed the only divergence from the parse-backed original was an UNPARSEABLE A2A equation -- confined, therefore, to projects that already fail to assemble. That claim was wrong, and `/code-review` caught it. `parse_equation` builds an A2A as `ast.map(|ast| ApplyToAll(dims, ast))`, so an equation that yields no `Ast` yields no dimensions -- and `parser::parse` answers `Ok(None)` for exactly one reason: an input with no tokens. That is reachable on VALID, COMPILING models, because `variable.rs`'s empty-equation suppression exists precisely to make two such shapes legal: a standalone lookup-only table (an empty `ApplyToAll` equation plus a ``, issue #606) and a module input port whose own equation is dead. Measured on a fixture holding both, the derivation widened each from `variable_size` 1 to 3 and took the model's `n_slots` from 2 to 6 -- shifting every later variable's layout offset on a model that compiles. The A2A arm now gates on `parser::is_token_free`, a LEX and not a parse: it builds no AST and resolves nothing, so it keeps the point of deriving the dimensions at all. Measured cost on C-LEARN is -0.05% of a cold compile, inside the baseline arm's own 0.10% spread. The predicate is deliberately the MECHANISM rather than the two shapes that were found: any token-free `ApplyToAll` equation reaches it. That is not hypothetical -- `parse`'s own contract says `Ok(None)` covers "empty or comment-only input", so a comment-only equation is a third instance, and it is a test row. A gate spelled `eqn.trim().is_empty()` would have answered it wrongly. `is_token_free` lives next to `parse` so the two cannot drift, and its rustdoc records the asymmetry it keeps: a lexical error counts as having tokens, because `parse` answers `Err` for it and not `Ok(None)`. That leaves exactly one divergence, which is the one the rustdoc always claimed: an A2A equation that is not token-free but does not parse. Such a variable's error still reaches `compile_var_fragment`, which drops the fragment and fails the build. The claim and the code now agree. Blast radius, measured before fixing rather than assumed: across all 483 checked-in models under `test/`, the pre-fix derivation changed ZERO compile outcomes and ZERO `n_slots` values -- 55 variables in 25 files diverged, but every one of those models failed to compile under both implementations, so only the text of the failure differed. The artifact-identity claim held by corpus luck rather than by construction, which is why this was worth fixing rather than resting on. With the gate, the whole corpus is identical to the parse-backed implementation on both compile outcome and `n_slots`. Nothing in the suite caught this: the 13 `lookup_only_tests` pass under BOTH implementations, because a lookup-only holder produces no series and those tests assert its absence -- a variable silently taking three slots instead of one is invisible to them. Worth knowing what that suite does and does not pin. UNVERIFIED OBSERVATION, recorded so it is inherited rather than rediscovered from a suppressed error path: an arrayed module input port whose equation is empty may genuinely WANT its declared extent rather than one slot, since the caller binds an array to it -- in which case the parse-backed behaviour is a latent truncation bug and this commit preserves it. Settling that means showing what the VM and the module-input wiring actually read from those slots (a sub-model with an `ApplyToAll` input port over a multi-element dimension, bound from a parent array, checked for whether elements past the first are read), not reasoning from the equation shape. It is a hypothesis, not a finding. --- src/simlin-engine/src/db/query.rs | 52 +++++++-- .../src/db/variable_dimensions_tests.rs | 103 ++++++++++++++++++ src/simlin-engine/src/parser/mod.rs | 15 +++ 3 files changed, 159 insertions(+), 11 deletions(-) diff --git a/src/simlin-engine/src/db/query.rs b/src/simlin-engine/src/db/query.rs index aaf6c4d52..1896a97b7 100644 --- a/src/simlin-engine/src/db/query.rs +++ b/src/simlin-engine/src/db/query.rs @@ -801,18 +801,28 @@ pub fn variable_relevant_dimensions(db: &dyn Db, var: SourceVariable) -> BTreeSe /// early return and never depends on the project's dimensions at all /// (`db::dimension_invalidation_tests`). /// -/// **One arm differs from the parse, deliberately.** The parse builds -/// `Ast::ApplyToAll` as `ast.map(|ast| Ast::ApplyToAll(dims, ast))`, so an A2A -/// variable whose EQUATION does not parse yields no `Ast` and therefore -/// reported no dimensions; this reports its declared ones. Both the -/// unresolvable-dimension-name arm (`[]` on either path) and the `Arrayed` arm -/// (which the parse builds unconditionally once its dims resolve, however many -/// element equations failed) are unchanged. The divergence is confined to a -/// project that already fails to assemble -- the parse error still reaches +/// **`Ast::ApplyToAll` is built as `ast.map(|ast| ApplyToAll(dims, ast))`**, so +/// an A2A equation that yields no `Ast` yields no dimensions -- and `parse` +/// answers `Ok(None)` for exactly one reason, an input with no tokens. That is +/// reachable on VALID, COMPILING models: a standalone lookup-only table (an +/// empty A2A equation plus a ``, issue #606) and a module input port whose +/// own equation is dead are both empty-equation A2A variables, and reporting +/// their declared extent would widen `variable_size` from 1 and shift every +/// later variable's layout offset. The A2A arm therefore gates on +/// `parser::is_token_free`, which is a LEX rather than a parse. +/// +/// **One arm still differs from the parse, deliberately.** An A2A equation that +/// is not token-free but does not PARSE (`Err`, not `Ok(None)`) reported no +/// dimensions and now reports its declared ones. That divergence is confined to +/// a project which already fails to assemble -- the parse error still reaches /// `compile_var_fragment`, which drops the fragment and accumulates the /// diagnostic -- and it moves the reported size from a wrong 1 toward the -/// declared extent, so nothing that compiled before reads a different slot. -/// Every arm is enumerated in `db::variable_dimensions_tests`. +/// declaration, so nothing that compiled before reads a different slot. The +/// unresolvable-dimension-name arm (`[]` on either path) and the `Arrayed` arm +/// (built unconditionally once its dims resolve, however many element equations +/// failed) are unchanged. Every arm is enumerated in +/// `db::variable_dimensions_tests`, asserted against the previous +/// implementation as an oracle rather than against hand-written expectations. #[salsa::tracked(returns(ref))] pub fn variable_dimensions( db: &dyn Db, @@ -821,7 +831,27 @@ pub fn variable_dimensions( ) -> Vec { let dimension_names: &[String] = match var.equation(db) { datamodel::Equation::Scalar(_) => return Vec::new(), - datamodel::Equation::ApplyToAll(dim_names, _) => dim_names, + datamodel::Equation::ApplyToAll(dim_names, eqn) => { + // `parse_equation`'s A2A arm is `ast.map(|ast| ApplyToAll(dims, ast))`, + // so an equation that produces no `Ast` produces no dimensions -- + // and `parse` returns `Ok(None)` for exactly one reason: the input + // contains no tokens. That is the case for a STANDALONE LOOKUP-ONLY + // table (an empty `ApplyToAll` equation plus a ``) and for a + // module input port whose dead equation is empty, both of which are + // VALID and both of which compile -- so answering with the declared + // dimensions here would widen their `variable_size` from 1 and shift + // every later variable's layout offset on a working model. + // + // `is_token_free` is a lex, not a parse: it neither builds an AST + // nor resolves anything, so this keeps the whole point of deriving + // the dimensions instead of demanding `parse_source_variable_*`. + if crate::parser::is_token_free(eqn, crate::lexer::LexerType::Equation) { + return Vec::new(); + } + dim_names + } + // `Arrayed` needs no such check: the parse builds its `Ast` whenever the + // dimension names resolve, however many element equations failed. datamodel::Equation::Arrayed(dim_names, _, _, _) => dim_names, }; // A module variable carries a synthesized equation but has no array shape diff --git a/src/simlin-engine/src/db/variable_dimensions_tests.rs b/src/simlin-engine/src/db/variable_dimensions_tests.rs index 8ecefa756..669afd399 100644 --- a/src/simlin-engine/src/db/variable_dimensions_tests.rs +++ b/src/simlin-engine/src/db/variable_dimensions_tests.rs @@ -167,6 +167,109 @@ fn variable_dimensions_matches_the_parse_on_every_agreeing_arm() { ); } +/// The two VALID shapes whose A2A equation is empty, which is the whole reason +/// the derivation gates on the equation having tokens. +/// +/// Both are legal and both COMPILE, so a divergence here is not confined to +/// broken projects the way the unparseable arm below is -- it would widen +/// `variable_size` from 1 to the declared extent and shift every later +/// variable's layout offset on a working model. `parse_equation` builds an A2A +/// as `ast.map(|ast| ApplyToAll(dims, ast))` and `parser::parse` answers +/// `Ok(None)` for a token-free input, so the parse reports no dimensions for +/// them; the derivation must agree, and is asserted against the oracle rather +/// than against a hand-written expectation. +/// +/// Enumerated from that MECHANISM rather than from the two shapes: any +/// `Equation::ApplyToAll` with a token-free equation reaches it. These two are +/// the ones `variable.rs`'s empty-equation suppression makes valid -- a +/// standalone lookup-only table (issue #606) and a module input port whose own +/// equation is dead -- but a third would be covered by the same gate. +#[test] +fn an_empty_a2a_equation_reports_no_dimensions_on_both_paths() { + let lookup_only = datamodel::Variable::Aux(datamodel::Aux { + ident: "a2a_table".to_string(), + equation: a2a(&["DimA"], ""), + documentation: String::new(), + units: None, + gf: Some(datamodel::GraphicalFunction { + kind: datamodel::GraphicalFunctionKind::Continuous, + x_points: None, + y_points: vec![0.0, 1.0], + x_scale: datamodel::GraphicalFunctionScale { min: 0.0, max: 1.0 }, + y_scale: datamodel::GraphicalFunctionScale { min: 0.0, max: 1.0 }, + }), + ai_state: None, + uid: None, + compat: datamodel::Compat::default(), + }); + let mut port_aux = datamodel::Aux { + ident: "a2a_port".to_string(), + // Comment-only, not merely blank: `parse`'s own contract says + // `Ok(None)` covers "empty or comment-only input", and the lexer skips + // a `{...}` comment rather than emitting a token for it. A gate written + // as `eqn.trim().is_empty()` would answer differently here, which is + // why the predicate is a lex. + equation: a2a(&["DimA"], "{just a comment}"), + documentation: String::new(), + units: None, + gf: None, + ai_state: None, + uid: None, + compat: datamodel::Compat::default(), + }; + port_aux.compat.can_be_module_input = true; + + let mut db = SimlinDb::default(); + let project = datamodel::Project { + name: "empty_a2a".to_string(), + sim_specs: datamodel::SimSpecs::default(), + dimensions: dims(), + units: vec![], + models: vec![datamodel::Model { + name: "main".to_string(), + sim_specs: None, + variables: vec![lookup_only, datamodel::Variable::Aux(port_aux)], + views: vec![], + loop_metadata: vec![], + groups: vec![], + macro_spec: None, + }], + source: None, + ai_information: None, + }; + let state = sync_from_datamodel_incremental(&mut db, &project, None); + let sync = state.to_sync_result(); + for name in ["a2a_table", "a2a_port"] { + let sv = sync.models["main"].variables[name].source; + let derived: Vec = crate::db::query::variable_dimensions(&db, sv, sync.project) + .iter() + .map(|d| d.name().to_string()) + .collect(); + assert_eq!( + derived, + oracle_dimension_names(&db, sv, sync.project), + "{name}: the derivation must agree with the parse on an empty A2A equation" + ); + assert_eq!( + derived, + Vec::::new(), + "{name}: an empty A2A equation yields no Ast and hence no dimensions" + ); + assert_eq!( + crate::db::query::variable_size(&db, sv, sync.project), + 1, + "{name}: reporting the declared extent here would shift every later \ + variable's layout offset" + ); + } + // The layout is the thing the divergence was observable in, so pin it too. + assert_eq!( + crate::db::compute_layout(&db, sync.models["main"].source, sync.project).n_slots, + 2, + "two size-1 variables occupy two slots; the pre-gate derivation made this 6" + ); +} + /// The ONE arm that changed, pinned in the direction it changed to. /// /// The parse builds `Ast::ApplyToAll` as `ast.map(|ast| ApplyToAll(dims, ast))`, diff --git a/src/simlin-engine/src/parser/mod.rs b/src/simlin-engine/src/parser/mod.rs index 52d0f0830..be2eb9607 100644 --- a/src/simlin-engine/src/parser/mod.rs +++ b/src/simlin-engine/src/parser/mod.rs @@ -791,6 +791,21 @@ impl<'input> Parser<'input> { } } +/// Whether `input` contains no tokens at all. +/// +/// This is exactly the condition on which [`parse`] returns `Ok(None)` rather +/// than an expression -- `parse_equation`'s `is_at_end()` early return -- and +/// it lives here so the two cannot drift apart. It is a LEX, not a parse: no +/// AST is built and nothing is resolved. +/// +/// A caller uses it to answer "would this equation have produced an `Ast`?" +/// without paying for one. Note the asymmetry it deliberately keeps: an input +/// whose first token is a lexical ERROR is reported as having tokens, because +/// `parse` answers `Err` for it and not `Ok(None)`. +pub(crate) fn is_token_free(input: &str, lexer_type: LexerType) -> bool { + Lexer::new(input, lexer_type).next().is_none() +} + /// Parse an equation string into an AST. /// /// Returns: From 4b47f5e5374313697566110a3b5cf00f46236b9d Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 10:06:19 -0700 Subject: [PATCH 45/59] engine: make the LTM fragment index-to-variable coupling checkable `compile_ltm_fragment_at` is keyed by index alone, and that is deliberate: a name argument would join the salsa cache key and defeat the firewall its rustdoc describes. Both walkers get their index from `vars.iter().enumerate()`, so the key is a valid identity today -- but nothing in the signature or the types ties a caller's index to the `LtmSyntheticVar` it walked out of. A third caller, or any reordering of `LtmVariablesResult::vars` between the walk and the call, files a fragment under the wrong name, and both consumers read a mismatch as an ordinary "no fragment" rather than as an error. Nothing reports it. `compile_ltm_fragment_for` wraps the query with a debug-only assertion that the index still resolves to the variable the caller holds. Both callers already have it in hand, so the coupling becomes checkable rather than conventional, at no release cost and with the index-only key intact. The raw query is now private to the module rather than `pub(crate)`, which is what makes this structural instead of a convention a fourth caller can miss: the checked wrapper is the only way in. Its rustdoc says so, since widening the visibility back would silently reopen the hole. --- src/simlin-engine/src/db/assemble.rs | 3 +- src/simlin-engine/src/db/ltm/compile.rs | 48 +++++++++++++++++++++++-- src/simlin-engine/src/db/ltm/mod.rs | 2 +- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/simlin-engine/src/db/assemble.rs b/src/simlin-engine/src/db/assemble.rs index 0ccd619cf..96a772eaf 100644 --- a/src/simlin-engine/src/db/assemble.rs +++ b/src/simlin-engine/src/db/assemble.rs @@ -1530,7 +1530,8 @@ pub fn assemble_module<'db>( // drop. Both walkers reach it through the memoized per-index query, // so the diagnostic pass reuses these fragments instead of // recompiling the ones the direct path does not otherwise cache. - let fragment_result = compile_ltm_fragment_at(db, model, project, ltm_index).clone(); + let fragment_result = + compile_ltm_fragment_for(db, model, project, ltm_index, ltm_var).clone(); if let Some(result) = fragment_result { // Drop LTM fragments whose symbolic variable references can't diff --git a/src/simlin-engine/src/db/ltm/compile.rs b/src/simlin-engine/src/db/ltm/compile.rs index 6e8264c4d..85717c3bc 100644 --- a/src/simlin-engine/src/db/ltm/compile.rs +++ b/src/simlin-engine/src/db/ltm/compile.rs @@ -1933,8 +1933,13 @@ pub(crate) fn compile_ltm_synthetic_fragment( /// An out-of-range index yields `None`, which is also what a variable whose /// fragment failed to compile yields; callers treat both as "no fragment", /// exactly as they treated a `None` from the direct path. +/// +/// PRIVATE on purpose: [`compile_ltm_fragment_for`] is the only way in, so the +/// index-to-variable coupling is checked at every call site rather than relied +/// on. Widening this back to `pub(crate)` re-opens the hole that wrapper exists +/// to close. #[salsa::tracked(returns(ref))] -pub(crate) fn compile_ltm_fragment_at( +fn compile_ltm_fragment_at( db: &dyn Db, model: SourceModel, project: SourceProject, @@ -1945,6 +1950,45 @@ pub(crate) fn compile_ltm_fragment_at( compile_ltm_synthetic_fragment(db, ltm_var, model, project) } +/// [`compile_ltm_fragment_at`] plus a debug-only check that `index` still names +/// the variable the caller believes it does. +/// +/// The index IS the identity, deliberately: a name argument would join the +/// salsa cache key and defeat the firewall the query's rustdoc describes. But +/// nothing in the signature or the types ties a caller's `index` to the +/// `LtmSyntheticVar` it walked it out of, so a third caller -- or any +/// reordering of `LtmVariablesResult::vars` between the walk and the call -- +/// would file a fragment under the wrong name, and both consumers treat a +/// mismatch as an ordinary "no fragment" rather than as an error. Nothing would +/// report it. +/// +/// Both callers already hold the variable, so they can pay a debug-only +/// assertion and make the coupling CHECKABLE rather than conventional. The +/// check costs nothing in release, and the query keeps its index-only key. +pub(crate) fn compile_ltm_fragment_for<'db>( + db: &'db dyn Db, + model: SourceModel, + project: SourceProject, + index: usize, + expected: &LtmSyntheticVar, +) -> &'db Option { + #[cfg(debug_assertions)] + { + let resolved = model_ltm_variables(db, model, project) + .vars + .get(index) + .map(|v| v.name.as_str()); + debug_assert_eq!( + resolved, + Some(expected.name.as_str()), + "compile_ltm_fragment_at is keyed by index alone, so a caller's \ + index and its LtmSyntheticVar must come from the same walk of the \ + same `vars` vector; index {index} resolves to {resolved:?}" + ); + } + compile_ltm_fragment_at(db, model, project, index) +} + #[cfg(test)] thread_local! { /// Test-only forced-failure pattern for @@ -2037,7 +2081,7 @@ pub fn model_ltm_fragment_diagnostics(db: &dyn Db, model: SourceModel, project: for (index, ltm_var) in ltm_vars.vars.iter().enumerate() { // Through the memoized per-index query, so this pass READS assembly's // fragments rather than compiling its own copies. - let fragment = compile_ltm_fragment_at(db, model, project, index); + let fragment = compile_ltm_fragment_for(db, model, project, index, ltm_var); // A fragment is usable only if it compiled *and* produced // flow-phase bytecodes. `compile_ltm_equation_fragment` returns // `Some(_)` with `flow_bytecodes: None` when the synthetic diff --git a/src/simlin-engine/src/db/ltm/mod.rs b/src/simlin-engine/src/db/ltm/mod.rs index 2f8812232..a36dd1611 100644 --- a/src/simlin-engine/src/db/ltm/mod.rs +++ b/src/simlin-engine/src/db/ltm/mod.rs @@ -51,7 +51,7 @@ pub use equation::{LtmArm, LtmEquation}; pub(crate) use compile::ForcePartialEquationErrorGuard; pub use compile::{ShapedLinkScore, compile_ltm_var_fragment, link_score_equation_text_shaped}; pub(crate) use compile::{ - compile_ltm_fragment_at, compile_ltm_implicit_var_fragment, model_ltm_fragment_diagnostics, + compile_ltm_fragment_for, compile_ltm_implicit_var_fragment, model_ltm_fragment_diagnostics, }; // Production reaches an LTM fragment only through the memoized // `compile_ltm_fragment_at`; the unmemoized selector below it is re-exported From 721c3e435e39cf3d7ab248804cafe817e32d497d Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 10:15:22 -0700 Subject: [PATCH 46/59] engine: pin the double-lag residual and drop a workaround the fix retired `colliding_index_name_model`'s boston arm needed a zero-coefficient `0 * pop[nyc]` term to stay materialized: with the omission stopping at the first PREVIOUS, an arm whose every occurrence was frozen was dropped, and the GH #986 assertions had no text to inspect. The lag-alignment check retires that need -- the arm freezes a runtime subscript index inside an already-frozen head, so it is double-lagged, not a structural zero, and it survives on its own merits. The fixture goes back to its original form. The term stays on the sibling control, and the asymmetry is now the clearest statement of what the check distinguishes: `s1` there is a STATIC selector, so `PREVIOUS(q[slot.s1])` reads `q` at `t-1` at a fixed slot and really does equal `PREVIOUS(share)`. That arm is a genuine structural zero and is correctly omitted, which is why it still needs a live reference to be inspectable. Removing the term reds the control and leaves the sibling green. `pinned_double_lag_residual_is_not_a_structural_zero` pins -1.06 / +0.73 / -1.03 / +0.82 as values. Those numbers had been recorded in `colliding_index_boston_series`' rustdoc for some time, framed only as an unadjudicated semantics question about what ceteris paribus means for an index read under a freeze. They are also the measurement that an arm can look entirely frozen and still be worth -1.06, which is the soundness input the omission needs -- and prose in a rustdoc does not fail when someone stops believing it. Reverting the nested-PREVIOUS descent reds this test on the first value. --- src/simlin-engine/src/db/ltm_tests.rs | 127 +++++++++++++++++++------- 1 file changed, 93 insertions(+), 34 deletions(-) diff --git a/src/simlin-engine/src/db/ltm_tests.rs b/src/simlin-engine/src/db/ltm_tests.rs index 87689b549..6c2e17432 100644 --- a/src/simlin-engine/src/db/ltm_tests.rs +++ b/src/simlin-engine/src/db/ltm_tests.rs @@ -1041,31 +1041,28 @@ fn collect_agg_petals_groups_single_agg_circuits() { // which still compiles and reads a different slot than the anchor did. // --------------------------------------------------------------------------- -/// `share[boston]` reads `q`/`gtab` at the runtime index `ctr`, and its only -/// dependence on `pop[nyc]` is the zero-coefficient term `0 * pop[nyc]`. +/// `share[boston]` reads `q`/`gtab` at the runtime index `ctr`, and has no +/// causal dependence on `pop[nyc]` whatsoever. /// /// `declare_bucket` adds a dimension **no equation references**, whose first /// element is named `ctr` -- the same canonical name as the model variable. It /// changes nothing about the simulation; before the fix it changed the emitted /// link score. /// -/// **The `0 * pop[nyc]` term is load-bearing for the TEST, not for the model**, -/// and it is the same idiom the neighbouring `0 * ctr` already uses. Its job is -/// to give the `boston` arm a live reference to the link's SOURCE, which is what -/// makes the arm materialize at all: since GH #977 a slot whose transformed -/// partial is provably `PREVIOUS(target)` is omitted from the `Arrayed` element -/// map and lowered to a constant zero, and without this term every occurrence in -/// this arm is frozen, so the arm this test reads would not exist. Its -/// coefficient is zero, so it changes no value the test asserts on: the residual -/// series below is bit-identical with and without it. +/// The `boston` arm survives the GH #977 omission on its own merits, and WHY it +/// survives is the whole distinction between this fixture and the control below. +/// Freezing the runtime index inside an already-frozen head yields +/// `PREVIOUS(q[PREVIOUS(ctr, ctr)])` -- `q` read at `t-1` indexed at `t-2`, +/// where the `PREVIOUS(share)` anchor indexed at `t-1`. That is not +/// `PREVIOUS(target)`, so the lag-alignment check rejects it and the arm is +/// materialized. The control below has a genuinely STATIC selector, so its arm +/// really is a structural zero and really is omitted. /// -/// That materialization matters because "the slot is absent under both variants" -/// would NOT be an adequate stand-in for the assertions below. Both readings of -/// `ctr` -- frozen (`PREVIOUS(ctr, ctr)`, correct) and qualified onto the -/// unrelated dimension (`bucket·ctr`, the defect) -- leave the arm provably -/// `PREVIOUS(target)`, so an omission-based assertion passes on the defect too. -/// The sibling control below demonstrates exactly that: a genuinely static -/// `q[slot·s1]` selector produces an omitted arm as well. +/// That also settles what "the slot is absent under both variants" would be +/// worth as a stand-in for the assertions below: nothing. Both readings of `ctr` +/// -- frozen (`PREVIOUS(ctr, ctr)`, correct) and qualified onto the unrelated +/// dimension (`bucket·ctr`, the defect) -- leave the arm looking entirely +/// frozen, so an omission-based assertion cannot tell a selector from a freeze. /// /// `indexed_name` only varies the subscripted variable's NAME. Both iterations /// exercise the SAME path -- an ordinary arrayed variable subscripted directly -- @@ -1109,10 +1106,7 @@ fn colliding_index_name_model(declare_bucket: bool, second_name: bool) -> datamo "share[Region]", vec![ ("nyc", "pop[nyc] * 0.01"), - ( - "boston", - &format!("{indexed}[ctr] * 0.002 + 0 * ctr + 0 * pop[nyc]"), - ), + ("boston", &format!("{indexed}[ctr] * 0.002 + 0 * ctr")), ("la", "pop[la] * 0.03"), ], ) @@ -1218,10 +1212,9 @@ fn a_colliding_index_name_is_resolved_against_the_axis_it_indexes() { /// The simulated `boston` slot of the `pop[nyc] -> share` link score. /// /// NOTE what this deliberately does NOT assert: that the series is ZERO. -/// `share[boston]`'s only dependence on `pop[nyc]` carries a zero coefficient, -/// so a fully ceteris-paribus partial would be identically zero -- and it is -/// not; it runs -1.06 / +0.73 / -1.03 / +0.82 on this fixture, bit-identically -/// with and without that term. That residual is a SEPARATE +/// `share[boston]` has no causal dependence on `pop[nyc]`, so a fully +/// ceteris-paribus partial would be identically zero -- and it is not; it runs +/// -1.06 / +0.73 / -1.03 / +0.82 on this fixture. That residual is a SEPARATE /// defect from the one above and predates this branch: an index frozen inside an /// already-frozen head is DOUBLE-lagged (the partial reads `q` at `t-1` indexed /// by `ctr` at `t-2`, where the anchor `PREVIOUS(share)` used `ctr` at `t-1`). @@ -1241,6 +1234,14 @@ fn a_colliding_index_name_is_resolved_against_the_axis_it_indexes() { /// (`if frozen { return index; }`, which disables the entire index pass, not just /// the re-freeze) takes this fixture to exactly 0 and reds 5 tests; that is an /// UPPER BOUND on the cost of the narrow change, not a measurement of it. +/// +/// It is ALSO a soundness input, and THAT half is settled. These numbers are the +/// measurement that an arm can look entirely frozen and still be worth -1.06, so +/// the GH #977 omission must not claim it as a structural zero. They sat here +/// framed only as a semantics question until a code review found the same class +/// from the other direction; `pinned_double_lag_residual_is_not_a_structural_zero` +/// below pins them as VALUES, so the next reader inherits the number rather than +/// the framing. fn colliding_index_boston_series(project: &datamodel::Project) -> Vec { let mut db = SimlinDb::default(); let sync = sync_from_datamodel(&db, project); @@ -1270,20 +1271,78 @@ fn colliding_index_boston_series(project: &datamodel::Project) -> Vec { .collect() } +/// The double-lag residual, pinned as VALUES rather than described in prose. +/// +/// `colliding_index_boston_series`' rustdoc has recorded -1.06 / +0.73 / -1.03 / +/// +0.82 for this slot for some time, framed as an unadjudicated semantics +/// question about what ceteris paribus means for an index read under a freeze. +/// It is that. It is ALSO the measurement showing this arm is not a structural +/// zero -- every occurrence in it is frozen, it looks entirely inert, and it is +/// worth -1.06 -- which is the soundness input the GH #977 omission needs and +/// which nobody connected until a code review found the same class from the +/// other direction. +/// +/// Prose in a rustdoc does not fail. This does: a change that lets the omission +/// claim this arm reds here on the first value, and a change that alters the +/// residual reds on the specific numbers rather than on a vague "it moved". +/// +/// The lag-alignment check in `ltm_augment_zero_slot` is what keeps the arm +/// alive; `db::ltm_value_gate_tests::a_nested_freeze_arm_is_not_a_structural_zero` +/// pins the same mechanism on a minimal fixture. This row exists because THESE +/// numbers are the ones that were already on disk and read past. +#[test] +fn pinned_double_lag_residual_is_not_a_structural_zero() { + let series: Vec = colliding_index_boston_series(&colliding_index_name_model(true, false)) + .into_iter() + .map(f64::from_bits) + .collect(); + + // The first two steps are the guard form's own warm-up (TIME = INITIAL_TIME, + // then the first live step), so the residual starts at index 2. + assert!( + series.len() >= 6, + "fixture must run long enough to show the residual; got {series:?}" + ); + assert_eq!( + (series[0], series[1]), + (0.0, 0.0), + "the guard form's warm-up steps; got {series:?}" + ); + for (i, expected) in [ + (2usize, -1.0588235294117647f64), + (3, 0.7297297297297297), + (4, -1.0285714285714287), + (5, 0.8181818181818182), + ] { + assert_eq!( + series[i], expected, + "step {i} of the documented double-lag residual moved; full series {series:?}" + ); + } + // The load-bearing half, stated on its own so a future reader cannot miss + // which property is the soundness one: this arm is NOT zero. + assert!( + series[2].abs() > 1.0, + "an arm whose every occurrence is frozen is still worth {}; it must never \ + be omitted as a structural zero", + series[2] + ); +} + #[test] fn an_index_naming_the_axis_own_element_stays_a_static_selector() { // The control that keeps the fix from being "freeze every bare index": // `s1` IS an element of `gtab`'s own `Slot` axis, so it is a selector and // must stay unwrapped (and qualified onto its own dimension). // - // `0 * pop[nyc]` plays the same role it does in `colliding_index_name_model` - // and for the same reason: a `boston` arm holding only frozen reads is - // provably `PREVIOUS(target)` and GH #977 omits it, and an omitted arm has - // no text to inspect. This control is also the direct evidence that - // omission cannot substitute for the assertions here -- WITHOUT the term - // this correctly-qualified static selector produces an omitted arm, exactly - // as the frozen runtime index does, so "absent under both variants" cannot - // tell a selector from a freeze. + // `0 * pop[nyc]` gives this arm a live source reference so there is text to + // inspect. It is needed HERE and not in `colliding_index_name_model`, and + // that asymmetry IS the point: `s1` is a static selector, so + // `PREVIOUS(q[slot·s1])` reads `q` at `t-1` at a fixed slot and really + // does equal `PREVIOUS(share)` -- a genuine structural zero, correctly + // omitted. The sibling's runtime index is double-lagged and is not. + // Removing this term reds this test and leaves the sibling green, which is + // the sharpest statement of what the lag-alignment check distinguishes. let project = TestProject::new("axis_element_index") .named_dimension("Region", &["nyc", "boston", "la"]) .named_dimension("Slot", &["s1", "s2"]) From 5595a0ae542229b0efb456e9e43c2641cff4de4e Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 10:31:59 -0700 Subject: [PATCH 47/59] build: drop the jq dependency from the cargo target-dir lookup `scripts/cargo-target-dir.sh` is on the primary build path -- both `src/engine/build.sh` and `scripts/pysimlin-tests.sh` call it, so it runs on every `pnpm build`, every pre-commit, and in CI. It parsed `cargo metadata` with jq, which is otherwise used only by release and CI-support scripts (release-{ts,serve,mcp}.sh, codex-review, await-reviews.sh and the three release workflows). No workflow installs jq -- the GitHub runner images happen to ship it -- and `scripts/dev-init.sh` did not check for it. So on a machine without jq, `set -euo pipefail` turned the lookup into `jq: command not found`, exit 127, partway through a commit. Isolated by running the previous script in place with a PATH-shadowed jq: exit 0 with jq, exit 127 without, nothing else changed. That traded the `cp: cannot stat` this script exists to prevent for an equally opaque failure one step earlier, which is a bad deal. Parse with python3 instead. Neither of the two obvious fixes was necessary: a fallback to `/target` would be silent and WRONG in exactly the case the script exists for -- a genuinely moved target directory -- reintroducing the original `cp: cannot stat` for its only real users; and adding jq to dev-init would make a needless dependency legible rather than removing it. python3 costs nothing, because it is already required strictly earlier on the same path: `scripts/pre-commit` shells to it in phase 1 (check-deps.py, check-docs.py) before any build runs, and `scripts/pysimlin-tests.sh` is built on it. `cargo metadata` remains the single source of truth, so no resolution rules are reconstructed and there is nothing to drift. Checked that the container path is not affected: `src/simlin-serve/build.rs` runs `pnpm install`/`pnpm build` with `current_dir(web_dir)`, i.e. the SPA's own build, so Dockerfile.cross never reaches this script. Verified both resolutions still work -- the default (`/target`) and, more to the point, `CARGO_TARGET_DIR=/tmp/ctd-probe`, which is the case that justifies the script at all. dev-init.sh now checks python3, which it should have all along: pre-commit already depended on it in phase 1, and a missing python3 surfaced as a bare "python3: command not found" mid-hook instead of at the one place whose job is to report missing tools. --- scripts/cargo-target-dir.sh | 19 ++++++++++++++++++- scripts/dev-init.sh | 7 +++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/scripts/cargo-target-dir.sh b/scripts/cargo-target-dir.sh index c374bc968..f2e6e8c23 100755 --- a/scripts/cargo-target-dir.sh +++ b/scripts/cargo-target-dir.sh @@ -13,6 +13,23 @@ # the single copy: a second, hand-maintained resolution drifts exactly where the # real one is non-trivial. # +# The JSON is parsed with python3, NOT jq, and that is deliberate. This script +# is on the primary build path -- `src/engine/build.sh` and +# `scripts/pysimlin-tests.sh` both call it, so it runs on every `pnpm build` +# and so on every pre-commit and in CI. jq is otherwise used only by release +# and CI-support scripts, no workflow installs it (the GitHub runner images +# happen to ship it), and `scripts/dev-init.sh` does not check for it. +# Depending on it here would turn a missing jq into `jq: command not found` +# under `set -e` -- trading the `cp: cannot stat` this script exists to +# prevent for an equally opaque failure one step earlier. python3 adds +# nothing: `scripts/pre-commit` already shells to it in phase 1, before any +# build runs, and `scripts/pysimlin-tests.sh` is python by definition. +# +# There is deliberately NO fallback to `/target` when the lookup fails. +# A fallback would be silent and wrong in exactly the case this script exists +# for -- a genuinely moved target directory -- reintroducing the original +# `cp: cannot stat` for the only callers who need the resolution at all. +# # Usage: TARGET_DIR="$(scripts/cargo-target-dir.sh)" set -euo pipefail @@ -20,4 +37,4 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null 2>&1 && pwd)" cargo metadata --format-version 1 --no-deps \ --manifest-path "$REPO_ROOT/Cargo.toml" \ - | jq -r '.target_directory' + | python3 -c 'import json, sys; print(json.load(sys.stdin)["target_directory"])' diff --git a/scripts/dev-init.sh b/scripts/dev-init.sh index f1b4635c3..beff8e097 100755 --- a/scripts/dev-init.sh +++ b/scripts/dev-init.sh @@ -58,12 +58,19 @@ command -v rustc >/dev/null 2>&1 || missing+=("rustc") command -v cargo >/dev/null 2>&1 || missing+=("cargo") command -v node >/dev/null 2>&1 || missing+=("node") command -v pnpm >/dev/null 2>&1 || missing+=("pnpm") +# python3 is not optional and is not only a pysimlin concern: scripts/pre-commit +# shells to it in phase 1 (check-deps.py / check-docs.py) and +# scripts/cargo-target-dir.sh parses `cargo metadata` with it on every +# `pnpm build`. Without this line a missing python3 surfaces as a bare +# "python3: command not found" partway through a commit rather than here. +command -v python3 >/dev/null 2>&1 || missing+=("python3") if [ ${#missing[@]} -gt 0 ]; then errors+=("Missing required tools: ${missing[*]}") errors+=(" rustc/cargo: https://rustup.rs/") errors+=(" node: https://nodejs.org/") errors+=(" pnpm: npm install -g pnpm") + errors+=(" python3: https://www.python.org/downloads/") fi # cbindgen (auto-install if cargo is available) From 6ee73fd69794f6b6ec0c96d826cab82107b3122e Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 10:33:42 -0700 Subject: [PATCH 48/59] build: correct what the REQUIRE_WASM_OPT guard claims to cover The comment said "Only the deploy scripts set it" and justified itself with "the deploy is a local command with no CI gate", which reads as: the deploy path is covered by this check. It is not. `scripts/deploy-web.sh` -- the production deploy the root CLAUDE.md documents as `pnpm deploy:web` -- does not invoke `verify-deploy-build.sh` at all. Only `deploy-web-staged.sh` does, and it is the only caller that sets REQUIRE_WASM_OPT. A comment asserting coverage that does not exist is worse than no comment, because it is what stops the next person checking. GH #1020 tracks giving the production path an assembly check; deliberately not done here, since adding a gate to the production deploy on a path nobody has exercised end-to-end this round could block a real deploy for an unrelated pre-existing reason. Also records why deploy-web.sh is nonetheless safe from the cache bug this guard backstops, because the reason is not reassuring: it runs `pnpm clean` first, and src/engine's clean removes `core/`, so the staging cache is never consulted. That is two callers happening to clean first for unrelated reasons, not a property of the cache -- the same distinction the fix's own message draws, and the one that makes it worth writing down rather than leaving as "it works". --- scripts/verify-deploy-build.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/verify-deploy-build.sh b/scripts/verify-deploy-build.sh index 3379c3739..ba2568d5d 100755 --- a/scripts/verify-deploy-build.sh +++ b/scripts/verify-deploy-build.sh @@ -180,7 +180,19 @@ fi # REQUIRE_WASM_OPT=1 rather than always-on, because CI's frontend job runs # this same script after a deliberate `DISABLE_WASM_OPT=1 pnpm build` -- # its subject is the deploy ASSEMBLY, not the artifact's optimization. -# Only the deploy scripts set it. +# `scripts/deploy-web-staged.sh` is the only caller that sets it. +# +# WHAT THIS DOES NOT COVER: `scripts/deploy-web.sh` -- the production +# deploy the root CLAUDE.md documents as `pnpm deploy:web` -- does not +# invoke this script at all, so neither this check nor any other assembly +# check runs on it. GH #1020 tracks closing that; until it does, the +# production path is verified by nothing. +# +# That path is nonetheless safe from the specific bug below, but only +# incidentally: it runs `pnpm clean` before `pnpm build`, and +# src/engine's clean removes `core/`, so the staging cache cannot be +# consulted at all. Do not read that as protection -- it is two callers +# happening to clean first for unrelated reasons. # # This exists because the failure it catches is silent and user-facing: an # unoptimized browser bundle is ~24% larger (5.0MB -> 6.2MB) and nothing @@ -188,7 +200,7 @@ fi # src/engine/build.sh cache-key bug -- a pre-commit build staging an # unoptimized blob that then satisfied the next optimizing build's cache # check -- which is fixed at the source but is worth a tripwire here too, -# since the deploy is a local command with no CI gate. +# since a deploy is a local command with no CI gate. if [ "1" = "${REQUIRE_WASM_OPT-0}" ]; then for wasm in src/engine/core/libsimlin.wasm src/engine/core/libsimlin-browser.wasm; do if [ ! -f "$wasm" ]; then From 14a04ca9386cce38cad1710acf58f23339c8ffee Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 10:36:11 -0700 Subject: [PATCH 49/59] engine: split the two corpus sweeps into per-model tests, and gate the poison hook `only_documented_classes_carry_across_a_step` and `fusion_never_raises_peak_stack_depth` each looped the whole corpus inside one `#[test]`. Both are now one test per (check, model) through `corpus_tests!`, which already generates the simulation corpus that way. The reason is diagnosis, not wall time, and it is worth being exact about which because the two are easy to conflate. Measured on this binary: the longest test is `clearn_ltm_var_count_guardrail` at 3.181s, total test time is 48.30s across 648 tests, and with 32 threads `max(longest, total/threads)` is `max(3.181, 1.51)` -- the wall is pinned by the single longest test. The two sweeps ran 0.145s and 0.150s, roughly 21x shorter, and did not appear in the top twelve. Splitting them moves the wall by nothing, and the measurement after the change agrees: 3.63s before, 3.71s after, the difference being 114 extra test harness entries. What it does buy is that a failure names the model instead of the sweep, and one bad model fails alone instead of aborting the corpus at the first bad entry -- which for a sweep is the difference between one red test and no information about the other 57 models. The fusion sweep's corpus-wide `assert!(checked > 100)` becomes a per-model `assert!(checked > 0)`, which is strictly stronger. The aggregate could stay satisfied while an individual model silently stopped contributing any check; all 58 models now assert their own non-vacuity and name themselves if they stop. The extra check modules are named inside the macro's `array:` arm rather than passed at the invocation. An `$extra:ident`-style parameter ahead of the path list is a `macro_rules!` local ambiguity -- both alternatives begin with an ident -- and that arm has exactly one caller. Separately, `poison_next_chunk_for_test` and everything it drives -- the `Vm` field, its initializer, the hoisted local, the Euler-loop branch and `POISON_SENTINEL` -- move behind `any(test, feature = "test-support")`. It was `pub`, so every downstream crate could reach a method whose only effect is to corrupt every non-implicit slot on every step. The feature already exists and is already enabled on the crate's self dev-dependency, so the corpus test keeps running in the default suite and a production build now carries neither the flag nor the branch that reads it. --- src/simlin-engine/src/vm.rs | 9 ++- .../tests/integration/simulate.rs | 56 ++++++++++++------- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/src/simlin-engine/src/vm.rs b/src/simlin-engine/src/vm.rs index 53fd51f7e..75bf002cb 100644 --- a/src/simlin-engine/src/vm.rs +++ b/src/simlin-engine/src/vm.rs @@ -366,7 +366,9 @@ pub struct Vm { // RK stages advance TIME away from INITIAL_TIME. prev_values_valid: bool, // Test-only: fill the `next` chunk with a sentinel at the top of every - // Euler step. See `poison_next_chunk_for_test`. + // Euler step. See `poison_next_chunk_for_test`. Gated with its setter so a + // production build carries neither the flag nor the branch that reads it. + #[cfg(any(test, feature = "test-support"))] poison_next: bool, // Conveyor support (docs/design/conveyors.md §9.3). Empty for every // non-conveyor model, and all conveyor logic is guarded on a non-empty @@ -789,6 +791,7 @@ pub(crate) fn increment_indices(indices: &mut [u16], dims: &[u16]) { /// Sentinel written into the `next` chunk by `poison_next_chunk_for_test`. A /// distinctive finite value rather than NaN, so a slot that carries forward is /// distinguishable from a model's own NaN. +#[cfg(any(test, feature = "test-support"))] #[doc(hidden)] pub const POISON_SENTINEL: f64 = -1.234567e123; @@ -856,6 +859,7 @@ impl Vm { stock_offsets, rk_scratch, prev_values_valid: false, + #[cfg(any(test, feature = "test-support"))] poison_next: false, conveyor_plans: Vec::new(), conveyors: Vec::new(), @@ -924,6 +928,7 @@ impl Vm { /// to a `curr[DT_OFF]` read inside every stock update, so poisoning it /// corrupts every stock and hides the property under test. See /// `only_documented_classes_carry_across_a_step`. + #[cfg(any(test, feature = "test-support"))] #[doc(hidden)] // test-support: used by tests/integration/simulate.rs pub fn poison_next_chunk_for_test(&mut self) { self.poison_next = true; @@ -1018,6 +1023,7 @@ impl Vm { }}; } + #[cfg(any(test, feature = "test-support"))] let poison_next = self.poison_next; match self.specs.method { @@ -1026,6 +1032,7 @@ impl Vm { if curr[TIME_OFF] > end { break; } + #[cfg(any(test, feature = "test-support"))] if poison_next { next[IMPLICIT_VAR_COUNT..].fill(POISON_SENTINEL); } diff --git a/src/simlin-engine/tests/integration/simulate.rs b/src/simlin-engine/tests/integration/simulate.rs index 0cdc0d50c..de5e190e7 100644 --- a/src/simlin-engine/tests/integration/simulate.rs +++ b/src/simlin-engine/tests/integration/simulate.rs @@ -39,17 +39,40 @@ macro_rules! corpus_tests { $($name:ident => $path:literal),* $(,)? ) => { static $arr: &[&str] = &[$($path),*]; - corpus_tests! { module: $module; $($name => $path),* } + corpus_tests! { module: $module; fn: simulate_path; $($name => $path),* } + // The whole-corpus checks that are not simulation comparisons get one + // test per (check, model) here rather than a `#[test]` looping the + // corpus: the failing model lands in the test NAME, and one bad model + // fails alone instead of taking the entire sweep with it. + // + // They are named here rather than passed in at the invocation because + // an `$extra:ident`-style parameter ahead of the path list is a local + // ambiguity for `macro_rules!` -- both alternatives start with an + // ident -- and this arm has a single caller, the main corpus. + corpus_tests! { + module: carry_across_a_step; fn: assert_poisoned_next_matches; + $($name => $path),* + } + corpus_tests! { + module: fusion_depth; fn: assert_fusion_depth_never_rises; + $($name => $path),* + } }; ( module: $module:ident; $($name:ident => $path:literal),* $(,)? + ) => { + corpus_tests! { module: $module; fn: simulate_path; $($name => $path),* } + }; + ( + module: $module:ident; fn: $check:ident; + $($name:ident => $path:literal),* $(,)? ) => { mod $module { $( #[test] fn $name() { - super::simulate_path(concat!("../../", $path)); + super::$check(concat!("../../", $path)); } )* } @@ -6522,16 +6545,13 @@ fn assert_poisoned_next_matches(xmile_path: &str) { // // Scope: every corpus model, both executed phases, all modules. Initials are // excluded because `Vm::new` leaves them unfused. -#[test] -fn fusion_never_raises_peak_stack_depth() { +fn assert_fusion_depth_never_rises(path: &str) { let mut checked = 0usize; - for path in TEST_MODELS.iter() { - let path = format!("../../{path}"); - let Ok(f) = File::open(&path) else { continue }; + { + let f = File::open(path).unwrap_or_else(|e| panic!("{path}: {e}")); let mut f = BufReader::new(f); - let Ok(datamodel_project) = xmile::project_from_reader(&mut f) else { - continue; - }; + let datamodel_project = + xmile::project_from_reader(&mut f).unwrap_or_else(|e| panic!("{path}: {e}")); for check in compile_vm(&datamodel_project).fusion_depth_audit() { let (module, phase) = (&check.module, check.phase); let pre = check @@ -6551,15 +6571,13 @@ fn fusion_never_raises_peak_stack_depth() { checked += 1; } } + // Per-model rather than a corpus-wide count. The aggregate this replaces + // (`checked > 100` over the whole sweep) could stay satisfied while an + // individual model silently stopped contributing any check at all; here a + // model that produces none fails by name. assert!( - checked > 100, - "expected a real corpus sweep, checked {checked}" + checked > 0, + "{path}: compiled but produced no fusion-depth checks -- the audit \ + found no fused phase, so this model verifies nothing" ); } - -#[test] -fn only_documented_classes_carry_across_a_step() { - for path in TEST_MODELS.iter() { - assert_poisoned_next_matches(&format!("../../{path}")); - } -} From f8a00ff51467a5449f5fe1e2cc2b0645af2b6207 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 10:39:54 -0700 Subject: [PATCH 50/59] engine: quantize the C-LEARN LTM digest relatively, as documented `clearn_ltm_slot_maxima_digest`'s rustdoc promised a tolerance of nine SIGNIFICANT digits -- a relative one, chosen so the pin would not red on the last-bit drift a benign allocator, layout or FP-association change produces, because "a digest that reds on that is a digest people learn to re-capture without reading". The code scaled by a fixed `1e9` and rounded, which quantizes ABSOLUTELY, at 1e-9 in value units. On this model those are not close. Measured over the 1,369 non-zero LTM slots: the largest peaks at 1.53e15, 30 slots sit above 1e12, and one ULP of the top slot is 0.25 -- which the `* 1e9` scale turns into a digest movement of 2.5e8. The pin was therefore maximally sensitive to exactly the noise its own rustdoc said it tolerated, and would have trained the re-capture reflex it was written to prevent. The same scale had a sensitivity hole at the other end, which is why this is a rewrite rather than a rescale. Summing raw magnitudes lets the three 1e15 slots dominate an aggregate that 743 slots near 1.0 also contribute to, and one slot whose maximum is 1e-14 quantized to exactly zero -- it could not move the digest at all, at any value below 5e-10. `nine_significant_digits` splits each slot's maximum into a 9-significant-digit mantissa in `[1e8, 1e9)` and a decimal exponent, summed separately. Every non-zero slot now contributes comparably regardless of scale, so a small slot is as visible as a large one; the exponent sum catches order-of-magnitude moves the mantissa alone would miss; and last-bit drift changes neither. It also removes the overflow hazard by construction rather than by clamping. The old form fed a saturating `f64 -> i128` cast, silent above ~1.7e29; the sums here are bounded by 7e12 and ~2.2e6 at 7,000 slots, both far inside `i64`. The new pins are `(1369, 7000, 371_710_864_477, 1019)`. The exponent digest was cross-checked against a decade histogram taken by a separate probe -- 3 slots at 1e15, 20 at 1e14, ..., 743 at 1e0, 144 at 1e-1, ..., 1 at 1e-14 -- which sums to exactly 1019 by hand, so the implementation agrees with an independent derivation rather than only with itself. The gate still catches what it exists for, and more loudly. Forcing `partial_is_provably_previous_target` to `true`, which drops 82 arms carrying real scores, moves it to `(1287, 7000, 350_469_799_138, 979)` -- a 5.7% shift in the mantissa digest where the old absolute form moved by 2.2e-11 of its own magnitude. The four lag-alignment coverage rows are unaffected and their per-clause mutation table is unchanged: reverting the original-`PREVIOUS` check reds `an_original_previous_arm_is_not_a_structural_zero` alone, and reverting the nested descent reds `a_nested_freeze_arm_is_not_a_structural_zero` and `pinned_double_lag_residual_is_not_a_structural_zero`. --- .../tests/integration/simulate_ltm.rs | 81 ++++++++++++++++--- 1 file changed, 71 insertions(+), 10 deletions(-) diff --git a/src/simlin-engine/tests/integration/simulate_ltm.rs b/src/simlin-engine/tests/integration/simulate_ltm.rs index 5875b1098..9dc69b453 100644 --- a/src/simlin-engine/tests/integration/simulate_ltm.rs +++ b/src/simlin-engine/tests/integration/simulate_ltm.rs @@ -11206,15 +11206,24 @@ fn test_whole_rhs_mapped_reducer_routes_through_synthetic_agg() { /// materializing structural zeros as small residuals moves it UP. /// * `finite_slots` -- how many are finite throughout, so a regression that /// replaces values with NaN cannot hide behind an unchanged non-zero count. -/// * `magnitude_digest` -- an order-independent sum over each slot's maximum -/// magnitude, quantized to 1e-9 relative. Two slots swapping values keeps the -/// first two numbers and moves this. +/// * `mantissa_digest` / `exponent_digest` -- an order-independent sum over each +/// slot's maximum magnitude, split into a 9-significant-digit mantissa and its +/// decimal exponent (`nine_significant_digits`). Two slots swapping values +/// keeps the first two numbers and moves these. /// /// Quantizing is what makes the pin usable rather than a per-run coin flip: raw /// f64 maxima carry last-bit noise across allocator and layout changes, and a /// digest that reds on that is a digest people learn to re-capture without /// reading. A real zeroing moves it far outside the quantum. /// +/// The quantization is RELATIVE, and it has to be: this model's largest LTM slot +/// peaks at 1.53e15 and 30 slots sit above 1e12, so a fixed `* 1e9` scale +/// quantizes at 1e-9 in VALUE units and one ULP of the top slot moves the sum by +/// 2.5e8 -- the pin would red on exactly the benign changes the paragraph above +/// says it tolerates. Splitting mantissa from exponent also gives every slot +/// equal weight, so the 743 slots whose maxima sit near 1.0 are visible at all; +/// under a raw magnitude sum the three 1e15 slots drown them. +/// /// **"It passes" and "it constrains the code" are different claims, so both /// were measured.** Three runs of this digest, same binary, differing only in /// `ltm_augment_zero_slot`: @@ -11270,7 +11279,8 @@ fn clearn_ltm_slot_maxima_digest() { let mut nonzero_slots = 0usize; let mut finite_slots = 0usize; - let mut magnitude_digest: i128 = 0; + let mut mantissa_digest: i64 = 0; + let mut exponent_digest: i64 = 0; for &off in <m_offsets { let mut max_mag = 0.0f64; let mut all_finite = true; @@ -11294,17 +11304,23 @@ fn clearn_ltm_slot_maxima_digest() { if all_finite { finite_slots += 1; } - // Nine significant digits: far finer than any real zeroing, far coarser - // than last-bit drift. - magnitude_digest += (max_mag * 1e9).round() as i128; + let (mantissa, exponent) = nine_significant_digits(max_mag); + mantissa_digest += mantissa; + exponent_digest += exponent; } assert_eq!( - (nonzero_slots, finite_slots, magnitude_digest), + ( + nonzero_slots, + finite_slots, + mantissa_digest, + exponent_digest + ), ( CLEARN_LTM_NONZERO_SLOTS, CLEARN_LTM_FINITE_SLOTS, - CLEARN_LTM_MAGNITUDE_DIGEST + CLEARN_LTM_MANTISSA_DIGEST, + CLEARN_LTM_EXPONENT_DIGEST ), "C-LEARN's LTM slot values moved. A DROP in nonzero_slots is the \ silent-zeroing regression this gate exists for; re-derive before \ @@ -11312,7 +11328,52 @@ fn clearn_ltm_slot_maxima_digest() { ); } +/// Split `x` into a 9-significant-digit decimal mantissa and its exponent: +/// `x ~= mantissa * 10^(exponent - 8)`, with `mantissa` in `[1e8, 1e9)`. +/// Zero maps to `(0, 0)`. +/// +/// RELATIVE quantization, which is the whole point. Scaling by a fixed `1e9` +/// and rounding -- the obvious spelling -- quantizes ABSOLUTELY, and on this +/// model that is not a tolerance at all: the largest LTM slot peaks at +/// 1.53e15, where one ULP is 0.25, so a single last-bit difference moves such a +/// digest by 2.5e8 and any benign allocator, layout or FP-association change +/// reds the pin. A gate that reds on nothing is a gate people re-capture +/// without reading, which is exactly what this digest's rustdoc promises to +/// avoid. +/// +/// Splitting the mantissa from the exponent also fixes a SENSITIVITY problem +/// that the absolute form had in the other direction. Summing raw magnitudes +/// lets the three 1e15 slots dominate: the 743 slots whose maxima sit near 1.0 +/// contribute ~15 orders of magnitude less, so a change to any of them is far +/// below the aggregate's own resolution. Here every non-zero slot contributes a +/// mantissa in `[1e8, 1e9)` regardless of scale, so a small slot is exactly as +/// visible as a large one, and the exponent sum catches the order-of-magnitude +/// moves the mantissa alone would miss. +/// +/// It removes the overflow hazard by construction rather than by clamping: at +/// 7,000 slots the sums are bounded by 7e12 and ~2.2e6, both far inside `i64`, +/// where the absolute form fed a saturating `f64 -> i128` cast that would have +/// failed silently. +fn nine_significant_digits(x: f64) -> (i64, i64) { + if x == 0.0 || !x.is_finite() { + return (0, 0); + } + let exponent = x.abs().log10().floor(); + let mantissa = x.abs() / 10f64.powf(exponent); + // `log10`/`powf` are not exact, so the quotient can land a hair outside + // [1, 10). Renormalise rather than trusting it: a mantissa that rounded to + // 1e9 is 10 significant digits and belongs in the next decade. + let mut mantissa = (mantissa * 1e8).round() as i64; + let mut exponent = exponent as i64; + if mantissa >= 1_000_000_000 { + mantissa /= 10; + exponent += 1; + } + (mantissa, exponent) +} + /// Pinned by `clearn_ltm_slot_maxima_digest`; see its rustdoc before changing. const CLEARN_LTM_NONZERO_SLOTS: usize = 1369; const CLEARN_LTM_FINITE_SLOTS: usize = 7000; -const CLEARN_LTM_MAGNITUDE_DIGEST: i128 = 10_248_673_492_482_445_132_733_301; +const CLEARN_LTM_MANTISSA_DIGEST: i64 = 371_710_864_477; +const CLEARN_LTM_EXPONENT_DIGEST: i64 = 1019; From 2afe6265fe7734757b086bb08994048dba9dab3e Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 13:30:17 -0700 Subject: [PATCH 51/59] engine: make the harness allocator claim true, and pin the pysimlin archive Four corrections from review, each a case of an assertion the code did not implement. A dev-dependency does not select an allocator. The Cargo.toml comment claimed the profiling examples and criterion benches all ran on mimalloc; only clearn_profile did. backend_bench and ltm_mem_bench kept System as their counting allocator's backing and the four benches declared none at all, so the compile-path timings they report came from the allocator the comment calls unrepresentative -- over-crediting exactly the changes it warns about. All seven harnesses now install it, and the comment says a new harness has to do the same. pysimlin-tests.sh resolved the static library for its freshness check but did not pass it to the CFFI build, and _ffi_build.py::_get_library_path searches only the workspace and crate-local target directories unless SIMLIN_STATIC_LIB pins it. Under CARGO_TARGET_DIR that links a stale archive or fails -- the silent-stale-engine case GH #682 added the pin for. The build's failures are no longer discarded either: an unbuilt extension surfaced later as an import error that said nothing about the cause. The optimized-WASM lane did not run on rust-toolchain.toml. The compiler selects what WASM is emitted and whether binaryen can still read it, and the ordinary frontend lane runs with DISABLE_WASM_OPT=1, so a toolchain bump could reach a release without either blob having been optimized once. The diagnostic probe's comment still described compile_implicit_var_fragment as untracked and quoted a ~15ms per-revision helper recompile. It is a tracked query now, so the walk costs a memo lookup per helper; the comment pointed anyone investigating warm-edit latency at a cost that no longer exists. Also silences a release-only unused-variable warning: compile_ltm_fragment_for reads its expected argument solely from a debug assertion. --- .github/workflows/wasm-opt.yml | 10 ++++++++ scripts/pysimlin-tests.sh | 11 ++++++++- src/simlin-engine/Cargo.toml | 23 +++++++++++++------ src/simlin-engine/benches/array_ops.rs | 7 ++++++ src/simlin-engine/benches/compiler.rs | 7 ++++++ src/simlin-engine/benches/rapidhash_bench.rs | 7 ++++++ src/simlin-engine/benches/simulation.rs | 7 ++++++ src/simlin-engine/examples/backend_bench.rs | 3 ++- src/simlin-engine/examples/ltm_mem_bench.rs | 7 +++--- src/simlin-engine/src/db/diagnostic.rs | 24 ++++++++++++-------- src/simlin-engine/src/db/ltm/compile.rs | 4 ++++ 11 files changed, 88 insertions(+), 22 deletions(-) diff --git a/.github/workflows/wasm-opt.yml b/.github/workflows/wasm-opt.yml index 38b1a47f7..bef138fb5 100644 --- a/.github/workflows/wasm-opt.yml +++ b/.github/workflows/wasm-opt.yml @@ -44,6 +44,11 @@ name: WASM optimized-bundle check - 'Cargo.lock' - 'Cargo.toml' - '.cargo/config.toml' + # The compiler selects what WASM gets emitted, and with it whether + # binaryen can still read it -- and the ordinary frontend lane runs with + # DISABLE_WASM_OPT=1, so a toolchain bump would otherwise reach a release + # without either blob having been optimized once. + - 'rust-toolchain.toml' - '.github/workflows/wasm-opt.yml' pull_request: branches: @@ -55,6 +60,11 @@ name: WASM optimized-bundle check - 'Cargo.lock' - 'Cargo.toml' - '.cargo/config.toml' + # The compiler selects what WASM gets emitted, and with it whether + # binaryen can still read it -- and the ordinary frontend lane runs with + # DISABLE_WASM_OPT=1, so a toolchain bump would otherwise reach a release + # without either blob having been optimized once. + - 'rust-toolchain.toml' - '.github/workflows/wasm-opt.yml' env: diff --git a/scripts/pysimlin-tests.sh b/scripts/pysimlin-tests.sh index 95ac73cae..5d0c7cc7e 100755 --- a/scripts/pysimlin-tests.sh +++ b/scripts/pysimlin-tests.sh @@ -31,7 +31,16 @@ if [ -z "$CFFI_SO" ] || [ "$LIBSIMLIN_A" -nt "$CFFI_SO" ] || [ "$SIMLIN_H" -nt " rm -rf build/ uv sync --extra dev uv pip install setuptools - uv run python setup.py build_ext --inplace 2>/dev/null || true + # Pin the archive rather than letting `_ffi_build.py::_get_library_path` + # search: its candidate list covers the workspace and crate-local `target/` + # directories only, so under CARGO_TARGET_DIR it would either link a stale + # default-target archive or fail -- and its own docs say guessing wrong here + # silently links a stale engine into the extension (GH #682). This is the + # same archive the freshness check above compared against. + # + # Failures are NOT suppressed: a build error here leaves no extension for the + # suite to import, and the import error that follows says nothing about why. + SIMLIN_STATIC_LIB="$LIBSIMLIN_A" uv run python setup.py build_ext --inplace else # Ensure deps are up to date (uv fast-paths when nothing changed) uv sync --extra dev diff --git a/src/simlin-engine/Cargo.toml b/src/simlin-engine/Cargo.toml index 55b018e60..072517526 100644 --- a/src/simlin-engine/Cargo.toml +++ b/src/simlin-engine/Cargo.toml @@ -171,13 +171,22 @@ harness = false # Every native binary that embeds this engine (simlin-cli, simlin-serve, # simlin-mcp, and libsimlin's `mimalloc` feature, which pysimlin's build turns -# on) installs mimalloc as its global allocator. The profiling examples and -# criterion benches therefore back their allocator with mimalloc too: the -# compile path is allocation-bound, so a harness on system malloc measures an -# allocator no shipped native build actually runs, and over-credits any change -# that only moves malloc traffic. Allocation *counts* stay the -# allocator-independent metric (and the one that carries over to the wasm -# bundle, which links neither mimalloc nor this dependency). +# on) installs mimalloc as its global allocator. The compile path is +# allocation-bound, so a harness on system malloc measures an allocator no +# shipped native build actually runs, and over-credits any change that only +# moves malloc traffic. +# +# A dev-dependency does NOT select an allocator -- each harness has to install +# one -- so every harness that reports a timing or a memory figure does: +# `examples/clearn_profile.rs`, `examples/backend_bench.rs` and +# `examples/ltm_mem_bench.rs` back their counting allocators with it, and all +# four benches declare it directly. Adding a harness here means adding the +# `#[global_allocator]` too, or its numbers describe a different allocator than +# the rest. +# +# Allocation *counts* stay the allocator-independent metric, and the one that +# carries over to the wasm bundle, which links neither mimalloc nor this +# dependency. [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] mimalloc = "0.1" diff --git a/src/simlin-engine/benches/array_ops.rs b/src/simlin-engine/benches/array_ops.rs index fb53f6673..c0d5e5fe3 100644 --- a/src/simlin-engine/benches/array_ops.rs +++ b/src/simlin-engine/benches/array_ops.rs @@ -16,6 +16,13 @@ use simlin_engine::datamodel::{ }; use simlin_engine::db::{SimlinDb, compile_project_incremental, sync_from_datamodel_incremental}; +// Back this harness with mimalloc, the allocator every native binary that +// embeds the engine installs. The compile path is allocation-bound, so timing +// it against the system allocator measures one no shipped build runs and +// over-credits any change that only moves malloc traffic. +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + /// Create a project with a single large 1D array and a sum reduction fn create_sum_project(array_size: u32) -> Project { let dim_name = "Idx"; diff --git a/src/simlin-engine/benches/compiler.rs b/src/simlin-engine/benches/compiler.rs index 07959ed43..e45ecd06f 100644 --- a/src/simlin-engine/benches/compiler.rs +++ b/src/simlin-engine/benches/compiler.rs @@ -37,6 +37,13 @@ use simlin_engine::db::{ }; use simlin_engine::open_vensim; +// Back this harness with mimalloc, the allocator every native binary that +// embeds the engine installs. The compile path is allocation-bound, so timing +// it against the system allocator measures one no shipped build runs and +// over-credits any change that only moves malloc traffic. +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + /// Model metadata for benchmark parameterization. struct ModelFixture { name: &'static str, diff --git a/src/simlin-engine/benches/rapidhash_bench.rs b/src/simlin-engine/benches/rapidhash_bench.rs index 36252b906..527185ce4 100644 --- a/src/simlin-engine/benches/rapidhash_bench.rs +++ b/src/simlin-engine/benches/rapidhash_bench.rs @@ -17,6 +17,13 @@ use std::hint::black_box; use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; use simlin_engine::rapidhash::{hash_bytes, hash_u32_slice}; +// Back this harness with mimalloc, the allocator every native binary that +// embeds the engine installs. The compile path is allocation-bound, so timing +// it against the system allocator measures one no shipped build runs and +// over-credits any change that only moves malloc traffic. +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + /// Reference FNV-1a 64-bit hash, u32-at-a-time. /// /// This is a verbatim copy of the pre-rapidhash implementation that diff --git a/src/simlin-engine/benches/simulation.rs b/src/simlin-engine/benches/simulation.rs index 47aa12a3b..aba2fb10a 100644 --- a/src/simlin-engine/benches/simulation.rs +++ b/src/simlin-engine/benches/simulation.rs @@ -10,6 +10,13 @@ use simlin_engine::db::{SimlinDb, compile_project_incremental, sync_from_datamod use simlin_engine::test_common::TestProject; use simlin_engine::{CompiledSimulation, Vm}; +// Back this harness with mimalloc, the allocator every native binary that +// embeds the engine installs. The compile path is allocation-bound, so timing +// it against the system allocator measures one no shipped build runs and +// over-credits any change that only moves malloc traffic. +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + fn build_population_project(stop: f64) -> TestProject { TestProject::new("bench_pop") .with_sim_time(0.0, stop, 1.0) diff --git a/src/simlin-engine/examples/backend_bench.rs b/src/simlin-engine/examples/backend_bench.rs index b63f0e7bf..272706058 100644 --- a/src/simlin-engine/examples/backend_bench.rs +++ b/src/simlin-engine/examples/backend_bench.rs @@ -40,7 +40,8 @@ //! slow case (a large model under a non-JIT wasm interpreter); the adaptive //! budget falls back to a single iteration for any phase that exceeds it. -use std::alloc::{GlobalAlloc, Layout, System as Backing}; +use mimalloc::MiMalloc as Backing; +use std::alloc::{GlobalAlloc, Layout}; use std::hint::black_box; use std::io::BufReader; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; diff --git a/src/simlin-engine/examples/ltm_mem_bench.rs b/src/simlin-engine/examples/ltm_mem_bench.rs index b2a1ed359..2610a7aeb 100644 --- a/src/simlin-engine/examples/ltm_mem_bench.rs +++ b/src/simlin-engine/examples/ltm_mem_bench.rs @@ -32,7 +32,8 @@ //! LTM enumeration algorithm without having to reason about the full //! salsa pipeline. -use std::alloc::{GlobalAlloc, Layout, System}; +use mimalloc::MiMalloc as Backing; +use std::alloc::{GlobalAlloc, Layout}; use std::collections::{BTreeSet, HashMap}; use std::fs; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; @@ -61,10 +62,10 @@ unsafe impl GlobalAlloc for CountingAlloc { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { ALLOC_COUNT.fetch_add(1, Ordering::Relaxed); ALLOC_BYTES.fetch_add(layout.size(), Ordering::Relaxed); - unsafe { System.alloc(layout) } + unsafe { Backing.alloc(layout) } } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - unsafe { System.dealloc(ptr, layout) } + unsafe { Backing.dealloc(ptr, layout) } } } diff --git a/src/simlin-engine/src/db/diagnostic.rs b/src/simlin-engine/src/db/diagnostic.rs index 809c2dfea..30e651082 100644 --- a/src/simlin-engine/src/db/diagnostic.rs +++ b/src/simlin-engine/src/db/diagnostic.rs @@ -208,16 +208,20 @@ pub fn model_all_diagnostics(db: &dyn Db, model: SourceModel, project: SourcePro // choice, with the sub-model divergence disclosed rather than claimed // away. // - // Unlike `compile_var_fragment` this is NOT a tracked query (the - // parent's parse result provides the caching) and unlike the LTM - // implicit probe (which sits inside the tracked - // `model_ltm_fragment_diagnostics`) it lives in THIS query's body, which - // `report_untracked_read` above forces to re-execute every revision -- - // so the helpers recompile on every revision's FIRST collection, - // including the per-edit paths that call `collect_all_diagnostics` - // (libsimlin `get_errors`, MCP `edit_model`). Measured on C-LEARN that - // is ~15ms per first collection; same-revision re-collections recompile - // nothing. + // `compile_implicit_var_fragment` is a tracked query keyed per helper, so + // what this loop costs is a memo lookup per helper rather than a compile. + // That matters because `report_untracked_read` above forces THIS query's + // body to re-execute every revision: the walk repeats on every revision's + // first collection, but the compiles behind it do not. A helper + // recompiles only when its own key is invalidated -- its parse, its + // dimensions, or the input set it is instantiated at -- so an edit to an + // unrelated variable leaves every other helper's memo intact. + // + // This is the reason the per-edit paths that call `collect_all_diagnostics` + // (libsimlin `get_errors`, MCP `edit_model`) no longer pay a whole-model + // helper recompile per revision. Do not "optimize" the walk away on the + // assumption it is doing the compiling; it is the accumulator replay that + // needs it, and the compiles are already shared with assembly. { let implicit_info = crate::db::model_implicit_var_info(db, model, project); let mut sorted_implicit: Vec<&String> = implicit_info.keys().collect(); diff --git a/src/simlin-engine/src/db/ltm/compile.rs b/src/simlin-engine/src/db/ltm/compile.rs index 85717c3bc..4ecd8697e 100644 --- a/src/simlin-engine/src/db/ltm/compile.rs +++ b/src/simlin-engine/src/db/ltm/compile.rs @@ -1965,6 +1965,10 @@ fn compile_ltm_fragment_at( /// Both callers already hold the variable, so they can pay a debug-only /// assertion and make the coupling CHECKABLE rather than conventional. The /// check costs nothing in release, and the query keeps its index-only key. +// `expected` is read only by the debug assertion below, so a release build +// sees it as unused. Keep it in the signature regardless: it is what forces a +// caller to have the variable in hand, which is the coupling being checked. +#[cfg_attr(not(debug_assertions), allow(unused_variables))] pub(crate) fn compile_ltm_fragment_for<'db>( db: &'db dyn Db, model: SourceModel, From f49ed5cd28ebb09338ca7fb7d4c7c2b25d9b9a77 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 16:50:43 -0700 Subject: [PATCH 52/59] engine: disclose and pin the LTM omission's non-finite value change The GH #977 omission is bit-exact modulo sign-of-zero given lag alignment, with one exception that was not disclosed: when the target slot is NON-FINITE, it changes a value rather than a representation. A materialized arm over a `NaN` target computes `partial - PREVIOUS(target)` = `NaN - NaN`. The zero guards do not rescue it -- `NaN = 0` is false -- and `SAFEDIV`'s fallback fires on a zero denominator rather than a `NaN` one, so it returns `NaN / NaN`. The arm evaluates to `NaN`, where an omitted slot is `AssignCurr(off, Const(0.0))` and therefore `+0.0`. An infinite target collapses to the same case, since `inf - inf` is also `NaN`. Reproduced both ways rather than argued, on a fixture whose target really does go non-finite (a flow-less stock holding 0, so `zed / zed` is a runtime 0/0 that constant folding cannot reach): materialized gives `[0, NaN, NaN, NaN, NaN, NaN]`, omitted gives all zeros. Preserving the arms is not available. Non-finiteness is a runtime property, so declining to omit any target that COULD go non-finite means declining to omit at all. A cheap runtime sentinel is not available either: the nearest candidate, `0 * (target - PREVIOUS(target)) * SIGN(source - PREVIOUS(source))`, takes its sign-of-zero from the wrong delta, yields `NaN` at `TIME = INITIAL_TIME` where a materialized arm yields exactly `0`, and costs four to five opcodes against the one an omitted slot lowers to. Building a partially-equivalent form against a mechanism just learned is how the 2026-07 attempt failed seven times. So this discloses and pins rather than preserving, and records that whether `0` is the better answer is open (GH #1022). The two relevant positions disagree, which is why it is not settled here. `src/float.rs` holds that an engine-manufactured NaN is noise in a channel practitioners already debug by hand, and this NaN is engine-made -- the guard form's own subtraction -- on an arm with no causal dependence on its source, so `0` is the structurally known answer. GH #542 points the other way: `ltm_post::denom_summand` excludes a `NaN` summand from its partition denominator specifically so the bad entry's own numerator can stay `NaN`, described there as "the honest per-loop 'undefined here' signal" -- a deliberate decision that NaN scores carry meaning per entry. Two earlier looks at this qualifier cited only `float.rs` and neither weighed #542, which is the reason it goes to an issue rather than into this change's judgement. #542 also disposes of the argument that a NaN score poisons its partition's relative-score denominator: it does not, and has not since #542. `a_nonfinite_target_arm_is_omitted_to_zero_not_nan` is the one row in the value gate that pins the omission CHANGING a value rather than preserving one, and its rustdoc says so, since it reads as an anomaly otherwise. It asserts the fixture premise -- the target really is `NaN` -- so it cannot pass on a model that never went non-finite, and it asserts the counterweight that a live arm over the same `NaN` target still scores `NaN`. That bounds what changed: the signal survives on the target's own series and on every arm with a live source; only arms with no causal dependence move. Mutation-tested -- disabling the omission reds it. --- docs/design/engine-performance.md | 13 +++ .../src/db/ltm_value_gate_tests.rs | 92 +++++++++++++++++++ .../src/ltm_augment_zero_slot.rs | 22 +++++ 3 files changed, 127 insertions(+) diff --git a/docs/design/engine-performance.md b/docs/design/engine-performance.md index 461163635..43ded4e76 100644 --- a/docs/design/engine-performance.md +++ b/docs/design/engine-performance.md @@ -852,6 +852,19 @@ short of it rather than taking a ~2-3%. its own row in `db::ltm_value_gate_tests`, and rejecting them costs zero arms on C-LEARN — the win above is measured with both checks in place. + One disclosed **value** change remains, on a model that produces non-finite + values: a materialized arm over a `NaN` (or infinite) target computes + `NaN - NaN` and evaluates to `NaN`, where an omitted slot is `+0.0`. It is + reproduced both ways by + `db::ltm_value_gate_tests::a_nonfinite_target_arm_is_omitted_to_zero_not_nan`. + Whether `0` is the better answer is **open** and tracked as #1022: + `src/float.rs` argues an + engine-manufactured NaN is noise, while GH #542 built the `denom_summand` + exclusion specifically to preserve a `NaN` score as a per-loop "undefined + here" signal. The signal survives on the target's own series and on every + live arm, so what changes is confined to arms with no causal dependence on + their source. + Larger run-side swings identified during round 2 — all three were taken to a data verdict in round 3 (2026-06-04): diff --git a/src/simlin-engine/src/db/ltm_value_gate_tests.rs b/src/simlin-engine/src/db/ltm_value_gate_tests.rs index f6cad4eaa..63cd43142 100644 --- a/src/simlin-engine/src/db/ltm_value_gate_tests.rs +++ b/src/simlin-engine/src/db/ltm_value_gate_tests.rs @@ -401,3 +401,95 @@ fn a_nested_freeze_arm_is_not_a_structural_zero() { it must not be omitted as a structural zero; got {boston:?}" ); } + +/// Mechanism 5, and the one place the omission is NOT value-neutral: a frozen +/// arm whose TARGET is non-finite. +/// +/// Every other row here pins that the omission preserves a value. This one pins +/// that it CHANGES one, deliberately, and it exists so the change is executable +/// rather than a sentence in a PR body. +/// +/// When `growth[boston]` is `NaN`, the materialized guard form computes +/// `partial - PREVIOUS(growth)` = `NaN - NaN` = `NaN`. The zero guards do not +/// rescue it, because `NaN = 0` is false, and `SAFEDIV(NaN, ABS(NaN), 0)` is +/// `NaN` rather than the fallback (the fallback fires on a zero denominator, not +/// a `NaN` one). So the arm evaluates to `NaN`. An omitted slot is +/// `AssignCurr(off, Const(0.0))`, so it is `+0.0`. +/// +/// Measured on this fixture: materialized gives `[0, NaN, NaN, NaN, NaN, NaN]`, +/// omitted gives all zeros. An infinite target collapses to the same case, since +/// `inf - inf` is also `NaN`. +/// +/// **This is a semantics decision that has not been adjudicated. It is tracked +/// as GH #1022**, and the arguments do not point the same way: +/// +/// * `src/float.rs` holds that a NaN the ENGINE manufactures is noise in a +/// channel practitioners already debug by hand. This NaN is engine-made -- it +/// comes from the guard form's own `NaN - NaN`, not from the modeller's +/// equation -- and the arm has no causal dependence on the source at all, so +/// `0` is the structurally known answer rather than a guess. +/// * GH #542 points the other way. `ltm_post::denom_summand` excludes a `NaN` +/// summand from its partition denominator specifically so that one undefined +/// score does not poison its siblings, while the bad loop's OWN numerator +/// stays `NaN` -- described there as "the honest per-loop 'undefined here' +/// signal". That is a deliberate decision that NaN scores carry meaning, and +/// replacing some of them with `0` partially undoes it. +/// +/// What is NOT at stake: the NaN signal does not disappear from the model. The +/// target's own series is still `NaN`, and any LIVE arm reading it still scores +/// `NaN` -- only arms with no causal dependence on their source change. +/// +/// Blast radius is confined to models that already produce non-finite values. +/// This row's job is to make the current answer fail if it changes, so whoever +/// adjudicates GH #1022 does so on purpose rather than by re-pinning. +fn nonfinite_target_project() -> datamodel::Project { + TestProject::new("nonfinite_target") + .with_sim_time(0.0, 5.0, 1.0) + .named_dimension("Region", &["nyc", "boston", "la"]) + // A stock with no flows holds 0 and is not constant-foldable, so + // `zed / zed` really is evaluated as 0/0 at runtime. + .stock("zed", "0", &[], &[], None) + .aux("nan_src", "zed / zed", None) + .array_flow_with_ranges( + "growth[Region]", + vec![ + ("nyc", "pop[nyc] * 0.01"), + ("boston", "nan_src * 0.02"), + ("la", "0.03"), + ], + ) + .array_stock("pop[Region]", "10", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn a_nonfinite_target_arm_is_omitted_to_zero_not_nan() { + let series = ltm_slot_series(&nonfinite_target_project()); + + // The premise: the target element really is NaN. Without this the row + // could pass on a fixture that never went non-finite at all. + let stock_to_flow = slot(&series, "link_score\u{205A}growth\u{2192}pop", 1); + assert!( + stock_to_flow.iter().any(|v| v.is_nan()), + "fixture premise: `growth[boston]` must be NaN, so the NaN signal is \ + present in the model at all; got {stock_to_flow:?}" + ); + + // The omitted arm. Region declaration order: nyc=0, boston=1, la=2. + let boston = slot(&series, "link_score\u{205A}pop[nyc]\u{2192}growth", 1); + assert!( + boston.iter().all(|v| *v == 0.0), + "the omitted structural-zero arm reports 0 where a materialized one \ + reports NaN -- the one disclosed value change in the GH #977 \ + omission. If this is being changed, adjudicate it rather than \ + re-pinning; got {boston:?}" + ); + + // The counterweight: the NaN signal is NOT erased from the model. A live + // arm over the same NaN target still scores NaN, so what changed is + // confined to arms with no causal dependence on their source. + assert!( + stock_to_flow.iter().any(|v| v.is_nan()), + "a live arm over the NaN target must still carry NaN" + ); +} diff --git a/src/simlin-engine/src/ltm_augment_zero_slot.rs b/src/simlin-engine/src/ltm_augment_zero_slot.rs index 7656abbf9..8d1610953 100644 --- a/src/simlin-engine/src/ltm_augment_zero_slot.rs +++ b/src/simlin-engine/src/ltm_augment_zero_slot.rs @@ -46,6 +46,28 @@ pub(crate) enum ZeroSlotPolicy { /// row in `db::ltm_value_gate_tests`; skipping either omits an arm worth /// close to the canonical +/-1 attribution. /// + /// Bit-exactness has ONE disclosed exception, and it is a value change + /// rather than a representation one: when the target slot is NON-FINITE. + /// A materialized arm computes `NaN - NaN` (or `inf - inf`), the zero + /// guards do not fire because `NaN = 0` is false, `SAFEDIV`'s fallback is + /// for a zero denominator rather than a `NaN` one, and the arm evaluates to + /// `NaN`; an omitted slot is `+0.0`. Measured, not argued: + /// `db::ltm_value_gate_tests::a_nonfinite_target_arm_is_omitted_to_zero_not_nan` + /// reproduces both sides. + /// + /// That trade is NOT adjudicated -- it is tracked as GH #1022 -- and the two + /// relevant positions disagree. + /// `crate::float`'s module docs hold that an engine-manufactured NaN is + /// noise in a channel practitioners debug by hand, and this NaN is + /// engine-made -- the guard form's own subtraction -- on an arm with no + /// causal dependence on its source, so `0` is the structurally known answer. + /// GH #542 points the other way: `ltm_post::denom_summand` excludes a `NaN` + /// score from its partition denominator precisely so the bad entry's own + /// numerator can stay `NaN` as "the honest per-loop 'undefined here' + /// signal". Replacing some of those with `0` partially undoes that. + /// Confined to models already producing non-finite values, and the signal + /// survives on the target's own series and on every live arm. + /// /// The tempting negative test -- "the link's source /// stayed frozen" -- says nothing about what else the arm reads, and /// collapsing on it changes 187 C-LEARN result slots across 35 link-score From b8b3af8765317023c52095ac238519e34356f9d5 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 16:54:41 -0700 Subject: [PATCH 53/59] build: install a pinned binaryen instead of the distro's, and ignore the loro unmaintained cluster Two CI failures, one of them a latent break the new optimized-WASM lane caught on its first run. apt's binaryen is older than the flags src/engine/build.sh passes, so the optimized build dies on 'Unknown option --enable-bulk-memory-opt'. That recipe was copied from ts-release.yml, which publishes to npm and would have failed identically on the next release -- the lane exists to catch a binaryen incompatibility before a release, and the first thing it caught was its own install recipe. Both workflows now share scripts/install-binaryen.sh, which pins an upstream release, refuses to proceed if that version cannot accept the flags build.sh passes, and fails at install rather than after build.sh has staged an unoptimized blob. The three RUSTSEC advisories are unrelated to this branch and pre-date it: bitmaps, im and sized-chunks were all declared unmaintained together, all reached through one edge (simlin-serve -> loro -> loro-internal -> im), and each says no safe upgrade exists. im's own announcement points at the imbl fork, which is loro's migration to make. Ignored with the rationale and a revisit condition, matching the three entries already there for the same class; all three go together because bitmaps and sized-chunks are only present as im's dependencies. Also corrects the wasm Apply dispatch comment, which still described a fixed three-operand pop with codegen-supplied padding. Both are gone: the arity table is shared by codegen, the VM and wasmgen, and locals above the arity now hold whatever the previous Apply left there. --- .github/workflows/ts-release.yml | 8 ++-- .github/workflows/wasm-opt.yml | 9 ++-- deny.toml | 14 +++++++ scripts/install-binaryen.sh | 58 ++++++++++++++++++++++++++ src/simlin-engine/src/wasmgen/lower.rs | 9 ++-- 5 files changed, 88 insertions(+), 10 deletions(-) create mode 100755 scripts/install-binaryen.sh diff --git a/.github/workflows/ts-release.yml b/.github/workflows/ts-release.yml index 8126e2b7c..468aab263 100644 --- a/.github/workflows/ts-release.yml +++ b/.github/workflows/ts-release.yml @@ -55,10 +55,12 @@ jobs: rustup show rustup target add wasm32-unknown-unknown + # A pinned upstream release, NOT `apt-get install binaryen`: the distro + # version is older than the flags build.sh passes and fails with a bare + # "Unknown option '--enable-bulk-memory-opt'". Shared with ts-release.yml, + # which optimizes the bundle it publishes and would fail identically. - name: Install wasm-opt - run: | - sudo apt-get update -qq - sudo apt-get install -y -qq binaryen + run: ./scripts/install-binaryen.sh - uses: pnpm/action-setup@v4 diff --git a/.github/workflows/wasm-opt.yml b/.github/workflows/wasm-opt.yml index bef138fb5..2616effdf 100644 --- a/.github/workflows/wasm-opt.yml +++ b/.github/workflows/wasm-opt.yml @@ -85,11 +85,12 @@ jobs: - name: Install Rust toolchain run: rustup show - # Same recipe as ts-release.yml, the only other workflow that needs it. + # A pinned upstream release, NOT `apt-get install binaryen`: the distro + # version is older than the flags build.sh passes and fails with a bare + # "Unknown option '--enable-bulk-memory-opt'". Shared with ts-release.yml, + # which optimizes the bundle it publishes and would fail identically. - name: Install wasm-opt - run: | - sudo apt-get update -qq - sudo apt-get install -y -qq binaryen + run: ./scripts/install-binaryen.sh - name: Install pnpm uses: pnpm/action-setup@v4 diff --git a/deny.toml b/deny.toml index 1f2df5143..310783d79 100644 --- a/deny.toml +++ b/deny.toml @@ -24,6 +24,20 @@ ignore = [ # entry, purely a maintenance-status advisory -- no code-execution or # memory-safety risk. "RUSTSEC-2026-0206", + # bitmaps, im and sized-chunks are unmaintained -- one author's cluster, + # all three declared at once, all reached through the same single edge: + # simlin-serve -> loro -> loro-internal -> im -> {bitmaps, sized-chunks}. + # Each advisory states "No safe upgrade is available"; im's own + # announcement points at the imbl fork, which is loro's migration to make, + # not ours. Purely maintenance-status advisories -- no code-execution or + # memory-safety claim in any of the three. + # + # Revisit when loro drops im (watch loro-internal's dependency on it): all + # three go together, because bitmaps and sized-chunks are only here as im's + # dependencies. + "RUSTSEC-2026-0247", + "RUSTSEC-2026-0248", + "RUSTSEC-2026-0251", ] [licenses] diff --git a/scripts/install-binaryen.sh b/scripts/install-binaryen.sh new file mode 100755 index 000000000..f58a727b6 --- /dev/null +++ b/scripts/install-binaryen.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Install a pinned binaryen release and put `wasm-opt` on the PATH. +# +# `apt-get install binaryen` is NOT sufficient: the version Ubuntu ships is +# older than the flags `src/engine/build.sh` passes, and the failure is a bare +# `Unknown option '--enable-bulk-memory-opt'` from a `wasm-opt` that ran at all. +# Both workflows that optimize the bundle -- the optimized-WASM check and the +# npm publish -- therefore install from the upstream release rather than from +# the distro, so CI runs the same binaryen a developer does instead of whatever +# the runner image happens to carry. +# +# Bump VERSION when build.sh starts using a newer flag. Keep it at or below the +# version developers have locally, since this is the one that gates a release. +set -euo pipefail + +VERSION="${BINARYEN_VERSION:-125}" + +case "$(uname -s)-$(uname -m)" in + Linux-x86_64) ASSET="x86_64-linux" ;; + Linux-aarch64) ASSET="aarch64-linux" ;; + Darwin-arm64) ASSET="arm64-macos" ;; + Darwin-x86_64) ASSET="x86_64-macos" ;; + *) + echo "install-binaryen.sh: no pinned asset for $(uname -s)-$(uname -m)." >&2 + echo "Install binaryen >= $VERSION yourself and put wasm-opt on PATH." >&2 + exit 1 + ;; +esac + +PREFIX="${BINARYEN_PREFIX:-$HOME/.local/binaryen}" +URL="https://github.com/WebAssembly/binaryen/releases/download/version_${VERSION}/binaryen-version_${VERSION}-${ASSET}.tar.gz" + +echo "Installing binaryen $VERSION ($ASSET) into $PREFIX" +mkdir -p "$PREFIX" +curl --fail --location --silent --show-error "$URL" \ + | tar -xz -C "$PREFIX" --strip-components=1 + +BIN="$PREFIX/bin" +if [ ! -x "$BIN/wasm-opt" ]; then + echo "install-binaryen.sh: $BIN/wasm-opt missing after extraction" >&2 + exit 1 +fi + +# Fail here rather than mid-build if the pinned release cannot run our flags: +# a wasm-opt that rejects an option exits non-zero *after* build.sh has already +# staged the unoptimized blob. +"$BIN/wasm-opt" --version +"$BIN/wasm-opt" --help 2>&1 | grep -q -- '--enable-bulk-memory-opt' || { + echo "install-binaryen.sh: binaryen $VERSION does not support" >&2 + echo " --enable-bulk-memory-opt, which src/engine/build.sh passes." >&2 + exit 1 +} + +if [ -n "${GITHUB_PATH:-}" ]; then + echo "$BIN" >>"$GITHUB_PATH" +else + echo "Add to PATH: $BIN" +fi diff --git a/src/simlin-engine/src/wasmgen/lower.rs b/src/simlin-engine/src/wasmgen/lower.rs index 208ae4ee6..414572d76 100644 --- a/src/simlin-engine/src/wasmgen/lower.rs +++ b/src/simlin-engine/src/wasmgen/lower.rs @@ -1415,9 +1415,12 @@ fn emit_ops( emit_op2(*op, ctx, f)?; emit_assign(ctx.next_base, *off, ctx, f); } - // `Apply` always pops exactly three operands (codegen pads short - // builtins with `LoadConstant 0.0` / `LoadGlobalVar{FINAL_TIME}`), - // mirroring the VM (`vm.rs:1701`). See [`emit_apply`]. + // `Apply` pops `func.arity()` operands, not a fixed three: codegen, + // the VM and this backend all read that one table, so a builtin's + // operand count is decided in exactly one place. Codegen emits no + // padding, which means locals above the arity hold whatever the + // PREVIOUS `Apply` left in them -- do not read them. See + // [`emit_apply`], whose own comment names the tests that enforce it. Opcode::Apply { func } => emit_apply(*func, ctx, f), // `Lookup` pops `index` then `element_offset`, bounds-checks the // offset, and dispatches to the mode's helper over the table at From 2f9194b5baa5f1180dce02473630e44ab99a799f Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 17:37:08 -0700 Subject: [PATCH 54/59] engine: describe Apply's real arity at the three sites that still claimed three Three comments still described `Apply` as always receiving three operands with codegen supplying padding. That contract ended when builtins gained a real arity; the prose preserved the maintenance hazard the implementation had removed, telling a maintainer that unread `apply_locals` are freshly zeroed when they now hold whatever the previous `Apply` left there. - `EmitCtx::apply_locals` said the opcode "always pops exactly three operands (codegen pads)". Three is the WIDEST a builtin needs, not the number every `Apply` populates, so the locals are partially initialized in general. - `emit_apply`'s rustdoc said "the three operands are on the wasm stack". It is `BuiltinId::arity()` of them, with no padding. - `Opcode::stack_effect` still carried "Builtins always take 3 args (actual + padding)" immediately above the line replacing it. That one is mine: the edit that introduced the arity comment left the sentence it contradicted in place, so the stale claim read first. Both wasmgen sites now say what a maintainer adding a builtin needs: the arity table is shared by codegen, the VM and this backend, so the operand count is decided in one place; and the padding was accidentally load-bearing, since it guaranteed unread locals were `0.0`. Removing it moved that guarantee out of the data and into the `apply_*` parity tests in `lower_tests.rs`, which are now the only thing stopping an arm from reading past its arity -- the same pointer the obligation note at the pops already carries. Found by sweeping the class rather than the named sites: the third is in `bytecode.rs`, which a search scoped to `wasmgen/` would not have reached. Not changed, having checked: `vector.rs`'s several claims that the `Apply` scratch f64s are "free" are about availability to clobber, not about their contents. Those emitters write each local before reading it, so they never depended on the padding's zeroes and are unaffected. --- src/simlin-engine/src/bytecode.rs | 1 - src/simlin-engine/src/wasmgen/lower.rs | 29 +++++++++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/simlin-engine/src/bytecode.rs b/src/simlin-engine/src/bytecode.rs index 5979c237c..b883576b9 100644 --- a/src/simlin-engine/src/bytecode.rs +++ b/src/simlin-engine/src/bytecode.rs @@ -1522,7 +1522,6 @@ impl Opcode { // Assignment: pops 1 (the value to assign) Opcode::AssignCurr { .. } => (1, 0), - // Builtins always take 3 args (actual + padding), push 1 result // Builtins pop exactly the operands `vm::apply` reads (see // `BuiltinId::arity`), not a fixed 3 with discarded padding. Opcode::Apply { func } => (func.arity(), 1), diff --git a/src/simlin-engine/src/wasmgen/lower.rs b/src/simlin-engine/src/wasmgen/lower.rs index 414572d76..9f0a0a4dd 100644 --- a/src/simlin-engine/src/wasmgen/lower.rs +++ b/src/simlin-engine/src/wasmgen/lower.rs @@ -155,8 +155,15 @@ pub(crate) struct EmitCtx<'a> { /// [`max_condition_depth`]). pub condition_locals: Vec, /// Three dedicated scratch f64 local indices `[a, b, c]` for the `Apply` - /// opcode, which always pops exactly three operands (codegen pads). They - /// are distinct from [`scratch_local`](Self::scratch_local) and the + /// opcode. Three is the WIDEST a builtin needs, not the number every + /// `Apply` populates: codegen pushes exactly `BuiltinId::arity()` operands + /// and emits no padding, so a 1-arity builtin sets only `a` and leaves + /// `b`/`c` holding whatever an earlier `Apply` left there. They are + /// therefore partially initialized in general, and an arm must read only + /// the locals its own arity covers -- enforced by the `apply_*` tests in + /// `lower_tests.rs`, since nothing in the types requires it. + /// + /// Distinct from [`scratch_local`](Self::scratch_local) and the /// [`condition_locals`](Self::condition_locals) so an `Apply` inside an /// `If` arm (sharing the function) cannot clobber the condition register. /// Reserved unconditionally by the function builders (3 unused f64 locals @@ -2459,11 +2466,19 @@ fn emit_cmp(f: &mut Function, cmp: &Instruction) { f.instruction(&Instruction::F64ConvertI32U); } -/// Lower the `Apply { func }` opcode, mirroring the VM's `apply()` -/// (`vm.rs:2938`). The three operands are on the wasm stack in push order -/// `[a, b, c]` (`c` on top, matching the VM popping `c` then `b` then `a`); -/// they are parked in the dedicated `ctx.apply_locals` so each builtin can read -/// them any number of times in any order. The result is left on the stack. +/// Lower the `Apply { func }` opcode, mirroring the VM's `apply()`. +/// +/// `BuiltinId::arity()` operands are on the wasm stack in push order -- one, +/// two or three of `[a, b, c]` with the last on top, matching the order the VM +/// pops them. Codegen emits no padding, so only the arity's worth are present. +/// They are parked in the dedicated `ctx.apply_locals` so each builtin can read +/// them any number of times in any order; the locals above the arity keep +/// whatever an earlier `Apply` left in them. The result is left on the stack. +/// +/// The arity table is shared by codegen, the VM and this backend, so a +/// builtin's operand count is decided in exactly one place and the three +/// cannot disagree. See the obligation on the pops below for what a new +/// builtin has to respect and what enforces it. /// /// `time`/`dt` for the time-driven builtins are read from `curr[TIME_OFF]` / /// `curr[DT_OFF]` (absolute global slots, like `LoadGlobalVar`), matching the From d9b575a8b51f61cc98b9a2be721939cbfe5ca845 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 17:51:54 -0700 Subject: [PATCH 55/59] build: invalidate the WASM mode stamp before restaging, not just after Writing the stamp last protects a FIRST build -- an abort leaves no stamp, so the next run redoes the work. It does not protect an UPDATE, because a valid stamp from the previous build is still on disk, and that is the case the original reasoning missed. Reproduced before fixing, on a real failing wasm-opt (a shim on PATH, so `command -v` still succeeds and the invocation aborts under `set -e`): 1. good optimized build blob 6829333 raw 8327392 mode opt 2. Rust changes, wasm-opt dies after the copies blob 8327384 raw 8327384 mode opt <-- stale 3. next run, real wasm-opt EARLY-OUT, no wasm-opt, exit 0 blob 8327384 raw 8327384 mode opt Step 3 exits 0 with the raw cargo output staged and stamped `opt`. That is worse than the bug the stamp was added for: it is a green build whose artifact is wrong, and it also defeats verify-deploy-build.sh's REQUIRE_WASM_OPT check, which reads the stamp. Post-fix the same sequence leaves no stamp at step 2 and re-optimizes at step 3 (blob 6829390). The fix is `rm -f core/$out.mode` before restaging, which makes the whole window self-invalidating: no stamp means indeterminate, and indeterminate means rebuild. Paired with the existing write-last, the stamp now exists only while it is true of what is on disk. The `.raw` window asked about in review is already safe, and the comment now says why rather than leaving it to be re-derived: the blob is copied BEFORE `.raw`, so a failure between the two leaves `.raw` holding the previous output and the next `cmp` rebuilds. Reversed, it would leave a `.raw` describing the new source beside a blob built from the old one, which `cmp` cannot detect because it only ever compares `.raw` against cargo. verify-deploy-build.sh's check no longer trusts the stamp alone. It now also requires the blob to differ from its `.raw`, which is independent evidence that wasm-opt transformed something -- the same thing the CI lane asserts. A guard that can only be as correct as the thing it guards is not a guard, and this one demonstrably was: the forged stale-stamp state passed it before and fails it now. Mutation-tested both ways. --- scripts/verify-deploy-build.sh | 12 +++++++++++- src/engine/build.sh | 26 ++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/scripts/verify-deploy-build.sh b/scripts/verify-deploy-build.sh index ba2568d5d..937b6fedd 100755 --- a/scripts/verify-deploy-build.sh +++ b/scripts/verify-deploy-build.sh @@ -209,8 +209,18 @@ if [ "1" = "${REQUIRE_WASM_OPT-0}" ]; then fail "$wasm.mode missing -- src/engine/build.sh did not stage $wasm, or predates the mode stamp" elif [ "opt" != "$(cat "$wasm.mode")" ]; then fail "$wasm was built WITHOUT wasm-opt (mode: $(cat "$wasm.mode")). Deploying it would ship a ~24% larger bundle. Is wasm-opt installed, and is DISABLE_WASM_OPT unset?" + elif [ ! -f "$wasm.raw" ]; then + fail "$wasm.raw missing -- cannot corroborate the mode stamp, so $wasm may not actually be optimized" + elif cmp -s "$wasm" "$wasm.raw"; then + # Independent of the stamp on purpose. The stamp records INTENT and + # can outlive the artifact it describes -- a wasm-opt that fails + # after the blob is staged used to leave a stale `opt` stamp on a + # raw blob, which passed this check on the stamp alone. That window + # is closed in src/engine/build.sh, but a guard that can only be as + # correct as the thing it guards is not a guard. + fail "$wasm is byte-identical to $wasm.raw, so wasm-opt did not transform it despite a '$(cat "$wasm.mode")' stamp -- the staged artifact and its stamp disagree" else - pass "$wasm is wasm-opt'd ($(wc -c < "$wasm") bytes)" + pass "$wasm is wasm-opt'd ($(wc -c < "$wasm") bytes, stamp and artifact agree)" fi done fi diff --git a/src/engine/build.sh b/src/engine/build.sh index df9424ce2..74971e400 100755 --- a/src/engine/build.sh +++ b/src/engine/build.sh @@ -65,6 +65,27 @@ build_wasm() { if [ ! -f "core/$out_name" ] \ || [ "$have_mode" != "$want_mode" ] \ || ! cmp -s "$WASM_SRC" "core/$out_name.raw"; then + # Invalidate the stamp BEFORE restaging, not merely write it after. + # + # Writing it last protects a FIRST build: an abort leaves no stamp, so the + # next run redoes the work. It does NOT protect an update, because a valid + # stamp from the previous build is still on disk. If wasm-opt then fails or + # is interrupted after the copies below, `.raw` already matches the new + # cargo output while the staged blob is raw -- and the surviving `opt` + # stamp makes the next run early-out, treat the raw blob as optimized, and + # exit 0. That is the wrong answer the stamp exists to prevent, arriving + # through the update path, and it also defeats verify-deploy-build.sh's + # REQUIRE_WASM_OPT check, which reads this stamp. + # + # Removing it here makes the whole window self-invalidating: no stamp means + # indeterminate, and indeterminate means rebuild. + rm -f "core/$out_name.mode" + + # Blob before `.raw`, deliberately. If the second copy fails, `.raw` still + # holds the PREVIOUS output, so the `cmp` above fails next run and the work + # is redone. Reversed, a failure between the two would leave a `.raw` + # describing the new source beside a blob built from the old one -- which + # `cmp` cannot detect, since it only ever compares `.raw` to cargo. cp "$WASM_SRC" "core/$out_name" cp "$WASM_SRC" "core/$out_name.raw" @@ -80,8 +101,9 @@ build_wasm() { echo "Skipping wasm-opt (not installed or disabled)" fi - # Written LAST so an interrupted build leaves no stamp and the next run - # redoes the work rather than trusting a half-staged artifact. + # Written LAST, and only now that the artifact matches it. Paired with the + # `rm -f` above this makes the stamp transactional: it exists only while it + # is true of what is on disk. printf '%s\n' "$want_mode" > "core/$out_name.mode" fi } From ef16665d7b173eda4399fe58549cb4472ea48b8e Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 17:53:10 -0700 Subject: [PATCH 56/59] ci: trigger the wasm-opt lane on changes to its binaryen installer `scripts/install-binaryen.sh` was not in either path list, so a version bump, a changed release-asset name or a dead URL did not run the one workflow that executes it. Its only other consumer is ts-release.yml, which runs on `ts-v*` tags and manual dispatch -- so a break would first surface DURING an npm release. Same shape as the rust-toolchain.toml gap fixed earlier, and the same fix. I checked the lane's other inputs rather than assuming, and deliberately did NOT add the rest. The test is whether a change to a file can break something only THIS lane would catch: - scripts/install-binaryen.sh yes -- nothing else on a PR runs it - rust-toolchain.toml yes -- already listed, for the same reason - src/engine/build.sh yes -- already covered by src/engine/** - package.json, pnpm-lock.yaml, pnpm-workspace.yaml no -- ci.yaml's frontend job has NO path filter, so it runs `pnpm install`, `pnpm build` and `pnpm test` on every PR and reds first. Listing them here would fire a ~4-minute wasm lane on every dependency bump for no signal this lane alone provides. The push and pull_request lists are maintained by hand and read as one filter, so a path added to only one silently means "runs on merge but not on the PR" -- a gap that looks like coverage, and the same shape as the one this commit closes. scripts/lint-project.sh (pre-commit phase 1) now fails if any workflow's two lists differ, reporting which entries are on which side. Mutation-tested: dropping the installer from the pull_request list alone reds it, naming that entry. --- .github/workflows/wasm-opt.yml | 10 +++++++++ scripts/lint-project.sh | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/.github/workflows/wasm-opt.yml b/.github/workflows/wasm-opt.yml index 2616effdf..21dd9881c 100644 --- a/.github/workflows/wasm-opt.yml +++ b/.github/workflows/wasm-opt.yml @@ -49,6 +49,11 @@ name: WASM optimized-bundle check # DISABLE_WASM_OPT=1, so a toolchain bump would otherwise reach a release # without either blob having been optimized once. - 'rust-toolchain.toml' + # This lane and ts-release.yml are the installer's only consumers, and + # ts-release runs on tags and manual dispatch -- so a version bump, a + # changed asset name or a dead URL would otherwise first surface DURING + # an npm release. + - 'scripts/install-binaryen.sh' - '.github/workflows/wasm-opt.yml' pull_request: branches: @@ -65,6 +70,11 @@ name: WASM optimized-bundle check # DISABLE_WASM_OPT=1, so a toolchain bump would otherwise reach a release # without either blob having been optimized once. - 'rust-toolchain.toml' + # This lane and ts-release.yml are the installer's only consumers, and + # ts-release runs on tags and manual dispatch -- so a version bump, a + # changed asset name or a dead URL would otherwise first surface DURING + # an npm release. + - 'scripts/install-binaryen.sh' - '.github/workflows/wasm-opt.yml' env: diff --git a/scripts/lint-project.sh b/scripts/lint-project.sh index c79697835..69eadc3e4 100755 --- a/scripts/lint-project.sh +++ b/scripts/lint-project.sh @@ -63,6 +63,43 @@ if ! python3 scripts/check-copyright.py > "$COPYRIGHT_OUTPUT"; then fi rm -f "$COPYRIGHT_OUTPUT" +# Rule 4: a path-filtered workflow's push and pull_request `paths` lists must +# match. They are maintained by hand and read as one filter, so a path added to +# only one of them silently means "runs on merge but not on the PR" (or the +# reverse) -- a gap that looks like coverage. `.github/workflows/wasm-opt.yml` +# is the only such workflow today; the loop covers any future one. +PATHS_OUTPUT=$(mktemp) +if ! python3 - > "$PATHS_OUTPUT" <<'PYEOF'; then +import glob, sys +import yaml + +status = 0 +for path in sorted(glob.glob(".github/workflows/*.y*ml")): + with open(path) as fh: + wf = yaml.safe_load(fh) + triggers = (wf or {}).get(True) or (wf or {}).get("on") or {} + if not isinstance(triggers, dict): + continue + push = (triggers.get("push") or {}).get("paths") + pull = (triggers.get("pull_request") or {}).get("paths") + if push is None and pull is None: + continue + if push != pull: + only_push = [p for p in (push or []) if p not in (pull or [])] + only_pull = [p for p in (pull or []) if p not in (push or [])] + print(f"{path}: push and pull_request `paths` differ; " + f"push-only={only_push} pull_request-only={only_pull}") + status = 1 +sys.exit(status) +PYEOF + while IFS= read -r line; do + [ -z "$line" ] && continue + echo "ERROR: workflow paths: $line" + ERRORS=$((ERRORS + 1)) + done < "$PATHS_OUTPUT" +fi +rm -f "$PATHS_OUTPUT" + if [ "$ERRORS" -gt 0 ]; then echo "" echo "Project lint check failed with $ERRORS error(s)." From db0301b160707f04f17a4c9c772f27495c82b9a0 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 17:55:32 -0700 Subject: [PATCH 57/59] engine: cover every LTM slot in the C-LEARN digest, and bind it to identity Two defects in the gate that carries this branch's primary evidence, both of them the claim outrunning the code. **Coverage.** `Results::offsets` is `HashMap` -- one entry per VARIABLE, with no extent: `calc_flattened_offsets_incremental` computes a size and `CompiledSimulation` drops it. Reading one slot per entry therefore sampled only the FIRST element of every arrayed score, while the rustdoc called itself a digest over every LTM slot. Measured: 7,000 of **20,892** LTM slots, across 1,088 arrayed variables out of 7,153. Extents now come from each variable's own declared dimensions resolved through the project's dimension context, which is the same derivation `db::ltm_value_gate_tests` already used. The gap was not theoretical. Widening it raises `nonzero_slots` from 1,369 to **3,141**, so 1,772 slots carrying real scores were invisible; and the positive control -- forcing the predicate true -- now zeroes **614** real scores where the old walk saw 82. The gate was seeing an eighth of the damage it was built to detect, which made it weaker than the throwaway probe it was meant to replace. **Permutation-invariance.** The magnitude sums and both counts are unchanged when two slots exchange their maxima, so an offset or remapping regression that attached correct values to the wrong links would pass -- and the rustdoc claimed such a swap moved the digest. `slot_digests` adds an FNV-1a over the canonically ordered `(name, element, mantissa, exponent)` stream, binding each contribution to the slot it came from. The sums are kept because they are interpretable: a drop in one localises a regression faster than a hash does. The property is demonstrated rather than asserted. `permuting_two_slots_moves_only_the_identity_digest` constructs the swap and pins BOTH halves -- that the sums really are unchanged, so the blindness is not a strawman, and that the identity digest really moves. It covers a swap between variables and a swap between elements of one arrayed variable, the likelier remapping bug. It is a fast default-suite test rather than part of the `#[ignore]`d run, since the property belongs to the digest function and needs no model. Both digests stay stable across runs and across allocator or layout changes, because the order and the inputs derive from names and relatively-quantized values rather than from addresses -- preserving the relative-tolerance property the mantissa split exists for. `CLEARN_LTM_SLOTS` and `CLEARN_LTM_UNKNOWN_EXTENT` are pinned too, so a change that silently narrows what is examined fails here instead of passing quietly. **Value-neutrality re-established at the wider coverage, and this is the part that matters for the PR.** With the omission disabled, every pinned number is identical -- now over all 20,892 slots, where the previous run could only speak for the 7,000 it sampled. The controls are otherwise unchanged: the five lag-alignment coverage rows behave exactly as their per-clause mutation table says, with the original-`PREVIOUS` revert redding one row and the nested-descent revert redding two. New pins: 20,892 slots / 0 unknown extent / 3,141 non-zero / 20,892 finite / mantissa 798,101,758,590 / exponent 2,254 / identity 11438420344658315382. --- .../tests/integration/simulate_ltm.rs | 267 +++++++++++++++--- 1 file changed, 227 insertions(+), 40 deletions(-) diff --git a/src/simlin-engine/tests/integration/simulate_ltm.rs b/src/simlin-engine/tests/integration/simulate_ltm.rs index 9dc69b453..b9edc9c68 100644 --- a/src/simlin-engine/tests/integration/simulate_ltm.rs +++ b/src/simlin-engine/tests/integration/simulate_ltm.rs @@ -11190,26 +11190,43 @@ fn test_whole_rhs_mapped_reducer_routes_through_synthetic_agg() { /// per-step maximum magnitude. /// /// The sub-second half is `db::ltm_value_gate_tests`, which pins exact values on -/// a three-arm fixture built around the known ways an arm-level change zeroes a -/// score. It cannot show that the same change leaves 7,000 real variables alone, +/// small fixtures built around the known ways an arm-level change zeroes a +/// score. It cannot show that the same change leaves 7,153 real variables alone, /// and C-LEARN is the only model in the repo at that scale. Hence this: same -/// property, real model, `#[ignore]`d purely for runtime (~25 s release, against -/// the 3-minute debug-build cap in `docs/dev/rust.md`). +/// property, real model, `#[ignore]`d purely for runtime (~3 s release on top of +/// a release build, against the 3-minute debug-build cap in +/// `docs/dev/rust.md`). /// -/// The digest is deliberately NOT a checked-in series slab -- 30k slots x 251 -/// steps is 60 MB of golden nobody would read. It is three numbers that move -/// under exactly the failure this gate exists for: +/// It covers **every element of every LTM variable**, not one per variable. +/// `Results::offsets` is keyed by variable and carries no extent, so the obvious +/// walk samples only each arrayed score's FIRST element -- 7,000 of 20,892 LTM +/// slots here, blind to 1,772 slots that carry non-zero scores and to the other +/// 87% of the damage the positive control below inflicts. Extents come from each +/// variable's declared dimensions instead. /// +/// The digest is deliberately NOT a checked-in series slab -- 20,892 slots x 251 +/// steps is tens of MB of golden nobody would read. It is a small set of numbers +/// that move under exactly the failures this gate exists for: +/// +/// * `CLEARN_LTM_SLOTS` / `CLEARN_LTM_UNKNOWN_EXTENT` -- the coverage itself, so +/// a change that silently narrows what is examined fails here rather than +/// passing quietly. `unknown_extent` counts LTM-prefixed result slots the +/// variable metadata does not describe; it is 0 today. /// * `nonzero_slots` -- how many LTM slots are ever non-zero. Rewriting live /// arms to zero moves this DOWN, which is the GH #977 failure (a change that /// zeroed 149 C-LEARN LTM slots passed every named C-LEARN gate); wrongly /// materializing structural zeros as small residuals moves it UP. /// * `finite_slots` -- how many are finite throughout, so a regression that /// replaces values with NaN cannot hide behind an unchanged non-zero count. -/// * `mantissa_digest` / `exponent_digest` -- an order-independent sum over each -/// slot's maximum magnitude, split into a 9-significant-digit mantissa and its -/// decimal exponent (`nine_significant_digits`). Two slots swapping values -/// keeps the first two numbers and moves these. +/// * `mantissa_digest` / `exponent_digest` -- sums over each slot's maximum +/// magnitude, split into a 9-significant-digit mantissa and its decimal +/// exponent (`nine_significant_digits`). Interpretable, and therefore worth +/// keeping: a drop in one localises a regression faster than a hash does. +/// * `identity_digest` -- the same stream bound to slot IDENTITY. The sums above +/// are permutation-invariant, so two slots exchanging maxima leaves them and +/// both counts exactly unchanged; only this moves. See `slot_digests`, and +/// `permuting_two_slots_moves_only_the_identity_digest`, which constructs that +/// swap rather than asserting the property. /// /// Quantizing is what makes the pin usable rather than a per-run coin flip: raw /// f64 maxima carry last-bit noise across allocator and layout changes, and a @@ -11228,16 +11245,20 @@ fn test_whole_rhs_mapped_reducer_routes_through_synthetic_agg() { /// were measured.** Three runs of this digest, same binary, differing only in /// `ltm_augment_zero_slot`: /// -/// * predicate as shipped -- `(1369, 7000, 10_248_673_492_482_445_132_733_301)` +/// * predicate as shipped -- `nonzero_slots` 3,141 of 20,892. /// * `ZeroSlotPolicy::Materialize` forced everywhere, i.e. GH #977's omission -/// disabled -- **identical in all three numbers**. That is this gate's other +/// disabled -- **identical in every pinned number**. That is this gate's other /// job: it is the reproducible, checked-in form of the whole-slab differential /// that established the omission's value-neutrality on C-LEARN, which -/// previously existed only as a throwaway probe nobody could re-run. +/// previously existed only as a throwaway probe nobody could re-run. Note the +/// scope this now carries: value-neutrality is established over all 20,892 LTM +/// slots, where the pre-widening walk could only speak for the 7,000 it +/// sampled. /// * `partial_is_provably_previous_target` forced to `true`, so every arm is -/// omitted whether or not it is a structural zero -- `(1287, 7000, -/// 10_248_673_492_258_319_975_585_940)`. 82 slots that carry real scores go to -/// zero and the digest reds. +/// omitted whether or not it is a structural zero -- `nonzero_slots` falls to +/// 2,527 and every magnitude number moves. **614** slots carrying real scores +/// go to zero; the pre-widening walk saw 82 of them, an eighth of the +/// damage. /// /// The third run is what makes the second meaningful. Without it, "unchanged /// when the omission is disabled" would be equally consistent with a digest that @@ -11249,6 +11270,8 @@ fn test_whole_rhs_mapped_reducer_routes_through_synthetic_agg() { #[test] #[ignore] fn clearn_ltm_slot_maxima_digest() { + use simlin_engine::common::CanonicalDimensionName; + use simlin_engine::db::project_dimensions_context; use simlin_engine::open_vensim; let mdl_path = "../../test/xmutil_test_models/C-LEARN v77 for Vensim.mdl"; @@ -11257,31 +11280,83 @@ fn clearn_ltm_slot_maxima_digest() { let project = open_vensim(&contents).unwrap_or_else(|e| panic!("failed to parse {mdl_path}: {e}")); - let compiled = compile_ltm_discovery_incremental(&project); + // `compile_ltm_discovery_incremental` inlined, because the slot extents + // below need the same `db` and sync the compile used -- a second database + // would be a second derivation of the thing being pinned. + let mut db = SimlinDb::default(); + let sync = sync_from_datamodel_incremental(&mut db, &project, None); + set_project_ltm_enabled(&mut db, sync.project, true); + set_project_ltm_discovery_mode(&mut db, sync.project, true); + let compiled = compile_project_incremental(&db, sync.project, "main") + .expect("C-LEARN must compile with LTM enabled"); + let dim_ctx = project_dimensions_context(&db, sync.project); + let mut vm = Vm::new(compiled).expect("vm"); vm.run_to_end() .expect("C-LEARN must simulate with LTM enabled"); let results = vm.into_results(); - // Which result slots belong to LTM, taken from the run's own offset map - // rather than from a name list, so a renamed synthetic prefix fails loudly - // here instead of quietly shrinking the gate's scope. - let ltm_offsets: Vec = results - .offsets - .iter() - .filter(|(name, _)| name.as_str().starts_with("$\u{205A}ltm\u{205A}")) - .map(|(_, &off)| off) - .collect(); + // Which result slots belong to LTM. `Results::offsets` is one entry per + // VARIABLE and carries no extent -- `calc_flattened_offsets_incremental` + // computes a size but `CompiledSimulation` drops it -- so reading one slot + // per entry would sample only the FIRST element of every arrayed score. On + // C-LEARN that is ~7,000 of 21,045 LTM slots across 1,088 arrayed + // variables, and a regression in any later element would leave every pinned + // number unchanged. The extent therefore comes from each variable's own + // declared dimensions, resolved through the project's dimension context. + let mut ltm_widths: HashMap = HashMap::new(); + for m in sync.models.values() { + for v in model_ltm_variables(&db, m.source_model, sync.project) + .vars + .iter() + { + let width: usize = v + .dimensions + .iter() + .map(|d| { + let canonical = CanonicalDimensionName::from_raw(d); + dim_ctx.get(&canonical).map(|dim| dim.len()).unwrap_or(1) + }) + .product::() + .max(1); + ltm_widths.insert(v.name.clone(), width); + } + } + + // (name, element index, base offset), canonically ordered. The ORDER is + // what makes the digest below permutation-sensitive, and it must not come + // from a HashMap. + let mut ltm_slots: Vec<(&str, usize, usize)> = Vec::new(); + let mut unknown_extent = 0usize; + for (name, &base) in results.offsets.iter() { + let name = name.as_str(); + if !name.starts_with("$\u{205A}ltm\u{205A}") { + continue; + } + let width = match ltm_widths.get(name) { + Some(&w) => w, + None => { + // An LTM-prefixed slot the metadata does not describe: an + // implicit helper, which is scalar. Counted so a change in that + // population is visible rather than silently absorbed. + unknown_extent += 1; + 1 + } + }; + for elem in 0..width { + ltm_slots.push((name, elem, base + elem)); + } + } + ltm_slots.sort_unstable(); assert!( - !ltm_offsets.is_empty(), + !ltm_slots.is_empty(), "no LTM slots found in the results; the gate would pass vacuously" ); let mut nonzero_slots = 0usize; let mut finite_slots = 0usize; - let mut mantissa_digest: i64 = 0; - let mut exponent_digest: i64 = 0; - for &off in <m_offsets { + let mut maxima: Vec<(&str, usize, f64)> = Vec::with_capacity(ltm_slots.len()); + for &(name, elem, off) in <m_slots { let mut max_mag = 0.0f64; let mut all_finite = true; let mut ever_nonzero = false; @@ -11304,23 +11379,32 @@ fn clearn_ltm_slot_maxima_digest() { if all_finite { finite_slots += 1; } - let (mantissa, exponent) = nine_significant_digits(max_mag); - mantissa_digest += mantissa; - exponent_digest += exponent; + maxima.push((name, elem, max_mag)); } + let SlotDigests { + mantissa: mantissa_digest, + exponent: exponent_digest, + identity: identity_digest, + } = slot_digests(&maxima); assert_eq!( ( + ltm_slots.len(), + unknown_extent, nonzero_slots, finite_slots, mantissa_digest, - exponent_digest + exponent_digest, + identity_digest ), ( + CLEARN_LTM_SLOTS, + CLEARN_LTM_UNKNOWN_EXTENT, CLEARN_LTM_NONZERO_SLOTS, CLEARN_LTM_FINITE_SLOTS, CLEARN_LTM_MANTISSA_DIGEST, - CLEARN_LTM_EXPONENT_DIGEST + CLEARN_LTM_EXPONENT_DIGEST, + CLEARN_LTM_IDENTITY_DIGEST ), "C-LEARN's LTM slot values moved. A DROP in nonzero_slots is the \ silent-zeroing regression this gate exists for; re-derive before \ @@ -11372,8 +11456,111 @@ fn nine_significant_digits(x: f64) -> (i64, i64) { (mantissa, exponent) } +/// The three magnitude aggregates over a canonically ordered slot list. +struct SlotDigests { + mantissa: i64, + exponent: i64, + identity: u64, +} + +/// Reduce `(name, element, maximum magnitude)` triples to the aggregates +/// `clearn_ltm_slot_maxima_digest` pins. +/// +/// `mantissa` and `exponent` are plain sums, and they are useful precisely +/// because they are interpretable: a drop in one localises a regression far +/// faster than a hash does. But they are also permutation-INVARIANT -- two +/// slots exchanging their maxima leaves both of them, and the slot counts, +/// exactly unchanged -- so on their own they cannot see an offset or remapping +/// regression that attaches correct values to the wrong links. +/// +/// `identity` closes that: an FNV-1a over the ordered +/// `(name, element, mantissa, exponent)` stream, so every contribution is bound +/// to the slot it came from. It is stable across runs and across allocator or +/// layout changes because both the ORDER and the INPUTS derive from names and +/// relatively-quantized values, never from addresses -- which is what keeps the +/// relative-tolerance property the mantissa split exists for. +/// +/// The caller must pass `slots` in a canonical order; the C-LEARN caller sorts +/// by `(name, element)`. An unsorted list would make `identity` depend on +/// `HashMap` iteration order and flap per run. +/// +/// `permuting_two_slots_moves_only_the_identity_digest` is the discriminating +/// test: it constructs exactly the swap the sums cannot see. +fn slot_digests(slots: &[(&str, usize, f64)]) -> SlotDigests { + let mut mantissa_digest: i64 = 0; + let mut exponent_digest: i64 = 0; + let mut identity: u64 = 0xcbf2_9ce4_8422_2325; + let fold = |bytes: &[u8], acc: &mut u64| { + for b in bytes { + *acc ^= u64::from(*b); + *acc = acc.wrapping_mul(0x100_0000_01b3); + } + }; + for (name, elem, max_mag) in slots { + let (mantissa, exponent) = nine_significant_digits(*max_mag); + mantissa_digest += mantissa; + exponent_digest += exponent; + fold(name.as_bytes(), &mut identity); + fold(&(*elem as u64).to_le_bytes(), &mut identity); + fold(&mantissa.to_le_bytes(), &mut identity); + fold(&exponent.to_le_bytes(), &mut identity); + } + SlotDigests { + mantissa: mantissa_digest, + exponent: exponent_digest, + identity, + } +} + +/// The property the magnitude sums cannot have, demonstrated rather than +/// asserted: two slots exchanging their maxima. +/// +/// This is the regression class the digest was blind to before the identity +/// term -- an offset or remapping change that attaches correct values to the +/// wrong links, leaving every count and every sum intact. The test pins BOTH +/// halves: that the sums really are unchanged (so the blindness is real and not +/// a strawman) and that the identity digest really does move. +/// +/// It is a fast default-suite test rather than part of the `#[ignore]`d C-LEARN +/// run, because the property belongs to the digest function and needs no model. +#[test] +fn permuting_two_slots_moves_only_the_identity_digest() { + let baseline = [("alpha", 0usize, 1.5f64), ("beta", 0usize, 42.0f64)]; + let swapped = [("alpha", 0usize, 42.0f64), ("beta", 0usize, 1.5f64)]; + + let a = slot_digests(&baseline); + let b = slot_digests(&swapped); + + assert_eq!( + (a.mantissa, a.exponent), + (b.mantissa, b.exponent), + "the magnitude sums are permutation-invariant by construction; if this \ + ever fails the identity digest below is no longer the only thing \ + catching a swap, and this test's premise needs restating" + ); + assert_ne!( + a.identity, b.identity, + "two slots exchanging maxima must move the identity digest -- that is \ + the whole reason it exists" + ); + + // The element index is part of the identity too, not just the name: a value + // moving between elements of the SAME arrayed variable is the likelier + // remapping bug, and it must be visible. + let same_var = [("alpha", 0usize, 1.5f64), ("alpha", 1usize, 42.0f64)]; + let same_var_swapped = [("alpha", 0usize, 42.0f64), ("alpha", 1usize, 1.5f64)]; + assert_ne!( + slot_digests(&same_var).identity, + slot_digests(&same_var_swapped).identity, + "a swap WITHIN one arrayed variable must move the identity digest" + ); +} + /// Pinned by `clearn_ltm_slot_maxima_digest`; see its rustdoc before changing. -const CLEARN_LTM_NONZERO_SLOTS: usize = 1369; -const CLEARN_LTM_FINITE_SLOTS: usize = 7000; -const CLEARN_LTM_MANTISSA_DIGEST: i64 = 371_710_864_477; -const CLEARN_LTM_EXPONENT_DIGEST: i64 = 1019; +const CLEARN_LTM_SLOTS: usize = 20_892; +const CLEARN_LTM_UNKNOWN_EXTENT: usize = 0; +const CLEARN_LTM_NONZERO_SLOTS: usize = 3_141; +const CLEARN_LTM_FINITE_SLOTS: usize = 20_892; +const CLEARN_LTM_MANTISSA_DIGEST: i64 = 798_101_758_590; +const CLEARN_LTM_EXPONENT_DIGEST: i64 = 2_254; +const CLEARN_LTM_IDENTITY_DIGEST: u64 = 11_438_420_344_658_315_382; From b689e032799fd0730469932511a22724d7d8c469 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 18:09:39 -0700 Subject: [PATCH 58/59] engine: make the digest's permutation test constrain identity, not order `permuting_two_slots_moves_only_the_identity_digest` asserted that two slots exchanging their maxima moves the identity digest, and claimed that as evidence the fold binds each contribution to its slot. It is not: FNV-1a is an ORDERED fold, so exchanging the two maxima reorders the value stream and moves the hash whether or not `name` and `elem` are in it. Measured -- deleting both identity bytes from the fold left the test green. So the test passed for a different reason than it claimed, which is the same defect this gate has now been fixed for three times, this time in the test rather than in the code under it. The two properties are separate and both are now rows: * a VALUE SWAP -- maxima exchanged between slots, canonical order fixed. Moves an ordered fold with or without identity, so it constrains the ordering, not the identity. * a REBINDING -- the same maxima, in the same order, attached to a different slot identity: a renamed variable, or the same value at a different element index. Only the identity bytes catch this, and it is the closer analogue of the offset/remapping regression the term was added for. Each is mutation-tested against the fold it constrains: removing `name`/`elem` now reds the rebinding row (it previously red nothing), while the swap row stays green, which is exactly the discrimination that was missing. The C-LEARN digest itself is unchanged -- its real caller sorts by `(name, element)`, so a swap does move it. What was wrong was the evidence, not the pin: all seven pinned numbers are identical and the run still passes. --- .../tests/integration/simulate_ltm.rs | 77 ++++++++++++------- 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/src/simlin-engine/tests/integration/simulate_ltm.rs b/src/simlin-engine/tests/integration/simulate_ltm.rs index b9edc9c68..7c99fd868 100644 --- a/src/simlin-engine/tests/integration/simulate_ltm.rs +++ b/src/simlin-engine/tests/integration/simulate_ltm.rs @@ -11484,8 +11484,10 @@ struct SlotDigests { /// by `(name, element)`. An unsorted list would make `identity` depend on /// `HashMap` iteration order and flap per run. /// -/// `permuting_two_slots_moves_only_the_identity_digest` is the discriminating -/// test: it constructs exactly the swap the sums cannot see. +/// `the_digest_sees_both_a_value_swap_and_a_rebinding` is the discriminating +/// test. Note that it needs TWO rows: a value swap moves an ordered fold +/// whether or not identity is in it, so only the rebinding row constrains these +/// `name`/`elem` bytes. fn slot_digests(slots: &[(&str, usize, f64)]) -> SlotDigests { let mut mantissa_digest: i64 = 0; let mut exponent_digest: i64 = 0; @@ -11512,48 +11514,67 @@ fn slot_digests(slots: &[(&str, usize, f64)]) -> SlotDigests { } } -/// The property the magnitude sums cannot have, demonstrated rather than -/// asserted: two slots exchanging their maxima. +/// The two properties the magnitude sums cannot have, demonstrated rather than +/// asserted. They are SEPARATE, and conflating them is how the first version of +/// this test passed for the wrong reason. /// -/// This is the regression class the digest was blind to before the identity -/// term -- an offset or remapping change that attaches correct values to the -/// wrong links, leaving every count and every sum intact. The test pins BOTH -/// halves: that the sums really are unchanged (so the blindness is real and not -/// a strawman) and that the identity digest really does move. +/// * **Value swap** -- two slots exchange their maxima, canonical order fixed. +/// The sums are unchanged (an unordered multiset), and the digest moves +/// because FNV-1a is an ORDERED fold. This holds whether or not slot identity +/// is folded in, so it does NOT exercise the name/element bytes. +/// * **Rebinding** -- the same maxima, in the same order, attached to a +/// different slot identity: a renamed variable, or the same value at a +/// different element index. Only the identity bytes catch this, and it is the +/// closer analogue of the offset/remapping regression the identity term was +/// added for. /// -/// It is a fast default-suite test rather than part of the `#[ignore]`d C-LEARN -/// run, because the property belongs to the digest function and needs no model. +/// The first version of this test asserted only the swap and claimed it +/// demonstrated identity binding. It did not: removing `name` and `elem` from +/// the fold left it green, because reordering the value stream is enough to +/// move an ordered hash. Both rows exist now, and each was mutation-tested +/// against the fold it is supposed to constrain. +/// +/// Fast default-suite test rather than part of the `#[ignore]`d run, since the +/// property belongs to the digest function and needs no model. #[test] -fn permuting_two_slots_moves_only_the_identity_digest() { +fn the_digest_sees_both_a_value_swap_and_a_rebinding() { let baseline = [("alpha", 0usize, 1.5f64), ("beta", 0usize, 42.0f64)]; - let swapped = [("alpha", 0usize, 42.0f64), ("beta", 0usize, 1.5f64)]; + // Property 1: values exchanged between slots. + let swapped = [("alpha", 0usize, 42.0f64), ("beta", 0usize, 1.5f64)]; let a = slot_digests(&baseline); let b = slot_digests(&swapped); - assert_eq!( (a.mantissa, a.exponent), (b.mantissa, b.exponent), "the magnitude sums are permutation-invariant by construction; if this \ - ever fails the identity digest below is no longer the only thing \ - catching a swap, and this test's premise needs restating" + ever fails, the premise of this test needs restating" ); assert_ne!( a.identity, b.identity, - "two slots exchanging maxima must move the identity digest -- that is \ - the whole reason it exists" + "two slots exchanging maxima must move the identity digest" ); - // The element index is part of the identity too, not just the name: a value - // moving between elements of the SAME arrayed variable is the likelier - // remapping bug, and it must be visible. - let same_var = [("alpha", 0usize, 1.5f64), ("alpha", 1usize, 42.0f64)]; - let same_var_swapped = [("alpha", 0usize, 42.0f64), ("alpha", 1usize, 1.5f64)]; - assert_ne!( - slot_digests(&same_var).identity, - slot_digests(&same_var_swapped).identity, - "a swap WITHIN one arrayed variable must move the identity digest" - ); + // Property 2: same values, same order, different slot identity. This is + // the row that actually constrains the name/element bytes -- a fold over + // values alone reproduces `baseline` exactly here. + let renamed = [("alpha", 0usize, 1.5f64), ("gamma", 0usize, 42.0f64)]; + let reindexed = [("alpha", 0usize, 1.5f64), ("beta", 7usize, 42.0f64)]; + for (label, other) in [("renamed", &renamed), ("reindexed", &reindexed)] { + let c = slot_digests(other); + assert_eq!( + (a.mantissa, a.exponent), + (c.mantissa, c.exponent), + "{label}: the sums cannot see a rebinding, which is why the \ + identity digest exists" + ); + assert_ne!( + a.identity, c.identity, + "{label}: the same maxima bound to a different slot identity must \ + move the identity digest -- this is the offset/remapping \ + regression class" + ); + } } /// Pinned by `clearn_ltm_slot_maxima_digest`; see its rustdoc before changing. From 4ab02a75d480b4e6fa07b54b823596b8628274a2 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Mon, 10 Aug 2026 20:23:04 -0700 Subject: [PATCH 59/59] build: drop the undeclared PyYAML import from the workflow-paths lint The rule added to catch silently-diverging workflow path filters was itself silently not running. It imported PyYAML, which this repo neither declares nor installs, and the surrounding branch counted only stdout lines as errors -- so on a machine without it the traceback went to stderr, no error was counted, and `lint-project.sh` printed "Project lint check passed" and exited 0. Reproduced by shadowing the module: traceback, then a passing lint. It happened to work here because PyYAML is a system package on this machine, which is the whole hazard: an undeclared import makes a check's coverage a property of the developer's machine. check-workflow-paths.py parses the subset of YAML it needs with the standard library. It is strict in the direction that matters: a shape it does not model -- an inline `paths:` value, an unparsed line or a mapping key nested inside a paths list -- raises, so a workflow it cannot read fails loudly rather than reporting no difference. That property is tested, not asserted; the first version of the parser silently ignored a stray key inside a paths list while its docstring claimed otherwise. The bash side gains run_line_check, which treats a check that FAILS TO RUN as an error in its own right. That fixes the same latent defect in rule 3: if check-copyright.py crashed, its traceback went to stderr, its stdout was empty, and the lint passed. A rule that has silently stopped running looks exactly like a rule that found nothing. --- scripts/check-workflow-paths.py | 164 ++++++++++++++++++++++++++++++++ scripts/lint-project.sh | 75 +++++++-------- 2 files changed, 200 insertions(+), 39 deletions(-) create mode 100755 scripts/check-workflow-paths.py diff --git a/scripts/check-workflow-paths.py b/scripts/check-workflow-paths.py new file mode 100755 index 000000000..d49f2a761 --- /dev/null +++ b/scripts/check-workflow-paths.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# Copyright 2026 The Simlin Authors. All rights reserved. +# Use of this source code is governed by the Apache License, +# Version 2.0, that can be found in the LICENSE file. + +"""A path-filtered workflow's `push` and `pull_request` `paths` lists must match. + +They are maintained by hand and read as one filter, so an entry added to only one +of them silently means "runs on merge but not on the PR" (or the reverse) -- a gap +that looks like coverage. + +Writes one error per line to stdout and exits non-zero when any workflow differs. + +Deliberately parses the subset of YAML this needs with the standard library rather +than importing PyYAML, which this repo does not declare or install: an undeclared +import turns the check into a no-op on any machine that happens to lack it, which +is the same silent-non-coverage failure the rule exists to catch. The parser is +strict in the direction that matters -- it raises rather than returning nothing +when it meets a shape it does not understand, so a workflow it cannot read fails +loudly instead of passing vacuously. +""" + +from __future__ import annotations + +import glob +import re +import sys + +KEY_RE = re.compile(r"^(?P *)(?P\"[^\"]+\"|'[^']+'|[A-Za-z_][\w-]*)\s*:\s*(?P.*?)\s*$") +ITEM_RE = re.compile(r"^(?P *)-\s+(?P.*?)\s*$") + + +def _unquote(text: str) -> str: + text = text.strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'": + return text[1:-1] + return text + + +def _strip_comment(text: str) -> str: + # Only an unquoted `#` starts a comment. Workflow path entries are quoted + # or bare globs, neither of which contains one, so this stays simple. + if text.startswith(("'", '"')): + return text + return text.split("#", 1)[0].strip() + + +def _significant(lines: list[str]) -> list[tuple[int, str]]: + """(index, line) for lines that are neither blank nor whole-line comments.""" + out = [] + for i, line in enumerate(lines): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + out.append((i, line)) + return out + + +def trigger_paths(path: str) -> dict[str, list[str]]: + """`{trigger: paths}` for the `push`/`pull_request` triggers that declare `paths`. + + Raises ValueError on a shape this parser does not understand. + """ + with open(path, encoding="utf-8") as fh: + lines = fh.read().splitlines() + sig = _significant(lines) + + # Locate the top-level trigger block. GitHub reads bare `on` as the YAML + # boolean true, so workflows here quote it; accept both spellings. + on_at = None + for pos, (_, line) in enumerate(sig): + m = KEY_RE.match(line) + if m and len(m.group("indent")) == 0 and _unquote(m.group("key")) == "on": + on_at = pos + break + if on_at is None: + return {} + + # Everything indented under `on:` until the next top-level key. + block = [] + for _, line in sig[on_at + 1 :]: + m = KEY_RE.match(line) + if m and len(m.group("indent")) == 0: + break + block.append(line) + + result: dict[str, list[str]] = {} + trigger = None + trigger_indent = None + in_paths = False + paths_indent = None + + for line in block: + item = ITEM_RE.match(line) + if item and in_paths and len(item.group("indent")) > paths_indent: + result[trigger].append(_unquote(_strip_comment(item.group("value")))) + continue + + key_m = KEY_RE.match(line) + if not key_m: + if in_paths: + raise ValueError(f"{path}: unparsed line inside a `paths` list: {line!r}") + continue + + indent = len(key_m.group("indent")) + key = _unquote(key_m.group("key")) + + if trigger_indent is not None and indent <= trigger_indent: + trigger = None + in_paths = False + if in_paths and indent <= paths_indent: + in_paths = False + elif in_paths: + # `paths` is a flat list of strings, so a mapping key nested inside + # it is a shape this parser does not model. Raise rather than skip: + # silently ignoring it would let a workflow the parser cannot read + # report "no difference", which is the non-coverage this rule exists + # to catch. + raise ValueError(f"{path}: unexpected key inside a `paths` list: {line!r}") + + if key in ("push", "pull_request") and trigger is None: + trigger = key + trigger_indent = indent + continue + + if key == "paths" and trigger is not None: + if key_m.group("rest"): + raise ValueError(f"{path}: inline `paths:` value is not supported: {line!r}") + in_paths = True + paths_indent = indent + result.setdefault(trigger, []) + + if in_paths and not result.get(trigger): + raise ValueError(f"{path}: `paths:` under `{trigger}` parsed as empty") + return result + + +def main() -> int: + status = 0 + for path in sorted(glob.glob(".github/workflows/*.y*ml")): + try: + triggers = trigger_paths(path) + except (OSError, ValueError) as exc: + print(f"{path}: could not check trigger paths: {exc}") + status = 1 + continue + + push = triggers.get("push") + pull = triggers.get("pull_request") + if push is None and pull is None: + continue + if push != pull: + only_push = [p for p in (push or []) if p not in (pull or [])] + only_pull = [p for p in (pull or []) if p not in (push or [])] + print( + f"{path}: push and pull_request `paths` differ; " + f"push-only={only_push} pull_request-only={only_pull}" + ) + status = 1 + return status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/lint-project.sh b/scripts/lint-project.sh index 69eadc3e4..502aa1261 100755 --- a/scripts/lint-project.sh +++ b/scripts/lint-project.sh @@ -17,6 +17,40 @@ fi ERRORS=0 +# Run a check that writes one error per line to stdout, and count those lines. +# A check that FAILS TO RUN counts as an error in its own right: without that, +# a crashed script writes its traceback to stderr, contributes zero lines here, +# and the lint reports success -- a rule that silently stopped running looks +# exactly like a rule that found nothing. +run_line_check() { + local label="$1" + shift + local out err rc + out=$(mktemp) + err=$(mktemp) + set +e + "$@" > "$out" 2> "$err" + rc=$? + set -e + # Only a FAILING check's stdout is error lines; a passing one may print a + # summary there. + local found=0 + if [ "$rc" -ne 0 ]; then + while IFS= read -r line; do + [ -z "$line" ] && continue + echo "ERROR: $label: $line" + ERRORS=$((ERRORS + 1)) + found=1 + done < "$out" + fi + if [ "$rc" -ne 0 ] && [ "$found" -eq 0 ]; then + echo "ERROR: $label: check failed to run (exit $rc):" + sed 's/^/ /' < "$err" >&2 + ERRORS=$((ERRORS + 1)) + fi + rm -f "$out" "$err" +} + # Rule 1: No --no-verify in any script or config file (excluding this lint script itself). # This should always have zero occurrences. NOVERIFY_PATTERN='--no-verify' @@ -53,51 +87,14 @@ rm -f "$RS_FILES" # Rule 3: Copyright headers on all Rust and TypeScript source files # check-copyright.py writes one error per line to stdout; summary to stderr. -COPYRIGHT_OUTPUT=$(mktemp) -if ! python3 scripts/check-copyright.py > "$COPYRIGHT_OUTPUT"; then - while IFS= read -r line; do - [ -z "$line" ] && continue - echo "ERROR: copyright header: $line" - ERRORS=$((ERRORS + 1)) - done < "$COPYRIGHT_OUTPUT" -fi -rm -f "$COPYRIGHT_OUTPUT" +run_line_check "copyright header" python3 scripts/check-copyright.py # Rule 4: a path-filtered workflow's push and pull_request `paths` lists must # match. They are maintained by hand and read as one filter, so a path added to # only one of them silently means "runs on merge but not on the PR" (or the # reverse) -- a gap that looks like coverage. `.github/workflows/wasm-opt.yml` # is the only such workflow today; the loop covers any future one. -PATHS_OUTPUT=$(mktemp) -if ! python3 - > "$PATHS_OUTPUT" <<'PYEOF'; then -import glob, sys -import yaml - -status = 0 -for path in sorted(glob.glob(".github/workflows/*.y*ml")): - with open(path) as fh: - wf = yaml.safe_load(fh) - triggers = (wf or {}).get(True) or (wf or {}).get("on") or {} - if not isinstance(triggers, dict): - continue - push = (triggers.get("push") or {}).get("paths") - pull = (triggers.get("pull_request") or {}).get("paths") - if push is None and pull is None: - continue - if push != pull: - only_push = [p for p in (push or []) if p not in (pull or [])] - only_pull = [p for p in (pull or []) if p not in (push or [])] - print(f"{path}: push and pull_request `paths` differ; " - f"push-only={only_push} pull_request-only={only_pull}") - status = 1 -sys.exit(status) -PYEOF - while IFS= read -r line; do - [ -z "$line" ] && continue - echo "ERROR: workflow paths: $line" - ERRORS=$((ERRORS + 1)) - done < "$PATHS_OUTPUT" -fi +run_line_check "workflow paths" python3 scripts/check-workflow-paths.py rm -f "$PATHS_OUTPUT" if [ "$ERRORS" -gt 0 ]; then