From 99cd13da751f79ae0f4e662f6835e2f230deeeec Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 5 Oct 2026 03:15:35 -0500 Subject: [PATCH] Compile virtual extent queries in TypeScript - Own estimate-prefix arithmetic and exact offset-search decisions in TypeScript. - Preserve native parity and copied ABI ownership with bounded query facts. --- packages/core/src/runtime_policy.ts | 37 +++++++++- packages/core/test/virtual_extent.test.ts | 30 ++++++++ .../canvas/virtual_extent_policy.zig | 73 ++++++++++++++++++- src/primitives/canvas/virtual_extents.zig | 13 ++++ tests/ts-core/virtual_extent_e2e_tests.zig | 51 +++++++++++-- 5 files changed, 194 insertions(+), 10 deletions(-) diff --git a/packages/core/src/runtime_policy.ts b/packages/core/src/runtime_policy.ts index e85085c99..e653387ae 100644 --- a/packages/core/src/runtime_policy.ts +++ b/packages/core/src/runtime_policy.ts @@ -1316,8 +1316,7 @@ function nscvSemanticTree(request: Uint8Array): Uint8Array { out.setUint32(0, emitted, true); return result; } -/** Retained variable-window transitions. Cached estimate/index queries remain - * explicit native capabilities; sparse corrections cross once per measured +/** Retained variable-window transitions. Estimate arithmetic and search decisions use bounded copied facts; sparse corrections cross once per measured * batch and hot window coordination uses small scalar records. No retained * native address or borrowed model bytes escape this copied boundary. */ function nscvExtentPolicy(request: Uint8Array): Uint8Array { @@ -1394,6 +1393,40 @@ function nscvExtentPolicy(request: Uint8Array): Uint8Array { fixed(16);if(request[3]! > 1 || w.getUint32(12,true) !== 0)throw new Error("invalid extent pending shift"); const result=new Uint8Array(4);new DataView(result.buffer).setFloat32(0,request[3] === 1 ? f(v(4)-v(8)) : f(v(4)+v(8)),true);return result; } + if (op === 9 || op === 10) { + if (request.length < 32 || request[3] !== 0) throw new Error("invalid extent estimate header"); + const count=w.getUint32(4,true); + if (count > (op === 9 ? 1024 : 63) || request.length !== 32+count*4 || w.getUint32(28,true)!==0) throw new Error("invalid extent estimate batch"); + if (op === 9) { + const result=new Uint8Array(4*Math.ceil(count/64)),out=new DataView(result.buffer);let prefix=v(8); + for(let chunk=0;chunk0)prefix=f(prefix+f(v(16)*f(v(12)/v(20)))); + const result=new Uint8Array(4);new DataView(result.buffer).setFloat32(0,prefix,true);return result; + } + if (op === 11) { + fixed(24);if(request[3]!>1 || w.getUint32(20,true)!==0)throw new Error("invalid extent scalar query"); + const result=new Uint8Array(4),out=new DataView(result.buffer); + out.setFloat32(0,request[3]===1?max(0,f(v(4)+v(8))):f(f(v(4)+v(8))+f(v(12)*v(16))),true);return result; + } + if (op === 12) { + fixed(40);if(request[3]!>3 || w.getUint32(4,true)!==0)throw new Error("invalid extent search continuation"); + const checked=(request[3]!&1)!==0,continued=(request[3]!&2)!==0,count=integer(8),target=max(0,v(32)); + let low=continued?integer(16):nscvFlowSmall(0),high=continued?integer(24):nscvFlowSubtract(count,nscvFlowSmall(1)); + const midpoint=(a:NscFlowInteger,b:NscFlowInteger):NscFlowInteger=>{ + const sum=nscvFlowAdd(nscvFlowAdd(a,b,checked),nscvFlowSmall(1),checked); + return {low:Math.floor(sum.low/2)+(sum.high%2)*2147483648,high:Math.floor(sum.high/2)}; + }; + if(nscvFlowCompare(low,high)>0 || (!nscvFlowZero(count)&&nscvFlowCompare(high,count)>=0) || (continued&&nscvFlowCompare(low,high)>=0))throw new Error("invalid extent search bounds"); + if(continued){const mid=midpoint(low,high);if(v(36)<=target)low=mid;else high=nscvFlowSubtract(mid,nscvFlowSmall(1));} + const active=nscvFlowCompare(low,high)<0,result=new Uint8Array(32),out=new DataView(result.buffer); + nscvFlowWrite(out,0,low);nscvFlowWrite(out,8,high);nscvFlowWrite(out,16,active?midpoint(low,high):low);out.setUint32(24,active?1:0,true);out.setFloat32(28,target,true);return result; + } throw new Error("unknown extent policy operation"); } function nscvExtentClean(value:number):number{return !Number.isFinite(value) || value < 0 ? 0 : value;} diff --git a/packages/core/test/virtual_extent.test.ts b/packages/core/test/virtual_extent.test.ts index 8d5f6250f..281ef6edb 100644 --- a/packages/core/test/virtual_extent.test.ts +++ b/packages/core/test/virtual_extent.test.ts @@ -38,3 +38,33 @@ test("extent protocol rejects truncation padding flags invalid ordering and dirt test("extent output owns bytes across offset views repeated policies and later mutation",()=>{ const b=correction(3,[],[{physical:1n,estimate:1,extent:10}]);const padded=new Uint8Array(b.length+10);padded.set(b,7);const out=run(padded.subarray(7,7+b.length)),saved=out.slice();padded.fill(0);for(let i=0;i<12;i++)run(correction(3));assert.deepEqual(out,saved); }); +function estimates(op:number,values:readonly number[],prefix=0,total=0,extra=0,covered=0){const b=fixed(op,32+values.length*4),w=new DataView(b.buffer);w.setUint32(4,values.length,true);[prefix,total,extra,covered].forEach((v,i)=>w.setFloat32(8+i*4,v,true));values.forEach((v,i)=>w.setFloat32(32+i*4,v,true));return b;} +const float=(b:Uint8Array,at=0)=>new DataView(b.buffer,b.byteOffset,b.byteLength).getFloat32(at,true); +test("estimate batches preserve chunk grouping and partial-prefix rounding",()=>{ + const values=Array.from({length:1024},(_,i)=>Math.fround(i%5===0?16777216:i%3===0?0.125:1));let prefix=123.125; + const result=run(estimates(9,values,prefix));for(let c=0;c<16;c++){let sum=0;for(const value of values.slice(c*64,c*64+64))sum=Math.fround(sum+value);prefix=Math.fround(prefix+sum);assert.equal(float(result,c*4),prefix);} + assert.equal(float(run(estimates(10,[1,1],16777216))),16777216); + assert.equal(float(run(estimates(10,[],262144,262144,17,262144))),262161); + assert.equal(run(estimates(9,[],0)).length,0); + assert.throws(()=>run(estimates(9,new Array(1025).fill(1))),/batch/); + assert.throws(()=>run(estimates(10,new Array(64).fill(1))),/batch/); + const bad=estimates(10,[]);new DataView(bad.buffer).setUint32(28,1,true);assert.throws(()=>run(bad),/batch/); +}); +function search(count:bigint,low=0n,high=0n,flags=1,target=10,probe=0){const b=fixed(12,40),w=new DataView(b.buffer);b[3]=flags;[count,low,high].forEach((v,i)=>w.setBigUint64(8+i*8,v,true));w.setFloat32(32,target,true);w.setFloat32(36,probe,true);return b;} +test("query continuation owns exact uint64 midpoints ties zero and overflow modes",()=>{ + const count=9007199254740997n;let out=run(search(count)),w=new DataView(out.buffer);assert.equal(w.getBigUint64(16,true),(count)/2n); + let low=w.getBigUint64(0,true),high=w.getBigUint64(8,true),steps=0;const wanted=9007199254740987n; + while(w.getUint32(24,true)){const mid=w.getBigUint64(16,true);out=run(search(count,low,high,3,10,mid<=wanted?10:11));w=new DataView(out.buffer);low=w.getBigUint64(0,true);high=w.getBigUint64(8,true);steps++;assert.ok(steps<=54);} + assert.equal(low,wanted);assert.equal(w.getBigUint64(16,true),wanted); + assert.equal(new DataView(run(search(0n)).buffer).getUint32(24,true),0); + assert.equal(new DataView(run(search(1n)).buffer).getBigUint64(0,true),0n); + assert.throws(()=>run(search(0xffffffffffffffffn,0xfffffffffffffffdn,0xfffffffffffffffen,3)),/overflow/); + assert.throws(()=>run(search(4n,2n,1n,3)),/bounds/); + const wrapped=run(search(0xffffffffffffffffn,0xfffffffffffffffdn,0xfffffffffffffffen,2));assert.ok(wrapped.length===32); + const nan=run(search(2n,0n,0n,1,NaN));assert.equal(float(nan,28),0); +}); +test("extent scalar queries preserve f32 grouping and copied result ownership",()=>{ + const b=fixed(11,24),w=new DataView(b.buffer);[16777216,1,1,1].forEach((v,i)=>w.setFloat32(4+i*4,v,true));const out=run(b);assert.equal(float(out),16777216);const saved=out.slice();b.fill(0);run(search(99n));assert.deepEqual(out,saved); + const negative=fixed(11,24);negative[3]=1;new DataView(negative.buffer).setFloat32(4,-10,true);assert.equal(float(run(negative)),0); + for(const example of [estimates(9,[1]),estimates(10,[1]),fixed(11,24),search(2n)]){for(let n=1;nrun(example.subarray(0,n)));assert.throws(()=>run(new Uint8Array([...example,0])));} +}); diff --git a/src/primitives/canvas/virtual_extent_policy.zig b/src/primitives/canvas/virtual_extent_policy.zig index 28fed62f2..abbf04e49 100644 --- a/src/primitives/canvas/virtual_extent_policy.zig +++ b/src/primitives/canvas/virtual_extent_policy.zig @@ -6,7 +6,7 @@ pub const SyncPlan = struct { mode: u8, flags: u8, gap: f32, old_total: f32, fir /// Native owns every request/result allocation, callback and retained table. /// Compiled policy consumes copied sparse state once per measurement batch. -/// Cached estimate-prefix and index queries are explicit numeric capabilities; +/// Estimate samples and exact storage lookups are explicit native capabilities; /// no native address or borrowed model/view slice is stored by the reducer. pub fn sync(policy: Policy, table: anytype, args: anytype, old_total: f32) SyncPlan { var request = header(72, 0); @@ -154,6 +154,77 @@ pub fn shift(policy: Policy, pending: f32, delta: f32, subtract: bool) f32 { run(policy, &request, &result); return getFloat(&result, 0); } +/// Estimate callbacks are sampled by native code. Chunk arithmetic crosses +/// in batches of at most sixteen chunks, without copying retained sparse state. +pub fn rebuild(policy: Policy, table: anytype, first: usize) void { + var chunk = @min(first, table.chunk_count); + while (chunk < table.chunk_count) { + const end_chunk = @min(chunk + 16, table.chunk_count); + const start = chunk * 64; + const end = @min(table.covered_count, end_chunk * 64); + var request = header(32 + 1024 * 4, 9); + putWord(&request, 4, @intCast(end - start)); + putFloat(&request, 8, table.chunk_prefix[chunk]); + for (start..end) |i| putFloat(&request, 32 + (i - start) * 4, table.estimateAt(i)); + var result: [16 * 4]u8 = undefined; + run(policy, request[0 .. 32 + (end - start) * 4], result[0 .. (end_chunk - chunk) * 4]); + for (chunk..end_chunk) |i| table.chunk_prefix[i + 1] = getFloat(&result, (i - chunk) * 4); + chunk = end_chunk; + } +} +pub fn prefix(policy: Policy, table: anytype, index: usize) f32 { + const clamped = @min(index, table.item_count); + const tail = clamped > table.covered_count; + const chunk = if (tail) table.chunk_count else clamped / 64; + const start = chunk * 64; + const count = if (tail) 0 else clamped - start; + var request = header(32 + 63 * 4, 10); + putWord(&request, 4, @intCast(count)); + putFloat(&request, 8, table.chunk_prefix[chunk]); + if (tail) { + putFloat(&request, 12, table.chunk_prefix[table.chunk_count]); + putFloat(&request, 16, @floatFromInt(clamped - table.covered_count)); + putFloat(&request, 20, @floatFromInt(table.covered_count)); + } + for (0..count) |i| putFloat(&request, 32 + i * 4, table.estimateAt(start + i)); + var result: [4]u8 = undefined; + run(policy, request[0 .. 32 + count * 4], &result); + return getFloat(&result, 0); +} +pub fn scalar(policy: Policy, estimate: f32, delta: f32, gap: f32, count: usize, extent: bool) f32 { + var request = header(24, 11); + request[3] = if (extent) 1 else 0; + putFloat(&request, 4, estimate); + putFloat(&request, 8, delta); + putFloat(&request, 12, gap); + putFloat(&request, 16, @floatFromInt(count)); + var result: [4]u8 = undefined; + run(policy, &request, &result); + return getFloat(&result, 0); +} +pub fn search(policy: Policy, table: anytype, offset: f32) usize { + const builtin = @import("builtin"); + var request = header(40, 12); + request[3] = if (builtin.mode == .Debug or builtin.mode == .ReleaseSafe) 1 else 0; + putInt(&request, 8, table.item_count); + putFloat(&request, 32, offset); + var result: [32]u8 = undefined; + while (true) { + run(policy, &request, &result); + const active = getWord(&result, 24); + if (active > 1) @panic("invalid compiled extent search result"); + const low = getInt(&result, 0); + const high = getInt(&result, 8); + const mid = getInt(&result, 16); + if (low > high or (table.item_count != 0 and high >= table.item_count) or mid < low or mid > high) @panic("invalid compiled extent search bounds"); + if (active == 0) return low; + putInt(&request, 16, low); + putInt(&request, 24, high); + putFloat(&request, 32, getFloat(&result, 28)); + putFloat(&request, 36, table.offsetAtPhysical(mid)); + request[3] |= 2; + } +} fn header(comptime length: usize, operation: u8) [length]u8 { var request = [_]u8{0} ** length; request[0] = 10; diff --git a/src/primitives/canvas/virtual_extents.zig b/src/primitives/canvas/virtual_extents.zig index 1dc5183b0..581ca8f3c 100644 --- a/src/primitives/canvas/virtual_extents.zig +++ b/src/primitives/canvas/virtual_extents.zig @@ -185,6 +185,10 @@ pub const VirtualExtentTable = struct { fn rebuildChunkPrefixFrom(self: *VirtualExtentTable, first_chunk: usize) void { self.covered_count = @min(self.item_count, max_virtual_extent_items); self.chunk_count = (self.covered_count + virtual_extent_chunk - 1) / virtual_extent_chunk; + if (self.policy) |policy| { + compiled.rebuild(policy, self, first_chunk); + return; + } var chunk = first_chunk; if (chunk > self.chunk_count) chunk = self.chunk_count; while (chunk < self.chunk_count) : (chunk += 1) { @@ -208,6 +212,7 @@ pub const VirtualExtentTable = struct { /// Sum of estimates for physical items [0, index). fn estimatePrefix(self: *const VirtualExtentTable, index: usize) f32 { + if (self.policy) |policy| return compiled.prefix(policy, self, index); const clamped = @min(index, self.item_count); if (clamped > self.covered_count) { const extra = @as(f32, @floatFromInt(clamped - self.covered_count)); @@ -249,6 +254,11 @@ pub const VirtualExtentTable = struct { /// estimate otherwise. pub fn extentAtPhysical(self: *const VirtualExtentTable, physical: usize) f32 { const base = self.estimateAt(physical); + if (self.policy) |policy| { + const slot = self.measuredSlot(self.index_base + @as(u64, physical)); + // Unmeasured estimates retain signed zero without a max operation. + return if (slot) |i| compiled.scalar(policy, base, self.measured_delta[i], 0, 0, true) else base; + } if (self.measuredSlot(self.index_base + @as(u64, physical))) |slot| { return @max(0, base + self.measured_delta[slot]); } @@ -259,6 +269,7 @@ pub const VirtualExtentTable = struct { /// included). pub fn offsetAtPhysical(self: *const VirtualExtentTable, physical: usize) f32 { const clamped = @min(physical, self.item_count); + if (self.policy) |policy| return compiled.scalar(policy, self.estimatePrefix(clamped), self.measuredDeltaBefore(self.index_base + @as(u64, clamped)), self.gap, clamped, false); return self.estimatePrefix(clamped) + self.measuredDeltaBefore(self.index_base + @as(u64, clamped)) + self.gap * @as(f32, @floatFromInt(clamped)); @@ -267,6 +278,7 @@ pub const VirtualExtentTable = struct { /// Total content extent: every extent plus the gaps between rows. pub fn totalExtent(self: *const VirtualExtentTable) f32 { if (self.item_count == 0) return 0; + if (self.policy) |policy| return compiled.scalar(policy, self.estimatePrefix(self.item_count), self.measured_total_delta, self.gap, self.item_count - 1, false); return self.estimatePrefix(self.item_count) + self.measured_total_delta + self.gap * @as(f32, @floatFromInt(self.item_count - 1)); } @@ -275,6 +287,7 @@ pub const VirtualExtentTable = struct { /// a scroll offset). Monotone bisection over `offsetAtPhysical`. pub fn indexAtOffset(self: *const VirtualExtentTable, offset: f32) usize { if (self.item_count == 0) return 0; + if (self.policy) |policy| return compiled.search(policy, self, offset); const target = @max(0, offset); // Largest index whose leading edge is <= target. var low: usize = 0; diff --git a/tests/ts-core/virtual_extent_e2e_tests.zig b/tests/ts-core/virtual_extent_e2e_tests.zig index cdd50dfec..08fc38a1b 100644 --- a/tests/ts-core/virtual_extent_e2e_tests.zig +++ b/tests/ts-core/virtual_extent_e2e_tests.zig @@ -200,33 +200,70 @@ test "compiled extent hot queries use bounded scalar records and preserve borrow const owned = try std.testing.allocator.dupe(u8, borrowed); defer std.testing.allocator.free(owned); const Recorder = struct { - var counts = [_]usize{0} ** 9; - var bytes = [_]usize{0} ** 9; + var counts = [_]usize{0} ** 13; + var bytes = [_]usize{0} ** 13; + var largest: usize = 0; fn call(request: []const u8, output: []u8) usize { if (request.len > 1 and request[0] == 10) { + largest = @max(largest, request.len); counts[request[1]] += 1; bytes[request[1]] += request.len; } return core.nativeWindowPolicy(request, output); } }; - Recorder.counts = [_]usize{0} ** 9; - Recorder.bytes = [_]usize{0} ** 9; + Recorder.counts = [_]usize{0} ** 13; + Recorder.bytes = [_]usize{0} ** 13; var table = Table{ .policy = Recorder.call }; _ = table.sync(.{ .id = 7, .item_count = 3000, .uniform_estimate = 10, .gap = 1 }); + try std.testing.expectEqual(@as(usize, 3), Recorder.counts[9]); + try std.testing.expect(Recorder.largest <= 4128); var rows: [2048]Measure = undefined; for (&rows, 0..) |*row, i| row.* = .{ .physical = i, .extent = 12 }; table.applyMeasurements(20, null, &rows); try std.testing.expectEqual(@as(usize, 1), Recorder.counts[1]); try std.testing.expectEqual(@as(usize, 64 + 2048 * 16), Recorder.bytes[1]); - Recorder.counts = [_]usize{0} ** 9; - Recorder.bytes = [_]usize{0} ** 9; + Recorder.counts = [_]usize{0} ** 13; + Recorder.bytes = [_]usize{0} ** 13; _ = table.sync(.{ .id = 7, .item_count = 3000, .uniform_estimate = 10, .gap = 1 }); _ = canvas.virtualVariableListRange(.{ .item_count = 3000, .viewport_extent = 100, .scroll_offset = 1234, .overscan = 3 }, &table); try std.testing.expectEqual(@as(usize, 0), Recorder.counts[1]); + try std.testing.expect(Recorder.bytes[10] <= Recorder.counts[10] * 284); var total: usize = 0; for (Recorder.bytes) |count| total += count; - try std.testing.expect(total <= 256); + // Two logarithmic searches, each with <=63 estimate facts per step. + // No chunk/sparse table snapshot is allowed on a hot query. + try std.testing.expect(Recorder.counts[9] == 0); + try std.testing.expect(Recorder.counts[12] > 0 and Recorder.counts[12] <= 26); + try std.testing.expect(total <= 10000); try std.testing.expectEqualSlices(u8, owned, borrowed); try std.testing.expectEqualSlices(u8, owned, core.modelSnapshot()); } + +test "compiled estimate queries preserve adversarial f32 grouping and exact large counts" { + _ = core.initialModel(); + defer core.rt.frameReset(); + const Rounding = struct { + fn read(_: ?*const anyopaque, index: u64) f32 { + return switch (index % 5) { + 0 => 16777216, + 1 => 1, + 2 => 0.125, + 3 => -0.0, + else => std.math.inf(f32), + }; + } + }; + var tables = pair(); + var args = canvas.VirtualExtentSyncArgs{ .id = 71, .item_count = 2051, .estimate_fn = Rounding.read, .gap = 0.125 }; + try sync(&tables, args); + try batch(&tables, 1023, null, &.{ .{ .physical = 64, .extent = 0.5 }, .{ .physical = 1024, .extent = 19.75 } }); + args.item_count = 2115; + try sync(&tables, args); + args.item_count = 1025; + try sync(&tables, args); + args = .{ .id = 72, .item_count = 9007199254740997, .uniform_estimate = 0.125, .gap = 0.0625 }; + try sync(&tables, args); + for ([_]usize{ 9007199254740991, 9007199254740993, 9007199254740997 }) |index| try exact(tables[0].offsetAtPhysical(index), tables[1].offsetAtPhysical(index)); + for ([_]f32{ 1.0e12, 1.0e15, 1.0e16 }) |offset| try exact(tables[0].indexAtOffset(offset), tables[1].indexAtOffset(offset)); +}