Compile virtual extent queries in TypeScript

packages/core/src/runtime_policy.ts

@@ -1316,8 +1316,7 @@ function nscvSemanticTree(request: Uint8Array): Uint8Array {
   out.setUint32(0, emitted, true); return result;
 }
 
-/** Retained variable-window transitions. Cached estimate/index queries remain
- * explicit native capabilities; sparse corrections cross once per measured
+/** Retained variable-window transitions. Estimate arithmetic and search decisions use bounded copied facts; sparse corrections cross once per measured
  * batch and hot window coordination uses small scalar records. No retained
  * native address or borrowed model bytes escape this copied boundary. */
 function nscvExtentPolicy(request: Uint8Array): Uint8Array {
@@ -1394,6 +1393,40 @@ function nscvExtentPolicy(request: Uint8Array): Uint8Array {
     fixed(16);if(request[3]! > 1 || w.getUint32(12,true) !== 0)throw new Error("invalid extent pending shift");
     const result=new Uint8Array(4);new DataView(result.buffer).setFloat32(0,request[3] === 1 ? f(v(4)-v(8)) : f(v(4)+v(8)),true);return result;
   }
+  if (op === 9 || op === 10) {
+    if (request.length < 32 || request[3] !== 0) throw new Error("invalid extent estimate header");
+    const count=w.getUint32(4,true);
+    if (count > (op === 9 ? 1024 : 63) || request.length !== 32+count*4 || w.getUint32(28,true)!==0) throw new Error("invalid extent estimate batch");
+    if (op === 9) {
+      const result=new Uint8Array(4*Math.ceil(count/64)),out=new DataView(result.buffer);let prefix=v(8);
+      for(let chunk=0;chunk<Math.ceil(count/64);chunk++){
+        let sum=0;for(let i=chunk*64;i<Math.min(count,(chunk+1)*64);i++)sum=f(sum+v(32+i*4));
+        prefix=f(prefix+sum);out.setFloat32(chunk*4,prefix,true);
+      }
+      return result;
+    }
+    let prefix=v(8);for(let i=0;i<count;i++)prefix=f(prefix+v(32+i*4));
+    if(v(16)>0)prefix=f(prefix+f(v(16)*f(v(12)/v(20))));
+    const result=new Uint8Array(4);new DataView(result.buffer).setFloat32(0,prefix,true);return result;
+  }
+  if (op === 11) {
+    fixed(24);if(request[3]!>1 || w.getUint32(20,true)!==0)throw new Error("invalid extent scalar query");
+    const result=new Uint8Array(4),out=new DataView(result.buffer);
+    out.setFloat32(0,request[3]===1?max(0,f(v(4)+v(8))):f(f(v(4)+v(8))+f(v(12)*v(16))),true);return result;
+  }
+  if (op === 12) {
+    fixed(40);if(request[3]!>3 || w.getUint32(4,true)!==0)throw new Error("invalid extent search continuation");
+    const checked=(request[3]!&1)!==0,continued=(request[3]!&2)!==0,count=integer(8),target=max(0,v(32));
+    let low=continued?integer(16):nscvFlowSmall(0),high=continued?integer(24):nscvFlowSubtract(count,nscvFlowSmall(1));
+    const midpoint=(a:NscFlowInteger,b:NscFlowInteger):NscFlowInteger=>{
+      const sum=nscvFlowAdd(nscvFlowAdd(a,b,checked),nscvFlowSmall(1),checked);
+      return {low:Math.floor(sum.low/2)+(sum.high%2)*2147483648,high:Math.floor(sum.high/2)};
+    };
+    if(nscvFlowCompare(low,high)>0 || (!nscvFlowZero(count)&&nscvFlowCompare(high,count)>=0) || (continued&&nscvFlowCompare(low,high)>=0))throw new Error("invalid extent search bounds");
+    if(continued){const mid=midpoint(low,high);if(v(36)<=target)low=mid;else high=nscvFlowSubtract(mid,nscvFlowSmall(1));}
+    const active=nscvFlowCompare(low,high)<0,result=new Uint8Array(32),out=new DataView(result.buffer);
+    nscvFlowWrite(out,0,low);nscvFlowWrite(out,8,high);nscvFlowWrite(out,16,active?midpoint(low,high):low);out.setUint32(24,active?1:0,true);out.setFloat32(28,target,true);return result;
+  }
   throw new Error("unknown extent policy operation");
 }
 function nscvExtentClean(value:number):number{return !Number.isFinite(value) || value < 0 ? 0 : value;}

packages/core/test/virtual_extent.test.ts

@@ -38,3 +38,33 @@ test("extent protocol rejects truncation padding flags invalid ordering and dirt
 test("extent output owns bytes across offset views repeated policies and later mutation",()=>{
  const b=correction(3,[],[{physical:1n,estimate:1,extent:10}]);const padded=new Uint8Array(b.length+10);padded.set(b,7);const out=run(padded.subarray(7,7+b.length)),saved=out.slice();padded.fill(0);for(let i=0;i<12;i++)run(correction(3));assert.deepEqual(out,saved);
 });
+function estimates(op:number,values:readonly number[],prefix=0,total=0,extra=0,covered=0){const b=fixed(op,32+values.length*4),w=new DataView(b.buffer);w.setUint32(4,values.length,true);[prefix,total,extra,covered].forEach((v,i)=>w.setFloat32(8+i*4,v,true));values.forEach((v,i)=>w.setFloat32(32+i*4,v,true));return b;}
+const float=(b:Uint8Array,at=0)=>new DataView(b.buffer,b.byteOffset,b.byteLength).getFloat32(at,true);
+test("estimate batches preserve chunk grouping and partial-prefix rounding",()=>{
+ const values=Array.from({length:1024},(_,i)=>Math.fround(i%5===0?16777216:i%3===0?0.125:1));let prefix=123.125;
+ const result=run(estimates(9,values,prefix));for(let c=0;c<16;c++){let sum=0;for(const value of values.slice(c*64,c*64+64))sum=Math.fround(sum+value);prefix=Math.fround(prefix+sum);assert.equal(float(result,c*4),prefix);}
+ assert.equal(float(run(estimates(10,[1,1],16777216))),16777216);
+ assert.equal(float(run(estimates(10,[],262144,262144,17,262144))),262161);
+ assert.equal(run(estimates(9,[],0)).length,0);
+ assert.throws(()=>run(estimates(9,new Array(1025).fill(1))),/batch/);
+ assert.throws(()=>run(estimates(10,new Array(64).fill(1))),/batch/);
+ const bad=estimates(10,[]);new DataView(bad.buffer).setUint32(28,1,true);assert.throws(()=>run(bad),/batch/);
+});
+function search(count:bigint,low=0n,high=0n,flags=1,target=10,probe=0){const b=fixed(12,40),w=new DataView(b.buffer);b[3]=flags;[count,low,high].forEach((v,i)=>w.setBigUint64(8+i*8,v,true));w.setFloat32(32,target,true);w.setFloat32(36,probe,true);return b;}
+test("query continuation owns exact uint64 midpoints ties zero and overflow modes",()=>{
+ const count=9007199254740997n;let out=run(search(count)),w=new DataView(out.buffer);assert.equal(w.getBigUint64(16,true),(count)/2n);
+ let low=w.getBigUint64(0,true),high=w.getBigUint64(8,true),steps=0;const wanted=9007199254740987n;
+ while(w.getUint32(24,true)){const mid=w.getBigUint64(16,true);out=run(search(count,low,high,3,10,mid<=wanted?10:11));w=new DataView(out.buffer);low=w.getBigUint64(0,true);high=w.getBigUint64(8,true);steps++;assert.ok(steps<=54);}
+ assert.equal(low,wanted);assert.equal(w.getBigUint64(16,true),wanted);
+ assert.equal(new DataView(run(search(0n)).buffer).getUint32(24,true),0);
+ assert.equal(new DataView(run(search(1n)).buffer).getBigUint64(0,true),0n);
+ assert.throws(()=>run(search(0xffffffffffffffffn,0xfffffffffffffffdn,0xfffffffffffffffen,3)),/overflow/);
+ assert.throws(()=>run(search(4n,2n,1n,3)),/bounds/);
+ const wrapped=run(search(0xffffffffffffffffn,0xfffffffffffffffdn,0xfffffffffffffffen,2));assert.ok(wrapped.length===32);
+ const nan=run(search(2n,0n,0n,1,NaN));assert.equal(float(nan,28),0);
+});
+test("extent scalar queries preserve f32 grouping and copied result ownership",()=>{
+ const b=fixed(11,24),w=new DataView(b.buffer);[16777216,1,1,1].forEach((v,i)=>w.setFloat32(4+i*4,v,true));const out=run(b);assert.equal(float(out),16777216);const saved=out.slice();b.fill(0);run(search(99n));assert.deepEqual(out,saved);
+ const negative=fixed(11,24);negative[3]=1;new DataView(negative.buffer).setFloat32(4,-10,true);assert.equal(float(run(negative)),0);
+ for(const example of [estimates(9,[1]),estimates(10,[1]),fixed(11,24),search(2n)]){for(let n=1;n<example.length;n++)assert.throws(()=>run(example.subarray(0,n)));assert.throws(()=>run(new Uint8Array([...example,0])));}
+});

src/primitives/canvas/virtual_extent_policy.zig

@@ -6,7 +6,7 @@ pub const SyncPlan = struct { mode: u8, flags: u8, gap: f32, old_total: f32, fir
 
 /// Native owns every request/result allocation, callback and retained table.
 /// Compiled policy consumes copied sparse state once per measurement batch.
-/// Cached estimate-prefix and index queries are explicit numeric capabilities;
+/// Estimate samples and exact storage lookups are explicit native capabilities;
 /// no native address or borrowed model/view slice is stored by the reducer.
 pub fn sync(policy: Policy, table: anytype, args: anytype, old_total: f32) SyncPlan {
     var request = header(72, 0);
@@ -154,6 +154,77 @@ pub fn shift(policy: Policy, pending: f32, delta: f32, subtract: bool) f32 {
     run(policy, &request, &result);
     return getFloat(&result, 0);
 }
+/// Estimate callbacks are sampled by native code. Chunk arithmetic crosses
+/// in batches of at most sixteen chunks, without copying retained sparse state.
+pub fn rebuild(policy: Policy, table: anytype, first: usize) void {
+    var chunk = @min(first, table.chunk_count);
+    while (chunk < table.chunk_count) {
+        const end_chunk = @min(chunk + 16, table.chunk_count);
+        const start = chunk * 64;
+        const end = @min(table.covered_count, end_chunk * 64);
+        var request = header(32 + 1024 * 4, 9);
+        putWord(&request, 4, @intCast(end - start));
+        putFloat(&request, 8, table.chunk_prefix[chunk]);
+        for (start..end) |i| putFloat(&request, 32 + (i - start) * 4, table.estimateAt(i));
+        var result: [16 * 4]u8 = undefined;
+        run(policy, request[0 .. 32 + (end - start) * 4], result[0 .. (end_chunk - chunk) * 4]);
+        for (chunk..end_chunk) |i| table.chunk_prefix[i + 1] = getFloat(&result, (i - chunk) * 4);
+        chunk = end_chunk;
+    }
+}
+pub fn prefix(policy: Policy, table: anytype, index: usize) f32 {
+    const clamped = @min(index, table.item_count);
+    const tail = clamped > table.covered_count;
+    const chunk = if (tail) table.chunk_count else clamped / 64;
+    const start = chunk * 64;
+    const count = if (tail) 0 else clamped - start;
+    var request = header(32 + 63 * 4, 10);
+    putWord(&request, 4, @intCast(count));
+    putFloat(&request, 8, table.chunk_prefix[chunk]);
+    if (tail) {
+        putFloat(&request, 12, table.chunk_prefix[table.chunk_count]);
+        putFloat(&request, 16, @floatFromInt(clamped - table.covered_count));
+        putFloat(&request, 20, @floatFromInt(table.covered_count));
+    }
+    for (0..count) |i| putFloat(&request, 32 + i * 4, table.estimateAt(start + i));
+    var result: [4]u8 = undefined;
+    run(policy, request[0 .. 32 + count * 4], &result);
+    return getFloat(&result, 0);
+}
+pub fn scalar(policy: Policy, estimate: f32, delta: f32, gap: f32, count: usize, extent: bool) f32 {
+    var request = header(24, 11);
+    request[3] = if (extent) 1 else 0;
+    putFloat(&request, 4, estimate);
+    putFloat(&request, 8, delta);
+    putFloat(&request, 12, gap);
+    putFloat(&request, 16, @floatFromInt(count));
+    var result: [4]u8 = undefined;
+    run(policy, &request, &result);
+    return getFloat(&result, 0);
+}
+pub fn search(policy: Policy, table: anytype, offset: f32) usize {
+    const builtin = @import("builtin");
+    var request = header(40, 12);
+    request[3] = if (builtin.mode == .Debug or builtin.mode == .ReleaseSafe) 1 else 0;
+    putInt(&request, 8, table.item_count);
+    putFloat(&request, 32, offset);
+    var result: [32]u8 = undefined;
+    while (true) {
+        run(policy, &request, &result);
+        const active = getWord(&result, 24);
+        if (active > 1) @panic("invalid compiled extent search result");
+        const low = getInt(&result, 0);
+        const high = getInt(&result, 8);
+        const mid = getInt(&result, 16);
+        if (low > high or (table.item_count != 0 and high >= table.item_count) or mid < low or mid > high) @panic("invalid compiled extent search bounds");
+        if (active == 0) return low;
+        putInt(&request, 16, low);
+        putInt(&request, 24, high);
+        putFloat(&request, 32, getFloat(&result, 28));
+        putFloat(&request, 36, table.offsetAtPhysical(mid));
+        request[3] |= 2;
+    }
+}
 fn header(comptime length: usize, operation: u8) [length]u8 {
     var request = [_]u8{0} ** length;
     request[0] = 10;

src/primitives/canvas/virtual_extents.zig

@@ -185,6 +185,10 @@ pub const VirtualExtentTable = struct {
     fn rebuildChunkPrefixFrom(self: *VirtualExtentTable, first_chunk: usize) void {
         self.covered_count = @min(self.item_count, max_virtual_extent_items);
         self.chunk_count = (self.covered_count + virtual_extent_chunk - 1) / virtual_extent_chunk;
+        if (self.policy) |policy| {
+            compiled.rebuild(policy, self, first_chunk);
+            return;
+        }
         var chunk = first_chunk;
         if (chunk > self.chunk_count) chunk = self.chunk_count;
         while (chunk < self.chunk_count) : (chunk += 1) {
@@ -208,6 +212,7 @@ pub const VirtualExtentTable = struct {
 
     /// Sum of estimates for physical items [0, index).
     fn estimatePrefix(self: *const VirtualExtentTable, index: usize) f32 {
+        if (self.policy) |policy| return compiled.prefix(policy, self, index);
         const clamped = @min(index, self.item_count);
         if (clamped > self.covered_count) {
             const extra = @as(f32, @floatFromInt(clamped - self.covered_count));
@@ -249,6 +254,11 @@ pub const VirtualExtentTable = struct {
     /// estimate otherwise.
     pub fn extentAtPhysical(self: *const VirtualExtentTable, physical: usize) f32 {
         const base = self.estimateAt(physical);
+        if (self.policy) |policy| {
+            const slot = self.measuredSlot(self.index_base + @as(u64, physical));
+            // Unmeasured estimates retain signed zero without a max operation.
+            return if (slot) |i| compiled.scalar(policy, base, self.measured_delta[i], 0, 0, true) else base;
+        }
         if (self.measuredSlot(self.index_base + @as(u64, physical))) |slot| {
             return @max(0, base + self.measured_delta[slot]);
         }
@@ -259,6 +269,7 @@ pub const VirtualExtentTable = struct {
     /// included).
     pub fn offsetAtPhysical(self: *const VirtualExtentTable, physical: usize) f32 {
         const clamped = @min(physical, self.item_count);
+        if (self.policy) |policy| return compiled.scalar(policy, self.estimatePrefix(clamped), self.measuredDeltaBefore(self.index_base + @as(u64, clamped)), self.gap, clamped, false);
         return self.estimatePrefix(clamped) +
             self.measuredDeltaBefore(self.index_base + @as(u64, clamped)) +
             self.gap * @as(f32, @floatFromInt(clamped));
@@ -267,6 +278,7 @@ pub const VirtualExtentTable = struct {
     /// Total content extent: every extent plus the gaps between rows.
     pub fn totalExtent(self: *const VirtualExtentTable) f32 {
         if (self.item_count == 0) return 0;
+        if (self.policy) |policy| return compiled.scalar(policy, self.estimatePrefix(self.item_count), self.measured_total_delta, self.gap, self.item_count - 1, false);
         return self.estimatePrefix(self.item_count) + self.measured_total_delta +
             self.gap * @as(f32, @floatFromInt(self.item_count - 1));
     }
@@ -275,6 +287,7 @@ pub const VirtualExtentTable = struct {
     /// a scroll offset). Monotone bisection over `offsetAtPhysical`.
     pub fn indexAtOffset(self: *const VirtualExtentTable, offset: f32) usize {
         if (self.item_count == 0) return 0;
+        if (self.policy) |policy| return compiled.search(policy, self, offset);
         const target = @max(0, offset);
         // Largest index whose leading edge is <= target.
         var low: usize = 0;

tests/ts-core/virtual_extent_e2e_tests.zig

@@ -200,33 +200,70 @@ test "compiled extent hot queries use bounded scalar records and preserve borrow
     const owned = try std.testing.allocator.dupe(u8, borrowed);
     defer std.testing.allocator.free(owned);
     const Recorder = struct {
-        var counts = [_]usize{0} ** 9;
-        var bytes = [_]usize{0} ** 9;
+        var counts = [_]usize{0} ** 13;
+        var bytes = [_]usize{0} ** 13;
+        var largest: usize = 0;
         fn call(request: []const u8, output: []u8) usize {
             if (request.len > 1 and request[0] == 10) {
+                largest = @max(largest, request.len);
                 counts[request[1]] += 1;
                 bytes[request[1]] += request.len;
             }
             return core.nativeWindowPolicy(request, output);
         }
     };
-    Recorder.counts = [_]usize{0} ** 9;
-    Recorder.bytes = [_]usize{0} ** 9;
+    Recorder.counts = [_]usize{0} ** 13;
+    Recorder.bytes = [_]usize{0} ** 13;
     var table = Table{ .policy = Recorder.call };
     _ = table.sync(.{ .id = 7, .item_count = 3000, .uniform_estimate = 10, .gap = 1 });
+    try std.testing.expectEqual(@as(usize, 3), Recorder.counts[9]);
+    try std.testing.expect(Recorder.largest <= 4128);
     var rows: [2048]Measure = undefined;
     for (&rows, 0..) |*row, i| row.* = .{ .physical = i, .extent = 12 };
     table.applyMeasurements(20, null, &rows);
     try std.testing.expectEqual(@as(usize, 1), Recorder.counts[1]);
     try std.testing.expectEqual(@as(usize, 64 + 2048 * 16), Recorder.bytes[1]);
-    Recorder.counts = [_]usize{0} ** 9;
-    Recorder.bytes = [_]usize{0} ** 9;
+    Recorder.counts = [_]usize{0} ** 13;
+    Recorder.bytes = [_]usize{0} ** 13;
     _ = table.sync(.{ .id = 7, .item_count = 3000, .uniform_estimate = 10, .gap = 1 });
     _ = canvas.virtualVariableListRange(.{ .item_count = 3000, .viewport_extent = 100, .scroll_offset = 1234, .overscan = 3 }, &table);
     try std.testing.expectEqual(@as(usize, 0), Recorder.counts[1]);
+    try std.testing.expect(Recorder.bytes[10] <= Recorder.counts[10] * 284);
     var total: usize = 0;
     for (Recorder.bytes) |count| total += count;
-    try std.testing.expect(total <= 256);
+    // Two logarithmic searches, each with <=63 estimate facts per step.
+    // No chunk/sparse table snapshot is allowed on a hot query.
+    try std.testing.expect(Recorder.counts[9] == 0);
+    try std.testing.expect(Recorder.counts[12] > 0 and Recorder.counts[12] <= 26);
+    try std.testing.expect(total <= 10000);
     try std.testing.expectEqualSlices(u8, owned, borrowed);
     try std.testing.expectEqualSlices(u8, owned, core.modelSnapshot());
 }
+
+test "compiled estimate queries preserve adversarial f32 grouping and exact large counts" {
+    _ = core.initialModel();
+    defer core.rt.frameReset();
+    const Rounding = struct {
+        fn read(_: ?*const anyopaque, index: u64) f32 {
+            return switch (index % 5) {
+                0 => 16777216,
+                1 => 1,
+                2 => 0.125,
+                3 => -0.0,
+                else => std.math.inf(f32),
+            };
+        }
+    };
+    var tables = pair();
+    var args = canvas.VirtualExtentSyncArgs{ .id = 71, .item_count = 2051, .estimate_fn = Rounding.read, .gap = 0.125 };
+    try sync(&tables, args);
+    try batch(&tables, 1023, null, &.{ .{ .physical = 64, .extent = 0.5 }, .{ .physical = 1024, .extent = 19.75 } });
+    args.item_count = 2115;
+    try sync(&tables, args);
+    args.item_count = 1025;
+    try sync(&tables, args);
+    args = .{ .id = 72, .item_count = 9007199254740997, .uniform_estimate = 0.125, .gap = 0.0625 };
+    try sync(&tables, args);
+    for ([_]usize{ 9007199254740991, 9007199254740993, 9007199254740997 }) |index| try exact(tables[0].offsetAtPhysical(index), tables[1].offsetAtPhysical(index));
+    for ([_]f32{ 1.0e12, 1.0e15, 1.0e16 }) |offset| try exact(tables[0].indexAtOffset(offset), tables[1].indexAtOffset(offset));
+}