Skip to content

Compile virtual extent plans in TypeScript #567

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ctate merged 1 commit into from
Oct 5, 2026
Merged

Compile virtual extent plans in TypeScript #567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions packages/core/src/runtime_policy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1427,6 +1427,35 @@ function nscvExtentPolicy(request: Uint8Array): Uint8Array {
const active=nscvFlowCompare(low,high)<0,result=new Uint8Array(32),out=new DataView(result.buffer);
nscvFlowWrite(out,0,low);nscvFlowWrite(out,8,high);nscvFlowWrite(out,16,active?midpoint(low,high):low);out.setUint32(24,active?1:0,true);out.setFloat32(28,target,true);return result;
}
if (op === 13) {
fixed(48);if(request[3]!>6 || (request[3]!&3)>2 || w.getUint32(4,true)!==0)throw new Error("invalid extent query plan");
const mode=request[3]!&3,count=integer(8),index=mode===2?count:nscvFlowMin(integer(16),count),covered=integer(32),chunks=integer(40);
if(nscvFlowCompare(covered,nscvFlowSmall(262144))>0 || nscvFlowCompare(covered,count)>0 || chunks.high!==0 || chunks.low!==Math.ceil(covered.low/64))throw new Error("invalid extent cache shape");
const tail=nscvFlowCompare(index,covered)>0;
const chunk=tail?chunks.low:Math.floor(index.low/64),start=tail?covered:nscvFlowSmall(chunk*64),samples=tail?0:nscvFlowSubtract(index,start).low;
const extra=tail?nscvFlowSubtract(index,covered):nscvFlowSmall(0),gapCount=mode===2?nscvFlowSubtract(count,nscvFlowSmall(1)):index;
const logical=mode===1?nscvFlowAdd(integer(24),index,(request[3]!&4)!==0):nscvFlowSmall(0);
const result=new Uint8Array(64),out=new DataView(result.buffer);
nscvFlowWrite(out,0,index);nscvFlowWrite(out,8,logical);nscvFlowWrite(out,16,start);nscvFlowWrite(out,24,extra);nscvFlowWrite(out,32,gapCount);
out.setUint32(40,chunk,true);out.setUint32(44,samples,true);out.setUint32(48,tail?1:0,true);out.setUint32(52,mode===2&&nscvFlowZero(count)?1:0,true);return result;
}
if (op === 14) {
fixed(24);if(request[3]!==0 || w.getUint32(4,true)!==0)throw new Error("invalid extent rebuild plan");
const covered=nscvFlowMin(integer(8),nscvFlowSmall(262144)),chunks=Math.ceil(covered.low/64),first=nscvFlowMin(integer(16),nscvFlowSmall(chunks)).low,end=Math.min(first+16,chunks);
const result=new Uint8Array(40),out=new DataView(result.buffer);
out.setUint32(0,covered.low,true);out.setUint32(4,chunks,true);out.setUint32(8,first,true);out.setUint32(12,end,true);
out.setUint32(16,Math.min(covered.low,first*64),true);out.setUint32(20,Math.min(covered.low,end*64),true);out.setUint32(24,first<chunks?1:0,true);return result;
}
if (op === 15) {
if(request.length<40 || request[3]!>3)throw new Error("invalid extent composed query");
const count=w.getUint32(4,true);if(count>63 || request.length!==40+count*4 || w.getUint32(36,true)!==0)throw new Error("invalid extent composed facts");
const tail=(request[3]!&1)!==0,empty=(request[3]!&2)!==0;
if((tail&&count!==0) || (tail&&!(v(20)>0)))throw new Error("invalid extent tail facts");
let prefix=v(8);for(let i=0;i<count;i++)prefix=f(prefix+v(40+i*4));
if(tail)prefix=f(prefix+f(v(16)*f(v(12)/v(20))));
const gap=f(v(28)*v(32)),result=new Uint8Array(12),out=new DataView(result.buffer);
out.setFloat32(0,prefix,true);out.setFloat32(4,gap,true);out.setFloat32(8,empty?0:f(f(prefix+v(24))+gap),true);return result;
}
throw new Error("unknown extent policy operation");
}
function nscvExtentClean(value:number):number{return !Number.isFinite(value) || value < 0 ? 0 : value;}
Expand Down
33 changes: 33 additions & 0 deletions packages/core/test/virtual_extent.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,36 @@ test("extent scalar queries preserve f32 grouping and copied result ownership",(
const negative=fixed(11,24);negative[3]=1;new DataView(negative.buffer).setFloat32(4,-10,true);assert.equal(float(run(negative)),0);
for(const example of [estimates(9,[1]),estimates(10,[1]),fixed(11,24),search(2n)]){for(let n=1;n<example.length;n++)assert.throws(()=>run(example.subarray(0,n)));assert.throws(()=>run(new Uint8Array([...example,0])));}
});
function queryPlan(count:bigint,index:bigint,base=0n,covered=count<262144n?count:262144n,mode=1,checked=true){const b=fixed(13,48),w=new DataView(b.buffer);b[3]=mode|(checked?4:0);[count,index,base,covered,(covered+63n)/64n].forEach((v,i)=>w.setBigUint64(8+i*8,v,true));return b;}
function decodePlan(b:Uint8Array){const w=new DataView(b.buffer,b.byteOffset,b.byteLength);return {index:w.getBigUint64(0,true),logical:w.getBigUint64(8,true),start:w.getBigUint64(16,true),extra:w.getBigUint64(24,true),gapCount:w.getBigUint64(32,true),chunk:w.getUint32(40,true),samples:w.getUint32(44,true),tail:w.getUint32(48,true),empty:w.getUint32(52,true)};}
function rebuildPlan(count:bigint,first:bigint){const b=fixed(14,24),w=new DataView(b.buffer);w.setBigUint64(8,count,true);w.setBigUint64(16,first,true);return b;}
function composed(values:readonly number[],flags=0,prefix=0,total=0,extra=0,covered=0,delta=0,gap=0,gapCount=0){const b=fixed(15,40+values.length*4),w=new DataView(b.buffer);b[3]=flags;w.setUint32(4,values.length,true);[prefix,total,extra,covered,delta,gap,gapCount].forEach((v,i)=>w.setFloat32(8+i*4,v,true));values.forEach((v,i)=>w.setFloat32(40+i*4,v,true));return b;}
test("extent query plans own physical clamp cache addresses tails and exact logical identities",()=>{
for(const count of [0n,1n,63n,64n,65n,262143n,262144n,262145n,9007199254740997n])for(const index of [0n,1n,63n,64n,count,count+1n]){
const base=9007199254740993n,r=decodePlan(run(queryPlan(count,index,base)));const clamped=index<count?index:count,covered=count<262144n?count:262144n,tail=clamped>covered;
assert.deepEqual(r,{index:clamped,logical:base+clamped,start:tail?covered:clamped/64n*64n,extra:tail?clamped-covered:0n,gapCount:clamped,chunk:Number(tail?(covered+63n)/64n:clamped/64n),samples:tail?0:Number(clamped%64n),tail:tail?1:0,empty:0});
}
const zero=decodePlan(run(queryPlan(0n,99n,0n,0n,2)));assert.equal(zero.empty,1);assert.equal(zero.gapCount,0n);
const total=decodePlan(run(queryPlan(9007199254740997n,1n,0xffffffffffffffffn,262144n,2)));assert.equal(total.index,9007199254740997n);assert.equal(total.logical,0n);assert.equal(total.gapCount,9007199254740996n);
assert.throws(()=>run(queryPlan(3n,2n,0xffffffffffffffffn)),/overflow/);assert.equal(decodePlan(run(queryPlan(3n,2n,0xffffffffffffffffn,3n,1,false))).logical,1n);
});
test("extent rebuild planning admits bounded batches and covers all cached rows exactly",()=>{
for(const count of [0n,1n,64n,65n,1025n,262143n,262144n,0xffffffffffffffffn]){
const covered=Number(count<262144n?count:262144n);let first=0n,seen=0,calls=0;
while(true){const out=run(rebuildPlan(count,first)),w=new DataView(out.buffer);assert.equal(w.getUint32(0,true),covered);assert.equal(w.getUint32(4,true),Math.ceil(covered/64));if(!w.getUint32(24,true)){assert.equal(w.getUint32(16,true),covered);assert.equal(w.getUint32(20,true),covered);break;}const start=w.getUint32(16,true),end=w.getUint32(20,true);assert.equal(start,seen);assert.ok(end>start&&end-start<=1024);seen=end;first=BigInt(w.getUint32(12,true));assert.ok(++calls<=256);}
assert.equal(seen,covered);assert.equal(new DataView(run(rebuildPlan(count,0xffffffffffffffffn)).buffer).getUint32(24,true),0);
}
});
test("composed extent facts preserve prefix gap and measured f32 ordering without extra round trips",()=>{
let out=run(composed([1,1],0,16777216,0,0,0,1,1,1));assert.equal(float(out),16777216);assert.equal(float(out,4),1);assert.equal(float(out,8),16777216);
out=run(composed([],1,262144,262144,17,262144,0.25,0.125,262161));assert.equal(float(out),262161);assert.equal(float(out,4),Math.fround(0.125*262161));assert.equal(float(out,8),Math.fround(Math.fround(262161+0.25)+float(out,4)));
assert.equal(float(run(composed([],2,0,0,0,0,NaN,Infinity,0)),8),0);
});
test("extent planning rejects malformed shapes reserved lanes and composed fact mismatches",()=>{
const examples=[queryPlan(65n,64n),rebuildPlan(65n,0n),composed([1])];for(const b of examples){for(let n=1;n<b.length;n++)assert.throws(()=>run(b.subarray(0,n)));assert.throws(()=>run(new Uint8Array([...b,0])));}
const shape=queryPlan(64n,1n);new DataView(shape.buffer).setBigUint64(40,2n,true);assert.throws(()=>run(shape),/shape/);
const reserved=rebuildPlan(65n,0n);reserved[4]=1;assert.throws(()=>run(reserved),/plan/);
assert.throws(()=>run(composed([1],1,0,1,1,1)),/tail/);assert.throws(()=>run(composed([],1,0,1,1,0)),/tail/);
assert.throws(()=>run(composed(new Array(64).fill(1))),/facts/);
const output=run(queryPlan(65n,64n)),saved=output.slice();for(let i=0;i<8;i++)run(queryPlan(123n,BigInt(i)));assert.deepEqual(output,saved);
});
93 changes: 65 additions & 28 deletions src/primitives/canvas/virtual_extent_policy.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,39 +157,76 @@ pub fn shift(policy: Policy, pending: f32, delta: f32, subtract: bool) f32 {
/// Estimate callbacks are sampled by native code. Chunk arithmetic crosses
/// in batches of at most sixteen chunks, without copying retained sparse state.
pub fn rebuild(policy: Policy, table: anytype, first: usize) void {
var chunk = @min(first, table.chunk_count);
while (chunk < table.chunk_count) {
const end_chunk = @min(chunk + 16, table.chunk_count);
const start = chunk * 64;
const end = @min(table.covered_count, end_chunk * 64);
var request = header(32 + 1024 * 4, 9);
putWord(&request, 4, @intCast(end - start));
putFloat(&request, 8, table.chunk_prefix[chunk]);
for (start..end) |i| putFloat(&request, 32 + (i - start) * 4, table.estimateAt(i));
var cursor = first;
while (true) {
var request = header(24, 14);
putInt(&request, 8, table.item_count);
putInt(&request, 16, cursor);
var plan: [40]u8 = undefined;
run(policy, &request, &plan);
const covered = getWord(&plan, 0);
const chunks = getWord(&plan, 4);
const chunk = getWord(&plan, 8);
const end_chunk = getWord(&plan, 12);
const start = getWord(&plan, 16);
const end = getWord(&plan, 20);
const active = getWord(&plan, 24);
if (covered > 262144 or chunks > 4096 or chunk > end_chunk or end_chunk > chunks or end_chunk - chunk > 16 or end > covered or end < start or end - start > 1024 or active > 1) @panic("invalid compiled extent rebuild plan");
table.covered_count = covered;
table.chunk_count = chunks;
if (active == 0) return;
if (end_chunk == chunk) @panic("nonprogressing compiled extent rebuild plan");
var facts = header(32 + 1024 * 4, 9);
putWord(&facts, 4, end - start);
putFloat(&facts, 8, table.chunk_prefix[chunk]);
for (start..end) |i| putFloat(&facts, 32 + (i - start) * 4, table.estimateAt(i));
var result: [16 * 4]u8 = undefined;
run(policy, request[0 .. 32 + (end - start) * 4], result[0 .. (end_chunk - chunk) * 4]);
run(policy, facts[0 .. 32 + (end - start) * 4], result[0 .. (end_chunk - chunk) * 4]);
for (chunk..end_chunk) |i| table.chunk_prefix[i + 1] = getFloat(&result, (i - chunk) * 4);
chunk = end_chunk;
cursor = end_chunk;
}
}
pub const Query = struct { prefix: f32, gap: f32, offset: f32 };
/// TypeScript selects the physical/cache/logical plan. Native reads only the
/// requested storage and callback samples and supplies exact conversion facts.
pub fn query(policy: Policy, table: anytype, index: usize, mode: u8) Query {
const builtin = @import("builtin");
var request = header(48, 13);
request[3] = mode | @as(u8, if (builtin.mode == .Debug or builtin.mode == .ReleaseSafe) 4 else 0);
putInt(&request, 8, table.item_count);
putInt(&request, 16, index);
putInt(&request, 24, table.index_base);
putInt(&request, 32, table.covered_count);
putInt(&request, 40, table.chunk_count);
var plan: [64]u8 = undefined;
run(policy, &request, &plan);
const physical = getInt(&plan, 0);
const logical = getInt(&plan, 8);
const start = getInt(&plan, 16);
const extra = getInt(&plan, 24);
const gap_count = getInt(&plan, 32);
const chunk = getWord(&plan, 40);
const count = getWord(&plan, 44);
const tail = getWord(&plan, 48);
const empty = getWord(&plan, 52);
if (physical > table.item_count or chunk > table.chunk_count or count > 63 or tail > 1 or empty > 1 or start > table.covered_count or count > table.covered_count - start) @panic("invalid compiled extent query plan");
var facts = header(40 + 63 * 4, 15);
facts[3] = @intCast(tail | (empty << 1));
putWord(&facts, 4, count);
putFloat(&facts, 8, if (empty == 1) 0 else table.chunk_prefix[chunk]);
putFloat(&facts, 12, if (empty == 1) 0 else table.chunk_prefix[table.chunk_count]);
putFloat(&facts, 16, @floatFromInt(extra));
putFloat(&facts, 20, @floatFromInt(table.covered_count));
putFloat(&facts, 24, if (mode == 1) table.measuredDeltaBefore(logical) else if (mode == 2) table.measured_total_delta else 0);
putFloat(&facts, 28, table.gap);
putFloat(&facts, 32, @floatFromInt(gap_count));
for (0..count) |i| putFloat(&facts, 40 + i * 4, table.estimateAt(start + i));
var result: [12]u8 = undefined;
run(policy, facts[0 .. 40 + count * 4], &result);
return .{ .prefix = getFloat(&result, 0), .gap = getFloat(&result, 4), .offset = getFloat(&result, 8) };
}
pub fn prefix(policy: Policy, table: anytype, index: usize) f32 {
const clamped = @min(index, table.item_count);
const tail = clamped > table.covered_count;
const chunk = if (tail) table.chunk_count else clamped / 64;
const start = chunk * 64;
const count = if (tail) 0 else clamped - start;
var request = header(32 + 63 * 4, 10);
putWord(&request, 4, @intCast(count));
putFloat(&request, 8, table.chunk_prefix[chunk]);
if (tail) {
putFloat(&request, 12, table.chunk_prefix[table.chunk_count]);
putFloat(&request, 16, @floatFromInt(clamped - table.covered_count));
putFloat(&request, 20, @floatFromInt(table.covered_count));
}
for (0..count) |i| putFloat(&request, 32 + i * 4, table.estimateAt(start + i));
var result: [4]u8 = undefined;
run(policy, request[0 .. 32 + count * 4], &result);
return getFloat(&result, 0);
return query(policy, table, index, 0).prefix;
}
pub fn scalar(policy: Policy, estimate: f32, delta: f32, gap: f32, count: usize, extent: bool) f32 {
var request = header(24, 11);
Expand Down
21 changes: 11 additions & 10 deletions src/primitives/canvas/virtual_extents.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,12 +183,12 @@ pub const VirtualExtentTable = struct {
}

fn rebuildChunkPrefixFrom(self: *VirtualExtentTable, first_chunk: usize) void {
self.covered_count = @min(self.item_count, max_virtual_extent_items);
self.chunk_count = (self.covered_count + virtual_extent_chunk - 1) / virtual_extent_chunk;
if (self.policy) |policy| {
compiled.rebuild(policy, self, first_chunk);
return;
}
self.covered_count = @min(self.item_count, max_virtual_extent_items);
self.chunk_count = (self.covered_count + virtual_extent_chunk - 1) / virtual_extent_chunk;
var chunk = first_chunk;
if (chunk > self.chunk_count) chunk = self.chunk_count;
while (chunk < self.chunk_count) : (chunk += 1) {
Expand Down Expand Up @@ -226,7 +226,7 @@ pub const VirtualExtentTable = struct {
}

/// Sum of measured deltas for logical indices < `logical`.
fn measuredDeltaBefore(self: *const VirtualExtentTable, logical: u64) f32 {
pub fn measuredDeltaBefore(self: *const VirtualExtentTable, logical: u64) f32 {
std.debug.assert(!self.measured_prefix_dirty);
// Lowest measured slot whose index is >= logical.
var low: usize = 0;
Expand Down Expand Up @@ -268,17 +268,17 @@ pub const VirtualExtentTable = struct {
/// Leading edge of item `physical` (gaps between prior rows
/// included).
pub fn offsetAtPhysical(self: *const VirtualExtentTable, physical: usize) f32 {
if (self.policy) |policy| return compiled.query(policy, self, physical, 1).offset;
const clamped = @min(physical, self.item_count);
if (self.policy) |policy| return compiled.scalar(policy, self.estimatePrefix(clamped), self.measuredDeltaBefore(self.index_base + @as(u64, clamped)), self.gap, clamped, false);
return self.estimatePrefix(clamped) +
self.measuredDeltaBefore(self.index_base + @as(u64, clamped)) +
self.gap * @as(f32, @floatFromInt(clamped));
}

/// Total content extent: every extent plus the gaps between rows.
pub fn totalExtent(self: *const VirtualExtentTable) f32 {
if (self.policy) |policy| return compiled.query(policy, self, self.item_count, 2).offset;
if (self.item_count == 0) return 0;
if (self.policy) |policy| return compiled.scalar(policy, self.estimatePrefix(self.item_count), self.measured_total_delta, self.gap, self.item_count - 1, false);
return self.estimatePrefix(self.item_count) + self.measured_total_delta +
self.gap * @as(f32, @floatFromInt(self.item_count - 1));
}
Expand Down Expand Up @@ -403,8 +403,8 @@ pub const VirtualExtentTable = struct {
/// methods remain available with the same native/compiled ownership.
pub fn applyMeasurements(self: *VirtualExtentTable, anchor: usize, rendered_offset: ?f32, rows: []const VirtualExtentMeasurement) void {
if (self.policy) |policy| {
const clamped = @min(anchor, self.item_count);
compiled.corrections(policy, self, 3, anchor, rendered_offset, rows, self.estimatePrefix(clamped), self.gap * @as(f32, @floatFromInt(clamped)));
const facts = compiled.query(policy, self, anchor, 0);
compiled.corrections(policy, self, 3, anchor, rendered_offset, rows, facts.prefix, facts.gap);
return;
}
self.beginCorrections(anchor, rendered_offset);
Expand Down Expand Up @@ -442,8 +442,8 @@ pub const VirtualExtentTable = struct {
pub fn beginCorrections(self: *VirtualExtentTable, anchor_physical: usize, rendered_offset: ?f32) void {
std.debug.assert(!self.measured_prefix_dirty);
if (self.policy) |policy| {
const clamped = @min(anchor_physical, self.item_count);
compiled.corrections(policy, self, 0, anchor_physical, rendered_offset, &.{}, self.estimatePrefix(clamped), self.gap * @as(f32, @floatFromInt(clamped)));
const facts = compiled.query(policy, self, anchor_physical, 0);
compiled.corrections(policy, self, 0, anchor_physical, rendered_offset, &.{}, facts.prefix, facts.gap);
return;
}
self.anchor_physical = @min(anchor_physical, self.item_count);
Expand Down Expand Up @@ -510,7 +510,8 @@ pub const VirtualExtentTable = struct {
/// where the user sees it.
pub fn endCorrections(self: *VirtualExtentTable) void {
if (self.policy) |policy| {
compiled.corrections(policy, self, 2, self.anchor_physical, null, &.{}, self.estimatePrefix(self.anchor_physical), self.gap * @as(f32, @floatFromInt(self.anchor_physical)));
const facts = compiled.query(policy, self, self.anchor_physical, 0);
compiled.corrections(policy, self, 2, self.anchor_physical, null, &.{}, facts.prefix, facts.gap);
return;
}
if (self.measured_prefix_dirty) self.rebuildMeasuredPrefix();
Expand Down
Loading
Loading