fix(statistics): stop coercing unreported tokens to a measured zero

docs/statistics-contract.md

@@ -56,7 +56,7 @@ Pi's `setWidget` is the only fire-and-forget RPC push structured enough to carry
 
 ## Provenance and unavailable markers
 
-Every monetary and token figure carries `provenance`: `measured` while every component was reported, `partial` as soon as one component reported no cost, forever. An unavailable ratio is `null`, never a blank or a zero. Counts carry provenance too: the orchestrator's own tool calls are not in a usage record, so an aggregate that includes a parent record reports its `toolCalls` count as `partial` rather than a silently smaller number.
+Every monetary and token figure carries `provenance`: `measured` while every component was reported, `partial` as soon as one component reported no cost, forever. A token figure is `partial` when any contributing record's source omitted a token counter, so a summed lower bound is never presented as a measured zero. An unavailable ratio is `null`, never a blank or a zero. Counts carry provenance too: the orchestrator's own tool calls are not in a usage record, so an aggregate that includes a parent record reports its `toolCalls` count as `partial` rather than a silently smaller number.
 
 ## Bounds
 

lib/agents-protocol.ts

@@ -1,6 +1,6 @@
 import { createHash } from "node:crypto";
 import { sanitizeTerminalText } from "./terminal-theme.ts";
-import { accumulateTaskCost, childUsageCost, type UsageCost } from "./session-usage.ts";
+import { accumulateTaskCost, accumulateTaskTokens, childUsageCost, type UsageCost } from "./session-usage.ts";
 
 // Gentle Agents protocol. A child pi process streams RPC events; the host
 // turns each one into a small typed delta, applies it to an append-only,
@@ -70,7 +70,7 @@ export interface AgentSettledEvent { type: typeof TASK_EVENT.AGENT_SETTLED }
 export interface ErrorEvent { type: typeof TASK_EVENT.ERROR; message: string }
 export interface AskEvent { type: typeof TASK_EVENT.ASK; request: AskRequest }
 export interface NoteEvent { type: typeof TASK_EVENT.NOTE; text: string }
-export interface UsageEvent { type: typeof TASK_EVENT.USAGE; tokens: number; cost: UsageCost }
+export interface UsageEvent { type: typeof TASK_EVENT.USAGE; tokens: number; tokensComplete: boolean; cost: UsageCost }
 
 export type ChildUnavailable = Readonly<{ state: "unavailable" }>;
 export type ChildMetadata = Readonly<{ state: "observed"; value: string }> | ChildUnavailable;
@@ -137,6 +137,8 @@ export interface TaskRecord {
 	turns: number;
 	toolCalls: number;
 	tokens: number;
+	/** False once any usage event reported no token counters; absent means complete (legacy records). */
+	tokensComplete?: boolean;
 	cost: number;
 	/** False once any usage event reported no cost; absent means complete (legacy records). */
 	costComplete?: boolean;
@@ -215,6 +217,20 @@ function childTokens(value: unknown): ChildTokenMeasurement {
 		? Object.freeze({ state: "reported", value }) : CHILD_UNAVAILABLE;
 }
 
+/**
+ * True when a usage payload carried at least one positive token counter. An
+ * all-zero or empty usage is "not reported", exactly as `childTokens` decides
+ * for the observation channel: the SDK injects zeros, so zero is not evidence
+ * that the provider reported zero tokens.
+ */
+function providerTokensReported(usage: Raw | undefined): boolean {
+	for (const field of ["input", "output", "cacheRead", "cacheWrite", "totalTokens", "reasoning"] as const) {
+		const value = usage?.[field];
+		if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) return true;
+	}
+	return false;
+}
+
 function childResponse(message: Raw): ChildResponseObservation | undefined {
 	const reason = message.stopReason;
 	if (reason !== "stop" && reason !== "length" && reason !== "toolUse" && reason !== "error" && reason !== "aborted") return undefined;
@@ -317,7 +333,7 @@ export function normalizeRpcEvent(raw: unknown, options: { observeResponses?: bo
 			const usage = message?.role === "assistant" ? (message.usage as Raw | undefined) : undefined;
 			const events: TaskEvent[] = [];
 			if (usage) {
-				events.push({ type: TASK_EVENT.USAGE, tokens: Number(usage.totalTokens ?? 0) || 0, cost: childUsageCost(usage) });
+				events.push({ type: TASK_EVENT.USAGE, tokens: Number(usage.totalTokens ?? 0) || 0, tokensComplete: providerTokensReported(usage), cost: childUsageCost(usage) });
 			}
 			if (options.observeResponses === true && message?.role === "assistant") {
 				const observation = childResponse(message);
@@ -440,7 +456,7 @@ function recordPatch(task: TaskRecord, event: TaskEvent): Partial<TaskRecord> {
 		case TASK_EVENT.TEXT:
 			return { ...resumed, lastStep: task.lastStep === "queued" || task.lastStep === "starting" ? "writing" : task.lastStep };
 		case TASK_EVENT.USAGE:
-			return { ...resumed, tokens: task.tokens + event.tokens, ...accumulateTaskCost(task, event.cost) };
+			return { ...resumed, tokens: task.tokens + event.tokens, tokensComplete: accumulateTaskTokens(task, event.tokensComplete), ...accumulateTaskCost(task, event.cost) };
 		default:
 			return resumed;
 	}

lib/session-aggregate.ts

@@ -134,6 +134,7 @@ interface MutableBucket {
 	nanoUsd: number;
 	absent: number;
 	complete: boolean;
+	tokensComplete: boolean;
 	input: number;
 	output: number;
 	cacheRead: number;
@@ -143,7 +144,7 @@ interface MutableBucket {
 }
 
 function emptyBucket(): MutableBucket {
-	return { turns: 0, nanoUsd: 0, absent: 0, complete: true, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, reasoning: 0, total: 0 };
+	return { turns: 0, nanoUsd: 0, absent: 0, complete: true, tokensComplete: true, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, reasoning: 0, total: 0 };
 }
 
 function addRecord(bucket: MutableBucket, record: SessionUsageRecord): void {
@@ -154,6 +155,9 @@ function addRecord(bucket: MutableBucket, record: SessionUsageRecord): void {
 		bucket.absent += 1;
 	}
 	const tokens = record.tokens;
+	// One record whose source omitted a counter makes the whole token figure a
+	// lower bound; like cost, it never returns to measured.
+	if (record.tokensComplete === false) bucket.tokensComplete = false;
 	bucket.input += tokens.input;
 	bucket.output += tokens.output;
 	bucket.cacheRead += tokens.cacheRead;
@@ -174,7 +178,7 @@ function tokenFigure(bucket: MutableBucket): TokenFigure {
 		cacheWrite: bucket.cacheWrite,
 		reasoning: bucket.reasoning,
 		total: bucket.total,
-		provenance: "measured",
+		provenance: bucket.tokensComplete ? "measured" : "partial",
 	};
 }
 

lib/session-store.ts

@@ -100,6 +100,8 @@ export interface StoredUsageLine {
 			readonly cacheWrite: number;
 			readonly reasoning: number;
 			readonly totalTokens: number;
+			/** Present only when the source omitted a counter, so the round trip stays honest. */
+			readonly tokensComplete?: boolean;
 			readonly cost?: Readonly<Record<string, number>>;
 		};
 	};
@@ -160,6 +162,7 @@ export function usageLineFromRecord(record: TranscriptUsageRecord): StoredUsageL
 				cacheWrite: record.tokens.cacheWrite,
 				reasoning: record.tokens.reasoning,
 				totalTokens: record.tokens.total,
+				...(record.tokensComplete === false ? { tokensComplete: false } : {}),
 				...(Object.keys(cost).length > 0 ? { cost } : {}),
 			},
 		},

lib/session-transcript.ts

@@ -136,6 +136,14 @@ function count(value: unknown): number {
 	return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.trunc(value) : 0;
 }
 
+/**
+ * A token counter is reported only when the source carried a finite number; an
+ * omitted counter stays `undefined` instead of becoming a measured zero.
+ */
+function tokenCount(value: unknown): number | undefined {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.trunc(value) : undefined;
+}
+
 function text(value: unknown, fallback: string): string {
 	return typeof value === "string" && value.length > 0 ? value : fallback;
 }
@@ -183,14 +191,27 @@ export function parseTranscriptLine(line: string, context: TranscriptReadContext
 	if (timestamp === undefined) return { kind: "malformed" };
 	const costSource = usage.cost;
 	const total = isObject(costSource) ? costSource.total : undefined;
+	const counts = {
+		input: tokenCount(usage.input),
+		output: tokenCount(usage.output),
+		cacheRead: tokenCount(usage.cacheRead),
+		cacheWrite: tokenCount(usage.cacheWrite),
+		reasoning: tokenCount(usage.reasoning),
+		total: tokenCount(usage.totalTokens ?? usage.total),
+	};
 	const tokens: UsageTokens = {
-		input: count(usage.input),
-		output: count(usage.output),
-		cacheRead: count(usage.cacheRead),
-		cacheWrite: count(usage.cacheWrite),
-		reasoning: count(usage.reasoning),
-		total: count(usage.totalTokens ?? usage.total),
+		input: counts.input ?? 0,
+		output: counts.output ?? 0,
+		cacheRead: counts.cacheRead ?? 0,
+		cacheWrite: counts.cacheWrite ?? 0,
+		reasoning: counts.reasoning ?? 0,
+		total: counts.total ?? 0,
 	};
+	// A source that omitted a counter makes the sum a lower bound. A stored line
+	// states the flag explicitly, so an incomplete record survives the round trip;
+	// otherwise presence is derived from the counters themselves.
+	const declaredComplete = usage.tokensComplete;
+	const tokensComplete = declaredComplete === false ? false : Object.values(counts).every((value) => value !== undefined);
 	const effort = typeof message.thinkingLevel === "string" ? message.thinkingLevel : undefined;
 	const stopReason = typeof message.stopReason === "string" ? message.stopReason : undefined;
 	return {
@@ -203,6 +224,7 @@ export function parseTranscriptLine(line: string, context: TranscriptReadContext
 			provider: text(message.provider, UNKNOWN),
 			...(effort ? { effort } : {}),
 			tokens,
+			tokensComplete,
 			cost: reportedCostOrAbsent(total),
 			costBreakdown: breakdownOf(costSource),
 			sessionId: context.sessionId,

lib/session-usage.ts

@@ -83,6 +83,13 @@ export interface SessionUsageRecord {
 	readonly provider: string;
 	readonly effort?: string;
 	readonly tokens: UsageTokens;
+	/**
+	 * False when the source omitted at least one token counter, so `tokens` is a
+	 * lower bound and the aggregated figure must render as partial. Absent means
+	 * complete, for a legacy record written before this distinction existed. A
+	 * reported all-zero set stays complete.
+	 */
+	readonly tokensComplete?: boolean;
 	readonly cost: UsageCost;
 	/** Present when the source carries a per-component split; a transcript does. */
 	readonly costBreakdown?: UsageCostBreakdown;
@@ -202,6 +209,21 @@ export function accumulateTaskCost(current: TaskCostLike, cost: UsageCost): { co
 	return { cost: nanoUsd / NANO_USD_SCALE, costComplete: current.costComplete !== false && cost.state === "reported" };
 }
 
+/** The minimal slice of a task record the delegated token fold reads. */
+export interface TaskTokenLike {
+	readonly tokensComplete?: boolean;
+}
+
+/**
+ * Task accumulation for tokens (I1), the exact counterpart of
+ * `accumulateTaskCost`: one absent component makes the token total partial for
+ * good, and a legacy record without the flag stays complete until one reports
+ * absence.
+ */
+export function accumulateTaskTokens(current: TaskTokenLike, reported: boolean): boolean {
+	return current.tokensComplete !== false && reported;
+}
+
 /** Delegated ingestion (I2): fold every subagent's known cost into one partial-aware total for the bar. */
 export function delegatedCostFromTasks(tasks: Iterable<TaskCostLike>): SessionCostTotal {
 	const accumulator = new SessionCostAccumulator();

lib/statistics-view.ts

@@ -41,6 +41,7 @@ export interface StatisticsRow {
 	readonly costNanoUsd: number;
 	readonly costProvenance: FigureProvenance;
 	readonly tokens: number;
+	readonly tokensProvenance: FigureProvenance;
 }
 
 export interface StatisticsRatio {
@@ -101,7 +102,7 @@ function ratio(value: number | null): string {
 }
 
 function rowFrom(key: string, title: string, subtitle: string | undefined, bucket: UsageBucket): StatisticsRow {
-	return { key, title, ...(subtitle !== undefined && subtitle.length > 0 ? { subtitle } : {}), turns: bucket.turns, costNanoUsd: bucket.cost.nanoUsd, costProvenance: bucket.cost.provenance, tokens: bucket.tokens.total };
+	return { key, title, ...(subtitle !== undefined && subtitle.length > 0 ? { subtitle } : {}), turns: bucket.turns, costNanoUsd: bucket.cost.nanoUsd, costProvenance: bucket.cost.provenance, tokens: bucket.tokens.total, tokensProvenance: bucket.tokens.provenance };
 }
 
 /** Build the display model from the aggregate and the optional timeline. Pure. */
@@ -163,6 +164,11 @@ function costText(model: StatisticsModel): string {
 	return `${usdFromNano(model.costNanoUsd)}${marker}`;
 }
 
+/** The token total with the same partial marker the cost uses; a partial sum is never a measured zero. */
+function tokensText(tokens: UsageBucket["tokens"]): string {
+	return `${formatTokens(tokens.total)}${tokens.provenance === "partial" ? "+" : ""} tokens`;
+}
+
 function timeSplit(model: StatisticsModel): string {
 	if (!model.timeline) return "n/a";
 	return `model ${formatDuration(model.timeline.modelMs)} · tools ${formatDuration(model.timeline.toolMs)} · idle ${formatDuration(model.timeline.idleMs)} (est.)`;
@@ -171,7 +177,7 @@ function timeSplit(model: StatisticsModel): string {
 function rowLines(row: StatisticsRow, width: number, style: StatisticsStyle): string[] {
 	const lines: string[] = [`  ${style.accent(row.title)}${row.subtitle ? style.dim(`  ${row.subtitle}`) : ""}`];
 	const marker = row.costProvenance === "partial" ? "+" : "";
-	lines.push(`    ${wrapParts([`${row.turns} turn${row.turns === 1 ? "" : "s"}`, `${usdFromNano(row.costNanoUsd)}${marker}`, `${formatTokens(row.tokens)} tokens`], width - 4).join("\n    ")}`);
+	lines.push(`    ${wrapParts([`${row.turns} turn${row.turns === 1 ? "" : "s"}`, `${usdFromNano(row.costNanoUsd)}${marker}`, `${formatTokens(row.tokens)}${row.tokensProvenance === "partial" ? "+" : ""} tokens`], width - 4).join("\n    ")}`);
 	return lines;
 }
 
@@ -190,7 +196,7 @@ function footerText(options: StatisticsRenderOptions): string {
 function renderNarrow(model: StatisticsModel, width: number, style: StatisticsStyle, options: StatisticsRenderOptions): string[] {
 	const lines: string[] = [];
 	lines.push(style.title("Session statistics"));
-	for (const line of wrapParts([costText(model), `${model.turns} turns`, `${formatTokens(model.tokens.total)} tokens`, model.timeline ? formatDuration(model.timeline.wallClockMs) : "n/a"], width)) lines.push(line);
+	for (const line of wrapParts([costText(model), `${model.turns} turns`, tokensText(model.tokens), model.timeline ? formatDuration(model.timeline.wallClockMs) : "n/a"], width)) lines.push(line);
 	if (model.timeline) for (const line of wrapParts([`model ${formatDuration(model.timeline.modelMs)}`, `tools ${formatDuration(model.timeline.toolMs)}`, `idle ${formatDuration(model.timeline.idleMs)} (est.)`], width)) lines.push(style.dim(line));
 	if (!options.compact) lines.push("");
 	lines.push(style.accent("Helpers"));
@@ -215,7 +221,7 @@ function renderWide(model: StatisticsModel, width: number, style: StatisticsStyl
 	const inner = Math.max(1, width - 4);
 	const content: string[] = [];
 	if (!options.compact) content.push("");
-	for (const line of wrapParts([costText(model), `${model.turns} turns`, `${formatTokens(model.tokens.total)} tokens`, model.timeline ? formatDuration(model.timeline.wallClockMs) : "n/a"], inner)) content.push(`  ${line}`);
+	for (const line of wrapParts([costText(model), `${model.turns} turns`, tokensText(model.tokens), model.timeline ? formatDuration(model.timeline.wallClockMs) : "n/a"], inner)) content.push(`  ${line}`);
 	if (model.timeline) content.push(`  ${style.dim(timeSplit(model))}`);
 	if (!options.compact) content.push("");
 	content.push(style.accent("  Helpers"));