diff --git a/docs/code/dashboard.md b/docs/code/dashboard.md index 4e884f4..e43aedf 100644 --- a/docs/code/dashboard.md +++ b/docs/code/dashboard.md @@ -28,13 +28,17 @@ The standalone command reads the database in read-only mode, so it is safe to ru ## What it shows -- **Run list**: every run with its status, task key or automation id, origin (tracker task, PR mention, or scheduled), agent harness, PR link, and duration. Filter by status or origin (`origin=scheduled` isolates automation runs). -- **Run detail**: a stage-by-stage timeline for one run: the feasibility verdict, the implementation summary, each self-review iteration, each human change request and how it was handled, and the final outcome. -- **Stats**: runs per week, success and escalation rates, median run duration, and a per-harness breakdown over a selectable window (7, 30, or 90 days, or all time). +- **Run list**: every run with its status, task key or automation id, origin (tracker task, PR mention, or scheduled), agent harness, PR link, duration, and known cost. Filter by status or origin (`origin=scheduled` isolates automation runs). +- **Run detail**: a stage-by-stage timeline for one run: the feasibility verdict, the implementation summary, each self-review iteration, each human change request and how it was handled, and the final outcome. Runs that reported token usage also show the model, token totals (input / output), and cost (only when the harness reported one). +- **Stats**: runs per week, success and escalation rates, median run duration, aggregate token usage and known spend, and a per-harness breakdown (including per-harness spend) over a selectable window (7, 30, or 90 days, or all time). - **Worker status**: whether the daemon is running, queued and failed events, open agent PRs, and per-source poll cursors. Success and escalation rates are computed over finished runs only. Run duration is measured from pickup to PR creation and is a proxy for ticket-to-PR time. Merge rate is not shown yet: the worker records PRs as open or closed but does not track merges separately. +### Usage and cost data quality + +Token/cost accounting depends on what each harness CLI reports: usage is captured only from structured JSON output (e.g. `--json` / `--output-format json` modes), and CLIs that don't emit it show no usage at all rather than zeros. Wherever data is partial, the dashboard says so explicitly — unknown costs render as "unknown" (never `$0.00`) and incomplete accounting is marked "(partial)". Aggregate stats sum only *known* values and show an explicit partial-data notice when runs in the window have missing or unpriced usage. + ## Options | Option | Description | diff --git a/packages/agent-harness/src/index.ts b/packages/agent-harness/src/index.ts index 580c07f..49c172e 100644 --- a/packages/agent-harness/src/index.ts +++ b/packages/agent-harness/src/index.ts @@ -128,6 +128,18 @@ export { type UsageLimitResult, } from "./detect-usage-limit.js"; +// Normalized token/cost usage extraction +export { + extractAgentUsage, + findJsonUsageObjects, + mergeAgentUsages, + normalizeJsonUsage, + type AgentUsage, + type MergedAgentUsage, + type UsageExtractionInput, + type UsageSource, +} from "./usage.js"; + // Incomplete implementation detection export { detectIncompleteImplementation, diff --git a/packages/agent-harness/src/usage.ts b/packages/agent-harness/src/usage.ts new file mode 100644 index 0000000..b4bbbcd --- /dev/null +++ b/packages/agent-harness/src/usage.ts @@ -0,0 +1,522 @@ +/** + * Normalized token/cost usage contract. + * + * Every harness can report consumption through structured output (a JSON + * result document or JSONL event stream — e.g. `codex exec --json`, + * `opencode run --format json`, `grok -p --output-format json`). + * {@link extractAgentUsage} normalizes whatever structured data a finished + * session's captured output contains into one {@link AgentUsage} value: + * + * - Unavailable values stay `null` — never zero — so callers can distinguish + * "the provider did not report this" from "this really was 0". + * - Only structured output is trusted. Human-readable summary lines are + * deliberately ignored: heuristics over free text misread echoed task + * content, test output, and agent prose, so unstructured output means + * unknown usage, not flaky guesses. + * + * Extractors are defensive by design: harness CLIs change their output + * between versions, so any parse failure degrades to fewer populated fields + * (or `null` overall) instead of throwing into the run pipeline. + */ + +import { stripAnsi } from "./output-lines.js"; + +/** Where a usage reading was extracted from. */ +export type UsageSource = + | "structured_output" // embedded/piped JSON result object or JSONL events + | "session_artifacts" // on-disk session transcripts written by the CLI + | "mixed"; // merged from sessions with differing sources + +/** Normalized usage for one agent session. All numeric fields may be null. */ +export interface AgentUsage { + /** Prompt (+ cache-write) input tokens as reported by the provider. */ + inputTokens: number | null; + /** Completion/output tokens (includes reasoning tokens when not separate). */ + outputTokens: number | null; + /** Cache-read / cached-input tokens, when reported separately. */ + cachedInputTokens: number | null; + /** Reasoning/thinking tokens, when reported separately. */ + reasoningTokens: number | null; + /** + * Total tokens across categories. Populated only when the provider reports + * a total directly; never inferred from incomplete parts. + */ + totalTokens: number | null; + /** + * Model the session ran on, as reported by the CLI (may be an alias). + * Null when the harness does not surface it in headless output. + */ + model: string | null; + /** Provider-computed cost (USD unless {@link costCurrency} says otherwise). */ + reportedCost: number | null; + costCurrency: string | null; + /** Primary source of this extraction. */ + source: UsageSource; + /** + * True when both input and output tokens were found and the model (or a + * provider cost) is known. Partial reads are still returned, just with + * `complete: false`. + */ + complete: boolean; +} + +/** Inputs available to a usage extractor after a session exits. */ +export interface UsageExtractionInput { + harness: string; + stdout: string; + stderr: string; +} + +function emptyUsage(source: UsageSource): AgentUsage { + return { + inputTokens: null, + outputTokens: null, + cachedInputTokens: null, + reasoningTokens: null, + totalTokens: null, + model: null, + reportedCost: null, + costCurrency: null, + source, + complete: false, + }; +} + +/** + * A usage value counts as complete when token totals exist for both prompt + * and completion sides and we know what model (or cost) produced them. + */ +function isComplete(usage: AgentUsage): boolean { + const hasTokenPair = usage.inputTokens !== null && usage.outputTokens !== null; + if (!hasTokenPair && usage.totalTokens === null) { + return false; + } + return usage.model !== null || usage.reportedCost !== null; +} + +function finalize(usage: AgentUsage): AgentUsage { + return { ...usage, complete: isComplete(usage) }; +} + +// --------------------------------------------------------------------------- +// JSON scanning (structured output embedded in stdout/stderr) +// --------------------------------------------------------------------------- + +interface RawUsageFields { + inputTokens?: number; + outputTokens?: number; + cachedInputTokens?: number; + reasoningTokens?: number; + totalTokens?: number; + model?: string; + cost?: number; + currency?: string; +} + +const JSON_USAGE_FIELD_ALIASES: Record = { + inputTokens: ["input_tokens", "inputTokens", "prompt_tokens", "promptTokenCount", "tokens.input"], + outputTokens: [ + "output_tokens", + "outputTokens", + "completion_tokens", + "candidatesTokenCount", + "tokens.output", + ], + cachedInputTokens: [ + "cache_read_input_tokens", + "cached_input_tokens", + "cachedContentTokenCount", + "cache_read_tokens", + "prompt_tokens_details.cached_tokens", + "tokens.cache.read", + ], + reasoningTokens: [ + "reasoning_tokens", + "completion_tokens_details.reasoning_tokens", + "tokens.reasoning", + ], + totalTokens: ["total_tokens", "totalTokens", "total_token_count", "tokens.total"], + model: ["model", "modelId", "model_id"], + cost: ["total_cost_usd", "cost_usd", "costUSD", "total_cost", "cost"], + currency: ["currency", "cost_currency"], +}; + +/** Read a dotted alias ("a.b") out of a nested object. */ +function readDotted(object: Record, path: string): unknown { + let current: unknown = object; + for (const part of path.split(".")) { + if (current === null || typeof current !== "object") { + return undefined; + } + current = (current as Record)[part]; + } + return current; +} + +function firstNumber(object: Record, aliases: readonly string[]): number | null { + for (const alias of aliases) { + const value = readDotted(object, alias); + if (typeof value === "number" && Number.isFinite(value) && value >= 0) { + return value; + } + } + return null; +} + +function firstString(object: Record, aliases: readonly string[]): string | null { + for (const alias of aliases) { + const value = readDotted(object, alias); + if (typeof value === "string" && value.trim()) { + return value.trim(); + } + } + return null; +} + +/** Map loose JSON keys onto the normalized shape; undefined fields stay absent. */ +export function normalizeJsonUsage(object: Record): RawUsageFields | null { + const raw: RawUsageFields = {}; + const input = firstNumber(object, JSON_USAGE_FIELD_ALIASES.inputTokens); + const output = firstNumber(object, JSON_USAGE_FIELD_ALIASES.outputTokens); + const total = firstNumber(object, JSON_USAGE_FIELD_ALIASES.totalTokens); + const cost = firstNumber(object, JSON_USAGE_FIELD_ALIASES.cost); + const model = firstString(object, JSON_USAGE_FIELD_ALIASES.model); + // A bare `model` key (configs, fixtures, API payloads) is not usage; at + // least one consumption number must be present. + if (input === null && output === null && total === null && cost === null) { + return null; + } + if (input !== null) { + raw.inputTokens = input; + } + if (output !== null) { + raw.outputTokens = output; + } + const cached = firstNumber(object, JSON_USAGE_FIELD_ALIASES.cachedInputTokens); + if (cached !== null) { + raw.cachedInputTokens = cached; + } + const reasoning = firstNumber(object, JSON_USAGE_FIELD_ALIASES.reasoningTokens); + if (reasoning !== null) { + raw.reasoningTokens = reasoning; + } + if (total !== null) { + raw.totalTokens = total; + } + if (model !== null) { + raw.model = model; + } + if (cost !== null) { + raw.cost = cost; + } + const currency = firstString(object, JSON_USAGE_FIELD_ALIASES.currency); + if (currency !== null) { + raw.currency = currency; + } + return raw; +} + +const NUMERIC_USAGE_KEYS = [ + "inputTokens", + "outputTokens", + "cachedInputTokens", + "reasoningTokens", + "totalTokens", + "cost", +] as const; + +/** + * Sum sibling contributions: per-model breakdowns describe disjoint token + * buckets, so their counts and costs add up. + */ +function sumContributions(list: RawUsageFields[]): RawUsageFields | null { + if (list.length === 0) { + return null; + } + if (list.length === 1) { + return list[0]!; + } + const out: RawUsageFields = {}; + for (const key of NUMERIC_USAGE_KEYS) { + let sum: number | undefined; + for (const item of list) { + const value = item[key]; + if (typeof value === "number") { + sum = (sum ?? 0) + value; + } + } + if (sum !== undefined) { + out[key] = sum; + } + } + for (const key of ["model", "currency"] as const) { + out[key] = list.find((item) => item[key] !== undefined)?.[key]; + } + return out; +} + +/** + * Keys whose direct-child object is an *aggregate* usage view. When present, + * it wins over sibling breakdowns (per-model maps describe the same tokens). + */ +const AGGREGATE_USAGE_KEYS = new Set(["usage"]); + +/** Reduce one parsed JSON document to a single usage contribution. + * + * - Per-model maps (`modelUsage`) are disjoint siblings → summed. + * - An aggregate `usage` sibling wins over breakdowns describing the same + * tokens (no double counting). + * - Parent-only fields (a top-level provider cost) survive. + * - Transcript arrays hold cumulative rows → last row wins. + */ +function collectFromParsed(root: unknown): RawUsageFields | null { + if (root === null || typeof root !== "object") { + return null; + } + if (Array.isArray(root)) { + let last: RawUsageFields | null = null; + for (const item of root) { + const contribution = collectFromParsed(item); + if (contribution) { + last = contribution; + } + } + return last; + } + + const record = root as Record; + let aggregate: RawUsageFields | null = null; + const breakdowns: RawUsageFields[] = []; + for (const [key, value] of Object.entries(record)) { + if (value === null || typeof value !== "object") { + continue; + } + const contribution = collectFromParsed(value); + if (!contribution) { + continue; + } + if (AGGREGATE_USAGE_KEYS.has(key)) { + aggregate ??= contribution; + } else { + breakdowns.push(contribution); + } + } + + const self = normalizeJsonUsage(record); + const children = aggregate ?? sumContributions(breakdowns); + if (self === null) { + return children; + } + if (children === null) { + return self; + } + // Both levels carry usage fields: token counts come from the deeper, + // more granular view; parent-exclusive totals (provider-reported cost) + // win over values derived from the breakdown. + const merged: RawUsageFields = { ...children }; + for (const key of NUMERIC_USAGE_KEYS) { + if (merged[key] === undefined && self[key] !== undefined) { + merged[key] = self[key]; + } + } + if (self.cost !== undefined) { + merged.cost = self.cost; + } + if (self.currency !== undefined) { + merged.currency = self.currency; + } + if (!merged.model && self.model !== undefined) { + merged.model = self.model; + } + return merged; +} + +const USAGE_MARKER_SENTINEL = + /"(?:input_tokens|inputTokens|prompt_tokens|output_tokens|outputTokens|completion_tokens|total_tokens|totalTokens|total_cost_usd|costUSD|modelUsage|usage|tokens)"/; + +/** + * Summary lines appear at the end of a run; scanning the whole transcript + * would risk matching source code, test output, or docs the agent printed. + */ +const TAIL_SCAN_LINES = 60; + +/** + * Find usage-bearing JSON objects in a captured stream. + * + * Handles streams that are entirely one JSON document (structured result + * mode) plus single-line JSON / JSONL rows near the end of plain output. + * Everything else is ignored, which keeps echoed fixtures, app logs, and + * test output from being mistaken for provider reporting. + */ +export function findJsonUsageObjects(text: string): RawUsageFields[] { + if (!text || !text.includes("{")) { + return []; + } + const found: RawUsageFields[] = []; + const trimmed = stripAnsi(text).trim(); + + const whole = collectFromParsed(safeParse(trimmed)); + if (whole) { + return [whole]; + } + + for (const line of trimmed.split(/\r?\n/).slice(-TAIL_SCAN_LINES)) { + const candidate = line.trim(); + if (!candidate.startsWith("{") || !USAGE_MARKER_SENTINEL.test(candidate)) { + continue; + } + const contribution = collectFromParsed(safeParse(candidate)); + if (contribution) { + found.push(contribution); + } + } + return found; +} + +function safeParse(text: string): unknown { + try { + return JSON.parse(text); + } catch { + return null; + } +} + +function mergeRawIntoUsage(usage: AgentUsage, raw: RawUsageFields): void { + if (raw.inputTokens !== undefined) { + usage.inputTokens = raw.inputTokens; + } + if (raw.outputTokens !== undefined) { + usage.outputTokens = raw.outputTokens; + } + if (raw.cachedInputTokens !== undefined) { + usage.cachedInputTokens = raw.cachedInputTokens; + } + if (raw.reasoningTokens !== undefined) { + usage.reasoningTokens = raw.reasoningTokens; + } + if (raw.totalTokens !== undefined) { + usage.totalTokens = raw.totalTokens; + } + if (raw.model !== undefined) { + usage.model = raw.model; + } + if (raw.cost !== undefined) { + usage.reportedCost = raw.cost; + usage.costCurrency = raw.currency ?? "USD"; + } +} + +// --------------------------------------------------------------------------- +// Per-harness extractors +// --------------------------------------------------------------------------- + +/** + * Extract usage from a finished session's captured output. + * + * Only structured output counts: whole-document JSON results and usage-bearing + * JSONL rows near the end of the stream. Human-readable summary lines are + * deliberately ignored — heuristics over free text misread echoed task + * content, test output, and agent prose, so anything not structured stays + * unknown instead of flaky. + */ +function extractStructuredUsage(input: UsageExtractionInput): AgentUsage | null { + const jsonCandidates = [ + ...findJsonUsageObjects(input.stdout), + ...findJsonUsageObjects(input.stderr), + ]; + if (jsonCandidates.length === 0) { + return null; + } + const usage = emptyUsage("structured_output"); + for (const raw of jsonCandidates) { + mergeRawIntoUsage(usage, raw); + } + return finalize(usage); +} + +/** + * Normalize a finished agent session's output into usage data. + * + * Returns `null` when no structured usage was found — that is an explicit + * "usage unknown" rather than zeros or a guess. + * + * @param input - Harness id plus the session's captured stdout/stderr. + */ +export function extractAgentUsage(input: UsageExtractionInput): AgentUsage | null { + try { + return extractStructuredUsage(input); + } catch { + // Extraction must never break the run pipeline. + return null; + } +} + +// --------------------------------------------------------------------------- +// Multi-session merging (implementation + feasibility + review sessions) +// --------------------------------------------------------------------------- + +export interface MergedAgentUsage extends AgentUsage { + /** Number of contributing sessions. */ + sessions: number; + /** Sessions that yielded no usage signal at all (unknown exposure). */ + sessionsWithoutUsage: number; + /** True when at least one contributing session had a different model. */ + mixedModels: boolean; +} + +/** + * Merge usage from every session attributable to one run without double + * counting: token categories sum across sessions, `null` survives when no + * session reported a category, costs are summed only over sessions that + * reported them. + */ +export function mergeAgentUsages(usages: (AgentUsage | null)[]): MergedAgentUsage | null { + const present = usages.filter((usage): usage is AgentUsage => usage !== null); + const merged: MergedAgentUsage = { + ...emptyUsage(present.length === 0 ? "structured_output" : present[0]!.source), + sessions: usages.length, + sessionsWithoutUsage: usages.length - present.length, + mixedModels: false, + }; + + if (present.length === 0) { + return merged.sessions > 0 ? merged : null; + } + + const referenceSource = present[0]!.source; + merged.source = present.every((usage) => usage.source === referenceSource) + ? referenceSource + : "mixed"; + + const models = new Set(); + for (const usage of present) { + merged.inputTokens = sumNullable(merged.inputTokens, usage.inputTokens); + merged.outputTokens = sumNullable(merged.outputTokens, usage.outputTokens); + merged.cachedInputTokens = sumNullable(merged.cachedInputTokens, usage.cachedInputTokens); + merged.reasoningTokens = sumNullable(merged.reasoningTokens, usage.reasoningTokens); + merged.totalTokens = sumNullable(merged.totalTokens, usage.totalTokens); + merged.reportedCost = sumNullable(merged.reportedCost, usage.reportedCost); + if (usage.costCurrency) { + merged.costCurrency = merged.costCurrency ?? usage.costCurrency; + } + if (usage.model) { + models.add(usage.model); + merged.model = merged.model ?? usage.model; + } + } + merged.mixedModels = models.size > 1; + + const allComplete = present.every((usage) => usage.complete); + merged.complete = allComplete && merged.sessionsWithoutUsage === 0; + return merged; +} + +function sumNullable(a: number | null, b: number | null): number | null { + if (a === null) { + return b; + } + if (b === null) { + return a; + } + return a + b; +} diff --git a/packages/agent-harness/tests/sandbox-providers.test.ts b/packages/agent-harness/tests/sandbox-providers.test.ts index 3ba0630..f92e867 100644 --- a/packages/agent-harness/tests/sandbox-providers.test.ts +++ b/packages/agent-harness/tests/sandbox-providers.test.ts @@ -14,7 +14,30 @@ import type { SandboxPolicy } from "../src/sandbox/types.js"; mock.module("child_process", () => ({ execSync, spawn: nodeSpawn, - spawnSync: () => ({ status: 0, stdout: "", stderr: "" }), + spawnSync: ( + command: string, + args: string[] = [], + options: { cwd?: string; env?: Record } = {}, + ) => { + if (command === "nono") { + return { status: 0, stdout: "", stderr: "" }; + } + try { + const result = Bun.spawnSync([command, ...args], { + cwd: options.cwd, + env: options.env, + stdout: "pipe", + stderr: "pipe", + }); + return { + status: result.exitCode, + stdout: result.stdout.toString(), + stderr: result.stderr.toString(), + }; + } catch (error) { + return { status: null, stdout: "", stderr: "", error }; + } + }, })); const { NonoSandboxProvider, parseDenyOverlaps, refineGrantsAgainstDenies } = diff --git a/packages/agent-harness/tests/usage.test.ts b/packages/agent-harness/tests/usage.test.ts new file mode 100644 index 0000000..cac15d9 --- /dev/null +++ b/packages/agent-harness/tests/usage.test.ts @@ -0,0 +1,224 @@ +import { describe, expect, test } from "bun:test"; + +import { + extractAgentUsage, + findJsonUsageObjects, + mergeAgentUsages, +} from "../src/usage.js"; +import type { AgentUsage, UsageExtractionInput } from "../src/usage.js"; + + +function extract(harness: string, stdout = "", stderr = ""): ReturnType { + const input: UsageExtractionInput = { harness, stdout, stderr }; + return extractAgentUsage(input); +} + +describe("findJsonUsageObjects", () => { + test("parses a whole-document JSON result with nested usage", () => { + const text = JSON.stringify({ + type: "result", + total_cost_usd: 0.42, + modelUsage: { + "claude-sonnet-4-5": { + inputTokens: 1000, + outputTokens: 500, + cacheReadInputTokens: 200, + }, + }, + }); + const found = findJsonUsageObjects(text); + expect(found.length).toBeGreaterThan(0); + const merged = found.reduce>((acc, raw) => { + Object.assign(acc, raw); + return acc; + }, {}); + expect(merged.inputTokens).toBe(1000); + expect(merged.outputTokens).toBe(500); + expect(merged.cost).toBe(0.42); + }); + + test("does not double count aggregate + per-model views of the same tokens", () => { + const text = JSON.stringify({ + costUSD: 1.5, + usage: { input_tokens: 800, output_tokens: 400, total_tokens: 1200 }, + modelUsage: { "gpt-5": { input_tokens: 800, output_tokens: 400 } }, + }); + const found = findJsonUsageObjects(text); + // The deepest subtree wins; the aggregate parent is skipped. + const inputs = found.map((raw) => raw.inputTokens ?? null).filter((value) => value !== null); + expect(inputs).toEqual([800]); + }); + + test("scans JSONL rows mixed into plain output", () => { + const text = [ + "working...", + `{"type":"assistant","message":{"model":"claude-sonnet-4-5","usage":{"input_tokens":10,"output_tokens":5}}}`, + "done", + ].join("\n"); + const found = findJsonUsageObjects(text); + expect(found.length).toBe(1); + expect(found[0]?.inputTokens).toBe(10); + expect(found[0]?.outputTokens).toBe(5); + }); + + test("ignores prose mentioning costs", () => { + expect(findJsonUsageObjects("The total cost is $5 and input tokens are large")).toEqual([]); + }); + + test("returns empty for non-JSON garbage", () => { + expect(findJsonUsageObjects("{not json")).toEqual([]); + }); + + test("JSONL usage rows far from the end of a transcript are ignored", () => { + const filler = Array.from({ length: 200 }, (_, i) => `doing work ${i}`).join("\n"); + const after = Array.from({ length: 80 }, (_, i) => `more work ${i}`).join("\n"); + const text = `${filler}\n{"usage": {"prompt_tokens": 120, "completion_tokens": 30}}\n${after}`; + // Only the whole-document parse is trusted; mid-transcript JSONL rows + // look like app/test output, not provider reporting. + expect(findJsonUsageObjects(text)).toEqual([]); + }); +}); + +describe("extractAgentUsage", () => { + test("claude-code structured result yields complete usage", () => { + const usage = extract( + "claude-code", + JSON.stringify({ + type: "result", + subtype: "success", + total_cost_usd: 0.12, + modelUsage: { "claude-sonnet-4-5": { inputTokens: 900, outputTokens: 300 } }, + }), + ); + expect(usage).not.toBeNull(); + expect(usage?.source).toBe("structured_output"); + expect(usage?.inputTokens).toBe(900); + expect(usage?.outputTokens).toBe(300); + expect(usage?.reportedCost).toBe(0.12); + expect(usage?.costCurrency).toBe("USD"); + expect(usage?.complete).toBe(true); + }); + + test("claude-code plain-text mode reports explicit unknown (null)", () => { + expect(extract("claude-code", "Implemented the feature.\n\nDone.")).toBeNull(); + }); + + test("codex --jsonl turn events yield token counts", () => { + const usage = extract( + "codex", + '{"type":"turn.completed","usage":{"input_tokens":17439,"cached_input_tokens":11008,"cache_write_input_tokens":0,"output_tokens":5,"reasoning_output_tokens":0}}', + ); + expect(usage?.inputTokens).toBe(17439); + expect(usage?.cachedInputTokens).toBe(11008); + expect(usage?.outputTokens).toBe(5); + }); + + test("opencode --format json step_finish events yield full token breakdown", () => { + const usage = extract( + "opencode", + '{"type":"step_finish","part":{"tokens":{"total":9527,"input":2151,"output":11,"reasoning":5,"cache":{"write":0,"read":7360}},"cost":0}}', + ); + expect(usage?.inputTokens).toBe(2151); + expect(usage?.cachedInputTokens).toBe(7360); + expect(usage?.outputTokens).toBe(11); + expect(usage?.reasoningTokens).toBe(5); + expect(usage?.totalTokens).toBe(9527); + }); + + test("grok --output-format json result yields tokens and provider cost", () => { + const usage = extract( + "grok", + JSON.stringify({ + text: "ok", + usage: { input_tokens: 3472, cache_read_input_tokens: 11520, output_tokens: 37 }, + total_cost_usd: 0.00219742, + modelUsage: { "grok-4": { inputTokens: 3472, outputTokens: 37 } }, + }), + ); + expect(usage?.inputTokens).toBe(3472); + expect(usage?.cachedInputTokens).toBe(11520); + expect(usage?.totalTokens).toBeNull(); + expect(usage?.reportedCost).toBeCloseTo(0.00219742); + expect(usage?.complete).toBe(true); + }); + + test("unstructured output (prose, summaries, source) yields explicit null", () => { + // Real default-mode outputs: no structured usage → unknown, never guesses. + expect(extract("codex", "ok\n", "tokens used\n6,530\n")).toBeNull(); + expect(extract("grok", "ok\n")).toBeNull(); + expect(extract("opencode", "ok\n")).toBeNull(); + const stdout = ["const inputTokens = config.input;", "+ total cost: $999", "done"].join("\n"); + expect(extract("goose", stdout)).toBeNull(); + expect(extract("codex", "", "done\ntokens used: 500\ntotal cost: $0.42")).toBeNull(); + }); + + test("model-only JSON (configs, fixtures) is not usage", () => { + expect(extract("opencode", '{"name":"x","model":"y","scripts":{"t":"t"}}')).toBeNull(); + }); + + test("malformed JSON degrades to null instead of throwing", () => { + expect(extract("opencode", '{"usage": {"input_truncated')).toBeNull(); + }); + + test("zero-token sessions stay zero, not null", () => { + const usage = extract( + "claude-code", + JSON.stringify({ modelUsage: { "claude-haiku-4": { inputTokens: 0, outputTokens: 0 } } }), + ); + expect(usage?.inputTokens).toBe(0); + expect(usage?.outputTokens).toBe(0); + }); +}); + +describe("mergeAgentUsages", () => { + const base: AgentUsage = { + inputTokens: 100, + outputTokens: 50, + cachedInputTokens: null, + reasoningTokens: null, + totalTokens: null, + model: "claude-sonnet-4-5", + reportedCost: 0.01, + costCurrency: "USD", + source: "structured_output", + complete: true, + }; + + test("sums tokens and costs across sessions without double counting", () => { + const merged = mergeAgentUsages([base, { ...base, reportedCost: 0.02 }]); + expect(merged?.inputTokens).toBe(200); + expect(merged?.outputTokens).toBe(100); + expect(merged?.reportedCost).toBeCloseTo(0.03); + expect(merged?.sessions).toBe(2); + expect(merged?.complete).toBe(true); + }); + + test("null survives when no session reported a category", () => { + const merged = mergeAgentUsages([base, { ...base, cachedInputTokens: 7 }]); + expect(merged?.cachedInputTokens).toBe(7); // first session had none + }); + + test("all-null usages produce an explicit unknown-exposure record", () => { + const merged = mergeAgentUsages([null, null]); + expect(merged).not.toBeNull(); + expect(merged?.sessions).toBe(2); + expect(merged?.sessionsWithoutUsage).toBe(2); + expect(merged?.complete).toBe(false); + expect(merged?.inputTokens).toBeNull(); + }); + + test("empty list merges to null", () => { + expect(mergeAgentUsages([])).toBeNull(); + }); + + test("mixed models are flagged", () => { + const merged = mergeAgentUsages([base, { ...base, model: "gpt-5" }]); + expect(merged?.mixedModels).toBe(true); + expect(merged?.model).toBe("claude-sonnet-4-5"); + }); + + test("incomplete sessions mark the merge incomplete", () => { + const merged = mergeAgentUsages([{ ...base, complete: false }]); + expect(merged?.complete).toBe(false); + }); +}); diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index efa39b0..d897af6 100755 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -78,7 +78,14 @@ import { import { normalizeTaskKeys } from "./lib/normalize-task-keys"; import { LockManager } from "./lib/lock-manager"; import { PRManager } from "./lib/pr-client"; -import { RunStore, beginRun, endRun, recordRunPr, recordRunStage } from "./lib/run-recorder"; +import { + RunStore, + beginRun, + endRun, + recordRunPr, + recordRunStage, + recordSessionOutput, +} from "./lib/run-recorder"; import { clearRetryState, getRetryState, recordIncompleteAttempt } from "./lib/retry-state"; import { shouldSkipRetry } from "./lib/retry-gate"; import { formatProcessingFailureMarkdown } from "./lib/trackers/shared/markdown-comment-formatter"; @@ -712,6 +719,11 @@ if (process.argv[2] === "init") { }); requireLicense(licenseResult); + // Mark this process tree as unattended so run records can distinguish + // worker-originated runs from manual CLI runs. Workspace task and event + // subprocesses inherit this marker through their composed repo env. + process.env.DEVINTERN_WORKER = "1"; + const { runWorkspaceWorker } = await import("./lib/workspace/workspace-worker"); await runWorkspaceWorker({ workspacePath, @@ -1389,6 +1401,7 @@ async function processSingleTask(taskKey: string, taskIndex = 0, totalTasks = 1) origin: scheduledAutomationId ? "scheduled" : "task", taskKey: workflowKey, tracker: process.env.TASK_TRACKER || "jira", + unattended: process.env.DEVINTERN_WORKER === "1" || undefined, ...(scheduledAutomationId ? { automationId: scheduledAutomationId } : {}), }); @@ -2410,6 +2423,8 @@ async function runClarityCheck( clarityAgent.on("close", async (code: number | null) => { clearTimeout(timeout); sandboxCleanup().catch(() => {}); + // Attribute this feasibility session's usage to the current run. + recordSessionOutput(harness.name, stdoutOutput, stderrOutput); if (timedOut) { reject( new Error( @@ -2808,6 +2823,8 @@ async function runEstimation( estimationAgent.on("close", async (code: number | null) => { clearTimeout(timeout); sandboxCleanup().catch(() => {}); + // Attribute this estimation session's usage to the current run. + recordSessionOutput(harness.name, stdoutOutput, stderrOutput); if (timedOut) { reject(new Error(`Agent estimation timed out after ${timeoutMinutes} minutes`)); @@ -3267,6 +3284,8 @@ async function runAgentHarness( codeAgent.on("close", async (code: number | null) => { clearTimeout(timeout); sandboxCleanup().catch(() => {}); + // Attribute this implementation session's usage to the current run. + recordSessionOutput(harness.name, stdoutOutput, stderrOutput); console.log("\n" + "=".repeat(60)); if (timedOut) { @@ -3805,6 +3824,8 @@ async function runAgentHarness( retryProcess.on("close", async (retryCode: number | null) => { retrySandboxCleanup().catch(() => {}); + // Attribute this plan-retry session's usage to the run. + recordSessionOutput(harness.name, retryStdoutOutput, retryStderrOutput); console.log("\n" + "=".repeat(60)); if (retryCode === 0) { diff --git a/packages/code/src/lib/address-review.ts b/packages/code/src/lib/address-review.ts index e7972c8..b1c711a 100644 --- a/packages/code/src/lib/address-review.ts +++ b/packages/code/src/lib/address-review.ts @@ -17,7 +17,7 @@ import { resolveAgentModel } from "./agent-model"; import { getSandbox } from "./sandbox"; import { GitHubReviewsClient } from "./github-reviews"; import { GitHubAppAuth } from "./github-app-auth"; -import { beginRun, endRun, recordRunStage } from "./run-recorder"; +import { beginRun, endRun, recordRunStage, recordSessionOutput } from "./run-recorder"; import { formatReviewPrompt } from "./review-formatter"; import { GIT_CLEAN_ARGS, Utils } from "./utils"; import { isCommitAlreadyComplete, runAgentHarnessToFixGitHook } from "./git-hook-fixer"; @@ -213,6 +213,8 @@ export async function runAgent( agent.on("close", (code: number | null) => { clearTimeout(timeout); sandboxCleanup().catch(() => {}); + // Attribute this change-request session's usage to the current run. + recordSessionOutput(harness.name, stdoutOutput, stderrOutput); const maxTurnsReached = detectMaxTurnsReached( stdoutOutput, stderrOutput, @@ -485,6 +487,8 @@ export async function addressReview( repo: `${owner}/${repo}`, prNumber, branch: pr.head.ref, + harness: resolveHarness().harness.name, + unattended: process.env.DEVINTERN_WORKER === "1" || undefined, }); recordRunStage("change_request", { status: "succeeded", diff --git a/packages/code/src/lib/auto-review-loop.ts b/packages/code/src/lib/auto-review-loop.ts index c314113..dce024e 100644 --- a/packages/code/src/lib/auto-review-loop.ts +++ b/packages/code/src/lib/auto-review-loop.ts @@ -23,6 +23,7 @@ import { parseAgentJsonObject } from "./agent-json"; import { buildHeadlessAgentArgs, HEADLESS_AGENT_STDIO } from "./agent-spawn"; import { resolveAgentModel } from "./agent-model"; import { getSandbox } from "./sandbox"; +import { recordSessionOutput } from "./run-recorder"; import type { AutoReviewLoopOptions, AutoReviewLoopResult, @@ -391,6 +392,8 @@ async function runAgentPrompt( agentProcess.on("close", (code) => { clearTimeout(timeout); sandboxCleanup().catch(() => {}); + // Attribute this auto-review session's usage to the current run. + recordSessionOutput(harness.name, stdout, stderr); if (timedOut) { reject(new Error(`${harness.displayName} timed out after ${timeoutMinutes} minutes`)); } else if (usageLimit?.limited) { diff --git a/packages/code/src/lib/git-hook-fixer.ts b/packages/code/src/lib/git-hook-fixer.ts index d699e4c..54c5a04 100644 --- a/packages/code/src/lib/git-hook-fixer.ts +++ b/packages/code/src/lib/git-hook-fixer.ts @@ -15,6 +15,7 @@ import type { AgentHarness } from "@devintern/agent-harness"; import { buildHeadlessAgentArgs, HEADLESS_AGENT_STDIO } from "./agent-spawn"; import { resolveAgentModel } from "./agent-model"; import { getSandbox } from "./sandbox"; +import { recordSessionOutput } from "./run-recorder"; import { Utils } from "./utils"; import { resolveOutputDir } from "./output-dir"; @@ -333,6 +334,8 @@ ${hookType === "push" ? "- Make sure to amend the commit (git commit --amend --n agent.on("close", async (code: number | null) => { clearTimeout(timeout); sandboxCleanup().catch(() => {}); + // Attribute this hook-fix session's usage to the current run. + recordSessionOutput(harness.name, stdoutOutput, stderrOutput); if (timedOut) { console.error( `❌ ${harness.displayName} timed out after ${timeoutMinutes} minutes while fixing git hook`, diff --git a/packages/code/src/lib/run-recorder.ts b/packages/code/src/lib/run-recorder.ts index 45c6a17..ef982c9 100644 --- a/packages/code/src/lib/run-recorder.ts +++ b/packages/code/src/lib/run-recorder.ts @@ -13,6 +13,8 @@ */ import { Database } from "bun:sqlite"; +import { extractAgentUsage, mergeAgentUsages } from "@devintern/agent-harness"; +import type { AgentUsage, MergedAgentUsage } from "@devintern/agent-harness"; import { prepareQueueDbDirectory, resolveQueueDbPath } from "./webhook-queue"; export type RunOrigin = "task" | "pr_mention" | "conflict_resolution" | "scheduled"; @@ -50,11 +52,40 @@ export interface RunMeta { branch?: string; repo?: string; prNumber?: number; + /** + * True when the run was started by the unattended worker (polling, relay, + * webhook, mention, or workspace paths) rather than a manual CLI run. + */ + unattended?: boolean; automationId?: string; /** Explicit attempt for non-task durable events. */ attempt?: number; } +/** + * Normalized token/cost usage persisted with one run. Null means the + * provider (or extraction) did not supply the value — never zero. + */ +export interface RunUsage { + /** Where usage numbers came from ("mixed" when sessions disagree). */ + source: MergedAgentUsage["source"] | null; + /** False when any session's accounting was partial or missing entirely. */ + complete: boolean; + model: string | null; + inputTokens: number | null; + outputTokens: number | null; + cachedInputTokens: number | null; + reasoningTokens: number | null; + totalTokens: number | null; + /** Known cost for the whole run in `costCurrency` (USD unless stated). */ + costUsd: number | null; + costCurrency: string | null; + /** Number of agent sessions attributable to this run. */ + sessionCount: number; + /** Sessions that yielded no usage signal at all (unknown exposure). */ + sessionsWithoutUsage: number; +} + export interface RunRecord extends RunMeta { id: number; /** 1-based attempt number for the task (null-ish for pr_mention runs). */ @@ -66,6 +97,7 @@ export interface RunRecord extends RunMeta { outcomeReason?: string; startedAt: number; finishedAt?: number; + usage?: RunUsage | null; } export interface RunStageRecord { @@ -101,6 +133,29 @@ export interface RunStatsHarness { escalated: number; /** Median duration of succeeded runs, or null when none finished. */ medianDurationMs: number | null; + /** Known spend for this harness in USD; null when no run reported cost. */ + spendUsd: number | null; + /** Runs in this harness with usage but unknown cost (never counted as $0). */ + runsWithUnknownCost: number; +} + +/** Aggregate token/cost usage over a stats window. */ +export interface RunStatsUsage { + /** Sum of known per-category tokens; categories never inferred. */ + inputTokens: number | null; + outputTokens: number | null; + cachedInputTokens: number | null; + reasoningTokens: number | null; + totalTokens: number | null; + /** Known reported spend in USD; null when nothing is priced. */ + knownSpendUsd: number | null; + currency: "USD"; + /** Runs with at least one usage-bearing session. */ + runsWithUsage: number; + /** Terminal-ish runs without usage data (unknown exposure). */ + runsWithoutUsage: number; + /** Runs whose sessions had partial accounting or unpriced models. */ + runsWithIncompleteUsage: number; } export interface RunStats { @@ -114,11 +169,33 @@ export interface RunStats { medianDurationMs: number | null; byHarness: RunStatsHarness[]; byOrigin: Record; + usage: RunStatsUsage; } /** Statuses that no longer change (excluded: in_progress and deferred-for-retry). */ const TERMINAL_STATUSES: RunStatus[] = ["succeeded", "failed", "escalated", "abandoned"]; +/** + * Additive usage/cost columns, applied idempotently on every open. All are + * nullable: historical rows (and runs whose harness reported nothing) keep + * unknown usage rather than zeros. + */ +const USAGE_SCHEMA_MIGRATIONS: { column: string; decl: string }[] = [ + { column: "unattended", decl: "INTEGER" }, + { column: "usage_source", decl: "TEXT" }, + { column: "usage_complete", decl: "INTEGER" }, + { column: "model", decl: "TEXT" }, + { column: "input_tokens", decl: "INTEGER" }, + { column: "output_tokens", decl: "INTEGER" }, + { column: "cached_input_tokens", decl: "INTEGER" }, + { column: "reasoning_tokens", decl: "INTEGER" }, + { column: "total_tokens", decl: "INTEGER" }, + { column: "cost_usd", decl: "REAL" }, + { column: "cost_currency", decl: "TEXT" }, + { column: "session_count", decl: "INTEGER" }, + { column: "sessions_without_usage", decl: "INTEGER" }, +]; + /** ISO date (UTC) of the Monday starting the week that contains `epochMs`. */ function weekStartIso(epochMs: number): string { const date = new Date(epochMs); @@ -140,6 +217,17 @@ function median(values: number[]): number | null { return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; } +/** Sum two optional numbers; null survives when both sides are null. */ +function sumNullable(a: number | null, b: number | null): number | null { + if (a === null) { + return b; + } + if (b === null) { + return a; + } + return a + b; +} + /** * SQLite-backed store for run records. */ @@ -195,7 +283,8 @@ export class RunStore { // Additive migration for databases created before the attempt column. const columns = this.db.query("PRAGMA table_info(runs)").all() as Array<{ name: string }>; - if (!columns.some((c) => c.name === "attempt")) { + const hasColumn = (name: string): boolean => columns.some((c) => c.name === name); + if (!hasColumn("attempt")) { this.db.run("ALTER TABLE runs ADD COLUMN attempt INTEGER"); } if (!columns.some((c) => c.name === "automation_id")) { @@ -209,6 +298,14 @@ export class RunStore { } this.db.run("CREATE INDEX IF NOT EXISTS idx_runs_automation_id ON runs(automation_id)"); + // Additive usage/cost migration (DEV-78). Nullable columns keep + // pre-migration rows readable with unknown (null) usage. + for (const statement of USAGE_SCHEMA_MIGRATIONS) { + if (!hasColumn(statement.column)) { + this.db.run(`ALTER TABLE runs ADD COLUMN ${statement.column} ${statement.decl}`); + } + } + this.db.run(` CREATE INDEX IF NOT EXISTS idx_runs_task_key ON runs(task_key) `); @@ -240,8 +337,8 @@ export class RunStore { const attempt = meta.attempt ?? (meta.taskKey ? this.countRuns(meta.taskKey) + 1 : null); const result = this.db.run( `INSERT INTO runs (origin, task_key, tracker, harness, branch, repo, pr_number, - automation_id, status, started_at, attempt) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'in_progress', ?, ?)`, + automation_id, status, started_at, attempt, unattended) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'in_progress', ?, ?, ?)`, [ meta.origin, meta.taskKey ?? null, @@ -253,6 +350,7 @@ export class RunStore { meta.automationId ?? null, Date.now(), attempt, + meta.unattended === true ? 1 : null, ], ); return Number(result.lastInsertRowid); @@ -313,6 +411,47 @@ export class RunStore { ); } + /** + * Persist normalized token/cost usage for a run (one row per run; the + * latest call wins). Null fields stay null — never coerced to zero. + * + * @param runId - Run id + * @param usage - Merged usage across all sessions attributable to the run + */ + recordRunUsage(runId: number, usage: RunUsage): void { + this.db.run( + `UPDATE runs SET + usage_source = ?, + usage_complete = ?, + model = ?, + input_tokens = ?, + output_tokens = ?, + cached_input_tokens = ?, + reasoning_tokens = ?, + total_tokens = ?, + cost_usd = ?, + cost_currency = ?, + session_count = ?, + sessions_without_usage = ? + WHERE id = ?`, + [ + usage.source ?? null, + usage.complete === undefined ? null : usage.complete ? 1 : 0, + usage.model ?? null, + usage.inputTokens ?? null, + usage.outputTokens ?? null, + usage.cachedInputTokens ?? null, + usage.reasoningTokens ?? null, + usage.totalTokens ?? null, + usage.costUsd ?? null, + usage.costCurrency ?? null, + usage.sessionCount ?? null, + usage.sessionsWithoutUsage ?? null, + runId, + ], + ); + } + /** * Mark a run terminal and record an `outcome` stage row. * @@ -429,7 +568,9 @@ export class RunStore { const since = windowMs === null ? 0 : Date.now() - windowMs; const rows = this.db .query( - `SELECT origin, harness, status, pr_url, started_at, finished_at + `SELECT origin, harness, status, pr_url, started_at, finished_at, + input_tokens, output_tokens, cached_input_tokens, reasoning_tokens, + total_tokens, cost_usd, usage_complete, session_count FROM runs WHERE started_at >= ? ORDER BY started_at ASC`, ) .all(since) as { @@ -439,6 +580,14 @@ export class RunStore { pr_url: string | null; started_at: number; finished_at: number | null; + input_tokens: number | null; + output_tokens: number | null; + cached_input_tokens: number | null; + reasoning_tokens: number | null; + total_tokens: number | null; + cost_usd: number | null; + usage_complete: number | null; + session_count: number | null; }[]; const byStatus: Record = { @@ -458,10 +607,30 @@ export class RunStore { const weekCounts = new Map(); const harnesses = new Map< string, - { runs: number; byStatus: Map; durations: number[] } + { + runs: number; + byStatus: Map; + durations: number[]; + spendUsd: number; + pricedRuns: number; + unknownCostRuns: number; + } >(); const durations: number[] = []; + // Usage aggregation: known values sum; categories are never inferred. + const tokens = { + inputTokens: null as number | null, + outputTokens: null as number | null, + cachedInputTokens: null as number | null, + reasoningTokens: null as number | null, + totalTokens: null as number | null, + }; + let knownSpendUsd: number | null = null; + let runsWithUsage = 0; + let runsWithoutUsage = 0; + let runsWithIncompleteUsage = 0; + for (const row of rows) { byStatus[row.status] += 1; byOrigin[row.origin] = (byOrigin[row.origin] ?? 0) + 1; @@ -472,7 +641,14 @@ export class RunStore { const harnessKey = row.harness ?? "unknown"; let harness = harnesses.get(harnessKey); if (!harness) { - harness = { runs: 0, byStatus: new Map(), durations: [] }; + harness = { + runs: 0, + byStatus: new Map(), + durations: [], + spendUsd: 0, + pricedRuns: 0, + unknownCostRuns: 0, + }; harnesses.set(harnessKey, harness); } harness.runs += 1; @@ -485,6 +661,38 @@ export class RunStore { durations.push(duration); } } + + // --- per-run usage --- + const hasUsage = + row.session_count !== null && + row.session_count > 0 && + (row.input_tokens !== null || + row.output_tokens !== null || + row.total_tokens !== null || + row.cost_usd !== null); + if (hasUsage) { + runsWithUsage += 1; + } else if (row.finished_at !== null) { + runsWithoutUsage += 1; + } + if (hasUsage && row.usage_complete === 0) { + runsWithIncompleteUsage += 1; + } + + tokens.inputTokens = sumNullable(tokens.inputTokens, row.input_tokens); + tokens.outputTokens = sumNullable(tokens.outputTokens, row.output_tokens); + tokens.cachedInputTokens = sumNullable(tokens.cachedInputTokens, row.cached_input_tokens); + tokens.reasoningTokens = sumNullable(tokens.reasoningTokens, row.reasoning_tokens); + tokens.totalTokens = sumNullable(tokens.totalTokens, row.total_tokens); + + if (typeof row.cost_usd === "number") { + knownSpendUsd = (knownSpendUsd ?? 0) + row.cost_usd; + harness.spendUsd += row.cost_usd; + harness.pricedRuns += 1; + } else if (hasUsage) { + // Usage without a computable cost is unknown exposure — never $0. + harness.unknownCostRuns += 1; + } } const terminal = TERMINAL_STATUSES.reduce((sum, status) => sum + byStatus[status], 0); @@ -506,8 +714,18 @@ export class RunStore { failed: data.byStatus.get("failed") ?? 0, escalated: data.byStatus.get("escalated") ?? 0, medianDurationMs: median(data.durations), + spendUsd: data.pricedRuns > 0 ? data.spendUsd : null, + runsWithUnknownCost: data.unknownCostRuns, })), byOrigin, + usage: { + ...tokens, + knownSpendUsd, + currency: "USD", + runsWithUsage, + runsWithoutUsage, + runsWithIncompleteUsage, + }, }; } @@ -533,6 +751,7 @@ export class RunStore { /** Map a SQLite row to a {@link RunRecord}. */ private rowToRun(row: Record): RunRecord { + const usage = this.rowToUsage(row); return { id: row.id as number, origin: row.origin as RunOrigin, @@ -548,9 +767,42 @@ export class RunStore { startedAt: row.started_at as number, finishedAt: (row.finished_at as number | null) ?? undefined, attempt: (row.attempt as number | null) ?? undefined, + unattended: row.unattended === 1 ? true : undefined, automationId: (row.automation_id as string | null) ?? undefined, ticketKey: (row.ticket_key as string | null) ?? undefined, ticketUrl: (row.ticket_url as string | null) ?? undefined, + usage, + }; + } + + /** + * Map usage columns to a {@link RunUsage}. Rows from before the migration + * have all-null columns and surface `usage: null` so the dashboard can + * distinguish "no data" from zeros. + */ + private rowToUsage(row: Record): RunUsage | null { + if ( + row.usage_source === undefined || + (row.session_count === null && + row.input_tokens === null && + row.output_tokens === null && + row.cost_usd === null) + ) { + return null; + } + return { + source: (row.usage_source as RunUsage["source"] | null) ?? null, + complete: row.usage_complete === 1, + model: (row.model as string | null) ?? null, + inputTokens: (row.input_tokens as number | null) ?? null, + outputTokens: (row.output_tokens as number | null) ?? null, + cachedInputTokens: (row.cached_input_tokens as number | null) ?? null, + reasoningTokens: (row.reasoning_tokens as number | null) ?? null, + totalTokens: (row.total_tokens as number | null) ?? null, + costUsd: (row.cost_usd as number | null) ?? null, + costCurrency: (row.cost_currency as string | null) ?? null, + sessionCount: (row.session_count as number | null) ?? 0, + sessionsWithoutUsage: (row.sessions_without_usage as number | null) ?? 0, }; } @@ -570,6 +822,8 @@ export class RunStore { let currentStore: RunStore | null = null; let currentRunId: number | null = null; +/** Agent sessions recorded for the current run (implementation, feasibility, reviews). */ +let currentSessions: (AgentUsage | null)[] = []; /** Log a recording failure without ever propagating it into the pipeline. */ function warnOnce(action: string, error: unknown): void { @@ -585,12 +839,89 @@ export function beginRun(meta: RunMeta): void { try { currentStore ??= new RunStore(); currentRunId = currentStore.createRun(meta); + currentSessions = []; } catch (error) { currentRunId = null; + currentSessions = []; warnOnce("begin", error); } } +/** + * Attribute one finished agent session to the current run (best-effort). + * + * Called after every harness spawn in the pipeline — implementation, + * feasibility, auto-review, change-request sessions. Usage is merged at + * {@link endRun} time so multi-session runs record one normalized total + * without double counting session artifacts. + * + * @param usage - Extracted usage, or `null` when the harness reported nothing + */ +export function recordAgentSession(usage: AgentUsage | null): void { + if (currentStore === null || currentRunId === null) { + return; + } + currentSessions.push(usage); +} + +/** + * Extract normalized usage from a finished session's captured output and + * attribute it to the current run. Never throws — extraction/persistence + * failures must not affect the agent run. + * + * @param harness - Harness id (e.g. "claude-code") + * @param stdout - Captured session stdout + * @param stderr - Captured session stderr + */ +export function recordSessionOutput(harness: string, stdout: string, stderr: string): void { + if (currentStore === null || currentRunId === null) { + return; + } + try { + const usage = extractAgentUsage({ harness, stdout, stderr }); + currentSessions.push(usage); + } catch { + // Best-effort by contract. + } +} + +/** + * Merge the run's agent-session usage into a persisted {@link RunUsage}. + * + * Cost rule: a run's cost is reported only when every agent session + * properly reported its own provider-computed cost. Anything less stays + * unknown — never estimated, never fabricated. + */ +function buildRunUsage(): RunUsage | null { + if (currentSessions.length === 0) { + return null; + } + const merged = mergeAgentUsages(currentSessions); + if (!merged) { + return null; + } + + const allReportedCost = + merged.sessionsWithoutUsage === 0 && + currentSessions.every((session) => session !== null && session.reportedCost !== null); + const costUsd = allReportedCost ? merged.reportedCost : null; + + return { + source: merged.source, + complete: merged.complete && allReportedCost && !merged.mixedModels, + model: merged.model, + inputTokens: merged.inputTokens, + outputTokens: merged.outputTokens, + cachedInputTokens: merged.cachedInputTokens, + reasoningTokens: merged.reasoningTokens, + totalTokens: merged.totalTokens, + costUsd, + costCurrency: costUsd === null ? null : (merged.costCurrency ?? "USD"), + sessionCount: merged.sessions, + sessionsWithoutUsage: merged.sessionsWithoutUsage, + }; +} + /** * Record a pipeline stage for the current run (no-op when no run is active). * @@ -631,18 +962,45 @@ export function recordRunPr(pr: { repo?: string; prNumber?: number; url?: string /** * Finish the current run and clear the context (no-op when no run is active). * + * Usage from every session attributed to the run is merged and persisted + * before the terminal status is written. + * * @param status - Terminal status * @param reason - Optional human-readable reason */ export function endRun(status: Exclude, reason?: string): void { if (currentStore === null || currentRunId === null) { + currentSessions = []; return; } + const usage = buildRunUsage(); + try { + if (usage) { + currentStore.recordRunUsage(currentRunId, usage); + } + } catch (error) { + warnOnce("usage", error); + } try { currentStore.finishRun(currentRunId, status, reason); } catch (error) { warnOnce("end", error); } finally { currentRunId = null; + currentSessions = []; + } +} + +/** Test hook: drop the ambient recorder context between scenarios. */ +export function resetRunRecorderForTests(): void { + if (currentStore) { + try { + currentStore.close(); + } catch { + // Already closed. + } } + currentStore = null; + currentRunId = null; + currentSessions = []; } diff --git a/packages/code/src/webhook-server.ts b/packages/code/src/webhook-server.ts index ec738c7..ad19c8d 100644 --- a/packages/code/src/webhook-server.ts +++ b/packages/code/src/webhook-server.ts @@ -34,6 +34,7 @@ import { formatReviewPrompt } from "./lib/review-formatter"; import { Utils } from "./lib/utils"; import { isCommitAlreadyComplete, runAgentHarnessToFixGitHook } from "./lib/git-hook-fixer"; import { runAutoReviewLoop } from "./lib/auto-review-loop"; +import { recordSessionOutput } from "./lib/run-recorder"; import { handlePingEvent, isGitHubIP, @@ -1337,6 +1338,8 @@ async function runAgentHarnessForReview( agent.on("close", (code: number | null) => { clearTimeout(timeout); sandboxCleanup().catch(() => {}); + // Attribute this review session's usage when a run is being recorded. + recordSessionOutput(currentHarnessName(), stdoutOutput, stderrOutput); const maxTurnsReached = detectMaxTurnsReached( stdoutOutput, stderrOutput, diff --git a/packages/code/tests/dashboard-usage-api.test.ts b/packages/code/tests/dashboard-usage-api.test.ts new file mode 100644 index 0000000..caffacc --- /dev/null +++ b/packages/code/tests/dashboard-usage-api.test.ts @@ -0,0 +1,133 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +import { DashboardData, handleRuns, handleRunDetail, handleStats } from "../src/lib/dashboard-api"; +import { RunStore } from "../src/lib/run-recorder"; +import type { RunUsage } from "../src/lib/run-recorder"; + +function usage(overrides: Partial = {}): RunUsage { + return { + source: "structured_output", + complete: true, + model: "claude-sonnet-4-5", + inputTokens: 1000, + outputTokens: 500, + cachedInputTokens: null, + reasoningTokens: null, + totalTokens: null, + costUsd: 0.75, + costCurrency: "USD", + sessionCount: 1, + sessionsWithoutUsage: 0, + ...overrides, + }; +} + +describe("dashboard API usage exposure", () => { + let dir: string; + let dbPath: string; + let store: RunStore; + let data: DashboardData; + + beforeEach(() => { + dir = join(tmpdir(), `du-test-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(dir, { recursive: true }); + dbPath = join(dir, "queue.db"); + store = new RunStore(dbPath); + data = new DashboardData({ dbPath }); + }); + + afterEach(() => { + data.close(); + store.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + test("GET /api/runs returns per-run usage fields", () => { + const id = store.createRun({ origin: "task", taskKey: "API-1", unattended: true }); + store.recordRunUsage(id, usage()); + store.finishRun(id, "succeeded"); + + const response = handleRuns(data, new URLSearchParams()); + expect(response.status).toBe(200); + const body = response.body as { runs: Array> }; + const run = body.runs.find((r) => r.id === id); + expect(run).toBeDefined(); + const record = run as never as { usage: RunUsage; unattended?: boolean }; + expect(record.usage.costUsd).toBeCloseTo(0.75); + expect(record.usage.model).toBe("claude-sonnet-4-5"); + expect(record.unattended).toBe(true); + }); + + test("historical rows without usage stay compatible (usage null)", () => { + const id = store.createRun({ origin: "task", taskKey: "API-OLD" }); + store.finishRun(id, "failed", "boom"); + + const detail = handleRunDetail(data, String(id)); + expect(detail.status).toBe(200); + const body = detail.body as { run: { status: string; usage: RunUsage | null } }; + expect(body.run.status).toBe("failed"); + expect(body.run.usage).toBeNull(); + }); + + test("GET /api/stats exposes token totals, known spend, and coverage counts", () => { + const priced = store.createRun({ + origin: "task", + taskKey: "S-1", + harness: "claude-code", + unattended: true, + }); + store.recordRunUsage(priced, usage({ costUsd: 2 })); + store.finishRun(priced, "succeeded"); + + const unknownCost = store.createRun({ + origin: "task", + taskKey: "S-2", + harness: "codex", + unattended: true, + }); + // Usage present but unpriceable: unknown model → cost stays null. + store.recordRunUsage( + unknownCost, + usage({ inputTokens: 300, outputTokens: 200, costUsd: null, model: null, complete: false }), + ); + store.finishRun(unknownCost, "succeeded"); + + const bare = store.createRun({ origin: "task", taskKey: "S-3" }); + store.finishRun(bare, "escalated"); + + const response = handleStats(data, new URLSearchParams("window=all")); + expect(response.status).toBe(200); + const payload = response.body as { + stats: { + totals: { runs: number }; + usage: { + inputTokens: number | null; + outputTokens: number | null; + knownSpendUsd: number | null; + runsWithUsage: number; + runsWithoutUsage: number; + runsWithIncompleteUsage: number; + }; + byHarness: Array<{ harness: string; spendUsd: number | null; runsWithUnknownCost: number }>; + }; + }; + + expect(payload.stats.totals.runs).toBe(3); + expect(payload.stats.usage.inputTokens).toBe(1300); + expect(payload.stats.usage.outputTokens).toBe(700); + // Only known spend sums; the unknown-cost run is counted, not zeroed. + expect(payload.stats.usage.knownSpendUsd).toBeCloseTo(2); + expect(payload.stats.usage.runsWithUsage).toBe(2); + expect(payload.stats.usage.runsWithoutUsage).toBe(1); + expect(payload.stats.usage.runsWithIncompleteUsage).toBe(1); + + const claude = payload.stats.byHarness.find((h) => h.harness === "claude-code"); + const codex = payload.stats.byHarness.find((h) => h.harness === "codex"); + expect(claude?.spendUsd).toBeCloseTo(2); + expect(codex?.spendUsd).toBeNull(); + expect(codex?.runsWithUnknownCost).toBe(1); + }); +}); diff --git a/packages/code/tests/run-usage.test.ts b/packages/code/tests/run-usage.test.ts new file mode 100644 index 0000000..aa13140 --- /dev/null +++ b/packages/code/tests/run-usage.test.ts @@ -0,0 +1,328 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "fs"; +import { Database } from "bun:sqlite"; +import { tmpdir } from "os"; +import { join } from "path"; + +import { + RunStore, + beginRun, + endRun, + recordSessionOutput, + resetRunRecorderForTests, +} from "../src/lib/run-recorder"; +import type { RunUsage } from "../src/lib/run-recorder"; + +function usage(overrides: Partial = {}): RunUsage { + return { + source: "structured_output", + complete: true, + model: "claude-sonnet-4-5", + inputTokens: 1000, + outputTokens: 500, + cachedInputTokens: null, + reasoningTokens: null, + totalTokens: null, + costUsd: 0.02, + costCurrency: "USD", + sessionCount: 1, + sessionsWithoutUsage: 0, + ...overrides, + }; +} + +describe("RunStore usage recording", () => { + let dbPath: string; + let store: RunStore; + + beforeEach(() => { + dbPath = join(tmpdir(), `ru-test-${Date.now()}-${Math.random().toString(36).slice(2)}.db`); + store = new RunStore(dbPath); + }); + + afterEach(() => { + store.close(); + for (const suffix of ["", "-wal", "-shm"]) { + rmSync(`${dbPath}${suffix}`, { force: true }); + } + }); + + test("recordRunUsage persists every normalized field and round-trips", () => { + const id = store.createRun({ origin: "task", taskKey: "USE-1", unattended: true }); + store.recordRunUsage( + id, + usage({ + cachedInputTokens: 250, + reasoningTokens: 75, + totalTokens: 1825, + }), + ); + store.finishRun(id, "succeeded"); + + const run = store.getRun(id); + expect(run?.usage).not.toBeNull(); + expect(run?.usage?.inputTokens).toBe(1000); + expect(run?.usage?.outputTokens).toBe(500); + expect(run?.usage?.cachedInputTokens).toBe(250); + expect(run?.usage?.reasoningTokens).toBe(75); + expect(run?.usage?.totalTokens).toBe(1825); + expect(run?.usage?.model).toBe("claude-sonnet-4-5"); + expect(run?.usage?.costUsd).toBeCloseTo(0.02); + expect(run?.usage?.costCurrency).toBe("USD"); + expect(run?.usage?.complete).toBe(true); + expect(run?.unattended).toBe(true); + }); + + test("null usage fields stay null (unknown, never zero)", () => { + const id = store.createRun({ origin: "task", taskKey: "USE-2" }); + store.recordRunUsage( + id, + usage({ + complete: false, + model: null, + inputTokens: null, + outputTokens: null, + costUsd: null, + costCurrency: null, + }), + ); + + const run = store.getRun(id); + expect(run?.usage?.inputTokens).toBeNull(); + expect(run?.usage?.outputTokens).toBeNull(); + expect(run?.usage?.costUsd).toBeNull(); + expect(run?.usage?.model).toBeNull(); + expect(run?.usage?.complete).toBe(false); + }); + + test("historical rows without usage read as usage: null", () => { + const id = store.createRun({ origin: "task", taskKey: "OLD-1" }); + store.finishRun(id, "succeeded"); + const run = store.getRun(id); + expect(run?.usage).toBeNull(); + }); + + test("opening a pre-usage database migrates additively and stays readable", () => { + const legacyPath = join( + tmpdir(), + `ru-legacy-${Date.now()}-${Math.random().toString(36).slice(2)}.db`, + ); + const legacy = new Database(legacyPath); + legacy.run(` + CREATE TABLE runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + origin TEXT NOT NULL, + task_key TEXT, + tracker TEXT, + harness TEXT, + branch TEXT, + repo TEXT, + pr_number INTEGER, + pr_url TEXT, + status TEXT NOT NULL DEFAULT 'in_progress', + outcome_reason TEXT, + started_at INTEGER NOT NULL, + finished_at INTEGER + ) + `); + legacy.run(`INSERT INTO runs (origin, task_key, started_at) VALUES ('task', 'LEGACY-1', 1)`); + legacy.close(); + + const migrated = new RunStore(legacyPath); + // Historical row readable with null usage. + const runs = migrated.listRuns({ taskKey: "LEGACY-1" }); + expect(runs).toHaveLength(1); + expect(runs[0]?.usage).toBeNull(); + + // New writes work. + const id = migrated.createRun({ origin: "task", taskKey: "LEGACY-2" }); + migrated.recordRunUsage(id, usage()); + expect(migrated.getRun(id)?.usage?.costUsd).toBeCloseTo(0.02); + migrated.close(); + + // Re-opening is idempotent (no duplicate ALTER TABLE errors). + const reopened = new RunStore(legacyPath); + expect(reopened.getRun(id)?.usage?.inputTokens).toBe(1000); + reopened.close(); + + for (const suffix of ["", "-wal", "-shm"]) { + rmSync(`${legacyPath}${suffix}`, { force: true }); + } + }); +}); + +describe("RunStats usage aggregation", () => { + let dbPath: string; + let store: RunStore; + + beforeEach(() => { + dbPath = join(tmpdir(), `rs-test-${Date.now()}-${Math.random().toString(36).slice(2)}.db`); + store = new RunStore(dbPath); + }); + + afterEach(() => { + store.close(); + for (const suffix of ["", "-wal", "-shm"]) { + rmSync(`${dbPath}${suffix}`, { force: true }); + } + }); + + function seedFinished(meta: { + taskKey: string; + harness?: string; + unattended?: boolean; + runUsage?: Partial; + }): void { + const id = store.createRun({ + origin: "task", + taskKey: meta.taskKey, + harness: meta.harness ?? "claude-code", + unattended: meta.unattended, + }); + if (meta.runUsage) { + store.recordRunUsage(id, usage(meta.runUsage)); + } + store.finishRun(id, "succeeded"); + } + + test("aggregates tokens and known spend without treating unknown as zero", () => { + seedFinished({ + taskKey: "AGG-1", + runUsage: { inputTokens: 1000, outputTokens: 500, costUsd: 1 }, + }); + seedFinished({ + taskKey: "AGG-2", + runUsage: { inputTokens: 400, outputTokens: null, costUsd: null, complete: false }, + }); + seedFinished({ taskKey: "AGG-3" }); // no usage at all + + const stats = store.getStats(null); + expect(stats.usage.inputTokens).toBe(1400); + expect(stats.usage.outputTokens).toBe(500); + expect(stats.usage.cachedInputTokens).toBeNull(); // nothing reported → null, not 0 + expect(stats.usage.knownSpendUsd).toBeCloseTo(1); // unknown run adds nothing + expect(stats.usage.currency).toBe("USD"); + expect(stats.usage.runsWithUsage).toBe(2); + expect(stats.usage.runsWithoutUsage).toBe(1); + expect(stats.usage.runsWithIncompleteUsage).toBe(1); + }); + + test("per-harness spend splits by harness with unknown-cost counters", () => { + seedFinished({ + taskKey: "H-1", + harness: "claude-code", + runUsage: { costUsd: 0.5 }, + }); + seedFinished({ + taskKey: "H-2", + harness: "codex", + runUsage: { costUsd: 2, model: "gpt-5" }, + }); + seedFinished({ + taskKey: "H-3", + harness: "codex", + runUsage: { costUsd: null, complete: false }, + }); + + const stats = store.getStats(null); + const claude = stats.byHarness.find((h) => h.harness === "claude-code"); + const codex = stats.byHarness.find((h) => h.harness === "codex"); + expect(claude?.spendUsd).toBeCloseTo(0.5); + expect(codex?.spendUsd).toBeCloseTo(2); + expect(codex?.runsWithUnknownCost).toBe(1); + }); + + test("window filtering only includes runs started within the window", () => { + seedFinished({ taskKey: "WIN-1", runUsage: { costUsd: 3 } }); + // Backdate a second run beyond the 7d window. + const oldId = store.createRun({ origin: "task", taskKey: "WIN-2", unattended: true }); + store.recordRunUsage(oldId, usage({ costUsd: 100 })); + const db = (store as unknown as { db: { run: (sql: string, ...params: unknown[]) => void } }) + .db; + db.run(`UPDATE runs SET started_at = ? WHERE id = ?`, [ + Date.now() - 8 * 24 * 60 * 60 * 1000, + oldId, + ]); + store.finishRun(oldId, "succeeded"); + + const week = store.getStats(7 * 24 * 60 * 60 * 1000); + expect(week.totals.runs).toBe(1); + expect(week.usage.knownSpendUsd).toBeCloseTo(3); + + const all = store.getStats(null); + expect(all.totals.runs).toBe(2); + expect(all.usage.knownSpendUsd).toBeCloseTo(103); + }); +}); + +describe("module-level usage recording", () => { + let dir: string; + let dbPath: string; + let savedQueueDb: string | undefined; + + beforeEach(() => { + resetRunRecorderForTests(); + dir = join(tmpdir(), `ru-rec-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(dir, { recursive: true }); + dbPath = join(dir, "queue.db"); + savedQueueDb = process.env.WEBHOOK_QUEUE_DB; + process.env.WEBHOOK_QUEUE_DB = dbPath; + }); + + afterEach(() => { + resetRunRecorderForTests(); + if (savedQueueDb === undefined) { + delete process.env.WEBHOOK_QUEUE_DB; + } else { + process.env.WEBHOOK_QUEUE_DB = savedQueueDb; + } + rmSync(dir, { recursive: true, force: true }); + }); + + test("sessions merge across stages and persist on endRun without double counting", () => { + beginRun({ + origin: "task", + taskKey: "MERGE-1", + harness: "codex", + unattended: true, + }); + + // Feasibility session: structured result. + recordSessionOutput( + "codex", + JSON.stringify({ + modelUsage: { "gpt-5": { input_tokens: 100, output_tokens: 40 } }, + total_cost_usd: 0.01, + }), + "", + ); + // Implementation session: unstructured output — no usage signal. + recordSessionOutput("codex", "", "tokens used: 500\n"); + + endRun("succeeded"); + + const store = new RunStore(dbPath); + const run = store.getRun(1); + expect(run?.usage).not.toBeNull(); + expect(run?.usage?.sessionCount).toBe(2); + expect(run?.usage?.sessionsWithoutUsage).toBe(1); + // Structured tokens (100 in / 40 out) survive; the unstructured session + // contributes nothing. + expect(run?.usage?.inputTokens).toBe(100); + expect(run?.usage?.outputTokens).toBe(40); + // Cost is unknown: not every session reported one — never a partial or + // fabricated sum. + expect(run?.usage?.costUsd).toBeNull(); + expect(run?.usage?.complete).toBe(false); + store.close(); + }); + + test("runs with no agent sessions persist no usage row data", () => { + beginRun({ origin: "task", taskKey: "NOUSAGE-1" }); + endRun("failed", "branch creation failed"); + const store = new RunStore(dbPath); + expect(store.getRun(1)?.status).toBe("failed"); + expect(store.getRun(1)?.usage).toBeNull(); + store.close(); + }); +}); diff --git a/packages/dashboard-ui/src/lib/api.ts b/packages/dashboard-ui/src/lib/api.ts index 9600f11..2ef5085 100644 --- a/packages/dashboard-ui/src/lib/api.ts +++ b/packages/dashboard-ui/src/lib/api.ts @@ -15,6 +15,24 @@ export type RunStatus = | "escalated" | "abandoned"; +export interface RunUsage { + /** Where usage numbers came from ("mixed" when sessions disagree). */ + source: string | null; + /** False when any session's accounting was partial or missing entirely. */ + complete: boolean; + model: string | null; + inputTokens: number | null; + outputTokens: number | null; + cachedInputTokens: number | null; + reasoningTokens: number | null; + totalTokens: number | null; + /** Known cost in USD (null = unknown, never zero). */ + costUsd: number | null; + costCurrency: string | null; + sessionCount: number; + sessionsWithoutUsage: number; +} + export interface RunRecord { id: number; origin: RunOrigin; @@ -32,6 +50,8 @@ export interface RunRecord { outcomeReason?: string; startedAt: number; finishedAt?: number; + unattended?: boolean; + usage?: RunUsage | null; } /** Outcome stages carry the run's terminal status (escalated, deferred, …). */ @@ -63,6 +83,32 @@ export interface RunDetailResponse { stages: RunStageRecord[]; } +export interface RunStatsUsage { + inputTokens: number | null; + outputTokens: number | null; + cachedInputTokens: number | null; + reasoningTokens: number | null; + totalTokens: number | null; + /** Known spend in USD; null when no run in the window had a computable cost. */ + knownSpendUsd: number | null; + currency: "USD"; + runsWithUsage: number; + runsWithoutUsage: number; + runsWithIncompleteUsage: number; +} + +export interface RunStatsHarness { + harness: string; + runs: number; + succeeded: number; + failed: number; + escalated: number; + medianDurationMs: number | null; + /** Known spend for this harness; null when nothing priced. */ + spendUsd: number | null; + runsWithUnknownCost: number; +} + export interface StatsResponse { window: string; stats: { @@ -71,15 +117,9 @@ export interface StatsResponse { escalationRate: number | null; runsPerWeek: { weekStart: string; count: number }[]; medianDurationMs: number | null; - byHarness: { - harness: string; - runs: number; - succeeded: number; - failed: number; - escalated: number; - medianDurationMs: number | null; - }[]; + byHarness: RunStatsHarness[]; byOrigin: Record; + usage: RunStatsUsage; } | null; } diff --git a/packages/dashboard-ui/src/views/RunDetailView.tsx b/packages/dashboard-ui/src/views/RunDetailView.tsx index 96e4da9..a91e6b0 100644 --- a/packages/dashboard-ui/src/views/RunDetailView.tsx +++ b/packages/dashboard-ui/src/views/RunDetailView.tsx @@ -152,6 +152,37 @@ export function RunDetailView({ runId, onBack }: { runId: number; onBack: () => : "in progress" } /> + ${data.run.usage.costUsd.toFixed(4)} + ) : data.run.usage ? ( + unknown + ) : ( + "–" + ) + } + /> + {data.run.usage?.model ? ( + {data.run.usage.model}} + /> + ) : null} + {data.run.usage?.inputTokens != null || data.run.usage?.outputTokens != null ? ( + + {data.run.usage.inputTokens ?? "?"} / {data.run.usage.outputTokens ?? "?"} + {data.run.usage.complete === false ? ( + (partial) + ) : null} + + } + /> + ) : null} diff --git a/packages/dashboard-ui/src/views/RunsView.tsx b/packages/dashboard-ui/src/views/RunsView.tsx index 74a8bbf..7780a85 100644 --- a/packages/dashboard-ui/src/views/RunsView.tsx +++ b/packages/dashboard-ui/src/views/RunsView.tsx @@ -98,6 +98,7 @@ export function RunsView({ onOpenRun }: { onOpenRun: (id: number) => void }) { Harness Result Duration + Cost Started @@ -124,6 +125,13 @@ export function RunsView({ onOpenRun }: { onOpenRun: (id: number) => void }) { {run.finishedAt ? formatDuration(run.finishedAt - run.startedAt) : "…"} + + {run.usage?.costUsd != null + ? `$${run.usage.costUsd.toFixed(4)}${run.usage.complete === false ? "*" : ""}` + : run.usage + ? "unknown" + : "–"} + {formatTime(run.startedAt)} diff --git a/packages/dashboard-ui/src/views/StatsView.tsx b/packages/dashboard-ui/src/views/StatsView.tsx index 8748185..ee44742 100644 --- a/packages/dashboard-ui/src/views/StatsView.tsx +++ b/packages/dashboard-ui/src/views/StatsView.tsx @@ -18,6 +18,25 @@ import { formatDuration, formatRate } from "@/lib/utils"; const WINDOWS = ["7d", "30d", "90d", "all"] as const; const RUN_ORIGINS: RunOrigin[] = ["task", "pr_mention", "conflict_resolution", "scheduled"]; +/** Format a token count compactly (12.3k, 4.5M). */ +function formatTokens(count: number): string { + if (count >= 1_000_000) { + return `${(count / 1_000_000).toFixed(1)}M`; + } + if (count >= 1_000) { + return `${(count / 1_000).toFixed(1)}k`; + } + return String(count); +} + +/** + * Known-spend label that never presents unknown data as $0. + * Returns null when nothing is priced in the window. + */ +function formatSpend(knownSpendUsd: number | null): string | null { + return knownSpendUsd === null ? null : `$${knownSpendUsd.toFixed(2)}`; +} + /** Hand-rolled SVG bar chart of runs per week (keeps dependencies minimal). */ function WeeklyBars({ weeks }: { weeks: { weekStart: string; count: number }[] }) { if (weeks.length === 0) { @@ -105,6 +124,72 @@ export function StatsView() { /> + {(() => { + const usage = stats.usage; + const spend = formatSpend(usage.knownSpendUsd); + const unknownExposure = + usage.runsWithoutUsage + usage.runsWithIncompleteUsage > 0 || spend === null; + return ( + + + + Token & cost usage + + + +
+ + + + +
+ {unknownExposure ? ( +

+ ⚠️ Partial data:{" "} + {usage.runsWithoutUsage > 0 + ? `${usage.runsWithoutUsage} finished run${usage.runsWithoutUsage === 1 ? "" : "s"} without usage reporting` + : null} + {usage.runsWithoutUsage > 0 && usage.runsWithIncompleteUsage > 0 + ? "; " + : null} + {usage.runsWithIncompleteUsage > 0 + ? `${usage.runsWithIncompleteUsage} with incomplete accounting or unpriced model${usage.runsWithIncompleteUsage === 1 ? "" : "s"}` + : null} + . Totals cover known values only — unknown runs are not counted as $0. +

+ ) : null} +
+
+ ); + })()} + @@ -127,6 +212,7 @@ export function StatsView() { Failed Escalated Median duration + Spend (USD) @@ -146,6 +232,13 @@ export function StatsView() { ? "–" : formatDuration(harness.medianDurationMs)} + + {harness.spendUsd === null + ? harness.runsWithUnknownCost > 0 + ? `unknown (${harness.runsWithUnknownCost})` + : "–" + : `$${harness.spendUsd.toFixed(2)}`} + ))}