diff --git a/apps/web/components/app/feed/ActivityCard.tsx b/apps/web/components/app/feed/ActivityCard.tsx index 3e437bff..70c70132 100644 --- a/apps/web/components/app/feed/ActivityCard.tsx +++ b/apps/web/components/app/feed/ActivityCard.tsx @@ -48,6 +48,21 @@ function prettifyModel(model: string): string { } if (/^o4/i.test(model)) return "o4"; if (/^o3/i.test(model)) return "o3"; + // Gemini models + if (/^gemini-2\.5-pro/i.test(normalized)) return "Gemini Pro"; + if (/^gemini-2\.5-flash/i.test(normalized)) return "Gemini Flash"; + if (/^gemini-2\.0-flash/i.test(normalized)) return "Gemini Flash"; + if (/^gemini/i.test(normalized)) return "Gemini"; + // Qwen models + if (/^qwen3-coder/i.test(normalized)) return "Qwen Coder"; + if (/^qwen-coder/i.test(normalized)) return "Qwen Coder"; + if (/^qwen-max/i.test(normalized)) return "Qwen Max"; + if (/^qwen/i.test(normalized)) return "Qwen"; + // Mistral models + if (/^devstral/i.test(normalized)) return "Devstral"; + if (/^codestral/i.test(normalized)) return "Codestral"; + if (/^mistral-large/i.test(normalized)) return "Mistral Large"; + if (/^mistral/i.test(normalized)) return "Mistral"; // Legacy: broader Claude matching if (model.includes("opus")) return "Claude Opus"; if (model.includes("sonnet")) return "Claude Sonnet"; diff --git a/apps/web/lib/utils/post-share.ts b/apps/web/lib/utils/post-share.ts index 2a30c3a7..f5879bdb 100644 --- a/apps/web/lib/utils/post-share.ts +++ b/apps/web/lib/utils/post-share.ts @@ -23,6 +23,26 @@ function prettifyModel(model: string): string { if (/^o4/i.test(normalized)) return "o4"; if (/^o3/i.test(normalized)) return "o3"; + + // Gemini models + if (/^gemini-2\.5-pro/i.test(normalized)) return "Gemini Pro"; + if (/^gemini-2\.5-flash/i.test(normalized)) return "Gemini Flash"; + if (/^gemini-2\.0-flash/i.test(normalized)) return "Gemini Flash"; + if (/^gemini-exp/i.test(normalized)) return "Gemini Exp"; + if (/^gemini/i.test(normalized)) return "Gemini"; + + // Qwen models + if (/^qwen3-coder/i.test(normalized)) return "Qwen Coder"; + if (/^qwen-coder/i.test(normalized)) return "Qwen Coder"; + if (/^qwen-max/i.test(normalized)) return "Qwen Max"; + if (/^qwen/i.test(normalized)) return "Qwen"; + + // Mistral models + if (/^devstral/i.test(normalized)) return "Devstral"; + if (/^codestral/i.test(normalized)) return "Codestral"; + if (/^mistral-large/i.test(normalized)) return "Mistral Large"; + if (/^mistral/i.test(normalized)) return "Mistral"; + return normalized; } @@ -30,11 +50,19 @@ export function getShareModelLabel( models: string[] | null | undefined ): string | null { if (!models || models.length === 0) return null; + // Priority order: Claude > Gemini > Qwen > Mistral > Codex > first model if (models.some((model) => /claude-opus-4/i.test(model))) return "Claude Opus"; - if (models.some((model) => /claude-sonnet-4/i.test(model))) { - return "Claude Sonnet"; - } + if (models.some((model) => /claude-sonnet-4/i.test(model))) return "Claude Sonnet"; if (models.some((model) => /claude-haiku-4/i.test(model))) return "Claude Haiku"; + if (models.some((model) => /^gemini/i.test(model))) { + return prettifyModel(models.find((m) => /^gemini/i.test(m))!); + } + if (models.some((model) => /^qwen/i.test(model))) { + return prettifyModel(models.find((m) => /^qwen/i.test(m))!); + } + if (models.some((model) => /^devstral|^codestral|^mistral/i.test(model))) { + return prettifyModel(models.find((m) => /^devstral|^codestral|^mistral/i.test(m))!); + } return prettifyModel(models[0]!); } diff --git a/apps/web/lib/utils/recap.ts b/apps/web/lib/utils/recap.ts index 4789142c..38d8dde4 100644 --- a/apps/web/lib/utils/recap.ts +++ b/apps/web/lib/utils/recap.ts @@ -19,6 +19,16 @@ const MODEL_DISPLAY_NAMES: Record = { "claude-opus-4": "Claude Opus", "claude-sonnet-4": "Claude Sonnet", "claude-haiku-4": "Claude Haiku", + "gemini-2.5-pro": "Gemini Pro", + "gemini-2.5-flash": "Gemini Flash", + "gemini-2.0-flash": "Gemini Flash", + "gemini-exp": "Gemini Exp", + "qwen3-coder": "Qwen Coder", + "qwen-coder-plus": "Qwen Coder Plus", + "qwen-max": "Qwen Max", + "devstral": "Devstral", + "codestral": "Codestral", + "mistral-large": "Mistral Large", }; function resolveModelName(raw: string): string { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ca37957e..96f78c1c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- **Gemini CLI, Qwen Code, and Mistral Vibe provider support.** The CLI now reads usage data from Gemini CLI (via `gcusage`), Qwen Code (`~/.qwen/` session files), and Mistral Vibe (`~/.vibe/logs/session/` metadata). All providers run in parallel during `straude push` and merge into the existing daily usage pipeline. Model display names for all three ecosystems are recognized across the web UI (activity cards, recaps, social sharing). - **Open Graph image and social meta tags.** Created `/og-image` preview page matching the landing page design system (dark background, bolt icon, tagline, terminal snippet). Captured 1200x630 screenshot to `/public/og-image.png`. Updated root layout with `og:image` and `twitter:image` meta tags including dimensions and alt text. - **PWA manifest and mobile standalone support.** Created `manifest.ts` for Add-to-Home-Screen with standalone display mode. Added `viewportFit: "cover"` to viewport config and safe area inset utility classes for notched devices. - **Referral CTA in feed sidebar.** Added "Grow Your Crew" section with invite button to `RightSidebar`. diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 0e6d05db..8d27d896 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -1,5 +1,20 @@ # Architecture & Design Decisions +## Multi-Provider Usage Tracking: Gemini, Qwen Code, Mistral Vibe (2026-03-08) + +**Decision:** Added three new CLI provider modules alongside the existing Claude (ccusage) and Codex (@ccusage/codex) integrations. Gemini uses the `gcusage` npm package; Qwen Code and Mistral Vibe read local session files directly. + +**Alternatives considered:** +1. **Wait for `@ccusage/*` packages for each provider** — clean architecture but none exist yet for Gemini/Qwen/Mistral, so users would get no data. +2. **Build a unified session reader** — one generic parser for all tools. Rejected because each tool stores sessions differently (Gemini: telemetry.log via gcusage, Qwen: JSON session files forked from Gemini CLI, Mistral: meta.json with AgentStats in `~/.vibe/logs/session/`). +3. **Per-provider modules with silent fallback** (chosen) — each provider has its own module following the codex.ts pattern. All run in parallel during push. Missing providers return empty data silently. `mergeEntries` now accepts variadic sources instead of just Claude+Codex. + +**Data sources:** +- Gemini CLI: `gcusage --json --period day` (reads `~/.gemini/telemetry.log`) +- Qwen Code: direct read of `~/.qwen/tmp/*/chats/session-*.json` (forked from Gemini CLI, same token schema) +- Mistral Vibe: direct read of `~/.vibe/logs/session/session_*/meta.json` (AgentStats with prompt/completion tokens) + +**Cost tracking:** Gemini CLI and Qwen Code are free-tier — `costUSD` is 0. Mistral Vibe estimates cost from `input_price_per_million` / `output_price_per_million` in session metadata when available. ## Referral System: Column on Users, Not Separate Table (2026-03-07) **Decision:** Added a single `referred_by UUID` column to `users` instead of creating a separate `referrals` table. Crew stats (count, total spend) are derived via joins. diff --git a/packages/cli/__tests__/commands/push.test.ts b/packages/cli/__tests__/commands/push.test.ts index 7bca4945..06dabcd6 100644 --- a/packages/cli/__tests__/commands/push.test.ts +++ b/packages/cli/__tests__/commands/push.test.ts @@ -24,12 +24,30 @@ vi.mock("../../src/lib/codex.js", () => ({ parseCodexOutput: vi.fn(), })); +vi.mock("../../src/lib/gemini.js", () => ({ + runGeminiRawAsync: vi.fn(), + parseGeminiOutput: vi.fn(), +})); + +vi.mock("../../src/lib/qwen.js", () => ({ + runQwenRawAsync: vi.fn(), + parseQwenOutput: vi.fn(), +})); + +vi.mock("../../src/lib/mistral.js", () => ({ + runMistralRawAsync: vi.fn(), + parseMistralOutput: vi.fn(), +})); + import { pushCommand, mergeEntries } from "../../src/commands/push.js"; import { loadConfig, saveConfig } from "../../src/lib/auth.js"; import { loginCommand } from "../../src/commands/login.js"; import { apiRequest } from "../../src/lib/api.js"; import { runCcusageRawAsync, parseCcusageOutput } from "../../src/lib/ccusage.js"; import { runCodexRawAsync, parseCodexOutput } from "../../src/lib/codex.js"; +import { runGeminiRawAsync, parseGeminiOutput } from "../../src/lib/gemini.js"; +import { runQwenRawAsync, parseQwenOutput } from "../../src/lib/qwen.js"; +import { runMistralRawAsync, parseMistralOutput } from "../../src/lib/mistral.js"; const mockLoadConfig = vi.mocked(loadConfig); const mockLoginCommand = vi.mocked(loginCommand); @@ -39,6 +57,12 @@ const mockRunCcusageRawAsync = vi.mocked(runCcusageRawAsync); const mockParseCcusageOutput = vi.mocked(parseCcusageOutput); const mockRunCodexRawAsync = vi.mocked(runCodexRawAsync); const mockParseCodexOutput = vi.mocked(parseCodexOutput); +const mockRunGeminiRawAsync = vi.mocked(runGeminiRawAsync); +const mockParseGeminiOutput = vi.mocked(parseGeminiOutput); +const mockRunQwenRawAsync = vi.mocked(runQwenRawAsync); +const mockParseQwenOutput = vi.mocked(parseQwenOutput); +const mockRunMistralRawAsync = vi.mocked(runMistralRawAsync); +const mockParseMistralOutput = vi.mocked(parseMistralOutput); const fakeConfig = { token: "tok", username: "alice", api_url: "https://straude.com" }; @@ -61,9 +85,15 @@ function todayStr(): string { beforeEach(() => { vi.clearAllMocks(); mockLoadConfig.mockReturnValue(fakeConfig); - // Default: no Codex data + // Default: no Codex/Gemini/Qwen/Mistral data mockRunCodexRawAsync.mockResolvedValue(""); mockParseCodexOutput.mockReturnValue({ data: [] }); + mockRunGeminiRawAsync.mockResolvedValue(""); + mockParseGeminiOutput.mockReturnValue({ data: [] }); + mockRunQwenRawAsync.mockResolvedValue(""); + mockParseQwenOutput.mockReturnValue({ data: [] }); + mockRunMistralRawAsync.mockResolvedValue(""); + mockParseMistralOutput.mockReturnValue({ data: [] }); vi.spyOn(console, "log").mockImplementation(() => {}); vi.spyOn(console, "error").mockImplementation(() => {}); vi.spyOn(process, "exit").mockImplementation((code) => { diff --git a/packages/cli/__tests__/flows/cli-sync-flow.test.ts b/packages/cli/__tests__/flows/cli-sync-flow.test.ts index 29c41040..9d8db064 100644 --- a/packages/cli/__tests__/flows/cli-sync-flow.test.ts +++ b/packages/cli/__tests__/flows/cli-sync-flow.test.ts @@ -245,8 +245,8 @@ describe("sync flow", () => { // Re-syncs today's data (1 API call) expect(mockFetch).toHaveBeenCalledTimes(1); - // 2 calls: ccusage daily + codex attempt (no more --version probe) - expect(mockExecFileSync).toHaveBeenCalledTimes(2); + // 3 calls: ccusage daily + codex attempt + gcusage attempt + expect(mockExecFileSync).toHaveBeenCalledTimes(3); const saved = readPersistedConfig(); expect(saved.last_push_date).toBe(today); }); @@ -261,8 +261,8 @@ describe("sync flow", () => { await pushCommand({}); - // 2 calls: ccusage daily + codex attempt (no more --version probe) - expect(mockExecFileSync).toHaveBeenCalledTimes(2); + // 3 calls: ccusage daily + codex attempt + gcusage attempt + expect(mockExecFileSync).toHaveBeenCalledTimes(3); expect(mockFetch).toHaveBeenCalledTimes(1); const saved = readPersistedConfig(); @@ -278,8 +278,8 @@ describe("sync flow", () => { await pushCommand({}); - // 2 calls: ccusage daily + codex attempt (no more --version probe) - expect(mockExecFileSync).toHaveBeenCalledTimes(2); + // 3 calls: ccusage daily + codex attempt + gcusage attempt + expect(mockExecFileSync).toHaveBeenCalledTimes(3); // calls[0] = actual ccusage daily const args = mockExecFileSync.mock.calls[0]!; // ccusage is called with: ["daily", "--json", "--since", ..., "--until", ...] @@ -318,13 +318,18 @@ describe("Codex integration", () => { }); } - /** Mock that returns ccusage JSON for ccusage calls and codex JSON for codex calls. */ + /** Mock that returns ccusage JSON for ccusage calls and codex JSON for codex calls. + * Other providers (gcusage, etc.) throw so their async wrappers return empty. */ function mockBothSources(ccJson: string, codexJsonStr: string) { mockExecFileSync.mockImplementation(((cmd: string, args: string[]) => { if (args?.some?.((a: string) => typeof a === "string" && a.includes("@ccusage/codex"))) { return codexJsonStr; } - return ccJson; + if (args?.some?.((a: string) => typeof a === "string" && (a.includes("ccusage") || a === "daily"))) { + return ccJson; + } + // Unknown provider — throw so async wrappers return empty string + throw new Error("not installed"); }) as typeof execFileSync); } diff --git a/packages/cli/src/commands/push.ts b/packages/cli/src/commands/push.ts index 661d107e..cdce94aa 100644 --- a/packages/cli/src/commands/push.ts +++ b/packages/cli/src/commands/push.ts @@ -7,6 +7,9 @@ import { apiRequest } from "../lib/api.js"; import { runCcusageRawAsync, parseCcusageOutput } from "../lib/ccusage.js"; import type { CcusageDailyEntry, ModelBreakdownEntry } from "../lib/ccusage.js"; import { runCodexRawAsync, parseCodexOutput } from "../lib/codex.js"; +import { runGeminiRawAsync, parseGeminiOutput } from "../lib/gemini.js"; +import { runQwenRawAsync, parseQwenOutput } from "../lib/qwen.js"; +import { runMistralRawAsync, parseMistralOutput } from "../lib/mistral.js"; import { MAX_BACKFILL_DAYS } from "../config.js"; interface UsageSubmitRequest { @@ -96,42 +99,40 @@ function buildBreakdown(entry: CcusageDailyEntry): ModelBreakdownEntry[] { } /** - * Merge Claude and Codex daily entries by date. + * Merge daily entries from multiple providers by date. * Sums tokens/costs, unions models, builds model_breakdown. */ export function mergeEntries( claudeEntries: CcusageDailyEntry[], codexEntries: CcusageDailyEntry[], + ...additionalSources: CcusageDailyEntry[][] ): CcusageDailyEntry[] { - const byDate = new Map(); - - for (const e of claudeEntries) { - byDate.set(e.date, { ...byDate.get(e.date), claude: e }); - } - for (const e of codexEntries) { - byDate.set(e.date, { ...byDate.get(e.date), codex: e }); + const allSources = [claudeEntries, codexEntries, ...additionalSources]; + const byDate = new Map(); + + for (const source of allSources) { + for (const e of source) { + const existing = byDate.get(e.date) ?? []; + existing.push(e); + byDate.set(e.date, existing); + } } const merged: CcusageDailyEntry[] = []; - for (const [date, { claude, codex }] of byDate) { - const claudeBreakdown = claude ? buildBreakdown(claude) : []; - const codexBreakdown = codex ? buildBreakdown(codex) : []; - const modelBreakdown = [...claudeBreakdown, ...codexBreakdown]; + for (const [date, entries] of byDate) { + const allBreakdowns = entries.flatMap((e) => buildBreakdown(e)); merged.push({ date, - models: [ - ...(claude?.models ?? []), - ...(codex?.models ?? []), - ], - inputTokens: (claude?.inputTokens ?? 0) + (codex?.inputTokens ?? 0), - outputTokens: (claude?.outputTokens ?? 0) + (codex?.outputTokens ?? 0), - cacheCreationTokens: (claude?.cacheCreationTokens ?? 0) + (codex?.cacheCreationTokens ?? 0), - cacheReadTokens: (claude?.cacheReadTokens ?? 0) + (codex?.cacheReadTokens ?? 0), - totalTokens: (claude?.totalTokens ?? 0) + (codex?.totalTokens ?? 0), - costUSD: (claude?.costUSD ?? 0) + (codex?.costUSD ?? 0), - modelBreakdown: modelBreakdown.length > 0 ? modelBreakdown : undefined, + models: entries.flatMap((e) => e.models), + inputTokens: entries.reduce((sum, e) => sum + e.inputTokens, 0), + outputTokens: entries.reduce((sum, e) => sum + e.outputTokens, 0), + cacheCreationTokens: entries.reduce((sum, e) => sum + e.cacheCreationTokens, 0), + cacheReadTokens: entries.reduce((sum, e) => sum + e.cacheReadTokens, 0), + totalTokens: entries.reduce((sum, e) => sum + e.totalTokens, 0), + costUSD: entries.reduce((sum, e) => sum + e.costUSD, 0), + modelBreakdown: allBreakdowns.length > 0 ? allBreakdowns : undefined, }); } @@ -218,10 +219,13 @@ export async function pushCommand(options: PushOptions, apiUrlOverride?: string) : `Pushing usage for ${formatDate(sinceDate)} to ${formatDate(untilDate)}...`, ); - // Run ccusage + codex in parallel — the single biggest perf win - const [claudeResult, codexRaw] = await Promise.all([ + // Run all providers in parallel — the single biggest perf win + const [claudeResult, codexRaw, geminiRaw, qwenRaw, mistralRaw] = await Promise.all([ runCcusageRawAsync(sinceStr, untilStr).catch((err: Error) => err), runCodexRawAsync(sinceStr, untilStr), + runGeminiRawAsync(sinceStr, untilStr), + runQwenRawAsync(sinceStr, untilStr), + runMistralRawAsync(sinceStr, untilStr), ]); let claudeRaw = ""; @@ -250,7 +254,19 @@ export async function pushCommand(options: PushOptions, apiUrlOverride?: string) // Codex data — silent on fetch failure (empty string), but surface parser anomalies. const codexParsed = codexRaw ? parseCodexOutput(codexRaw) : { data: [], anomalies: [], entryMeta: [] }; - const allAnomalies = [...claudeAnomalies, ...(codexParsed.anomalies ?? [])]; + + // Gemini, Qwen, Mistral — silent on fetch failure + const geminiParsed = geminiRaw ? parseGeminiOutput(geminiRaw) : { data: [], anomalies: [], entryMeta: [] }; + const qwenParsed = qwenRaw ? parseQwenOutput(qwenRaw) : { data: [], anomalies: [], entryMeta: [] }; + const mistralParsed = mistralRaw ? parseMistralOutput(mistralRaw) : { data: [], anomalies: [], entryMeta: [] }; + + const allAnomalies = [ + ...claudeAnomalies, + ...(codexParsed.anomalies ?? []), + ...(geminiParsed.anomalies ?? []), + ...(qwenParsed.anomalies ?? []), + ...(mistralParsed.anomalies ?? []), + ]; const mediumLowCount = allAnomalies.filter((a) => a.confidence !== "high").length; if (mediumLowCount > 0) { const lowCount = allAnomalies.filter((a) => a.confidence === "low").length; @@ -277,8 +293,14 @@ export async function pushCommand(options: PushOptions, apiUrlOverride?: string) const codexEntries = codexParsed.data.filter((entry) => !blockedDates.has(entry.date)); - // Merge Claude + Codex entries by date - const entries = mergeEntries(claudeEntries, codexEntries); + // Merge all provider entries by date + const entries = mergeEntries( + claudeEntries, + codexEntries, + geminiParsed.data, + qwenParsed.data, + mistralParsed.data, + ); if (entries.length === 0) { console.log("No usage data found for the specified period."); @@ -300,8 +322,8 @@ export async function pushCommand(options: PushOptions, apiUrlOverride?: string) return; } - // Compute SHA-256 hash of concatenated raw JSONs - const hashInput = codexRaw ? claudeRaw + codexRaw : claudeRaw; + // Compute SHA-256 hash of concatenated raw JSONs from all providers + const hashInput = [claudeRaw, codexRaw, geminiRaw, qwenRaw, mistralRaw].filter(Boolean).join(""); const hash = createHash("sha256").update(hashInput).digest("hex"); const body: UsageSubmitRequest = { diff --git a/packages/cli/src/lib/ccusage.ts b/packages/cli/src/lib/ccusage.ts index 0620000d..eca5ba85 100644 --- a/packages/cli/src/lib/ccusage.ts +++ b/packages/cli/src/lib/ccusage.ts @@ -183,7 +183,7 @@ export interface CcusageOutput { export interface NormalizationAnomaly { date: string; - source: "ccusage" | "codex"; + source: "ccusage" | "codex" | "gemini" | "qwen" | "mistral"; mode: TokenNormalizationMode; confidence: TokenNormalizationConfidence; consistencyError: number; diff --git a/packages/cli/src/lib/gemini.ts b/packages/cli/src/lib/gemini.ts new file mode 100644 index 00000000..9d2a8551 --- /dev/null +++ b/packages/cli/src/lib/gemini.ts @@ -0,0 +1,306 @@ +import { execFile as execFileCb } from "node:child_process"; +import { readdir, readFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { CcusageDailyEntry, NormalizationAnomaly } from "./ccusage.js"; +import { + normalizeTokenBuckets, + summarizeNormalization, + type NormalizationMeta, + type NormalizationSummary, +} from "./token-normalization.js"; + +export interface GeminiOutput { + data: CcusageDailyEntry[]; + anomalies?: NormalizationAnomaly[]; + normalizationSummary?: NormalizationSummary; + entryMeta?: Array<{ date: string; meta: NormalizationMeta }>; +} + +const GCUSAGE_PKG = "gcusage@0"; + +/** + * Async — returns raw JSON string with Gemini usage data. + * Tries gcusage CLI first; falls back to reading session files directly. + * Returns empty string on failure. + */ +export async function runGeminiRawAsync(sinceDate: string, untilDate: string): Promise { + // Try gcusage CLI first (reads telemetry.log) + try { + const since = formatIsoDate(sinceDate); + const until = formatIsoDate(untilDate); + const result = await execGcusageAsync(["--json", "--period", "day", "--since", since, "--until", until]); + // gcusage returns "[]" when no telemetry.log exists — fall through to session reader + if (result.trim() !== "[]") return result; + } catch { + // gcusage not installed or failed — fall through + } + + // Fallback: read session files directly from ~/.gemini/tmp/*/chats/ + try { + const data = await readGeminiSessions(sinceDate, untilDate); + if (data.length === 0) return ""; + return JSON.stringify(data); + } catch { + return ""; + } +} + +/** Convert YYYYMMDD to YYYY-MM-DD if needed. */ +function formatIsoDate(d: string): string { + if (/^\d{8}$/.test(d)) return `${d.slice(0, 4)}-${d.slice(4, 6)}-${d.slice(6, 8)}`; + return d; +} + +function execGcusageAsync(args: string[]): Promise { + const cmd = process.versions.bun !== undefined ? "bunx" : "npx"; + const prefix = process.versions.bun !== undefined ? ["--bun"] : ["--yes"]; + + return new Promise((resolve, reject) => { + execFileCb(cmd, [...prefix, GCUSAGE_PKG, ...args], { + encoding: "utf-8", + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + }, (err, stdout) => { + if (err) reject(err); + else resolve(stdout); + }); + }); +} + +// --------------------------------------------------------------------------- +// gcusage CLI output parser +// --------------------------------------------------------------------------- + +/** + * gcusage daily JSON format: + * [{date: "YYYY-MM-DD", models: string[], input: number, output: number, thought: number, cache: number, tool: number}] + * + * Note: gcusage's `input` is net input (cache already subtracted). + * We add cache back for our normalized format where inputTokens is gross. + */ +interface GcusageEntry { + date: string; + models: string[]; + input: number; + output: number; + thought: number; + cache: number; + tool: number; +} + +// --------------------------------------------------------------------------- +// Direct session file reader (fallback when gcusage has no data) +// --------------------------------------------------------------------------- + +interface GeminiSessionMessage { + type?: string; + tokens?: { + input?: number; + output?: number; + cached?: number; + thoughts?: number; + tool?: number; + total?: number; + }; + model?: string; +} + +interface GeminiSession { + sessionId?: string; + startTime?: string; + messages?: GeminiSessionMessage[]; +} + +interface DailyAggregate { + models: Set; + input: number; + output: number; + thought: number; + cache: number; + tool: number; +} + +async function readGeminiSessions(sinceDate: string, untilDate: string): Promise { + const geminiDir = join(homedir(), ".gemini"); + if (!existsSync(geminiDir)) return []; + + const tmpDir = join(geminiDir, "tmp"); + if (!existsSync(tmpDir)) return []; + + const sinceIso = formatIsoDate(sinceDate); + const untilIso = formatIsoDate(untilDate); + + const byDate = new Map(); + + let projectDirs: string[]; + try { + projectDirs = await readdir(tmpDir); + } catch { + return []; + } + + for (const project of projectDirs) { + const chatsDir = join(tmpDir, project, "chats"); + if (!existsSync(chatsDir)) continue; + + let sessionFiles: string[]; + try { + sessionFiles = await readdir(chatsDir); + } catch { + continue; + } + + for (const file of sessionFiles) { + if (!file.endsWith(".json")) continue; + + // session-YYYY-MM-DDTHH-MM-{id}.json + const dateMatch = file.match(/session-(\d{4}-\d{2}-\d{2})/); + if (!dateMatch) continue; + + const sessionDate = dateMatch[1]!; + if (sessionDate < sinceIso || sessionDate > untilIso) continue; + + try { + const raw = await readFile(join(chatsDir, file), "utf-8"); + const session: GeminiSession = JSON.parse(raw); + if (!session.messages) continue; + + // Use startTime date or filename date + const date = session.startTime?.match(/^(\d{4}-\d{2}-\d{2})/)?.[1] ?? sessionDate; + if (date < sinceIso || date > untilIso) continue; + + let agg = byDate.get(date); + if (!agg) { + agg = { models: new Set(), input: 0, output: 0, thought: 0, cache: 0, tool: 0 }; + byDate.set(date, agg); + } + + for (const msg of session.messages) { + if (!msg.tokens) continue; + if (msg.model) agg.models.add(msg.model); + + agg.input += msg.tokens.input ?? 0; + agg.output += msg.tokens.output ?? 0; + agg.thought += msg.tokens.thoughts ?? 0; + agg.cache += msg.tokens.cached ?? 0; + agg.tool += msg.tokens.tool ?? 0; + } + } catch { + continue; + } + } + } + + // Return in gcusage format so parseGeminiOutput handles both paths + const entries: GcusageEntry[] = []; + for (const [date, agg] of byDate) { + // gcusage convention: input is net (cache subtracted from input total) + entries.push({ + date, + models: [...agg.models], + input: agg.input, + output: agg.output, + thought: agg.thought, + cache: agg.cache, + tool: agg.tool, + }); + } + + entries.sort((a, b) => a.date.localeCompare(b.date)); + return entries; +} + +// --------------------------------------------------------------------------- +// Parser (shared by both gcusage and session-reader paths) +// --------------------------------------------------------------------------- + +export function parseGeminiOutput(raw: string): GeminiOutput { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return { + data: [], + anomalies: [{ + date: "unknown", + source: "gemini", + mode: "unresolved", + confidence: "low", + consistencyError: 0, + warnings: ["Failed to parse Gemini JSON output."], + }], + normalizationSummary: { + total: 1, + anomalies: 1, + byMode: { unresolved: 1 }, + byConfidence: { low: 1 }, + }, + }; + } + + if (!Array.isArray(parsed)) { + return { data: [], anomalies: [], normalizationSummary: summarizeNormalization([]), entryMeta: [] }; + } + + const entries = parsed as GcusageEntry[]; + if (entries.length === 0) { + return { data: [], anomalies: [], normalizationSummary: summarizeNormalization([]), entryMeta: [] }; + } + + const normalizedRows = entries + .filter((e) => e.date) + .map((e) => { + // gcusage/session-reader: input is net (cache already subtracted by gcusage, + // or raw per-message input from session files where cached is separate). + // Gross input = input + cache for our normalized format. + const grossInput = e.input + e.cache; + const totalOutput = e.output + e.thought + e.tool; + const totalTokens = grossInput + totalOutput + e.cache; + + const normalized = normalizeTokenBuckets( + { + inputTokens: grossInput, + outputTokens: totalOutput, + cacheReadTokens: e.cache, + cacheCreationTokens: 0, + totalTokens, + }, + { source: "generic", cacheSemantics: "separate" }, + ); + + return { + date: e.date, + meta: normalized.meta, + entry: { + date: e.date, + models: e.models ?? [], + inputTokens: normalized.normalized.inputTokens, + outputTokens: normalized.normalized.outputTokens, + cacheCreationTokens: normalized.normalized.cacheCreationTokens, + cacheReadTokens: normalized.normalized.cacheReadTokens, + totalTokens: normalized.normalized.totalTokens, + costUSD: 0, // Gemini CLI is free tier; no cost data available + } satisfies CcusageDailyEntry, + }; + }); + + const anomalies: NormalizationAnomaly[] = normalizedRows + .filter((row) => row.meta.mode === "unresolved" || row.meta.confidence !== "high" || row.meta.warnings.length > 0) + .map((row) => ({ + date: row.date, + source: "gemini" as const, + mode: row.meta.mode, + confidence: row.meta.confidence, + consistencyError: row.meta.consistencyError, + warnings: row.meta.warnings, + })); + + return { + data: normalizedRows.map((row) => row.entry), + anomalies, + normalizationSummary: summarizeNormalization(normalizedRows.map((row) => row.meta)), + entryMeta: normalizedRows.map((row) => ({ date: row.date, meta: row.meta })), + }; +} diff --git a/packages/cli/src/lib/mistral.ts b/packages/cli/src/lib/mistral.ts new file mode 100644 index 00000000..29bad885 --- /dev/null +++ b/packages/cli/src/lib/mistral.ts @@ -0,0 +1,200 @@ +import { readdir, readFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { CcusageDailyEntry, NormalizationAnomaly } from "./ccusage.js"; +import { + summarizeNormalization, + type NormalizationMeta, + type NormalizationSummary, +} from "./token-normalization.js"; + +export interface MistralOutput { + data: CcusageDailyEntry[]; + anomalies?: NormalizationAnomaly[]; + normalizationSummary?: NormalizationSummary; + entryMeta?: Array<{ date: string; meta: NormalizationMeta }>; +} + +/** + * Mistral Vibe stores session data in: + * ~/.vibe/logs/session/session_{YYYYMMDD}_{HHMMSS}_{id}/ + * - meta.json: { session_id, start_time, stats: { session_prompt_tokens, session_completion_tokens, ... } } + * - messages.jsonl: JSONL with LLMMessage objects + * + * VIBE_HOME env var can override the ~/.vibe/ base directory. + */ + +/** Async — reads Mistral Vibe session metadata and aggregates by date. Empty on failure. */ +export async function runMistralRawAsync(sinceDate: string, untilDate: string): Promise { + try { + const data = await readMistralSessions(sinceDate, untilDate); + if (data.length === 0) return ""; + return JSON.stringify(data); + } catch { + return ""; + } +} + +interface MistralSessionMeta { + session_id?: string; + start_time?: string; + end_time?: string | null; + stats?: { + session_prompt_tokens?: number; + session_completion_tokens?: number; + context_tokens?: number; + input_price_per_million?: number; + output_price_per_million?: number; + }; + config?: { + active_model?: string; + provider?: { + model?: string; + }; + }; + agent_profile?: { + model?: string; + }; +} + +interface DailyAggregate { + models: Set; + inputTokens: number; + outputTokens: number; + costUSD: number; +} + +/** Convert YYYYMMDD to YYYY-MM-DD if needed. */ +function toIsoDate(d: string): string { + if (/^\d{8}$/.test(d)) return `${d.slice(0, 4)}-${d.slice(4, 6)}-${d.slice(6, 8)}`; + return d; +} + +/** Extract YYYY-MM-DD from an ISO timestamp or folder name. */ +function extractDate(s: string): string | null { + // ISO timestamp: 2026-03-05T13:11:24Z + const isoMatch = s.match(/^(\d{4}-\d{2}-\d{2})/); + if (isoMatch) return isoMatch[1]!; + + // Folder timestamp: 20260305_131124 + const folderMatch = s.match(/(\d{4})(\d{2})(\d{2})_\d{6}/); + if (folderMatch) return `${folderMatch[1]}-${folderMatch[2]}-${folderMatch[3]}`; + + return null; +} + +function getVibeHome(): string { + return process.env.VIBE_HOME || join(homedir(), ".vibe"); +} + +async function readMistralSessions(sinceDate: string, untilDate: string): Promise { + const vibeHome = getVibeHome(); + if (!existsSync(vibeHome)) return []; + + const sessionLogDir = join(vibeHome, "logs", "session"); + if (!existsSync(sessionLogDir)) return []; + + const sinceIso = toIsoDate(sinceDate); + const untilIso = toIsoDate(untilDate); + + const byDate = new Map(); + + let sessionDirs: string[]; + try { + sessionDirs = await readdir(sessionLogDir); + } catch { + return []; + } + + for (const dir of sessionDirs) { + // Folder pattern: session_YYYYMMDD_HHMMSS_{id} + const dateFromFolder = extractDate(dir.replace(/^session_/, "")); + + const metaPath = join(sessionLogDir, dir, "meta.json"); + if (!existsSync(metaPath)) continue; + + try { + const raw = await readFile(metaPath, "utf-8"); + const meta: MistralSessionMeta = JSON.parse(raw); + + // Determine session date from meta.start_time or folder name + const date = (meta.start_time ? extractDate(meta.start_time) : null) ?? dateFromFolder; + if (!date || date < sinceIso || date > untilIso) continue; + + const stats = meta.stats; + if (!stats) continue; + + const promptTokens = stats.session_prompt_tokens ?? 0; + const completionTokens = stats.session_completion_tokens ?? 0; + if (promptTokens === 0 && completionTokens === 0) continue; + + let agg = byDate.get(date); + if (!agg) { + agg = { models: new Set(), inputTokens: 0, outputTokens: 0, costUSD: 0 }; + byDate.set(date, agg); + } + + // Extract model name: config.active_model (primary), then fallbacks + const model = meta.config?.active_model ?? meta.agent_profile?.model ?? meta.config?.provider?.model; + if (model) agg.models.add(model); + + agg.inputTokens += promptTokens; + agg.outputTokens += completionTokens; + + // Estimate cost from pricing info if available + if (stats.input_price_per_million && stats.output_price_per_million) { + agg.costUSD += + (promptTokens / 1_000_000) * stats.input_price_per_million + + (completionTokens / 1_000_000) * stats.output_price_per_million; + } + } catch { + // Skip malformed session metadata + continue; + } + } + + const entries: CcusageDailyEntry[] = []; + for (const [date, agg] of byDate) { + entries.push({ + date, + models: [...agg.models], + inputTokens: agg.inputTokens, + outputTokens: agg.outputTokens, + cacheCreationTokens: 0, + cacheReadTokens: 0, + totalTokens: agg.inputTokens + agg.outputTokens, + costUSD: agg.costUSD, + }); + } + + entries.sort((a, b) => a.date.localeCompare(b.date)); + return entries; +} + +export function parseMistralOutput(raw: string): MistralOutput { + if (!raw) { + return { data: [], anomalies: [], normalizationSummary: summarizeNormalization([]), entryMeta: [] }; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return { data: [], anomalies: [], normalizationSummary: summarizeNormalization([]), entryMeta: [] }; + } + + if (!Array.isArray(parsed)) { + return { data: [], anomalies: [], normalizationSummary: summarizeNormalization([]), entryMeta: [] }; + } + + const data = parsed as CcusageDailyEntry[]; + + // Direct file reading produces pre-normalized data — no anomalies expected + return { + data, + anomalies: [], + normalizationSummary: summarizeNormalization([]), + entryMeta: [], + }; +} diff --git a/packages/cli/src/lib/qwen.ts b/packages/cli/src/lib/qwen.ts new file mode 100644 index 00000000..9a13e3d3 --- /dev/null +++ b/packages/cli/src/lib/qwen.ts @@ -0,0 +1,178 @@ +import { readdir, readFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { CcusageDailyEntry, NormalizationAnomaly } from "./ccusage.js"; +import { + summarizeNormalization, + type NormalizationMeta, + type NormalizationSummary, +} from "./token-normalization.js"; + +export interface QwenOutput { + data: CcusageDailyEntry[]; + anomalies?: NormalizationAnomaly[]; + normalizationSummary?: NormalizationSummary; + entryMeta?: Array<{ date: string; meta: NormalizationMeta }>; +} + +/** + * Qwen Code stores session data as JSONL in: + * ~/.qwen/projects/{project-path}/chats/{session-uuid}.jsonl + * + * Each line is a JSON object. Token usage is in entries with: + * - type: "assistant" → usageMetadata: { promptTokenCount, candidatesTokenCount, ... } + * - model field on assistant entries + */ + +/** Async — reads Qwen Code session files and aggregates by date. Empty on failure. */ +export async function runQwenRawAsync(sinceDate: string, untilDate: string): Promise { + try { + const data = await readQwenSessions(sinceDate, untilDate); + if (data.length === 0) return ""; + return JSON.stringify(data); + } catch { + return ""; + } +} + +interface DailyAggregate { + models: Set; + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + thoughtsTokens: number; + totalTokens: number; +} + +/** Convert YYYYMMDD to YYYY-MM-DD if needed. */ +function toIsoDate(d: string): string { + if (/^\d{8}$/.test(d)) return `${d.slice(0, 4)}-${d.slice(4, 6)}-${d.slice(6, 8)}`; + return d; +} + +async function readQwenSessions(sinceDate: string, untilDate: string): Promise { + const qwenDir = join(homedir(), ".qwen"); + if (!existsSync(qwenDir)) return []; + + const projectsDir = join(qwenDir, "projects"); + if (!existsSync(projectsDir)) return []; + + const sinceIso = toIsoDate(sinceDate); + const untilIso = toIsoDate(untilDate); + + const byDate = new Map(); + + // Scan all project directories under ~/.qwen/projects/ + let projectDirs: string[]; + try { + projectDirs = await readdir(projectsDir); + } catch { + return []; + } + + for (const project of projectDirs) { + const chatsDir = join(projectsDir, project, "chats"); + if (!existsSync(chatsDir)) continue; + + let chatFiles: string[]; + try { + chatFiles = await readdir(chatsDir); + } catch { + continue; + } + + for (const file of chatFiles) { + if (!file.endsWith(".jsonl")) continue; + + try { + const raw = await readFile(join(chatsDir, file), "utf-8"); + const lines = raw.split("\n").filter((l) => l.trim()); + + for (const line of lines) { + let entry: Record; + try { + entry = JSON.parse(line); + } catch { + continue; + } + + // Only process assistant entries with usageMetadata + if (entry.type !== "assistant") continue; + + const usage = entry.usageMetadata as Record | undefined; + if (!usage) continue; + + const timestamp = entry.timestamp as string | undefined; + if (!timestamp) continue; + + const dateMatch = timestamp.match(/^(\d{4}-\d{2}-\d{2})/); + if (!dateMatch) continue; + + const date = dateMatch[1]!; + if (date < sinceIso || date > untilIso) continue; + + let agg = byDate.get(date); + if (!agg) { + agg = { models: new Set(), inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, thoughtsTokens: 0, totalTokens: 0 }; + byDate.set(date, agg); + } + + const model = entry.model as string | undefined; + if (model) agg.models.add(model); + + agg.inputTokens += usage.promptTokenCount ?? 0; + agg.outputTokens += usage.candidatesTokenCount ?? 0; + agg.cacheReadTokens += usage.cachedContentTokenCount ?? 0; + agg.thoughtsTokens += usage.thoughtsTokenCount ?? 0; + agg.totalTokens += usage.totalTokenCount ?? 0; + } + } catch { + continue; + } + } + } + + const entries: CcusageDailyEntry[] = []; + for (const [date, agg] of byDate) { + entries.push({ + date, + models: [...agg.models], + inputTokens: agg.inputTokens, + outputTokens: agg.outputTokens + agg.thoughtsTokens, + cacheCreationTokens: 0, + cacheReadTokens: agg.cacheReadTokens, + totalTokens: agg.totalTokens, + costUSD: 0, // Qwen Code free tier via OAuth; no cost data + }); + } + + entries.sort((a, b) => a.date.localeCompare(b.date)); + return entries; +} + +export function parseQwenOutput(raw: string): QwenOutput { + if (!raw) { + return { data: [], anomalies: [], normalizationSummary: summarizeNormalization([]), entryMeta: [] }; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return { data: [], anomalies: [], normalizationSummary: summarizeNormalization([]), entryMeta: [] }; + } + + if (!Array.isArray(parsed)) { + return { data: [], anomalies: [], normalizationSummary: summarizeNormalization([]), entryMeta: [] }; + } + + const data = parsed as CcusageDailyEntry[]; + + return { + data, + anomalies: [], + normalizationSummary: summarizeNormalization([]), + entryMeta: [], + }; +}