diff --git a/docs/code/configuration.md b/docs/code/configuration.md index 569c463..0301782 100644 --- a/docs/code/configuration.md +++ b/docs/code/configuration.md @@ -253,6 +253,32 @@ SENTRY_DISABLED=1 Set this in your shell environment or in `.devintern-code/.env`. +## Anonymous Usage Analytics + +The CLI sends one anonymous usage event per run to DevIntern's PostHog project so we can understand popularity and which features are used. It never sends task content, code, repository names, file paths, or credentials — only: + +- CLI version, OS, architecture +- Active tracker type (e.g. `jira`, `linear`) and run mode (tasks / query / estimate) +- Task count and boolean feature flags (`--create-pr`, `--auto-review`, `--estimate`, sandbox provider) +- Whether the session runs in CI + +A random anonymous ID is generated once per project and stored in `.devintern-code/telemetry.json`. Analytics are disabled automatically when running from source. To opt out, either: + +```bash +# Shell or .devintern-code/.env +DEVINTERN_TELEMETRY_DISABLED=1 +``` + +or set in `.devintern-code/settings.json`: + +```json +{ + "analytics": { "enabled": false } +} +``` + +See [devintern.com/privacy](https://devintern.com/privacy/) for details. + ## Readiness Check Run `devintern doctor` for a one-screen answer to "is everything set up?": diff --git a/packages/code/build.ts b/packages/code/build.ts index 7a05c83..75ba490 100644 --- a/packages/code/build.ts +++ b/packages/code/build.ts @@ -9,6 +9,11 @@ await Bun.build({ minify: true, define: { __VERSION__: JSON.stringify(pkg.version), + // Analytics is permanently disabled in builds without a key (local dev). + __POSTHOG_API_KEY__: JSON.stringify(process.env.POSTHOG_API_KEY?.trim() ?? ""), + __POSTHOG_HOST__: JSON.stringify( + process.env.POSTHOG_HOST?.trim() || "https://us.i.posthog.com", + ), }, }); diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 63c43d0..994a071 100755 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -49,6 +49,8 @@ import { maybeOfferCliUpdate, resolveConfigDir, } from "@devintern/utils"; +import { flushAnalytics, isAnonymousIdNewlyCreated, track } from "./lib/analytics"; +import type { AnalyticsPropValue } from "./lib/analytics"; import { ReadonlyAnalysisError, runAnalysisWithFallback } from "./lib/analysis-mode"; import { resolveAgentModel } from "./lib/agent-model"; import { parseAgentJsonObject } from "./lib/agent-json"; @@ -108,6 +110,34 @@ async function checkForCliUpdate(): Promise { const __filename_resolved = fileURLToPath(import.meta.url); const __dirname_resolved = dirname(__filename_resolved); +const KNOWN_SANDBOX_PROVIDERS = new Set([ + "none", + "auto", + "native", + "nono", + "srt", + "docker", + "smolvm", +]); + +/** Allowlisted, non-identifying props for the `cli_run` analytics event. */ +function buildCliRunProps(tracker: string): Record { + const sandboxProvider = options.sandbox ?? process.env.AGENT_SANDBOX; + return { + cli_version: VERSION, + os: process.platform, + arch: process.arch, + ci: isAutomatedEnvironment(), + tracker, + run_mode: options.estimate ? "estimate" : options.query ? "query" : "tasks", + task_count: options.query ? undefined : taskKeys.length, + create_pr: options.createPr === true, + auto_review: options.autoReview === true, + estimate: options.estimate === true, + sandbox: KNOWN_SANDBOX_PROVIDERS.has(sandboxProvider ?? "") ? sandboxProvider : undefined, + }; +} + /** * Rename legacy `.claude-intern` project config to `.devintern-code` once. */ @@ -2283,6 +2313,18 @@ async function main(): Promise { process.exit(1); } + // Anonymous usage analytics (PostHog). Fire-and-forget; never blocks or + // fails the run. Opt out via DEVINTERN_TELEMETRY_DISABLED=1 or + // analytics.enabled: false in .devintern-code/settings.json. + const firstTelemetryRun = isAnonymousIdNewlyCreated(); + void track("cli_run", buildCliRunProps(activeTrackerType)); + if (firstTelemetryRun && !isAutomatedEnvironment()) { + console.log( + "ℹ️ devintern collects anonymous usage stats (never task content, code, or credentials)." + + "\n Disable with DEVINTERN_TELEMETRY_DISABLED=1 — see https://devintern.com/privacy/", + ); + } + // Validate environment — skip when every argument is a local markdown file path // (those tasks need no PM credentials). With missing credentials in an // interactive terminal, offer the setup wizard inline before failing. @@ -2532,6 +2574,7 @@ async function main(): Promise { if (lockManager) { lockManager.release(); } + await flushAnalytics(); if (estimationResults.failed > 0) { process.exit(1); } @@ -2603,6 +2646,7 @@ async function main(): Promise { if (lockManager) { lockManager.release(); } + await flushAnalytics(); process.exit(1); } } @@ -2611,6 +2655,7 @@ async function main(): Promise { if (lockManager) { lockManager.release(); } + await flushAnalytics(); } catch (error) { const err = error as Error; console.error(`❌ Error: ${err.message}`); @@ -2621,6 +2666,7 @@ async function main(): Promise { if (lockManager) { lockManager.release(); } + await flushAnalytics(); process.exit(1); } } diff --git a/packages/code/src/lib/analytics.ts b/packages/code/src/lib/analytics.ts new file mode 100644 index 0000000..a1ed483 --- /dev/null +++ b/packages/code/src/lib/analytics.ts @@ -0,0 +1,227 @@ +/** + * Anonymous product analytics for the CLI (PostHog). + * + * Sends one fire-and-forget event per run. Never sends task keys, prompts, + * repo names, paths, or credentials — only allowlisted enum/bool/number props + * (see ALLOWED_PROP_KEYS). Opt out via DEVINTERN_TELEMETRY_DISABLED=1 or + * `analytics.enabled: false` in .devintern-code/settings.json. + */ + +import { randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { resolveConfigDir } from "@devintern/utils"; + +// Injected at build time via --define; absent when running from source, +// which permanently disables analytics in dev builds. +declare const __POSTHOG_API_KEY__: string; +declare const __POSTHOG_HOST__: string; + +const CONFIG_DIR_NAME = ".devintern-code"; +const DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com"; + +export type AnalyticsPropValue = string | boolean | number; + +/** Curated event names — keep in sync with privacy copy. */ +export type AnalyticsEvent = "cli_run" | "analytics_opt_out"; + +const ALLOWED_PROP_KEYS = new Set([ + "cli_version", + "os", + "arch", + "ci", + "tracker", + "run_mode", + "task_count", + "create_pr", + "auto_review", + "estimate", + "sandbox", +]); + +/** Minimal send surface so tests can inject a mock without network access. */ +export interface AnalyticsSender { + send(payload: { + api_key: string; + event: string; + distinct_id: string; + properties: Record; + timestamp: string; + }): Promise; +} + +type RealSender = AnalyticsSender & { inflight: Promise[] }; + +let senderForTests: AnalyticsSender | null | undefined; +let realSender: RealSender | undefined; + +/** @internal Test override: `null` forces disabled capture; `undefined` restores the real sender. */ +export function setAnalyticsSenderForTests(value: AnalyticsSender | null | undefined): void { + senderForTests = value; + realSender = undefined; +} + +export function resolveApiKey(): string { + const baked = typeof __POSTHOG_API_KEY__ === "string" ? __POSTHOG_API_KEY__.trim() : ""; + return baked || process.env.POSTHOG_API_KEY?.trim() || ""; +} + +function resolveHost(): string { + if (typeof __POSTHOG_HOST__ === "string" && __POSTHOG_HOST__.trim().length > 0) { + return __POSTHOG_HOST__.trim(); + } + return process.env.POSTHOG_HOST?.trim() || DEFAULT_POSTHOG_HOST; +} + +/** + * Truthy env values disable telemetry; the variable only needs to exist for + * common CI conventions like `DEVINTERN_TELEMETRY_DISABLED=` to work too. + */ +export function isTelemetryDisabledByEnv( + env: Record = process.env, +): boolean { + const raw = env.DEVINTERN_TELEMETRY_DISABLED; + if (raw === undefined) return false; + const value = raw.trim().toLowerCase(); + return value === "" || value === "0" ? false : true; +} + +interface TelemetrySettingsShape { + analytics?: { enabled?: boolean }; +} + +/** + * Reads `analytics.enabled` from .devintern-code/settings.json. Returns + * `undefined` when unset or unreadable so env/config absence means opt-in. + */ +export function readAnalyticsEnabledFromSettings(configDir?: string): boolean | undefined { + try { + const dir = + configDir ?? resolveConfigDir({ startDir: process.cwd(), configDirName: CONFIG_DIR_NAME }); + const settingsPath = join(dir, "settings.json"); + if (!existsSync(settingsPath)) return undefined; + const parsed = JSON.parse(readFileSync(settingsPath, "utf8")) as TelemetrySettingsShape; + return parsed.analytics?.enabled; + } catch { + return undefined; + } +} + +export function isAnalyticsEnabled(configDir?: string): boolean { + if (!resolveApiKey()) return false; + if (isTelemetryDisabledByEnv()) return false; + if (readAnalyticsEnabledFromSettings(configDir) === false) return false; + return true; +} + +/** Scrub to allowlisted keys; drop nullish values. */ +export function scrubProps( + props: Record | undefined, +): Record { + if (!props) return {}; + const out: Record = {}; + for (const [key, value] of Object.entries(props)) { + if (!ALLOWED_PROP_KEYS.has(key)) continue; + if (value === undefined) continue; + out[key] = value; + } + return out; +} + +function getOrCreateAnonymousId(configDir?: string): string { + const dir = + configDir ?? resolveConfigDir({ startDir: process.cwd(), configDirName: CONFIG_DIR_NAME }); + const telemetryFile = join(dir, "telemetry.json"); + try { + if (existsSync(telemetryFile)) { + const parsed = JSON.parse(readFileSync(telemetryFile, "utf8")) as { anonymousId?: string }; + if (parsed.anonymousId) return parsed.anonymousId; + } + } catch { + // Corrupt file — fall through and regenerate. + } + const id = randomUUID(); + try { + mkdirSync(dir, { recursive: true }); + writeFileSync(telemetryFile, `${JSON.stringify({ anonymousId: id }, null, 2)}\n`, "utf8"); + } catch { + // Read-only config dir — use an ephemeral id for this run only. + } + return id; +} + +function getSender(): AnalyticsSender | null { + if (senderForTests !== undefined) return senderForTests ?? null; + if (!realSender) { + realSender = { + inflight: [], + async send(payload) { + const request = fetch(`${resolveHost()}/i/v0/e/`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(payload), + }).then( + () => {}, + () => {}, + ); + this.inflight.push(request); + await request; + }, + }; + } + return realSender; +} + +/** True when this is the first run that created telemetry.json (for disclosure). */ +export function isAnonymousIdNewlyCreated(configDir?: string): boolean { + try { + const dir = + configDir ?? resolveConfigDir({ startDir: process.cwd(), configDirName: CONFIG_DIR_NAME }); + return !existsSync(join(dir, "telemetry.json")); + } catch { + return false; + } +} + +/** + * Capture a product event without blocking or ever throwing. The returned + * promise resolves once the payload is handed to the network layer (or + * immediately when analytics is disabled). + */ +export async function track( + event: AnalyticsEvent, + props?: Record, + options: { configDir?: string } = {}, +): Promise { + try { + if (!isAnalyticsEnabled(options.configDir)) return; + const sender = getSender(); + if (!sender) return; + await sender.send({ + api_key: resolveApiKey(), + event, + distinct_id: getOrCreateAnonymousId(options.configDir), + properties: scrubProps(props), + timestamp: new Date().toISOString(), + }); + } catch { + // Swallow — product use must not fail because of analytics. + } +} + +/** + * Await pending sends so short-lived runs do not drop their event before + * exit. Bounded by `timeoutMs`; never throws. + */ +export async function flushAnalytics(timeoutMs = 1500): Promise { + try { + const sender = getSender(); + if (!sender || !("inflight" in sender)) return; + await Promise.race([ + Promise.all((sender as RealSender).inflight.splice(0)), + new Promise((resolve) => setTimeout(resolve, timeoutMs)), + ]); + } catch { + // ignore + } +} diff --git a/packages/code/src/types/settings.ts b/packages/code/src/types/settings.ts index f01d1bf..9494e13 100644 --- a/packages/code/src/types/settings.ts +++ b/packages/code/src/types/settings.ts @@ -69,6 +69,12 @@ export interface TrackerSection { }; } +/** Anonymous usage analytics preferences. */ +export interface AnalyticsSettings { + /** Set to false to disable anonymous usage analytics for this project. */ + enabled?: boolean; +} + /** * Per-project configuration settings. * @@ -80,6 +86,9 @@ export interface TrackerSection { * honored for JIRA when no `jira` section exists. */ export interface ProjectSettings { + /** Anonymous usage analytics preferences. */ + analytics?: AnalyticsSettings; + /** * Legacy project configurations (backward compatible). * diff --git a/packages/code/tests/analytics.test.ts b/packages/code/tests/analytics.test.ts new file mode 100644 index 0000000..d2f5574 --- /dev/null +++ b/packages/code/tests/analytics.test.ts @@ -0,0 +1,178 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { + flushAnalytics, + isAnonymousIdNewlyCreated, + isAnalyticsEnabled, + isTelemetryDisabledByEnv, + readAnalyticsEnabledFromSettings, + scrubProps, + setAnalyticsSenderForTests, + track, +} from "../src/lib/analytics"; + +const tmpDirs: string[] = []; + +function makeConfigDir(withSettings?: object): string { + const dir = mkdtempSync(join("/tmp", "devintern-analytics-")); + tmpDirs.push(dir); + if (withSettings) { + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "settings.json"), JSON.stringify(withSettings), "utf8"); + } + return dir; +} + +afterEach(() => { + setAnalyticsSenderForTests(undefined); + delete process.env.POSTHOG_API_KEY; + delete process.env.DEVINTERN_TELEMETRY_DISABLED; + for (const dir of tmpDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe("isTelemetryDisabledByEnv", () => { + test("unset means enabled", () => { + expect(isTelemetryDisabledByEnv({})).toBe(false); + }); + + test("truthy values disable", () => { + expect(isTelemetryDisabledByEnv({ DEVINTERN_TELEMETRY_DISABLED: "1" })).toBe(true); + expect(isTelemetryDisabledByEnv({ DEVINTERN_TELEMETRY_DISABLED: "true" })).toBe(true); + expect(isTelemetryDisabledByEnv({ DEVINTERN_TELEMETRY_DISABLED: "YES" })).toBe(true); + }); + + test("empty and zero keep analytics enabled", () => { + expect(isTelemetryDisabledByEnv({ DEVINTERN_TELEMETRY_DISABLED: "" })).toBe(false); + expect(isTelemetryDisabledByEnv({ DEVINTERN_TELEMETRY_DISABLED: "0" })).toBe(false); + }); +}); + +describe("readAnalyticsEnabledFromSettings", () => { + test("undefined when no settings file exists", () => { + expect(readAnalyticsEnabledFromSettings(makeConfigDir())).toBeUndefined(); + }); + + test("reads analytics.enabled=false", () => { + const dir = makeConfigDir({ analytics: { enabled: false } }); + expect(readAnalyticsEnabledFromSettings(dir)).toBe(false); + }); + + test("undefined when analytics section missing", () => { + const dir = makeConfigDir({ jira: {} }); + expect(readAnalyticsEnabledFromSettings(dir)).toBeUndefined(); + }); + + test("undefined on malformed settings instead of throwing", () => { + const dir = makeConfigDir(); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "settings.json"), "{not json", "utf8"); + expect(readAnalyticsEnabledFromSettings(dir)).toBeUndefined(); + }); +}); + +describe("isAnalyticsEnabled", () => { + test("disabled when the API key is absent (source/dev builds)", () => { + delete process.env.POSTHOG_API_KEY; + expect(isAnalyticsEnabled(makeConfigDir())).toBe(false); + }); +}); + +describe("scrubProps", () => { + test("drops non-allowlisted keys and undefined values", () => { + expect( + scrubProps({ + tracker: "jira", + task_count: 3, + task_key: "PROJ-123", + repo_url: "https://github.com/acme/webapp", + create_pr: undefined, + }), + ).toEqual({ tracker: "jira", task_count: 3 }); + }); + + test("returns empty object for undefined input", () => { + expect(scrubProps(undefined)).toEqual({}); + }); +}); + +describe("track", () => { + test("sends allowlisted payload with a stable anonymous id", async () => { + process.env.POSTHOG_API_KEY = "phc_test"; + const dir = makeConfigDir(); + + let received: unknown; + setAnalyticsSenderForTests({ + send: async (payload) => { + received = payload; + }, + }); + + await track("cli_run", { tracker: "linear", task_key: "ENG-42" }, { configDir: dir }); + + expect(received).toBeDefined(); + const payload = received as { + event: string; + distinct_id: string; + properties: Record; + }; + expect(payload.event).toBe("cli_run"); + expect(payload.properties).toEqual({ tracker: "linear" }); + expect(payload.distinct_id).toMatch(/[0-9a-f-]{36}/); + + await track("cli_run", {}, { configDir: dir }); + const second = received as { distinct_id: string }; + expect(second.distinct_id).toBe(payload.distinct_id); + }); + + test("no network when opted out via env", async () => { + process.env.POSTHOG_API_KEY = "phc_test"; + process.env.DEVINTERN_TELEMETRY_DISABLED = "1"; + let called = false; + setAnalyticsSenderForTests({ + send: async () => { + called = true; + }, + }); + await track("cli_run", {}, { configDir: makeConfigDir() }); + expect(called).toBe(false); + }); + + test("no network when disabled in settings", async () => { + process.env.POSTHOG_API_KEY = "phc_test"; + let called = false; + setAnalyticsSenderForTests({ + send: async () => { + called = true; + }, + }); + const dir = makeConfigDir({ analytics: { enabled: false } }); + await track("cli_run", {}, { configDir: dir }); + expect(called).toBe(false); + }); + + test("never throws when the sender fails", async () => { + process.env.POSTHOG_API_KEY = "phc_test"; + setAnalyticsSenderForTests({ + send: async () => { + throw new Error("network down"); + }, + }); + await expect(track("cli_run", {}, { configDir: makeConfigDir() })).resolves.toBeUndefined(); + }); +}); + +describe("anonymous id persistence", () => { + test("first run reports new, subsequent runs do not", async () => { + process.env.POSTHOG_API_KEY = "phc_test"; + setAnalyticsSenderForTests({ send: async () => {} }); + const dir = makeConfigDir(); + expect(isAnonymousIdNewlyCreated(dir)).toBe(true); + await track("cli_run", {}, { configDir: dir }); + expect(isAnonymousIdNewlyCreated(dir)).toBe(false); + }); + + test("flushAnalytics resolves without pending sends", async () => { + await expect(flushAnalytics(10)).resolves.toBeUndefined(); + }); +});