From 836ab12a08aed46f8d5bbfcc096611925328de89 Mon Sep 17 00:00:00 2001 From: glifocat Date: Thu, 17 Sep 2026 11:48:56 +0200 Subject: [PATCH 1/4] feat(typesafe-docs-drift): add phase-1 docs-drift detector with fixture replay Extract facts from a NanoClaw checkout with deterministic parsers (ncl verbs, flags and enums; environment variables; container_configs columns and cli_scope values; workspace skills; gateway selection; timestamp rules), find the top doc sections per fact with a BM25 index over the nanoclaw-docs pages, and send one TypeSafe fan-out request per fact x section asking contradicts, covers and staleness. Gate the answers and print ranked DRIFT / MISSING / UNSURE / OK rows with the doc page and heading, the model's numbers and the code evidence; save raw answers under a gitignored directory; --json for phase 2. --facts-only and --plan show the deterministic half without a key; --fixture replays recorded facts, sections and responses offline and --record saves a live run in that format. TYPESAFE_API_KEY is read from the environment only and never printed. Nothing is written to either repository. --- .claude-plugin/plugin.json | 2 +- .gitignore | 4 + CHANGELOG.md | 13 +- README.md | 1 + docs/skills-catalog.md | 36 +- docs/typesafe-docs-drift.md | 173 ++ skills/typesafe-docs-drift/SKILL.md | 109 ++ .../fixtures/nanoclaw-sample.json | 414 +++++ .../scripts/typesafe-docs-drift.py | 1398 +++++++++++++++++ tests/test_typesafe_docs_drift.py | 891 +++++++++++ 10 files changed, 3035 insertions(+), 6 deletions(-) create mode 100644 docs/typesafe-docs-drift.md create mode 100644 skills/typesafe-docs-drift/SKILL.md create mode 100644 skills/typesafe-docs-drift/fixtures/nanoclaw-sample.json create mode 100644 skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py create mode 100644 tests/test_typesafe_docs_drift.py diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index fbc3b25..a05a44f 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "nanoclaw-e2e", "description": "Install and verify NanoClaw on exe.dev, Proxmox, macOS or Windows WSL2, then triage failures against upstream issues and PRs with form-aware reporting.", - "version": "0.11.0", + "version": "0.12.0", "author": { "name": "Nano Co", "url": "https://github.com/nanocoai" diff --git a/.gitignore b/.gitignore index 5b78141..9ffdc46 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ __pycache__/ # Private live-run evidence and machine-specific terminal prototype. /evidence/ /tools/shared-terminal/ + +# typesafe-docs-drift raw answers (doc excerpts and code snippets) and this clone's worktrees. +/skills/typesafe-docs-drift/output/ +/.worktrees/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e688c0..cf240d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ instead of being backfilled as releases. ## Unreleased -Development milestone (plugin 0.11.0); not yet released. +Development milestone (plugin 0.12.0); supersedes the unreleased 0.11.0 +milestone and is not yet released. ### Added @@ -46,6 +47,16 @@ Development milestone (plugin 0.11.0); not yet released. verification and the Proxmox transport follow the per-source commits; single-block reports are unchanged. +- `typesafe-docs-drift` skill (phase 1 of a docs-drift detector): deterministic + extraction of facts from a NanoClaw checkout (`ncl` verbs, flags and enums, + environment variables, `container_configs` columns and `cli_scope` values, + workspace skills, gateway selection, timestamp rules), BM25 candidate search + over the nanoclaw-docs pages, one TypeSafe fan-out request per fact x section + (`contradicts`, `covers`, `staleness`), a confidence gate with `DRIFT` / + `MISSING` / `UNSURE` / `OK` verdicts, per-area summary with tokens and wall + time, raw answers under a gitignored directory, `--facts-only` and `--plan` + for the key-free half, and `--fixture` / `--record` replay. Reads + `TYPESAFE_API_KEY` from the environment only and writes to neither repository. - OpenCode public-wizard E2E for OpenRouter, DeepSeek, and custom/self-hosted API-key endpoints across exe.dev, Proxmox and Windows WSL2. Select the model with `--opencode-model`, the endpoint with `--opencode-base-url`, and its API diff --git a/README.md b/README.md index 0abf8cb..192a5c3 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ selected provider's authentication choices from that exact Git revision. | [e2e-wizard](skills/e2e-wizard/SKILL.md) | Drive the real public setup wizard through a PTY on a fresh exe.dev VM or Proxmox LXC, with a retained agent reply, service proof and sanitized evidence. | | [e2e-windows](skills/e2e-windows/SKILL.md) | Qualify Windows WSL2 with the local Docker Desktop engine, then run the public wizard with validated reply, service and sanitized evidence. | | [e2e-triage](skills/e2e-triage/SKILL.md) | Research unexpected E2E failures against upstream issues and PRs, recommend next actions, and prepare drafts using the target repository’s current issue forms. | +| [typesafe-docs-drift](skills/typesafe-docs-drift/SKILL.md) | Rank where the docs portal disagrees with the NanoClaw code using the TypeSafe decision API: deterministic fact extraction, lexical section search, contradicts / covers / staleness per pair, DRIFT / MISSING / UNSURE / OK report; offline fixture replay; nothing written. | | [shared-terminal](skills/shared-terminal/SKILL.md) | One local browser terminal shared by the human and agent for supervised installers and interactive handoffs on macOS or Linux. | Each skill follows the [Agent Skills format](https://agentskills.io). Use it with diff --git a/docs/skills-catalog.md b/docs/skills-catalog.md index ffb0574..fad55cf 100644 --- a/docs/skills-catalog.md +++ b/docs/skills-catalog.md @@ -1,11 +1,13 @@ # NanoClaw OSS Dev Tools: skill catalog -Verified: **2026-09-15**. This checkout contains **7 skills**, with plugin -manifest version **0.11.0** (development, unreleased). Three drive headless setup steps; `e2e-wizard` and +Verified: **2026-09-17**. This checkout contains **8 skills**, with plugin +manifest version **0.12.0** (development, unreleased). Three drive headless setup steps; `e2e-wizard` and `e2e-windows` drive the public interactive wizard. The shared `e2e-triage` skill researches unexpected failures and prepares reporting recommendations. The workflows and their qualifications are listed separately below. `shared-terminal` -provides a local terminal for human and agent handoffs. +provides a local terminal for human and agent handoffs. `typesafe-docs-drift` +ranks where the docs portal disagrees with the code through the TypeSafe +decision API and writes nothing. Installed skill copies must be updated separately, and the manifest version does not imply a tagged GitHub release. @@ -31,6 +33,7 @@ attempt exposed a harness tool-path bug and remains a retained failure. See the | [e2e-wizard](../skills/e2e-wizard/SKILL.md) | Public interactive setup in a fresh exe.dev VM or Proxmox LXC, driven through a real PTY and terminal emulator. | Fresh Proxmox wizard completed on 2026-09-11, with a retained agent's real reply and exact service verification. On 2026-09-15 a fresh exe.dev VM passed the distributed harness with OpenCode through a custom OpenAI-compatible endpoint. | Native macOS, OpenCode on Proxmox/WSL2 and the OpenRouter/DeepSeek backends, post-install channel/provider refresh and restart paths, reboot recovery, and real messaging channels remain unqualified. | | [e2e-windows](../skills/e2e-windows/SKILL.md) | Fresh Windows WSL2 distribution using the local Docker Desktop Linux engine. | Public wizard passed again on 2026-09-14 at current NanoClaw `main`; terminal-close and post-Windows-reboot inference also passed with locally validated sanitized evidence. | WSL-only restart still left the service/socket unavailable, and Docker Desktop did not start automatically after Windows reboot. | | [e2e-triage](../skills/e2e-triage/SKILL.md) | Agent workflow on the operator machine, using retained evidence and current upstream trackers. | Replayed retained Windows CA failure against live issues/PRs and the current NanoClaw bug form on 2026-09-12. | Instruction-driven; direct shell runs do not invoke it. No public submission was performed during validation. | +| [typesafe-docs-drift](../skills/typesafe-docs-drift/SKILL.md) | Ranks disagreements between a NanoClaw checkout and the nanoclaw-docs portal through the TypeSafe System One API: deterministic facts, lexical candidate sections, one fan-out request per pair, gated DRIFT / MISSING / UNSURE / OK verdicts. | Offline fixture replay and mocked-transport tests; one live run on 2026-09-16 (142 facts, 426 requests, 71.5 s: DRIFT 7, MISSING 1, UNSURE 11, OK 123; see [typesafe-docs-drift.md](typesafe-docs-drift.md)). | Detection only; candidate recall is lexical top-3; fixture answers are hand-written. | | [shared-terminal](../skills/shared-terminal/SKILL.md) | One local PTY with browser and agent control, for supervised interactive work. | macOS browser typing, interactive prompts, agent/human handoff and shutdown passed; real HTTP/PTY tests passed. | Native Windows, WSL and browser forwarding are unqualified. | The shared baseline for the three headless skills tested NanoClaw @@ -309,7 +312,7 @@ run inside the dedicated WSL2 distribution from the NanoClaw checkout. **Entry point:** [SKILL.md](../skills/e2e-triage/SKILL.md), invoked by the testing agent after an unexpected failure or directly against a retained run. Install it -alongside the E2E skill; the plugin includes all seven. This workflow adds no +alongside the E2E skill; the plugin includes all eight. This workflow adds no runtime or GitHub dependency to the shell/Python test drivers. - Captures distinct failures without rewriting installation/recovery outcomes. @@ -339,6 +342,31 @@ bug form and contribution rules were inspected at This is manual workflow validation with live reads, not a fresh E2E installation or automated proof of future agent behavior. No issue or comment was posted. +## typesafe-docs-drift + +**Entry point:** [typesafe-docs-drift.py](../skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py), +run from a NanoClaw checkout (`--code`, default `.`) with a nanoclaw-docs +checkout (`--docs`). + +- Needs Python 3.10+ and, for live mode, `TYPESAFE_API_KEY` exported in the + shell. Standard library only; no SDK. +- Extracts facts with code (57 `ncl`, 18 env, 8 container-config, 53 skills, + 3 gateway, 3 timestamp facts on NanoClaw `6e5008fe`), finds the top 3 doc + sections per fact with a BM25 index, and sends one fan-out request per pair + asking `contradicts`, `covers` and `staleness`. +- Gates on the nouls (0.7) and the score confidence (0.5), all overridable, + and prints one ranked row per fact with the doc page and heading, the + numbers and the code evidence; a per-area summary reports tokens and wall + time. +- `--facts-only` and `--plan` show the deterministic half without a key; + `--fixture` replays recorded facts, sections and responses; `--record` + saves a live run in that format. Raw answers go to a gitignored directory. +- Writes nothing anywhere. Details: [typesafe-docs-drift.md](typesafe-docs-drift.md). + +Validation: fixture replay through the full pipeline and the mocked tests in +`tests/test_typesafe_docs_drift.py`, plus one live run on 2026-09-16 whose +sanitized summary is recorded on the detail page. + ## Installation and maintenance Use the [README installation instructions](../README.md#1-install-the-skill). diff --git a/docs/typesafe-docs-drift.md b/docs/typesafe-docs-drift.md new file mode 100644 index 0000000..0ee29ad --- /dev/null +++ b/docs/typesafe-docs-drift.md @@ -0,0 +1,173 @@ +# TypeSafe docs drift detector (phase 1) + +`skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py` finds where the +[nanoclaw-docs](https://github.com/glifocat/nanoclaw-docs) portal disagrees +with the NanoClaw code and ranks the disagreements. It writes to neither +repository. This page records the pipeline, the questions, the gate, the JSON +that phase 2 consumes, and how to run and replay it. + +## Pipeline + +1. **Extract facts with code, not the model.** Deterministic parsers read a + NanoClaw checkout and emit `{id, area, statement, evidence: {file, line, + excerpt}, terms}` records: + + | Area | Source | Facts (main @ `6e5008fe`) | + |---|---|---| + | `ncl` | `src/cli/resources/*.ts`: per resource the verbs and their `open`/`approval` access; per custom verb the flags it declares in `args` (strict) or, without an `args` block, the `--flags` named in its help text (the statement says which); per column its `enum` | 57 | + | `env` | `src/config.ts` `readEnvFile([...])` keys with their `process.env` then `.env` precedence and default; `NANOCLAW_GATEWAY_PROVIDER` and `NANOCLAW_RUNTIME_DRIVER` defaults; `process.env.X \|\| 'default'` reads elsewhere under `src/`; `.env.example` keys | 18 | + | `container-config` | `CREATE TABLE container_configs` and `ALTER TABLE ... ADD COLUMN` across `src/db/migrations/`, column defaults, the `--cli-scope` value check in `groups.ts` | 8 | + | `skills` | `.claude/skills/*/SKILL.md` frontmatter `name` and `description`, plus one catalog fact listing them all | 53 | + | `gateway` | `docs/gateway-seam.md` (`## Selection`, `## How a gateway gets installed`) when present, else `src/gateway-providers/index.ts` | 3 | + | `timestamps` | the `## Timestamps` section of `CLAUDE.md`, one fact per bullet or paragraph | 3 | + +2. **Find candidate sections lexically.** Every page in `docs.json` (changelog + pages excluded) is split on `##`/`###`/`####` headings after stripping MDX + comments and component tags. A BM25 index over section text, page keywords + and headings, with extra weight for the fact's identifiers (backticked terms + such as `cli_scope` or `--rebuild`), returns the top 3 sections per fact. + Facts with no lexical hit are `MISSING` without a request. + +3. **One fan-out request per fact x section.** State is + `{fact: {area, statement, evidence}, doc_section: {page, title, heading, + text}}` (section text capped at 6,000 characters). Three questions: + + | Question | Type | Meaning | + |---|---|---| + | `contradicts` | noul | The section states something incompatible with the fact (a different name, value, default, option set, verb, flag or rule). Silence is not a contradiction. | + | `covers` | noul | This section is where a reader would expect the fact, judging by title, heading and content, whether or not it is currently correct. | + | `staleness` | score | Four levels: current / slightly outdated / wrong / dangerous if followed. | + + Per fact, `documented_anywhere` is the maximum `covers` over its sections. + +4. **Gate and rank.** With the default gates (`contradicts >= 0.7`, + `covers >= 0.7`, staleness confidence `>= 0.5`): + + | Pair verdict | Rule | + |---|---| + | `DRIFT` | `contradicts >= 0.7` | + | `OK` | `contradicts <= 0.3` and `covers >= 0.7` | + | `UNRELATED` | `contradicts <= 0.3` and `covers <= 0.3` | + | `UNSURE` | anything else, or a missing answer | + + A fact is `DRIFT` if any pair is (best = highest contradiction, then + staleness), else `OK` if any pair is (best = highest covers), else `UNSURE` + if any pair is or if any of its requests failed, else `MISSING`. Rows are ranked `DRIFT` by contradiction, + then `MISSING`, `UNSURE`, `OK`. The staleness label is shown with `?` when + the score's confidence is below the gate. + +## Confidence gate + +Noul answers are probabilities of yes with no separate confidence, so the gate +is symmetric: yes above `t`, no below `1 - t`, uncertain in between. Score +answers carry `confidence`; the staleness level is informational and only its +label is gated. Start values are 0.7 for the nouls and 0.5 for the score, +exposed as `--contradicts-threshold`, `--covers-threshold` and +`--staleness-threshold`; evaluate them on a live run before trusting a cut. + +## The key + +Live mode reads `TYPESAFE_API_KEY` from the environment when the first request +is built. If it is unset the script exits with status 2 and a message; it does +not look for a file. Export it for the shell session only, for example from a +password manager. Never commit it, never put it in `.env` or a fixture. The +script does not log request headers, error bodies that echo the key are +redacted, and the saved JSON contains only facts, sections, responses and +decisions. + +## Run + +Offline, using the shipped fixture (six real facts and six real doc sections +from 2026-09-17, section text truncated, with hand-written answers in the +documented shapes): + +```bash +python3 skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py \ + --fixture skills/typesafe-docs-drift/fixtures/nanoclaw-sample.json +``` + +Without a key, inspect the deterministic half: + +```bash +python3 skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py --code /path/to/nanoclaw --facts-only +python3 skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py --code /path/to/nanoclaw --docs /path/to/nanoclaw-docs --plan +``` + +Live: + +```bash +export TYPESAFE_API_KEY=... +python3 skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py \ + --code /path/to/nanoclaw --docs /path/to/nanoclaw-docs --record /tmp/drift-record.json +``` + +Requests run with `--concurrency` 4 workers by default; 429 and 529 are +retried with backoff. A pair that fails after retries is reported in `partial` +and counted in its fact's `failed_pairs`; the fact is judged on the pairs that +completed and is never called `MISSING` on that basis, and the exit status +is 1. With `--concurrency 1` the run stops after three consecutive failures. +`--record` writes facts, sections, candidates, lexical scores and raw +responses in the fixture format, dropping pairs that failed so the record +replays cleanly. Raw answers for every run are saved as +`drift-.json` under `skills/typesafe-docs-drift/output/` +(gitignored) or `--output-dir`; `--json` prints the same payload. + +## JSON for phase 2 + +Phase 2 (an agent that writes doc PRs) should read the saved payload, keep +`results` rows with `verdict == "DRIFT"` that a human confirmed, and for each +use: + +- `statement` and `evidence.file` / `evidence.line` / `evidence.excerpt` with + `code.commit`: what the docs must say and where the proof is. +- `best.section.page`, `best.section.heading`, `best.section.line`: the MDX + file (`.mdx` under the docs root) and the source line of the heading + to edit, with `docs.commit` as the base revision. +- `best.decision.contradicts`, `covers`, `staleness`, `staleness_label`: the + model's numbers for the PR description. +- `pairs[]`: every candidate section with its numbers and `lexical_score`, for + the cases where a second section also needs the change; `failed_pairs` says + how many candidates got no answer. + +`MISSING` rows list the closest section (highest `covers`) and are the input +for a "document this" pass rather than a fix. Nothing in the payload is +generated text; phase 2 writes the prose. + +## First live run + +Run on 2026-09-16 22:29 UTC from this repository against nanocoai/nanoclaw +`6e5008fe` and nanoclaw-docs `e3e1546`, model `jev-latest`, default gates, +`--concurrency 4`: + +| | | +|---|---| +| Facts / pairs / requests | 142 / 426 / 426 | +| Verdicts | DRIFT 7, MISSING 1, UNSURE 11, OK 123 | +| Per area (DRIFT / MISSING / UNSURE / OK) | ncl 5/0/7/45, env 1/0/2/15, skills 1/0/1/51, timestamps 0/1/1/1, container-config 0/0/0/8, gateway 0/0/0/3 | +| Tokens | 609,315 input, 22,578 output | +| Wall time | 71.5 s | + +The seven `DRIFT` rows, highest contradiction first: the `ncl approvals` +`status` enum (docs list an extra `awaiting_reason` value; `reference/ncl-cli#approvals`, +0.96), `WEBHOOK_PORT` precedence (`operate/troubleshooting#webhook-channel-is-silent`, +0.93), the `ncl dropped-messages` `reason` enum (`reference/ncl-cli#dropped-messages`, +0.92), `ncl tasks cancel` and `ncl tasks update` declared flags +(`reference/ncl-cli#tasks`, 0.88 and 0.87), the `/add-ollama-provider` +description (`extend/providers#ollama-add-ollama-provider`, 0.76) and `ncl tasks +resume` flags (0.71, staleness confidence below the gate). The raw payload is +in the gitignored output directory of the machine that ran it; the numbers +above are the only part recorded here. + +## Tests + +`tests/test_typesafe_docs_drift.py` runs in the offline suite: every extractor +against a synthetic checkout (resources, env, migrations, skills, gateway doc +present and absent, timestamps), section splitting over a synthetic `docs.json` +(code fences, duplicate headings, comments, component tags, changelog +exclusion), lexical ranking, request shape, gate and fold logic, ranking and +summary, HTTP transport with a mocked opener (bearer header, private keys +stripped from the body, retry on 429/529, no retry on 422, redaction), +concurrent and sequential pair execution with partial failures, fixture replay +end to end, a mocked live run that records a replayable fixture, and a check +that a key in the environment never reaches stdout, stderr or the saved JSON. +No test contacts the network. diff --git a/skills/typesafe-docs-drift/SKILL.md b/skills/typesafe-docs-drift/SKILL.md new file mode 100644 index 0000000..fa10391 --- /dev/null +++ b/skills/typesafe-docs-drift/SKILL.md @@ -0,0 +1,109 @@ +--- +name: typesafe-docs-drift +description: Detect and rank drift between the NanoClaw code and the nanoclaw-docs Mintlify portal with the TypeSafe System One decision API. Deterministic code extracts facts (ncl verbs and flags, environment variables, container_configs and cli_scope, workspace skills, gateway selection, timestamp rules), lexical search picks candidate doc sections, and one fan-out request per fact x section asks contradicts / covers / staleness. Prints DRIFT, MISSING, UNSURE and OK rows with the model's numbers; writes nothing to either repository. Use to find doc pages that need a fix, or replay the offline fixture without a key. +license: MIT +--- + +# TypeSafe docs drift (phase 1) + +Extract facts from a NanoClaw checkout with deterministic code, find the top +doc sections for each fact by lexical search over the docs portal, ask TypeSafe +one fan-out request per fact x section pair, gate on the answers, and print a +ranked drift report. The model only judges; it never writes text. Phase 2 (an +agent writing doc PRs from confirmed `DRIFT` rows) consumes the JSON this +script saves. + +Entry point: [typesafe-docs-drift.py](scripts/typesafe-docs-drift.py) (Python +3.10+, standard library only). Offline fixture: +[nanoclaw-sample.json](fixtures/nanoclaw-sample.json). Details of the facts, +questions, gate and JSON: [docs/typesafe-docs-drift.md](../../docs/typesafe-docs-drift.md). + +## Sources this skill drives + +- NanoClaw `src/cli/resources/*.ts` (`registerResource` definitions: plural, + `operations`, `customOperations` with `access`, declared `args` or the + `--flags` named in descriptions, column `enum`s), `src/config.ts` (`readEnvFile([...])` keys and defaults), + `src/gateway-providers/index.ts` and `src/drivers/index.ts` (default kinds), + `src/**/*.ts` `process.env.X || 'default'` reads, `.env.example`, + `src/db/migrations/*.ts` (`container_configs` columns), the `--cli-scope` + validation in `src/cli/resources/groups.ts`, `.claude/skills/*/SKILL.md` + frontmatter, `docs/gateway-seam.md` (`## Selection` and + `## How a gateway gets installed`, when present) and the `## Timestamps` + section of `CLAUDE.md`. Verified against nanocoai/nanoclaw `6e5008fe`. +- nanoclaw-docs: `docs.json` navigation and the `.mdx` pages it lists, split + on `##`/`###`/`####` headings (changelog pages excluded by default). +- TypeSafe HTTP API: `POST https://api.typesafe.ai/v1/systemone`, model + `jev-latest`, question types `noul` and `score` + ([API reference](https://docs.typesafe.ai/api.md), + [confidence](https://docs.typesafe.ai/confidence.md), + [speculative fan-out](https://docs.typesafe.ai/patterns/fan-out.md), + [citation check cookbook](https://docs.typesafe.ai/cookbooks/citation_check.md)). + +Re-check the extractors when `src/cli/crud.ts`, `src/config.ts` or the +migration layout changes. + +## Prerequisites + +- A NanoClaw checkout (`--code`, default: the current directory) and a + nanoclaw-docs checkout (`--docs`). Both are read only. +- `TYPESAFE_API_KEY` exported in the shell (live mode only). The script reads + it from the environment at request time, never from a file, and never prints + it. Do not put the key in `.env`, a fixture, or any file inside a repository. + +## Run + +Offline demonstration, no key and no network: + +```bash +python3 skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py \ + --fixture skills/typesafe-docs-drift/fixtures/nanoclaw-sample.json +``` + +See the facts, or the fact x section pairs, without calling the API: + +```bash +python3 skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py --code . --facts-only +python3 skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py --code . --docs ../nanoclaw-docs --plan +``` + +Live run from a NanoClaw checkout (about 140 facts, 430 requests, a few +minutes at the default concurrency of 4): + +```bash +export TYPESAFE_API_KEY=... # from your password manager, this shell only +python3 skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py \ + --code . --docs ../nanoclaw-docs --record /tmp/drift-record.json +``` + +`--record` saves facts, sections and raw responses so the run can be replayed +with `--fixture`. Raw answers for every run land in +`skills/typesafe-docs-drift/output/` (gitignored) or `--output-dir`. + +Flags: `--areas` (comma-separated subset of `ncl,env,container-config,skills, +gateway,timestamps`), `--limit-facts`, `--top-k` (3), `--include-changelog`, +`--contradicts-threshold` (0.7), `--covers-threshold` (0.7), +`--staleness-threshold` (0.5, score confidence), `--concurrency` (4), +`--model`, `--timeout`, `--json`. + +## Read the output + +One row per fact, ranked `DRIFT` (a section contradicts the fact, highest +contradiction first), `MISSING` (no candidate section covers the fact), +`UNSURE` (an answer sits between the gates), `OK` (a section covers the fact +and does not contradict it). Each row names the doc page and heading the +verdict rests on, the `contradicts` and `covers` probabilities, the staleness +score with its level (`?` when the score's confidence is below the gate) and +the code evidence as `file:line`. The summary counts verdicts per area and +reports tokens and wall time. + +## Limits + +- Detection only. Nothing is written to NanoClaw, nanoclaw-docs or GitHub. +- Candidate sections come from a small BM25 index over headings and + paragraphs; a fact whose true home is not in its top-k sections shows as + `MISSING` or `UNSURE`. Raise `--top-k` or read the `pairs` in the JSON. +- The fixture's answers are hand-written in the documented response shapes; + they exercise the pipeline and are not evidence of model accuracy. +- `docs/gateway-seam.md` and a non-empty `.env.example` are used when present; + on NanoClaw `main` at `6e5008fe` both are absent, so gateway facts come from + `src/gateway-providers/index.ts` and the run notes say so. diff --git a/skills/typesafe-docs-drift/fixtures/nanoclaw-sample.json b/skills/typesafe-docs-drift/fixtures/nanoclaw-sample.json new file mode 100644 index 0000000..69368f2 --- /dev/null +++ b/skills/typesafe-docs-drift/fixtures/nanoclaw-sample.json @@ -0,0 +1,414 @@ +{ + "schema_version": 1, + "recorded_at": "2026-09-17T00:00:00Z", + "code_commit": "6e5008fe", + "docs_commit": "e3e1546", + "note": "Facts and sections are real (nanocoai/nanoclaw and nanoclaw-docs on 2026-09-17, section text truncated); responses are hand-written in the documented TypeSafe answer shapes, not recorded from the API.", + "facts": [ + { + "id": "ncl:groups-config-update-flags", + "area": "ncl", + "statement": "`ncl groups config update` (approval) names these flags in its help text: `--id`, `--provider`, `--model`, `--effort`, `--speed`, `--image-tag`, `--assistant-name`, `--max-messages-per-prompt`, `--cli-scope`, `--timezone`.", + "evidence": { + "file": "src/cli/resources/groups.ts", + "line": 389, + "excerpt": "config update: { access: 'approval', description: ... --id --provider --model --effort --speed --image-tag --assistant-name --max-messages-per-prompt --cli-scope --timezone }" + }, + "terms": [ + "assistant-name", + "cli-scope", + "config", + "effort", + "groups", + "id", + "image-tag", + "max-messages-per-prompt", + "model", + "ncl", + "provider", + "speed", + "timezone" + ] + }, + { + "id": "container-config:cli-scope-values", + "area": "container-config", + "statement": "`cli_scope` accepts exactly these values via `ncl groups config update --cli-scope`: `disabled`, `group`, `global`.", + "evidence": { + "file": "src/cli/resources/groups.ts", + "line": 434, + "excerpt": "['disabled', 'group', 'global'].includes(scope)" + }, + "terms": [ + "cli-scope", + "cli_scope", + "disabled", + "global", + "group" + ] + }, + { + "id": "env:CONTAINER_PIDS_LIMIT", + "area": "env", + "statement": "`CONTAINER_PIDS_LIMIT` is read from the process environment first, then from `.env`; its default is `2048`. It is resolved with `??`, so an explicitly empty value wins over the default.", + "evidence": { + "file": "src/config.ts", + "line": 107, + "excerpt": "export const CONTAINER_PIDS_LIMIT = process.env.CONTAINER_PIDS_LIMIT ?? envConfig.CONTAINER_PIDS_LIMIT ?? '2048';" + }, + "terms": [ + "CONTAINER_PIDS_LIMIT" + ] + }, + { + "id": "gateway:selection-default", + "area": "gateway", + "statement": "One credential gateway provider is active per install, selected by `NANOCLAW_GATEWAY_PROVIDER`; when the variable is unset the `onecli` gateway is used, so an install that never sets it behaves as before.", + "evidence": { + "file": "src/gateway-providers/index.ts", + "line": 26, + "excerpt": "DEFAULT_GATEWAY_PROVIDER_KIND = 'onecli'" + }, + "terms": [ + "NANOCLAW_GATEWAY_PROVIDER", + "credential gateway", + "gateway", + "onecli" + ] + }, + { + "id": "gateway:selection-unknown", + "area": "gateway", + "statement": "Setting `NANOCLAW_GATEWAY_PROVIDER` to a name with no registered gateway provider throws at startup and lists the installed kinds; other gateways arrive as skills (overlays) and the host never silently falls back to another gateway.", + "evidence": { + "file": "src/gateway-providers/index.ts", + "line": 44, + "excerpt": "`NANOCLAW_GATEWAY_PROVIDER='${kind}' but no gateway provider is registered for '${kind}'; ` +" + }, + "terms": [ + "NANOCLAW_GATEWAY_PROVIDER", + "gateway", + "overlay", + "skill" + ] + }, + { + "id": "timestamps:rule-1", + "area": "timestamps", + "statement": "**Storage**: every timestamp written from JS is `new Date().toISOString()` (ISO-8601 UTC with `Z`). Central SQL receives that timestamp as a parameter; never use `datetime('now')` or `strftime(...)` there because they are SQLite-specific and the naive shape is misparsed as local time by `new Date()`. SQLite-only mailbox SQL may use `strftime('%Y-%m-%dT%H:%M:%fZ','now')`. SQLite-only comparisons wrap both sides in `datetime()`.", + "evidence": { + "file": "CLAUDE.md", + "line": 268, + "excerpt": "**Storage**: every timestamp written from JS is `new Date().toISOString()` (ISO-8601 UTC with `Z`). Central SQL receives that timestamp as a parameter; never use `datetime('now')` or `strftime(...)` there because they are SQLite-specific and the naive shape is misparsed as local time by `new Date()`. SQLite-only mailbox SQL may use `strftime('%Y-%m-%dT%H:%M:%fZ','now')`. SQLite-only comparisons w…" + }, + "terms": [ + "ISO", + "UTC", + "Z", + "datetime('now')", + "datetime()", + "new Date()", + "new Date().toISOString()", + "strftime('%Y-%m-%dT%H:%M:%fZ','now')", + "strftime(...)", + "timestamp", + "timezone" + ] + } + ], + "sections": [ + { + "id": "operate/configuration#read-from-env", + "page": "operate/configuration", + "title": "Configuration", + "heading": "Read from .env", + "level": 3, + "line": 22, + "text": "| Variable | Default | What it does |\n|---|---|---|\n| `ASSISTANT_NAME` | `Andy` | The assistant's name. Also sets the default trigger, `@`. |\n| `ASSISTANT_HAS_OWN_NUMBER` | `false` | Set `true` when the assistant runs on its own WhatsApp number instead of yours. |\n| `DEFAULT_AGENT_PROVIDER` | `claude` | Provider stamped on **new** agent groups at creation. Existing groups are never touched, and the per-group `provider` config still wins. |\n| `NANOCLAW_DEFAULT_MODEL` | provider SDK default | Model used at the next spawn for groups without their own model. Unlike the provider default, this is not stamped at group creation. |\n| `NANOCLAW_FAST_MODE` | `false` | Enables the faster, higher-priced serving tier for the Claude provider. Only `1` or `true` enables it, case-insensitively. |\n| `NANOCLAW_RUNTIME_DRIVER` | `docker` | Selects an installed session driver. Docker is the only driver shipped here; an unregistered value fails. |\n| `NANOCLAW_GATEWAY_PROVIDER` | `onecli` | Selects an installed credential-gateway provider. An unregistered value fails. |\n| `NANOCLAW_SESSION_MATERIAL_ROOT` | `data/session-materials` | Host directory admitted for session identity material. |\n| `ONECLI_URL` | — | URL of the OneCLI Agent Vault. |\n| `ONECLI_API_KEY` | — | API key for the vault. See [Credentials](/operate/credentials). |\n| `TZ` | system timezone, else `UTC` | IANA timezone for scheduled tasks and message timestamps. Invalid values are skipped, not errored. |\n| `CONTAINER_CPU_LIMIT` | _unset_ (unbounded) | Opt-in CPU cap per container, passed to `docker create --cpus` (e.g. `2`). Can be set in `.env` or the process environment. |\n| `CONTAINER_MEMORY_LIMIT` | _unset_ (unbounded) | Opt-in memory cap per container, passed to `docker create --memory` (e.g. `8g`). Hard cap only on a swapless host. Can be set in `.env` or the process environment. |\n| `NANOCLAW_EGRESS_LOCKDOWN` | `false` | Set `true` to force all agent traffic through the OneCLI gateway on an internal Docker network. Fails fast if the gateway isn't running. Can be set in `.env` or the process environment. See [Hardening](/operate/hardening). |\n| `NANOCLAW_EGRESS_NETWORK` | `nanoclaw-egress` | Name of the locked-down Docker network. Can be set in `.env` or the process environment. |\n| `ONECLI_GATEWAY_CONTAINER` | `onecli` | Name of the gateway container attached as the only egress hop. Can be set in `.env` or the process environment. |\n| `ANTHROPIC_BASE_URL` | — | Custom Anthropic-compatible endpoint for ", + "keywords": "[\"configuration\", \"environment variables\", \".env\", \"container config\", \"ASSISTANT_NAME\", \"LOG_LEVEL\", \"container_configs\"]" + }, + { + "id": "operate/credentials#accessing-the-onecli-dashboard", + "page": "operate/credentials", + "title": "Credentials", + "heading": "Accessing the OneCLI dashboard", + "level": 2, + "line": 83, + "text": "The vault ships a web dashboard at the same address as `ONECLI_URL` — the \"OneCLI web UI\". Open it in a browser to connect OAuth services, browse secrets, and watch gateway activity.\n\nTwo gotchas on a headless install:\n\n- The dashboard binds where OneCLI binds. On installs where containers reach the gateway over the Docker bridge, that address isn't routable from your laptop — reach it through an SSH tunnel or a private network such as a tailnet rather than exposing the port publicly.\n- OAuth **Connect** flows finish by redirecting the browser back into the dashboard at OneCLI's configured `APP_URL`, which defaults to `http://localhost:10254`. Started from a remote browser, the provider consent itself works (that redirect is derived from the request), but the final hop lands on localhost and dies unless `APP_URL` is set to the URL you actually browse from. The `connect_url` links the gateway hands to agents are built from the same value, so an unset `APP_URL` makes those unreachable too.\n\nDashboard specifics beyond this are OneCLI's territory — see the [OneCLI docs](https://onecli.sh/docs).", + "keywords": "[\"credentials\", \"OneCLI\", \"Agent Vault\", \"secrets\", \"approval\", \"onecli-managed\", \"credential proxy\", \"ANTHROPIC_BASE_URL\"]" + }, + { + "id": "operate/ncl-cli#agents-can-run-ncl-too", + "page": "operate/ncl-cli", + "title": "The ncl admin CLI", + "heading": "Agents can run ncl too", + "level": 2, + "line": 46, + "text": "The same command surface is exposed to agents inside containers. What an agent may do is controlled per group by `cli_scope` in its container config:\n\n- **`disabled`** — all `ncl` commands are rejected.\n- **`group`** (default) — the agent only sees the `groups`, `sessions`, `destinations`, `members`, and `tasks` resources, pinned to its own group: IDs are auto-filled, rows from other groups are filtered out, and it cannot change its own `cli_scope`. It also gets `wirings get`/`wirings update` for the current conversation's wiring only — the id is force-filled, and updates may only change `engage_mode`/`engage_pattern`.\n- **`global`** — full access to every resource.\n\nAcross most resources, mutating verbs (`create`, `update`, `delete`, `grant`, `restart`, ...) are approval-gated for agents: the agent gets `approval-pending` and an admin receives a card showing the exact command. The exception is `tasks` — every tasks verb, including `create` and `cancel`, is open to agents, since a task is scoped to the agent's own group. From the host socket, the same verbs run immediately — the socket file permissions are the auth boundary. Change the scope with:\n\n```bash\nncl groups config update --id --cli-scope disabled\n```", + "keywords": "[\"ncl\", \"cli\", \"admin\", \"unix socket\", \"cli_scope\", \"groups\", \"wirings\", \"sessions\", \"approvals\"]" + }, + { + "id": "reference/environment-variables#read-from-env", + "page": "reference/environment-variables", + "title": "Environment variables", + "heading": "Read from .env", + "level": 2, + "line": 13, + "text": "NanoClaw's own parser ([`src/env.ts`](https://github.com/nanocoai/nanoclaw/blob/main/src/env.ts)) returns only the keys requested by each caller from `.env` at the project root. The parser itself does not load values into `process.env`. For the core settings below, the process environment takes precedence over `.env` unless noted. `ANTHROPIC_BASE_URL` in the Claude provider is read from `.env` only. The shared parser strips matching outer single or double quotes, ignores blank values, and does not execute shell expressions. These parsing rules apply to callers of `readEnvFile`/`envValue`; some setup steps retain their own readers.\n\n| Variable | Default | Read in | Effect |\n|---|---|---|---|\n| `ASSISTANT_NAME` | `Andy` | `src/config.ts:29` | Assistant name. Also sets the default trigger, `@`. |\n| `ASSISTANT_HAS_OWN_NUMBER` | `false` | `src/config.ts:47` | Set `true` when the assistant runs on its own WhatsApp number instead of yours. |\n| `ONECLI_URL` | — | `src/config.ts:75` | URL of the OneCLI Agent Vault. |\n| `ONECLI_API_KEY` | — | `src/config.ts:76` | API key for the vault. See [Credentials](/operate/credentials). |\n| `TZ` | system timezone, else `UTC` | `src/config.ts:101` | IANA timezone for scheduled tasks and timestamps. Invalid values are skipped, not errored. Injected into every agent container at spawn. |\n| `CONTAINER_CPU_LIMIT` | _unset_ (unbounded) | `src/config.ts:80` | Opt-in CPU cap per agent container, passed through to `docker create --cpus` (e.g. `2`). Empty leaves containers unbounded — the current default. Can be set in `.env` or the process environment. |\n| `CONTAINER_MEMORY_LIMIT` | _unset_ (unbounded) | `src/config.ts:81` | Opt-in memory cap per agent container, passed through to `docker create --memory` (e.g. `8g`). A hard cap only on a swapless host — there the runaway is OOM-killed; with swap present the container can spill past it. NanoClaw doesn't manage swap. Can be set in `.env` or the process environment. |\n| `CONTAINER_PIDS_LIMIT` | `2048` | `src/config.ts:88` | Process/thread cap per agent container, passed through to `docker create --pids-limit` — a fork-bomb backstop, and the one container limit that's on by default. Set `0` (or any non-numeric value) to drop the cap. Resolved with `??` rather than `||`, so an explicitly empty value in the process environment wins and disables the cap instead of falling through to `.env` or the default. Can be set in `.env` or the process environment. |\n| `NANOCLAW_EGRESS_LOCKDOWN` |", + "keywords": "[\"environment variables\", \".env\", \"configuration\", \"ASSISTANT_NAME\", \"LOG_LEVEL\", \"NANOCLAW_SKIP\", \"NANOCLAW_EGRESS_LOCKDOWN\", \"setup\", \"reference\"]" + }, + { + "id": "reference/ncl-cli#groups", + "page": "reference/ncl-cli", + "title": "ncl CLI reference", + "heading": "groups", + "level": 2, + "line": 79, + "text": "Agent groups — logical agent identities with their own workspace folder, history, and container image.\n\n| Operation | Invocation | Access | Notes |\n| --- | --- | --- | --- |\n| list | `ncl groups list` | open | |\n| get | `ncl groups get --id ` | open | |\n| create | `ncl groups create --folder [--name ] [--timezone ]` or `ncl groups create --template [--name ] [--timezone ] [--id ] [--new] [--yes]` | approval | **Without `--template`:** `--folder` required and immutable, `--name` defaults to the folder, idempotent on `--folder` (creating an existing folder returns the existing row), and new ids are prefixed `ag-`. **With `--template `:** stamps from a local [template](/templates/overview) under `templates/`. Combining it with `--folder` is **rejected** — a bare group *is* its folder, a templated group *is* its plugin. The display name is `--name`, else the manifest's `agentName`, else the template ref's leaf folder; the group folder derives from that name, with a random `-XXXXXXXX` suffix on collision. Always creates a new agent, except when updating one this template already made: a group can only be targeted if its workspace holds that plugin's manifest, so a hand-built group or one stamped from another template is rejected. Convergent rather than folder-idempotent: if a group already carries that plugin, the command returns an in-place [update plan](/templates/updating) instead of creating anything — `--yes` applies it, `--id` picks the target when several groups carry it, and `--new` forces a fresh stamp. `--name` and `--timezone` apply only when a group is created; both are ignored on an in-place update. Reader notices come back as `templateReport` on a create, and as `report` on an update. |\n| update | `ncl groups update --id --name ` | approval | `name` is the only updatable column. |\n| delete | `ncl groups delete --id ` | approval | Custom cascading handler (not the generic single-table delete): removes sessions, destinations, approvals, role grants, memberships, and wirings in one FK-ordered transaction. Does **not** kill running containers or clean up `groups//` and `data/v2-sessions//` on disk. |\n| restart | `ncl groups restart --id [--rebuild] [--message ]` | approval | Host caller: restarts all running containers in the group. Agent caller: `--id` auto-filled, only the calling session restarts. `--rebuild` rebuilds the image first (required afte", + "keywords": "[\"ncl\", \"cli\", \"reference\", \"commands\", \"flags\", \"groups\", \"wirings\", \"sessions\", \"roles\", \"members\", \"destinations\", \"approvals\", \"cli_scope\"]" + }, + { + "id": "templates/building#skills", + "page": "templates/building", + "title": "Building a template", + "heading": "Skills", + "level": 2, + "line": 168, + "text": "Each immediate subfolder of `skills/` is one skill, named after the folder. The whole folder is copied, so put `SKILL.md` (with `name` and `description` frontmatter) and any `references/*.md` inside it, following the usual [skill conventions](/extend/writing-skills).\n\nSkills land in the group's private skills overlay — keyed to that group, never shared with others. Each skill's `name` and `description` are always in the prompt, and that's what the auto-trigger matches on; only the body is read on demand.\n\nA non-conforming skill is skipped with a named notice at stamp time, never silently dropped.", + "keywords": "[\"build template\", \"author template\", \"plugin.json\", \"mcp.json\", \"ai.nanoco.nanoclaw\", \"instructions.md\", \"additional_context\", \"template tasks\", \"agent plugins\", \"test template\"]" + } + ], + "candidates": { + "ncl:groups-config-update-flags": [ + "reference/ncl-cli#groups" + ], + "container-config:cli-scope-values": [ + "operate/ncl-cli#agents-can-run-ncl-too" + ], + "env:CONTAINER_PIDS_LIMIT": [ + "reference/environment-variables#read-from-env" + ], + "gateway:selection-default": [ + "operate/credentials#accessing-the-onecli-dashboard" + ], + "gateway:selection-unknown": [ + "templates/building#skills", + "operate/configuration#read-from-env" + ], + "timestamps:rule-1": [] + }, + "scores": { + "ncl:groups-config-update-flags|reference/ncl-cli#groups": 410.935, + "container-config:cli-scope-values|operate/ncl-cli#agents-can-run-ncl-too": 133.167, + "env:CONTAINER_PIDS_LIMIT|reference/environment-variables#read-from-env": 136.279, + "gateway:selection-default|operate/credentials#accessing-the-onecli-dashboard": 48.726, + "gateway:selection-unknown|templates/building#skills": 37.878, + "gateway:selection-unknown|operate/configuration#read-from-env": 67.252 + }, + "responses": { + "ncl:groups-config-update-flags|reference/ncl-cli#groups": { + "model": "jev-latest", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.78 + }, + "covers": { + "type": "noul", + "noul": 0.96 + }, + "staleness": { + "type": "score", + "score": 1.12, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.14, + "1": 0.62, + "2": 0.22, + "3": 0.02 + }, + "confidence": 0.71 + } + }, + "usage": { + "input_tokens": 2610, + "output_tokens": 96 + } + }, + "container-config:cli-scope-values|operate/ncl-cli#agents-can-run-ncl-too": { + "model": "jev-latest", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.95 + }, + "staleness": { + "type": "score", + "score": 0.08, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.93, + "1": 0.06, + "2": 0.01, + "3": 0.0 + }, + "confidence": 0.93 + } + }, + "usage": { + "input_tokens": 1480, + "output_tokens": 96 + } + }, + "env:CONTAINER_PIDS_LIMIT|reference/environment-variables#read-from-env": { + "model": "jev-latest", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.05 + }, + "covers": { + "type": "noul", + "noul": 0.9 + }, + "staleness": { + "type": "score", + "score": 0.2, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.83, + "1": 0.14, + "2": 0.03, + "3": 0.0 + }, + "confidence": 0.88 + } + }, + "usage": { + "input_tokens": 3020, + "output_tokens": 96 + } + }, + "gateway:selection-default|operate/credentials#accessing-the-onecli-dashboard": { + "model": "jev-latest", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.42 + }, + "covers": { + "type": "noul", + "noul": 0.55 + }, + "staleness": { + "type": "score", + "score": 0.9, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.4, + "1": 0.35, + "2": 0.2, + "3": 0.05 + }, + "confidence": 0.4 + } + }, + "usage": { + "input_tokens": 1210, + "output_tokens": 96 + } + }, + "gateway:selection-unknown|templates/building#skills": { + "model": "jev-latest", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.02 + }, + "covers": { + "type": "noul", + "noul": 0.08 + }, + "staleness": { + "type": "score", + "score": 0.05, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.95, + "1": 0.05, + "2": 0.0, + "3": 0.0 + }, + "confidence": 0.95 + } + }, + "usage": { + "input_tokens": 1160, + "output_tokens": 96 + } + }, + "gateway:selection-unknown|operate/configuration#read-from-env": { + "model": "jev-latest", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.1 + }, + "covers": { + "type": "noul", + "noul": 0.25 + }, + "staleness": { + "type": "score", + "score": 0.3, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.75, + "1": 0.2, + "2": 0.05, + "3": 0.0 + }, + "confidence": 0.8 + } + }, + "usage": { + "input_tokens": 1750, + "output_tokens": 96 + } + } + } +} diff --git a/skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py b/skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py new file mode 100644 index 0000000..00db5c3 --- /dev/null +++ b/skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py @@ -0,0 +1,1398 @@ +#!/usr/bin/env python3 +"""Docs-drift detector for NanoClaw (phase 1): rank where the docs portal disagrees with the code. + +Deterministic code extracts facts from a NanoClaw checkout (``ncl`` resources, +verbs and flags; environment variables; ``container_configs`` columns and +``cli_scope`` values; workspace skills; gateway selection rules; timestamp +rules). Each fact is matched lexically against the headings and paragraphs of +the nanoclaw-docs Mintlify site, and every fact x section pair is sent as one +speculative fan-out request to TypeSafe System One with three questions: +``contradicts`` (noul), ``covers`` (noul) and ``staleness`` (score). Code then +gates on the answers and prints a ranked DRIFT / MISSING / UNSURE / OK report. + +Nothing is written to either repository. The model never writes text, so this +script detects and ranks drift; a later phase writes the fix from the JSON. + +``--fixture `` replays recorded facts, sections and API responses so the +whole pipeline runs offline. ``TYPESAFE_API_KEY`` is read from the environment +only when the live API is used; it is never read from a file and never printed. +""" + +import argparse +import concurrent.futures +import datetime +import json +import math +import os +from pathlib import Path +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request + + +API_URL = "https://api.typesafe.ai/v1/systemone" +DEFAULT_MODEL = "jev-latest" +KEY_ENV = "TYPESAFE_API_KEY" +SKILL_DIR = Path(__file__).resolve().parents[1] +DEFAULT_OUTPUT_DIR = SKILL_DIR / "output" +SECTION_LIMIT = 6000 +EXCERPT_LIMIT = 400 +DEFAULT_TOP_K = 3 +DEFAULT_CONCURRENCY = 4 +MAX_CONSECUTIVE_FAILURES = 3 +EXCLUDED_PAGE_PREFIXES = ("changelog/",) +AREAS = ("ncl", "env", "container-config", "skills", "gateway", "timestamps") +VERDICTS = ("DRIFT", "MISSING", "UNSURE", "OK") +STALENESS_LEVELS = ("current", "slightly outdated", "wrong", "dangerous if followed") +STALENESS_CRITERIA = [ + "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, " + "but a reader following the section would still succeed.", + "Wrong: the section states something the code no longer does or never did on this point; " + "a reader following it would be misled, get an error, or miss a required step.", + "Dangerous if followed: following the section on this point could lose data, expose a credential, " + "weaken a security boundary, or leave the install broken.", +] +# Process-environment names that are not NanoClaw settings. +ENV_IGNORE = {"HOME", "PATH", "USER", "TMPDIR", "DISPLAY", "WAYLAND_DISPLAY", "COLORTERM", "NO_COLOR", "TZ"} +ENV_PREFIXES = ("NANOCLAW_", "CONTAINER_", "ONECLI_", "LOG_LEVEL", "WEBHOOK_PORT", "ASSISTANT_", "DEFAULT_AGENT_") + + +class DriftError(RuntimeError): + """A failure the operator can act on; printed without a traceback.""" + + +def utc_now(): + return datetime.datetime.now(datetime.timezone.utc) + + +def stamp(): + return utc_now().strftime("%Y%m%dT%H%M%SZ") + + +def read_text(path): + try: + return Path(path).read_text(encoding="utf-8", errors="replace") + except OSError as error: + raise DriftError("could not read %s: %s" % (path, error)) + + +def excerpt(text, limit=EXCERPT_LIMIT): + text = " ".join((text or "").split()) + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def line_of(text, offset): + return text.count("\n", 0, offset) + 1 + + +def git_commit(root): + try: + result = subprocess.run(["git", "-C", str(root), "rev-parse", "--short", "HEAD"], + capture_output=True, text=True, timeout=10) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + return result.stdout.strip() or None + + +# --------------------------------------------------------------------------- +# Fact extraction (deterministic, no model) +# --------------------------------------------------------------------------- + +def fact(area, key, statement, path, line, snippet, terms=()): + return { + "id": "%s:%s" % (area, key), + "area": area, + "statement": statement, + "evidence": {"file": str(path), "line": line, "excerpt": excerpt(snippet)}, + "terms": sorted({term for term in terms if term}), + } + + +def relative(root, path): + try: + return str(Path(path).resolve().relative_to(Path(root).resolve())) + except ValueError: + return str(path) + + +_VERB_LINE = re.compile(r"^ (?:'([^']+)'|([A-Za-z_][\w-]*)): \{\s*$") +_ACCESS_LINE = re.compile(r"^\s+access: '(open|approval)'") +_FLAG = re.compile(r"--[a-z][a-z0-9-]*") +_ARG_NAME = re.compile(r"name: '(\w+)'") + + +def parse_resource(text): + """Pull plural, standard operations, custom verbs and column enums out of one resource module.""" + start = text.find("registerResource({") + if start < 0: + return None + body = text[start:] + plural = re.search(r"^\s*plural: '([^']+)'", body, re.M) + if not plural: + return None + resource = {"plural": plural.group(1), "line": line_of(text, start), "operations": {}, "custom": [], "enums": []} + operations = re.search(r"^\s*operations: \{([^}]*)\}", body, re.M) + if operations: + for verb, access in re.findall(r"(\w+): '(open|approval)'", operations.group(1)): + resource["operations"][verb] = access + # Top-level keys of the definition sit at two spaces; a nested array's `],` is deeper. + columns = re.search(r"^ columns: \[(.*?)^ \],", body, re.M | re.S) + if columns: + for name, values in re.findall(r"name: '(\w+)'[^{}]*?enum: \[([^\]]*)\]", columns.group(1), re.S): + resource["enums"].append((name, re.findall(r"'([^']*)'", values))) + custom = re.search(r"^\s*customOperations: \{\s*$", body, re.M) + if custom: + lines = body[custom.end():].splitlines() + offset = line_of(text, start + custom.end()) + current = None + for index, line in enumerate(lines): + if line == " },": + break + verb_match = _VERB_LINE.match(line) + if verb_match: + current = {"verb": verb_match.group(1) or verb_match.group(2), "access": None, "flags": [], + "declared": None, "host_only": False, "line": offset + index + 1} + resource["custom"].append(current) + continue + if current is None: + continue + access_match = _ACCESS_LINE.match(line) + if access_match and current["access"] is None: + current["access"] = access_match.group(1) + if re.match(r"^\s+hostOnly: true", line): + current["host_only"] = True + if re.match(r"^ args: \[", line): + current["declared"] = [] + current["in_args"] = True + continue + if current.get("in_args"): + if line.startswith(" ],"): + current["in_args"] = False + else: + for name in _ARG_NAME.findall(line): + flag = "--" + name.replace("_", "-") + if flag not in current["declared"]: + current["declared"].append(flag) + continue + if re.match(r"^\s+handler:", line): + current["done"] = True + if not current.get("done"): + for flag in _FLAG.findall(line): + if flag not in current["flags"]: + current["flags"].append(flag) + return resource + + +def extract_ncl(root): + facts = [] + directory = Path(root) / "src" / "cli" / "resources" + for path in sorted(directory.glob("*.ts")) if directory.is_dir() else []: + if path.name.endswith(".test.ts") or path.name == "index.ts": + continue + text = read_text(path) + resource = parse_resource(text) + if not resource: + continue + plural = resource["plural"] + rel = relative(root, path) + verbs = ["%s (%s)" % (verb, access) for verb, access in resource["operations"].items()] + verbs += ["%s (%s%s)" % (entry["verb"], entry["access"] or "unknown access", ", host only" if entry["host_only"] else "") + for entry in resource["custom"]] + if verbs: + facts.append(fact( + "ncl", "%s-verbs" % plural, + "`ncl %s` exposes exactly these verbs, with their access level for agent callers: %s. " + "`open` runs inline; `approval` sends an approval card when called from inside a container." + % (plural, ", ".join(verbs)), + rel, resource["line"], + "registerResource plural '%s' operations %s custom %s" % ( + plural, resource["operations"], [entry["verb"] for entry in resource["custom"]]), + terms=["ncl", plural] + [entry["verb"].split()[0] for entry in resource["custom"]] + list(resource["operations"]), + )) + for entry in resource["custom"]: + if entry["declared"] is not None: + flags = entry["declared"] + wording = "declares exactly these flags (any other flag is rejected)" + snippet = "%s: { access: '%s', args: [%s] }" % (entry["verb"], entry["access"], ", ".join(flags)) + else: + flags = entry["flags"] + wording = "names these flags in its help text" + snippet = "%s: { access: '%s', description: ... %s }" % (entry["verb"], entry["access"], " ".join(flags)) + if not flags: + continue + facts.append(fact( + "ncl", "%s-%s-flags" % (plural, entry["verb"].replace(" ", "-")), + "`ncl %s %s` (%s) %s: %s." % ( + plural, entry["verb"], entry["access"] or "unknown access", wording, ", ".join("`%s`" % flag for flag in flags)), + rel, entry["line"], snippet, + terms=["ncl", plural, entry["verb"].split()[0]] + [flag.lstrip("-") for flag in flags], + )) + for column, values in resource["enums"]: + facts.append(fact( + "ncl", "%s-%s-enum" % (plural, column), + "`ncl %s` column `%s` accepts exactly these values: %s." % ( + plural, column, ", ".join("`%s`" % value for value in values)), + rel, line_of(text, text.find("name: '%s'" % column)), "name: '%s' enum: %s" % (column, values), + terms=["ncl", plural, column] + values, + )) + return facts + + +def extract_env(root): + facts = [] + config_path = Path(root) / "src" / "config.ts" + dotenv_keys = [] + if config_path.exists(): + text = read_text(config_path) + rel = relative(root, config_path) + block = re.search(r"readEnvFile\(\[(.*?)\]", text, re.S) + if block: + dotenv_keys = re.findall(r"'([A-Z][A-Z0-9_]+)'", block.group(1)) + for key in dotenv_keys: + match = re.search(r"process\.env\.%s\s*(\|\||\?\?)\s*envConfig\.%s\s*(?:(\|\||\?\?)\s*('([^']*)'|\"([^\"]*)\"))?" % (key, key), text) + if not match: + match = re.search(r"process\.env\.%s\b" % key, text) + if not match: + continue + facts.append(fact("env", key, "`%s` is read from the process environment first, then from `.env`." % key, + rel, line_of(text, match.start()), text[match.start(): match.start() + 160], terms=[key])) + continue + default = match.group(4) if match.group(4) is not None else match.group(5) + line = line_of(text, match.start()) + snippet = text[text.rfind("\n", 0, match.start()) + 1: text.find("\n", match.end())] + tail = text[match.end(): match.end() + 40] + if "=== 'true'" in tail or "=== \"true\"" in tail: + statement = "`%s` is read from the process environment first, then from `.env`, and is enabled only when set to the string `true`." % key + elif default is None: + statement = "`%s` is read from the process environment first, then from `.env`; it has no default." % key + elif default == "": + statement = "`%s` is read from the process environment first, then from `.env`; it is unset (empty) by default." % key + else: + statement = "`%s` is read from the process environment first, then from `.env`; its default is `%s`." % (key, default) + if match.group(1) == "??": + statement += " It is resolved with `??`, so an explicitly empty value wins over the default." + facts.append(fact("env", key, statement, rel, line, snippet, terms=[key])) + for path, constant, key, what in ( + ("src/gateway-providers/index.ts", "DEFAULT_GATEWAY_PROVIDER_KIND", "NANOCLAW_GATEWAY_PROVIDER", "gateway provider"), + ("src/drivers/index.ts", "DEFAULT_DRIVER_KIND", "NANOCLAW_RUNTIME_DRIVER", "session runtime driver"), + ): + full = Path(root) / path + if not full.exists(): + continue + text = read_text(full) + match = re.search(r"%s = '([^']+)'" % constant, text) + if match: + facts.append(fact( + "env", key, + "`%s` selects the %s; it is read from the process environment first, then from `.env`, and defaults to `%s`. " + "A configured name with no registered implementation is an error at startup, not a fallback." % (key, what, match.group(1)), + relative(root, full), line_of(text, match.start()), text[match.start(): match.end()], terms=[key, match.group(1)])) + seen = set(dotenv_keys) | {"NANOCLAW_GATEWAY_PROVIDER", "NANOCLAW_RUNTIME_DRIVER"} + src = Path(root) / "src" + for path in sorted(src.rglob("*.ts")) if src.is_dir() else []: + if path.name.endswith(".test.ts") or "/test" in str(path): + continue + text = read_text(path) + for match in re.finditer(r"process\.env\.([A-Z][A-Z0-9_]+)\s*(?:as \w+\))?\s*(\|\||\?\?)\s*'([^']*)'", text): + key, default = match.group(1), match.group(3) + if key in seen or key in ENV_IGNORE or not key.startswith(ENV_PREFIXES): + continue + seen.add(key) + facts.append(fact( + "env", key, + "`%s` is read from the process environment only (no `.env` fallback) and defaults to `%s`." % (key, default), + relative(root, path), line_of(text, match.start()), text[match.start(): match.end()], terms=[key])) + example = Path(root) / ".env.example" + if example.exists(): + text = read_text(example) + for match in re.finditer(r"(?m)^\s*#?\s*([A-Z][A-Z0-9_]+)=(.*)$", text): + key = match.group(1) + facts.append(fact( + "env", "example-%s" % key, + "`.env.example` lists `%s` (example value: `%s`)." % (key, match.group(2).strip() or "empty"), + relative(root, example), line_of(text, match.start()), match.group(0), terms=[key])) + return facts + + +def extract_container_config(root): + facts = [] + migrations = Path(root) / "src" / "db" / "migrations" + columns = [] + for path in sorted(migrations.glob("*.ts")) if migrations.is_dir() else []: + text = read_text(path) + rel = relative(root, path) + create = re.search(r"CREATE TABLE container_configs \((.*?)\);", text, re.S) + if create: + for line in create.group(1).splitlines(): + stripped = line.strip().rstrip(",") + match = re.match(r"([a-z_]+)\s+([A-Z]+)(.*)$", stripped) + if not match: + continue + columns.append((match.group(1), match.group(3).strip(), rel, line_of(text, create.start()))) + for match in re.finditer(r"ALTER TABLE container_configs ADD COLUMN ([a-z_]+) ([A-Z]+)([^;`\"]*)", text): + columns.append((match.group(1), match.group(3).strip(), rel, line_of(text, match.start()))) + if columns: + first = columns[0] + facts.append(fact( + "container-config", "columns", + "The `container_configs` table has exactly these columns: %s." % ", ".join("`%s`" % name for name, _, _, _ in columns), + first[2], first[3], "CREATE TABLE container_configs + ALTER TABLE ADD COLUMN across migrations", + terms=["container_configs", "container config"] + [name for name, _, _, _ in columns])) + for name, rest, rel, line in columns: + default = re.search(r"DEFAULT (\S+)", rest) + if default: + facts.append(fact( + "container-config", "%s-default" % name, + "`container_configs.%s` defaults to %s." % (name, default.group(1)), + rel, line, "%s %s" % (name, rest), terms=["container_configs", name, "default"])) + groups = Path(root) / "src" / "cli" / "resources" / "groups.ts" + if groups.exists(): + text = read_text(groups) + match = re.search(r"\[((?:'[\w-]+'(?:,\s*)?)+)\]\.includes\(scope\)", text) + if match: + values = re.findall(r"'([\w-]+)'", match.group(1)) + facts.append(fact( + "container-config", "cli-scope-values", + "`cli_scope` accepts exactly these values via `ncl groups config update --cli-scope`: %s." % ", ".join("`%s`" % value for value in values), + relative(root, groups), line_of(text, match.start()), match.group(0), terms=["cli_scope", "cli-scope"] + values)) + return facts + + +def parse_frontmatter(text): + if not text.startswith("---"): + return {} + end = text.find("\n---", 3) + if end < 0: + return {} + fields = {} + for line in text[3:end].splitlines(): + match = re.match(r"^([A-Za-z_-]+):\s*(.*)$", line) + if match: + value = match.group(2).strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + fields[match.group(1)] = value + return fields + + +def extract_skills(root): + facts = [] + directory = Path(root) / ".claude" / "skills" + names = [] + for skill_dir in sorted(directory.iterdir()) if directory.is_dir() else []: + skill_file = skill_dir / "SKILL.md" + if not skill_file.is_file(): + continue + text = read_text(skill_file) + front = parse_frontmatter(text) + name = front.get("name") or skill_dir.name + description = front.get("description") or "" + names.append(name) + facts.append(fact( + "skills", name, + "A workspace skill `/%s` exists under `.claude/skills/`; it is invoked when: %s" % (name, description or "(no description)"), + relative(root, skill_file), 2, "name: %s / description: %s" % (name, description), terms=[name, "/" + name])) + if names: + facts.insert(0, fact( + "skills", "catalog", + "There are exactly %d workspace skills under `.claude/skills/` with a SKILL.md: %s." % (len(names), ", ".join("`%s`" % name for name in names)), + relative(root, directory), 1, "directory listing", terms=["skills", "catalog", "workspace skills"])) + return facts + + +def extract_gateway(root): + facts = [] + doc = Path(root) / "docs" / "gateway-seam.md" + if doc.exists(): + text = read_text(doc) + rel = relative(root, doc) + for heading in ("Selection", "How a gateway gets installed"): + match = re.search(r"(?m)^## %s\s*$" % re.escape(heading), text) + if not match: + continue + end = re.search(r"(?m)^## ", text[match.end():]) + body = text[match.end(): match.end() + end.start() if end else len(text)] + for index, paragraph in enumerate(re.split(r"\n\s*\n", body)): + cleaned = " ".join(paragraph.split()) + if not cleaned or cleaned.startswith("```") or cleaned.startswith("{"): + continue + facts.append(fact( + "gateway", "%s-%d" % (heading.lower().replace(" ", "-"), index + 1), cleaned, rel, + line_of(text, match.end() + body.find(paragraph) + len(paragraph) - len(paragraph.lstrip())), paragraph, + terms=["gateway", "credential gateway", "NANOCLAW_GATEWAY_PROVIDER"] + re.findall(r"`([^`]+)`", paragraph))) + return facts + index = Path(root) / "src" / "gateway-providers" / "index.ts" + if not index.exists(): + return facts + text = read_text(index) + rel = relative(root, index) + default = re.search(r"DEFAULT_GATEWAY_PROVIDER_KIND = '([^']+)'", text) + if default: + facts.append(fact( + "gateway", "selection-default", + "One credential gateway provider is active per install, selected by `NANOCLAW_GATEWAY_PROVIDER`; when the " + "variable is unset the `%s` gateway is used, so an install that never sets it behaves as before." % default.group(1), + rel, line_of(text, default.start()), text[default.start(): default.end()], + terms=["gateway", "credential gateway", "NANOCLAW_GATEWAY_PROVIDER", default.group(1)])) + unknown = re.search(r"no gateway provider is registered", text) + if unknown: + facts.append(fact( + "gateway", "selection-unknown", + "Setting `NANOCLAW_GATEWAY_PROVIDER` to a name with no registered gateway provider throws at startup and lists " + "the installed kinds; other gateways arrive as skills (overlays) and the host never silently falls back to another gateway.", + rel, line_of(text, unknown.start()), text[text.rfind("\n", 0, unknown.start()) + 1: text.find("\n", unknown.end())], + terms=["gateway", "NANOCLAW_GATEWAY_PROVIDER", "overlay", "skill"])) + precedence = re.search(r"process\.env.*precedence", text) + if precedence: + facts.append(fact( + "gateway", "selection-precedence", + "`NANOCLAW_GATEWAY_PROVIDER` is read once at first use; the process environment takes precedence over `.env`, " + "and the value is lowercased.", + rel, line_of(text, precedence.start()), text[precedence.start(): precedence.end()], + terms=["gateway", "NANOCLAW_GATEWAY_PROVIDER", ".env"])) + return facts + + +def extract_timestamps(root): + facts = [] + claude = Path(root) / "CLAUDE.md" + if not claude.exists(): + return facts + text = read_text(claude) + match = re.search(r"(?m)^## Timestamps\s*$", text) + if not match: + return facts + end = re.search(r"(?m)^## ", text[match.end():]) + body = text[match.end(): match.end() + end.start() if end else len(text)] + index = 0 + for paragraph in re.split(r"\n\s*\n", body): + for piece in re.split(r"(?m)^- ", paragraph): + cleaned = " ".join(piece.split()) + if len(cleaned) < 40: + continue + index += 1 + facts.append(fact( + "timestamps", "rule-%d" % index, cleaned, relative(root, claude), + line_of(text, match.end() + body.find(piece) + len(piece) - len(piece.lstrip())), piece, + terms=["timestamp", "timezone", "ISO", "UTC"] + re.findall(r"`([^`]+)`", piece))) + return facts + + +EXTRACTORS = { + "ncl": extract_ncl, + "env": extract_env, + "container-config": extract_container_config, + "skills": extract_skills, + "gateway": extract_gateway, + "timestamps": extract_timestamps, +} + + +def extract_facts(root, areas=AREAS): + root = Path(root) + if not root.is_dir(): + raise DriftError("code checkout not found: %s" % root) + facts = [] + for area in AREAS: + if area in areas: + facts.extend(EXTRACTORS[area](root)) + seen = set() + for entry in facts: + if entry["id"] in seen: + raise DriftError("duplicate fact id %s" % entry["id"]) + seen.add(entry["id"]) + return facts + + +# --------------------------------------------------------------------------- +# Docs sections and lexical search +# --------------------------------------------------------------------------- + +def navigation_pages(config): + pages = [] + + def walk(node): + if isinstance(node, str): + pages.append(node) + elif isinstance(node, dict): + for key in ("pages", "groups", "tabs", "navigation", "anchors", "dropdowns", "versions", "languages"): + if key in node: + walk(node[key]) + elif isinstance(node, list): + for item in node: + walk(item) + + walk(config.get("navigation")) + ordered = [] + for page in pages: + if page not in ordered: + ordered.append(page) + return ordered + + +def blank(match): + """Replace a match with the newlines it contained so line numbers stay those of the source file.""" + return "\n" * match.group(0).count("\n") + + +def clean_mdx(text): + text = re.sub(r"\{/\*.*?\*/\}", blank, text, flags=re.S) + text = re.sub(r"(?m)^import .*$", "", text) + text = re.sub(r"(?m)^[ \t]*]*)?/?>[ \t]*$", "", text) + return text + + +def slugify(heading): + slug = re.sub(r"[^a-z0-9]+", "-", heading.lower()).strip("-") + return slug or "section" + + +def split_sections(page, text): + front = parse_frontmatter(text) + body = text + if text.startswith("---"): + end = text.find("\n---", 3) + if end >= 0: + body = "\n" * text.count("\n", 0, end + 4) + text[end + 4:] + body = clean_mdx(body) + title = front.get("title") or page + sections = [] + used = {} + current = {"heading": title, "level": 1, "line": 1, "lines": []} + in_code = False + for number, line in enumerate(body.splitlines(), 1): + if line.strip().startswith("```"): + in_code = not in_code + match = None if in_code else re.match(r"^(#{2,4})\s+(.+?)\s*#*\s*$", line) + if match: + sections.append(current) + current = {"heading": match.group(2).strip(), "level": len(match.group(1)), "line": number, "lines": []} + else: + current["lines"].append(line) + sections.append(current) + results = [] + for entry in sections: + content = "\n".join(entry["lines"]).strip() + if not content and entry["level"] == 1: + continue + anchor = slugify(entry["heading"]) + count = used.get(anchor, 0) + used[anchor] = count + 1 + if count: + anchor = "%s-%d" % (anchor, count + 1) + results.append({ + "id": "%s#%s" % (page, anchor), + "page": page, + "title": title, + "heading": entry["heading"], + "level": entry["level"], + "line": entry["line"], + "text": content, + "keywords": front.get("keywords", ""), + }) + return results + + +def load_sections(docs_root, include_changelog=False): + docs_root = Path(docs_root) + config_path = docs_root / "docs.json" + if not config_path.is_file(): + raise DriftError("docs.json not found under %s" % docs_root) + try: + config = json.loads(read_text(config_path)) + except json.JSONDecodeError as error: + raise DriftError("docs.json is not valid JSON: %s" % error) + sections = [] + for page in navigation_pages(config): + if not include_changelog and page.startswith(EXCLUDED_PAGE_PREFIXES): + continue + path = docs_root / (page + ".mdx") + if not path.is_file(): + path = docs_root / (page + ".md") + if not path.is_file(): + continue + sections.extend(split_sections(page, read_text(path))) + if not sections: + raise DriftError("no documentation sections found under %s" % docs_root) + return sections + + +_TOKEN = re.compile(r"[a-z0-9][a-z0-9_./-]*[a-z0-9]|[a-z0-9]") + + +def tokenize(text): + tokens = [] + for token in _TOKEN.findall((text or "").lower()): + tokens.append(token) + parts = [part for part in re.split(r"[_./-]", token) if len(part) > 1] + if len(parts) > 1: + tokens.extend(parts) + return tokens + + +STOPWORDS = set("the a an and or of to in on for is are be by with from that this it its as at when only then than into not no exactly these their via one per".split()) + + +class SectionIndex: + """Tiny in-memory lexical index: BM25 over section text with heading and identifier boosts.""" + + K1 = 1.2 + B = 0.75 + + def __init__(self, sections): + self.sections = sections + self.doc_frequency = {} + self.body_counts = [] + self.lowered = [] + self.lengths = [] + self.heading_sets = [] + for section in sections: + body = tokenize(section["text"] + " " + section.get("keywords", "")) + counts = {} + for token in body: + counts[token] = counts.get(token, 0) + 1 + heading = set(tokenize(section["heading"] + " " + section["title"])) + self.body_counts.append(counts) + self.lowered.append((section["heading"] + "\n" + section["text"]).lower()) + self.lengths.append(len(body)) + self.heading_sets.append(heading) + for token in set(counts) | heading: + self.doc_frequency[token] = self.doc_frequency.get(token, 0) + 1 + self.total = max(1, len(sections)) + self.average_length = max(1.0, sum(self.lengths) / self.total) + + def idf(self, token): + frequency = self.doc_frequency.get(token, 0) + return math.log(1 + (self.total - frequency + 0.5) / (frequency + 0.5)) + + def query_terms(self, entry): + """Token weights plus verbatim identifiers (backticked in the statement or listed as terms).""" + weights = {} + phrases = set() + for term in entry.get("terms") or []: + for token in tokenize(term): + weights[token] = max(weights.get(token, 0), 3.0) + if len(term) >= 4 and not term.isalpha(): + phrases.add(term.lower()) + for identifier in re.findall(r"`([^`]+)`", entry["statement"]): + for piece in tokenize(identifier): + weights[piece] = max(weights.get(piece, 0), 2.0) + if len(identifier) >= 4 and not identifier.isalpha(): + phrases.add(identifier.lower()) + for token in tokenize(entry["statement"]): + if token not in STOPWORDS and len(token) > 2: + weights.setdefault(token, 1.0) + return weights, phrases + + def search(self, entry, top_k=DEFAULT_TOP_K): + weights, phrases = self.query_terms(entry) + scored = [] + for index, section in enumerate(self.sections): + counts = self.body_counts[index] + heading = self.heading_sets[index] + norm = self.K1 * (1 - self.B + self.B * self.lengths[index] / self.average_length) + score = 0.0 + for token, weight in weights.items(): + idf = self.idf(token) + tf = counts.get(token, 0) + if tf: + score += weight * idf * (tf * (self.K1 + 1)) / (tf + norm) + if token in heading: + score += weight * idf * 1.5 + if score > 0 and phrases: + lowered = self.lowered[index] + for phrase in phrases: + if phrase in lowered: + score += 5.0 * max(self.idf(token) for token in tokenize(phrase) or ["-"]) + if score > 0: + scored.append((round(score, 3), index)) + scored.sort(key=lambda pair: (-pair[0], pair[1])) + return [(self.sections[index]["id"], score) for score, index in scored[:top_k]] + + +# --------------------------------------------------------------------------- +# Questions and requests +# --------------------------------------------------------------------------- + +def build_questions(): + return { + "contradicts": { + "type": "noul", + "instructions": ( + "Does `doc_section.text` state something incompatible with `fact.statement`? Judge only what the " + "section actually says about the point the fact makes: a different name, value, default, list of " + "options, verb, flag or rule is a contradiction. A section that does not mention the point at all, " + "or is silent on a detail, is not a contradiction." + ), + "criteria": { + "true": "The section asserts a name, value, default, option set, behavior or rule that cannot be true if the fact is true.", + "false": "The section agrees with the fact, only partly restates it, or does not address the point at all.", + }, + }, + "covers": { + "type": "noul", + "instructions": ( + "Is `doc_section` the place in this documentation site where a reader would expect `fact.statement` " + "to be stated, judging from the page title, the heading and the content? Whether the section is " + "currently correct does not matter; only whether this is where the fact belongs and is addressed." + ), + "criteria": { + "true": "The section is about this exact topic and addresses the point the fact makes (correctly or not).", + "false": "The section is about something else, or only mentions the topic in passing without addressing the point.", + }, + }, + "staleness": { + "type": "score", + "instructions": ( + "Compared with `fact.statement`, which is derived from the current code at `fact.evidence`, how out of " + "date is `doc_section.text` on that point?" + ), + "criteria": list(STALENESS_CRITERIA), + }, + } + + +def build_state(entry, section, code_commit=None): + return { + "fact": { + "area": entry["area"], + "statement": entry["statement"], + "evidence": dict(entry["evidence"], commit=code_commit) if code_commit else dict(entry["evidence"]), + }, + "doc_section": { + "page": section["page"], + "title": section["title"], + "heading": section["heading"], + "text": section["text"][:SECTION_LIMIT], + }, + } + + +def build_request(entry, section, model=DEFAULT_MODEL, code_commit=None): + return {"state": build_state(entry, section, code_commit), "model": model, "questions": build_questions()} + + +def pair_key(fact_id, section_id): + return "%s|%s" % (fact_id, section_id) + + +# --------------------------------------------------------------------------- +# Transports +# --------------------------------------------------------------------------- + +class HttpTransport: + """POST to the TypeSafe API with exponential backoff on 429/529.""" + + RETRY_STATUSES = (429, 529) + + def __init__(self, api_key, url=API_URL, timeout=60, attempts=4, sleep=time.sleep, opener=None): + if not api_key: + raise DriftError("%s is not set" % KEY_ENV) + if any(character.isspace() for character in api_key) or not api_key.isprintable(): + # Never echo the value: a newline inside a header would otherwise surface in a ValueError. + raise DriftError("%s contains whitespace or non-printable characters; export the bare key" % KEY_ENV) + self._api_key = api_key + self.url = url + self.timeout = timeout + self.attempts = attempts + self.sleep = sleep + self.opener = opener + + def __call__(self, payload): + body = json.dumps({name: value for name, value in payload.items() if not name.startswith("_")}).encode("utf-8") + delay = 1.0 + last_error = None + for attempt in range(1, self.attempts + 1): + request = urllib.request.Request( + self.url, + data=body, + method="POST", + headers={ + "Authorization": "Bearer " + self._api_key, + "Content-Type": "application/json", + "Accept": "application/json", + "User-Agent": "nanoclaw-oss-dev-tools typesafe-docs-drift", + }, + ) + opener = self.opener or urllib.request.urlopen + try: + with opener(request, timeout=self.timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + detail = "" + try: + # Redact before truncating: a cut through the key would defeat the replacement. + detail = self.redact(error.read().decode("utf-8", "replace"))[:300] + except Exception: # pragma: no cover - best effort + detail = "" + finally: + error.close() + last_error = "HTTP %s from TypeSafe: %s" % (error.code, detail.strip() or self.redact(str(error.reason))) + if error.code not in self.RETRY_STATUSES or attempt == self.attempts: + raise DriftError(last_error) + except urllib.error.URLError as error: + last_error = "could not reach TypeSafe: %s" % self.redact(str(error.reason)) + if attempt == self.attempts: + raise DriftError(last_error) + except Exception as error: # truncated bodies (http.client.IncompleteRead), bad JSON, socket errors + raise DriftError("TypeSafe response could not be read: %s: %s" % (type(error).__name__, self.redact(str(error)))) + self.sleep(delay) + delay *= 2 + raise DriftError(last_error or "TypeSafe request failed") + + def redact(self, text): + # Real keys are long; skip tiny values so redaction cannot mangle ordinary words. + return text.replace(self._api_key, "[redacted]") if len(self._api_key) >= 8 else text + + +class FixtureTransport: + """Replay recorded responses keyed by ``|
``.""" + + def __init__(self, responses): + self.responses = responses + self.requests = [] + + def __call__(self, payload): + self.requests.append(payload) + key = payload.get("_key") + try: + return self.responses[key] + except KeyError: + raise DriftError("fixture has no recorded response for %s" % key) + + +def load_fixture(path): + try: + data = json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError) as error: + raise DriftError("could not read fixture %s: %s" % (path, error)) + for key in ("facts", "sections", "candidates", "responses"): + if not isinstance(data, dict) or key not in data: + raise DriftError("fixture %s must be an object with 'facts', 'sections', 'candidates' and 'responses'" % path) + return data + + +# --------------------------------------------------------------------------- +# Decisions +# --------------------------------------------------------------------------- + +def number(value): + try: + result = float(value) + except (TypeError, ValueError): + return None + return result if math.isfinite(result) else None + + +def staleness_label(score): + if score is None: + return None + index = int(math.floor(score + 0.5)) + return STALENESS_LEVELS[max(0, min(len(STALENESS_LEVELS) - 1, index))] + + +def decide_pair(answers, thresholds): + contradicts = number((answers.get("contradicts") or {}).get("noul")) + covers = number((answers.get("covers") or {}).get("noul")) + staleness = answers.get("staleness") or {} + score = number(staleness.get("score")) + confidence = number(staleness.get("confidence")) + gate_c, gate_v = thresholds["contradicts"], thresholds["covers"] + if contradicts is None or covers is None: + verdict = "UNSURE" + note = "no answer" + elif contradicts >= gate_c: + verdict = "DRIFT" + note = "" + elif contradicts <= 1 - gate_c and covers >= gate_v: + verdict = "OK" + note = "" + elif contradicts <= 1 - gate_c and covers <= 1 - gate_v: + verdict = "UNRELATED" + note = "" + else: + verdict = "UNSURE" + note = "contradicts %.2f, covers %.2f between the gates" % (contradicts, covers) + staleness_gated = confidence is None or confidence < thresholds["staleness"] + return { + "verdict": verdict, + "contradicts": None if contradicts is None else round(contradicts, 3), + "covers": None if covers is None else round(covers, 3), + "staleness": None if score is None else round(score, 3), + "staleness_label": staleness_label(score), + "staleness_confidence": None if confidence is None else round(confidence, 3), + "staleness_gated": staleness_gated, + "note": note, + } + + +def decide_fact(entry, pairs, thresholds, failed=0): + """Fold the pair decisions of one fact into a fact-level verdict; ``failed`` counts pairs with no answer.""" + result = {"id": entry["id"], "area": entry["area"], "statement": entry["statement"], "evidence": entry["evidence"], + "pairs": pairs, "failed_pairs": failed, "documented_anywhere": None, "verdict": "MISSING", "best": None, "note": ""} + if not pairs: + if failed: + result["verdict"] = "UNSURE" + result["note"] = "%d request(s) failed; no answers" % failed + else: + result["note"] = "no doc section matched lexically" + return result + answered = [pair for pair in pairs if pair["decision"]["covers"] is not None] + if answered: + result["documented_anywhere"] = max(pair["decision"]["covers"] for pair in answered) + drift = [pair for pair in pairs if pair["decision"]["verdict"] == "DRIFT"] + ok = [pair for pair in pairs if pair["decision"]["verdict"] == "OK"] + unsure = [pair for pair in pairs if pair["decision"]["verdict"] == "UNSURE"] + if drift: + best = max(drift, key=lambda pair: (pair["decision"]["contradicts"], pair["decision"]["staleness"] or 0)) + result["verdict"] = "DRIFT" + elif ok: + best = max(ok, key=lambda pair: pair["decision"]["covers"]) + result["verdict"] = "OK" + elif unsure: + best = max(unsure, key=lambda pair: (pair["decision"]["covers"] or 0, pair["decision"]["contradicts"] or 0)) + result["verdict"] = "UNSURE" + result["note"] = best["decision"]["note"] + elif failed: + best = max(pairs, key=lambda pair: pair["decision"]["covers"] or 0) + result["verdict"] = "UNSURE" + result["note"] = "%d request(s) failed; the answered sections do not cover it" % failed + else: + best = max(pairs, key=lambda pair: pair["decision"]["covers"] or 0) + result["verdict"] = "MISSING" + result["note"] = "no candidate section covers it (max covers %.2f)" % (result["documented_anywhere"] or 0) + result["best"] = {"section": best["section"], "decision": best["decision"]} + return result + + +def rank(results): + order = {name: index for index, name in enumerate(VERDICTS)} + + def sort_key(result): + best = (result.get("best") or {}).get("decision") or {} + if result["verdict"] == "DRIFT": + secondary = (-(best.get("contradicts") or 0), -(best.get("staleness") or 0)) + elif result["verdict"] == "OK": + secondary = (-(best.get("covers") or 0), 0) + else: + secondary = (-(best.get("contradicts") or 0), -(best.get("covers") or 0)) + return (order[result["verdict"]], secondary, result["area"], result["id"]) + + return sorted(results, key=sort_key) + + +def summarize(results, pair_count, request_count): + areas = {} + totals = {name: 0 for name in VERDICTS} + for result in results: + entry = areas.setdefault(result["area"], {name: 0 for name in VERDICTS}) + entry[result["verdict"]] += 1 + totals[result["verdict"]] += 1 + return {"facts": len(results), "pairs": pair_count, "requests": request_count, "verdicts": totals, + "areas": {area: areas[area] for area in sorted(areas)}} + + +# --------------------------------------------------------------------------- +# Pipeline +# --------------------------------------------------------------------------- + +def plan_pairs(facts, sections, candidates, scores=None): + """Return the ordered (fact, section, lexical score) jobs from a candidate map.""" + by_id = {section["id"]: section for section in sections} + scores = scores or {} + jobs = [] + for entry in facts: + for section_id in candidates.get(entry["id"], []): + section = by_id.get(section_id) + if section is None: + raise DriftError("candidate section %s for %s is not in the section list" % (section_id, entry["id"])) + jobs.append((entry, section, scores.get(pair_key(entry["id"], section_id)))) + return jobs + + +def run_pairs(jobs, transport, thresholds, model=DEFAULT_MODEL, code_commit=None, concurrency=1, log=None): + """Send one request per job; return per-job outcomes (None where a request failed), usage and the errors.""" + outcomes = [None] * len(jobs) + usage = {"input_tokens": 0, "output_tokens": 0} + errors = [] + done = [0] + + def work(index): + entry, section, _score = jobs[index] + payload = build_request(entry, section, model, code_commit) + payload["_key"] = pair_key(entry["id"], section["id"]) + started = time.monotonic() + response = transport(payload) + return index, response, time.monotonic() - started + + def settle(index, response, elapsed): + entry, section, score = jobs[index] + answers = response.get("answers") if isinstance(response, dict) else None + if not isinstance(answers, dict): + raise DriftError("TypeSafe response for %s has no 'answers' map" % pair_key(entry["id"], section["id"])) + for name, value in (response.get("usage") or {}).items(): + if name in usage and isinstance(value, (int, float)): + usage[name] += int(value) + outcomes[index] = { + "key": pair_key(entry["id"], section["id"]), + "section": {name: section[name] for name in ("id", "page", "title", "heading", "line") if name in section}, + "lexical_score": score, + "response": response, + "decision": decide_pair(answers, thresholds), + "elapsed_seconds": round(elapsed, 3), + } + done[0] += 1 + if log: + log("[%d/%d] %s (%.1fs)" % (done[0], len(jobs), outcomes[index]["key"], elapsed)) + + def job_key(index): + entry, section, _score = jobs[index] + return pair_key(entry["id"], section["id"]) + + def describe(error): + if isinstance(error, DriftError): + return str(error) + # Not a transport message; keep the type only so no request detail can reach the terminal. + return "unexpected %s" % type(error).__name__ + + if concurrency <= 1 or len(jobs) <= 1: + consecutive = 0 + for index in range(len(jobs)): + try: + settle(*work(index)) + consecutive = 0 + except Exception as error: + errors.append("%s: %s" % (job_key(index), describe(error))) + consecutive += 1 + if consecutive >= MAX_CONSECUTIVE_FAILURES: + errors.append("stopped after %d consecutive failures" % consecutive) + break + return outcomes, usage, errors + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: + futures = {pool.submit(work, index): index for index in range(len(jobs))} + for future in concurrent.futures.as_completed(futures): + index = futures[future] + try: + settle(*future.result()) + except Exception as error: + errors.append("%s: %s" % (job_key(index), describe(error))) + return outcomes, usage, errors + + +def assemble(facts, jobs, outcomes, thresholds): + grouped = {entry["id"]: [] for entry in facts} + failed = {entry["id"]: 0 for entry in facts} + for index, (entry, _section, _score) in enumerate(jobs): + if outcomes[index] is not None: + grouped[entry["id"]].append(outcomes[index]) + else: + failed[entry["id"]] += 1 + return rank([decide_fact(entry, grouped[entry["id"]], thresholds, failed[entry["id"]]) for entry in facts]) + + +# --------------------------------------------------------------------------- +# Rendering and files +# --------------------------------------------------------------------------- + +def truncate(text, width): + text = " ".join((text or "").split()) + if len(text) <= width: + return text + return text[: max(0, width - 1)] + "…" + + +def format_number(value): + return "-" if value is None else "%.2f" % value + + +def table(columns, rows): + widths = [len(name) for name in columns] + for row in rows: + for index, cell in enumerate(row): + widths[index] = max(widths[index], len(cell)) + fmt = " ".join("{:<%d}" % width for width in widths) + lines = [fmt.format(*columns).rstrip(), fmt.format(*["-" * width for width in widths]).rstrip()] + lines.extend(fmt.format(*row).rstrip() for row in rows) + return "\n".join(lines) + + +def render_table(results): + rows = [] + for result in results: + best = result.get("best") or {} + section = best.get("section") or {} + decision = best.get("decision") or {} + where = section["id"] if section else "-" + stale = format_number(decision.get("staleness")) + if decision.get("staleness") is not None: + stale = "%s %s%s" % (stale, decision.get("staleness_label"), "?" if decision.get("staleness_gated") else "") + rows.append(( + result["verdict"], + result["area"], + truncate(result["statement"], 60), + truncate(where, 48), + format_number(decision.get("contradicts")), + format_number(decision.get("covers")), + stale, + "%s:%s" % (result["evidence"]["file"], result["evidence"]["line"]), + )) + return table(("Verdict", "Area", "Fact", "Doc section", "Contra", "Covers", "Stale", "Evidence"), rows) + + +def render_summary(summary, usage, wall_seconds, thresholds, output_path, notes=()): + lines = ["", "Summary"] + verdicts = summary["verdicts"] + lines.append(" facts: %d, pairs: %d, requests: %d" % (summary["facts"], summary["pairs"], summary["requests"])) + lines.append(" DRIFT %d, MISSING %d, UNSURE %d, OK %d" % tuple(verdicts[name] for name in VERDICTS)) + for area, counts in summary["areas"].items(): + lines.append(" %-17s DRIFT %d, MISSING %d, UNSURE %d, OK %d" % ((area,) + tuple(counts[name] for name in VERDICTS))) + lines.append(" gate: contradicts >= %.2f, covers >= %.2f, staleness confidence >= %.2f" % ( + thresholds["contradicts"], thresholds["covers"], thresholds["staleness"])) + lines.append(" tokens: input %d, output %d" % (usage["input_tokens"], usage["output_tokens"])) + lines.append(" wall time: %.1fs" % wall_seconds) + for note in notes: + lines.append(" note: %s" % note) + if output_path: + lines.append(" raw answers: %s" % output_path) + return "\n".join(lines) + + +def render_facts(facts): + rows = [(entry["id"], entry["area"], truncate(entry["statement"], 90), + "%s:%s" % (entry["evidence"]["file"], entry["evidence"]["line"])) for entry in facts] + return table(("Id", "Area", "Statement", "Evidence"), rows) + + +def render_plan(jobs): + lines = [] + last = None + for entry, section, score in jobs: + if entry["id"] != last: + lines.append("%s %s" % (entry["id"], truncate(entry["statement"], 90))) + last = entry["id"] + lines.append(" -> %s (%s) score %s" % (section["id"], truncate(section["heading"], 40), score)) + return "\n".join(lines) + + +def write_output(directory, payload, prefix="drift"): + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + base = "%s-%s" % (prefix, stamp()) + for suffix in [""] + ["-%d" % n for n in range(2, 1000)]: + path = directory / (base + suffix + ".json") + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + continue + with os.fdopen(fd, "w") as output: + output.write(json.dumps(payload, indent=2) + "\n") + return path + raise DriftError("could not find a free output name under %s" % directory) + + +def write_record(path, facts, sections, candidates, scores, outcomes, code_commit, docs_commit): + """Save a replayable fixture; pairs whose request failed are dropped so replay never hits a missing response.""" + answered = {outcome["key"] for outcome in outcomes if outcome} + candidates = {fact_id: [section_id for section_id in ids if pair_key(fact_id, section_id) in answered] + for fact_id, ids in candidates.items()} + scores = {key: value for key, value in scores.items() if key in answered} + used = {section_id for ids in candidates.values() for section_id in ids} + record = { + "schema_version": 1, + "recorded_at": utc_now().isoformat(), + "code_commit": code_commit, + "docs_commit": docs_commit, + "facts": facts, + "sections": [section for section in sections if section["id"] in used], + "candidates": candidates, + "scores": scores, + "responses": {outcome["key"]: outcome["response"] for outcome in outcomes if outcome}, + } + Path(path).write_text(json.dumps(record, indent=2) + "\n") + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--code", default=".", help="NanoClaw checkout to extract facts from (default: current directory)") + parser.add_argument("--docs", help="nanoclaw-docs checkout (Mintlify site with docs.json); required unless --fixture or --facts-only") + parser.add_argument("--areas", default=",".join(AREAS), help="comma-separated fact areas: %s" % ", ".join(AREAS)) + parser.add_argument("--limit-facts", type=int, default=0, help="stop after this many facts (0 = all)") + parser.add_argument("--top-k", type=int, default=DEFAULT_TOP_K, help="candidate doc sections per fact (default %d)" % DEFAULT_TOP_K) + parser.add_argument("--include-changelog", action="store_true", help="also search changelog pages (excluded by default)") + parser.add_argument("--facts-only", action="store_true", help="print the extracted facts and exit; no docs or key needed") + parser.add_argument("--plan", action="store_true", help="print the fact x section pairs that would be sent and exit; no key needed") + parser.add_argument("--fixture", help="replay recorded facts, sections and responses from this JSON file instead of the checkouts and the API") + parser.add_argument("--record", help="write facts, sections and raw responses to this JSON file for later --fixture replay") + parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR), help="where raw answers are saved (default: skill-local output/, gitignored)") + parser.add_argument("--model", default=DEFAULT_MODEL) + parser.add_argument("--timeout", type=float, default=60.0, help="HTTP timeout per request in seconds") + parser.add_argument("--concurrency", type=int, default=DEFAULT_CONCURRENCY, help="parallel requests in live mode (default %d)" % DEFAULT_CONCURRENCY) + parser.add_argument("--contradicts-threshold", type=float, default=0.7, help="yes probability needed to call DRIFT (default 0.7)") + parser.add_argument("--covers-threshold", type=float, default=0.7, help="yes probability needed to call a section covering (default 0.7)") + parser.add_argument("--staleness-threshold", type=float, default=0.5, help="score confidence below which the staleness label is marked '?' (default 0.5)") + parser.add_argument("--json", action="store_true", help="print the full result JSON to stdout instead of the table") + args = parser.parse_args(argv) + for name in ("contradicts_threshold", "covers_threshold"): + if not 0.5 <= getattr(args, name) <= 1: + parser.error("--%s must be between 0.5 and 1" % name.replace("_", "-")) + if not 0 <= args.staleness_threshold <= 1: + parser.error("--staleness-threshold must be between 0 and 1") + if args.top_k < 1: + parser.error("--top-k must be at least 1") + if args.concurrency < 1: + parser.error("--concurrency must be at least 1") + areas = [area.strip() for area in args.areas.split(",") if area.strip()] + unknown = [area for area in areas if area not in AREAS] + if unknown: + parser.error("unknown area(s): %s (choose from %s)" % (", ".join(unknown), ", ".join(AREAS))) + args.area_list = areas + if not args.fixture and not args.facts_only and not args.docs: + parser.error("--docs is required unless --fixture or --facts-only is used") + return args + + +def main(argv=None, environ=None, stdout=None, stderr=None): + environ = os.environ if environ is None else environ + stdout = sys.stdout if stdout is None else stdout + stderr = sys.stderr if stderr is None else stderr + args = parse_args(argv) + thresholds = {"contradicts": args.contradicts_threshold, "covers": args.covers_threshold, "staleness": args.staleness_threshold} + started = time.monotonic() + notes = [] + try: + if args.fixture: + fixture = load_fixture(args.fixture) + facts = [entry for entry in fixture["facts"] if entry["area"] in args.area_list] + if args.limit_facts > 0: + facts = facts[: args.limit_facts] + sections = fixture["sections"] + candidates = {entry["id"]: fixture["candidates"].get(entry["id"], []) for entry in facts} + code_commit, docs_commit = fixture.get("code_commit"), fixture.get("docs_commit") + scores = fixture.get("scores") or {} + transport = FixtureTransport(fixture["responses"]) + mode = "fixture" + if fixture.get("note"): + notes.append(fixture["note"]) + else: + mode = "live" + facts = extract_facts(args.code, args.area_list) + code_commit = git_commit(args.code) + example = Path(args.code) / ".env.example" + if not example.exists() or not read_text(example).strip(): + notes.append(".env.example is missing or empty; no facts came from it") + if not (Path(args.code) / "docs" / "gateway-seam.md").exists(): + notes.append("docs/gateway-seam.md is absent in this checkout; gateway facts come from src/gateway-providers/index.ts") + if args.limit_facts > 0: + facts = facts[: args.limit_facts] + if args.facts_only: + sections, candidates, scores, docs_commit = [], {}, {}, None + if args.facts_only: + if args.json: + print(json.dumps({"code_commit": code_commit, "facts": facts, "notes": notes}, indent=2), file=stdout) + else: + print(render_facts(facts), file=stdout) + print("\n%d facts from %s @ %s" % (len(facts), args.fixture or args.code, code_commit or "unknown"), file=stdout) + for note in notes: + print("note: %s" % note, file=stdout) + return 0 + if mode == "live": + sections = load_sections(args.docs, args.include_changelog) + docs_commit = git_commit(args.docs) + index = SectionIndex(sections) + candidates = {} + scores = {} + for entry in facts: + hits = index.search(entry, args.top_k) + candidates[entry["id"]] = [section_id for section_id, _score in hits] + for section_id, score in hits: + scores[pair_key(entry["id"], section_id)] = score + if not facts: + print("no facts extracted for areas: %s" % ", ".join(args.area_list), file=stderr) + return 1 + jobs = plan_pairs(facts, sections, candidates, scores) + if args.plan: + print(render_plan(jobs), file=stdout) + print("\n%d facts, %d pairs, %d sections from %s @ %s" % ( + len(facts), len(jobs), len(sections), args.fixture or args.docs, docs_commit or "unknown"), file=stdout) + for note in notes: + print("note: %s" % note, file=stdout) + return 0 + if mode == "live": + api_key = environ.get(KEY_ENV) + if not api_key: + print( + "%s is not set. Export it in this shell (for example `export %s=...` from your " + "password manager) and rerun. The key is read from the environment only; never put it " + "in a file inside the repository. Use --fixture to run offline, or --plan / --facts-only " + "to see the pairs and facts without calling the API." % (KEY_ENV, KEY_ENV), + file=stderr, + ) + return 2 + transport = HttpTransport(api_key, timeout=args.timeout) + outcomes, usage, errors = run_pairs( + jobs, transport, thresholds, args.model, code_commit, + concurrency=args.concurrency if mode == "live" else 1, + log=(lambda line: print(line, file=stderr)) if mode == "live" else None) + completed = sum(1 for outcome in outcomes if outcome) + error_text = None + if errors: + error_text = "; ".join(errors[:3]) + (" (+%d more)" % (len(errors) - 3) if len(errors) > 3 else "") + print("error: %s; keeping %d completed pair(s)" % (error_text, completed), file=stderr) + if not completed: + return 1 + results = assemble(facts, jobs, outcomes, thresholds) + except DriftError as error: + print("error: %s" % error, file=stderr) + return 1 + except Exception as error: # never let a traceback carry request details to the terminal + print("error: unexpected %s: %s" % (type(error).__name__, redact_env(str(error), environ)), file=stderr) + return 1 + wall = time.monotonic() - started + summary = summarize(results, len(jobs), completed) + payload = { + "schema_version": 1, + "generated_at": utc_now().isoformat(), + "mode": mode, + "code": {"root": None if mode == "fixture" else str(Path(args.code).resolve()), "commit": code_commit}, + "docs": {"root": None if mode == "fixture" else str(Path(args.docs).resolve()), "commit": docs_commit}, + "model": args.model, + "thresholds": thresholds, + "top_k": args.top_k, + "usage": usage, + "wall_seconds": round(wall, 3), + "partial": error_text, + "notes": notes, + "summary": summary, + "results": results, + } + output_path = write_output(args.output_dir, payload) + if args.record and mode == "live": + write_record(args.record, facts, sections, candidates, scores, outcomes, code_commit, docs_commit) + if args.json: + print(json.dumps(payload, indent=2), file=stdout) + else: + print("Docs drift (%s mode): %d facts, %d pairs. Nothing was written to either repository." % ( + mode, len(facts), len(jobs)), file=stdout) + print("", file=stdout) + print(render_table(results), file=stdout) + print(render_summary(summary, usage, wall, thresholds, output_path, notes), file=stdout) + return 1 if error_text else 0 + + +def redact_env(text, environ): + key = environ.get(KEY_ENV) or "" + return text.replace(key, "[redacted]") if len(key) >= 8 else text + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_typesafe_docs_drift.py b/tests/test_typesafe_docs_drift.py new file mode 100644 index 0000000..6d2ded6 --- /dev/null +++ b/tests/test_typesafe_docs_drift.py @@ -0,0 +1,891 @@ +"""TypeSafe docs-drift detector: extraction, sections, search, gate, fixture replay and transport, all offline.""" + +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import textwrap +import threading +import unittest +from unittest.mock import patch +import urllib.error + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py" +FIXTURE = ROOT / "skills/typesafe-docs-drift/fixtures/nanoclaw-sample.json" + + +def load(): + spec = importlib.util.spec_from_file_location("typesafe_docs_drift_test", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +drift = load() +THRESHOLDS = {"contradicts": 0.7, "covers": 0.7, "staleness": 0.5} + + +def write(root, relative_path, content): + path = Path(root) / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(content).lstrip("\n")) + return path + + +def make_code(root, with_gateway_doc=False): + write(root, "src/cli/resources/index.ts", "import './widgets.js';\n") + write(root, "src/cli/resources/widgets.test.ts", "registerResource({ plural: 'ghosts', operations: { list: 'open' } });\n") + write(root, "src/cli/resources/widgets.ts", """ + import { registerResource } from '../crud.js'; + + registerResource({ + name: 'widget', + plural: 'widgets', + table: 'widgets', + description: 'Widgets.', + idColumn: 'id', + columns: [ + { name: 'id', type: 'string', description: 'UUID.', generated: true }, + { + name: 'state', + type: 'string', + description: 'Live state.', + enum: [ + 'on', + 'off', + ], + }, + ], + operations: { list: 'open', get: 'open', update: 'approval' }, + customOperations: { + poke: { + access: 'approval', + description: 'Poke a widget. Use --id [--hard]. ' + + 'Pass --hard to poke harder.', + handler: async (args) => ({ poked: args.id, flag: '--not-a-flag-after-handler' }), + }, + 'config get': { + access: 'open', + hostOnly: true, + description: 'Show config. Use --id .', + handler: async () => ({}), + }, + run: { + access: 'open', + description: 'Run now. Unlike update --process-after now, it keeps the schedule.', + args: [ + { name: 'id', type: 'string', description: 'Id.', required: true }, + { + name: 'group_id', + type: 'string', + description: 'Group.', + }, + ], + handler: async () => ({}), + }, + }, + }); + """) + write(root, "src/cli/resources/groups.ts", """ + registerResource({ name: 'group', plural: 'groups', columns: [], operations: { list: 'open' } }); + if (!['disabled', 'group', 'global'].includes(scope)) { + throw new Error('--cli-scope must be one of: disabled, group, global'); + } + """) + write(root, "src/config.ts", """ + const envConfig = readEnvFile([ + 'ASSISTANT_NAME', + 'FLAG', + 'LIMIT', + 'NOTHING', + ]); + export const ASSISTANT_NAME = process.env.ASSISTANT_NAME || envConfig.ASSISTANT_NAME || 'Andy'; + export const FLAG = (process.env.FLAG || envConfig.FLAG) === 'true'; + export const LIMIT = process.env.LIMIT ?? envConfig.LIMIT ?? '2048'; + export const NOTHING = process.env.NOTHING || envConfig.NOTHING; + """) + write(root, "src/gateway-providers/index.ts", """ + /** + * Settings honor `.env` with `process.env` taking precedence, like every other setting. + */ + const DEFAULT_GATEWAY_PROVIDER_KIND = 'onecli'; + throw new Error(`NANOCLAW_GATEWAY_PROVIDER='${kind}' but no gateway provider is registered for '${kind}'`); + """) + write(root, "src/drivers/index.ts", "const DEFAULT_DRIVER_KIND = 'docker';\n") + write(root, "src/log.ts", "const threshold = LEVELS[(process.env.LOG_LEVEL as Level) || 'info'] ?? LEVELS.info;\nconst home = process.env.HOME || '/root';\n") + write(root, "src/db/migrations/014-container-configs.ts", """ + db.exec(` + CREATE TABLE container_configs ( + agent_group_id TEXT PRIMARY KEY REFERENCES agent_groups(id) ON DELETE CASCADE, + provider TEXT, + skills TEXT NOT NULL DEFAULT '"all"', + updated_at TEXT NOT NULL + ); + `); + """) + write(root, "src/db/migrations/015-cli-scope.ts", """ + db.prepare("ALTER TABLE container_configs ADD COLUMN cli_scope TEXT NOT NULL DEFAULT 'group'").run(); + """) + write(root, ".claude/skills/add-foo/SKILL.md", """ + --- + name: add-foo + description: "Add the Foo channel via Chat SDK." + --- + + # Add Foo + """) + write(root, ".claude/skills/bar/SKILL.md", "---\nname: bar\ndescription: Bar things.\n---\n") + (Path(root) / ".claude/skills/no-skill-file").mkdir(parents=True, exist_ok=True) + write(root, "CLAUDE.md", """ + # Project + + ## Timestamps + + Two rules, no exceptions: + + - **Storage**: every timestamp written from JS is `new Date().toISOString()` (ISO-8601 UTC with `Z`). + - **Display**: anything shown to a user renders in the install timezone via `formatLocalTime`. + + An agent group can override the install timezone with `ncl groups config update --timezone `. + + ## Next section + + Unrelated. + """) + write(root, ".env.example", "# comment\nFOO=bar\nEMPTY=\n") + if with_gateway_doc: + write(root, "docs/gateway-seam.md", """ + # Gateway Provider Seam + + ## Selection + + One provider per install, chosen by `NANOCLAW_GATEWAY_PROVIDER` and resolved + once at startup by `getGatewayProvider()`. + + With no provider registered, the host refuses to start: there is no implicit default. + + ## Other + + Nothing here. + """) + return root + + +def make_docs(root): + write(root, "docs.json", json.dumps({ + "navigation": {"tabs": [ + {"tab": "Documentation", "groups": [ + {"group": "Operate", "pages": ["operate/ncl-cli", "operate/ncl-cli"]}, + {"group": "Changelog", "pages": ["changelog/index"]}, + ]}, + {"tab": "Reference", "groups": [{"group": "Operating", "pages": ["reference/env", "missing/page"]}]}, + ]}, + })) + write(root, "operate/ncl-cli.mdx", """ + --- + title: "The ncl admin CLI" + description: "Administer a host." + keywords: ["ncl", "cli_scope"] + --- + + {/* verified-against: src/cli/dispatch.ts @ abc123 */} + + Intro paragraph about `ncl`. + + ## Agents can run ncl too + + + What an agent may do is controlled per group by `cli_scope`: `disabled`, `group` (default) or `global`. + + + ```bash + ## not a heading + ncl groups config update --id --cli-scope disabled + ``` + + ## Working with resources + + ### widgets + + Poke widgets with `ncl widgets poke --id `. + + ### widgets + + Duplicate heading. + """) + write(root, "reference/env.mdx", """ + --- + title: "Environment variables" + --- + + ## Read from .env + + | Variable | Default | + |---|---| + | `ASSISTANT_NAME` | `Andy` | + | `LIMIT` | `1024` | + """) + write(root, "changelog/index.mdx", "---\ntitle: Changelog\n---\n\n## 2026-09-01\n\nAdded `ASSISTANT_NAME` and `cli_scope` notes.\n") + return root + + +def noul(value): + return {"type": "noul", "noul": value} + + +def score(value, confidence): + return {"type": "score", "score": value, "legend": {}, "probabilities": {}, "confidence": confidence} + + +def answers(contradicts=0.1, covers=0.9, staleness=0.2, confidence=0.9): + return {"contradicts": noul(contradicts), "covers": noul(covers), "staleness": score(staleness, confidence)} + + +def response(contradicts=0.1, covers=0.9, staleness=0.2, confidence=0.9, tokens=100): + return {"model": "jev-latest", "answers": answers(contradicts, covers, staleness, confidence), + "usage": {"input_tokens": tokens, "output_tokens": 10}} + + +def pair(section_id, decision): + return {"key": "f|" + section_id, "section": {"id": section_id, "page": section_id.split("#")[0], "heading": "h"}, + "decision": decision} + + +class TempDirs(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="typesafe-docs-drift-") + self.addCleanup(self.temporary.cleanup) + self.base = Path(self.temporary.name) + + def code(self, **kwargs): + return make_code(self.base / "nanoclaw", **kwargs) + + def docs(self): + return make_docs(self.base / "nanoclaw-docs") + + +class ExtractionTests(TempDirs): + def by_id(self, facts): + return {entry["id"]: entry for entry in facts} + + def test_ncl_resources_yield_verbs_flags_and_enums_and_skip_test_modules(self): + facts = self.by_id(drift.extract_facts(self.code(), ["ncl"])) + self.assertEqual(set(facts), {"ncl:widgets-verbs", "ncl:widgets-poke-flags", "ncl:widgets-config-get-flags", + "ncl:widgets-run-flags", "ncl:widgets-state-enum"}) + verbs = facts["ncl:widgets-verbs"] + self.assertIn("list (open), get (open), update (approval), poke (approval), config get (open, host only), run (open)", verbs["statement"]) + self.assertEqual(verbs["evidence"]["file"], "src/cli/resources/widgets.ts") + self.assertEqual(verbs["evidence"]["line"], 3) + poke = facts["ncl:widgets-poke-flags"] + self.assertIn("names these flags in its help text: `--id`, `--hard`", poke["statement"]) + self.assertNotIn("not-a-flag", poke["statement"]) + run = facts["ncl:widgets-run-flags"] + self.assertIn("declares exactly these flags (any other flag is rejected): `--id`, `--group-id`", run["statement"]) + self.assertNotIn("process-after", run["statement"]) + self.assertIn("`on`, `off`", facts["ncl:widgets-state-enum"]["statement"]) + self.assertIn("widgets", facts["ncl:widgets-state-enum"]["terms"]) + self.assertNotIn("ncl:ghosts-verbs", facts) + + def test_env_facts_capture_precedence_defaults_and_boolean_flags(self): + facts = self.by_id(drift.extract_facts(self.code(), ["env"])) + self.assertIn("default is `Andy`", facts["env:ASSISTANT_NAME"]["statement"]) + self.assertIn("only when set to the string `true`", facts["env:FLAG"]["statement"]) + self.assertIn("default is `2048`", facts["env:LIMIT"]["statement"]) + self.assertIn("`??`", facts["env:LIMIT"]["statement"]) + self.assertIn("no default", facts["env:NOTHING"]["statement"]) + self.assertIn("defaults to `onecli`", facts["env:NANOCLAW_GATEWAY_PROVIDER"]["statement"]) + self.assertIn("defaults to `docker`", facts["env:NANOCLAW_RUNTIME_DRIVER"]["statement"]) + self.assertIn("process environment only", facts["env:LOG_LEVEL"]["statement"]) + self.assertNotIn("env:HOME", facts) + self.assertIn("example value: `bar`", facts["env:example-FOO"]["statement"]) + self.assertIn("example value: `empty`", facts["env:example-EMPTY"]["statement"]) + self.assertEqual(facts["env:ASSISTANT_NAME"]["evidence"]["line"], 7) + + def test_container_config_columns_defaults_and_cli_scope_values(self): + facts = self.by_id(drift.extract_facts(self.code(), ["container-config"])) + self.assertIn("`agent_group_id`, `provider`, `skills`, `updated_at`, `cli_scope`", facts["container-config:columns"]["statement"]) + self.assertIn("defaults to '\"all\"'", facts["container-config:skills-default"]["statement"]) + self.assertIn("defaults to 'group'", facts["container-config:cli_scope-default"]["statement"]) + self.assertEqual(facts["container-config:cli_scope-default"]["evidence"]["file"], "src/db/migrations/015-cli-scope.ts") + self.assertIn("`disabled`, `group`, `global`", facts["container-config:cli-scope-values"]["statement"]) + self.assertNotIn("container-config:provider-default", facts) + + def test_skills_come_from_frontmatter_and_directories_without_a_skill_file_are_ignored(self): + facts = self.by_id(drift.extract_facts(self.code(), ["skills"])) + self.assertEqual(set(facts), {"skills:catalog", "skills:add-foo", "skills:bar"}) + self.assertIn("Add the Foo channel via Chat SDK.", facts["skills:add-foo"]["statement"]) + self.assertIn("exactly 2 workspace skills", facts["skills:catalog"]["statement"]) + self.assertEqual(facts["skills:add-foo"]["evidence"]["file"], ".claude/skills/add-foo/SKILL.md") + self.assertIn("/add-foo", facts["skills:add-foo"]["terms"]) + + def test_gateway_facts_come_from_code_or_from_the_seam_doc_when_present(self): + from_code = self.by_id(drift.extract_facts(self.code(), ["gateway"])) + self.assertEqual(set(from_code), {"gateway:selection-default", "gateway:selection-unknown", "gateway:selection-precedence"}) + self.assertIn("`onecli`", from_code["gateway:selection-default"]["statement"]) + with_doc = self.by_id(drift.extract_facts(make_code(self.base / "with-doc", with_gateway_doc=True), ["gateway"])) + self.assertEqual(set(with_doc), {"gateway:selection-1", "gateway:selection-2"}) + self.assertEqual(with_doc["gateway:selection-1"]["evidence"]["file"], "docs/gateway-seam.md") + self.assertEqual(with_doc["gateway:selection-1"]["evidence"]["line"], 5) + self.assertIn("refuses to start", with_doc["gateway:selection-2"]["statement"]) + + def test_timestamp_rules_are_split_by_bullet_and_paragraph(self): + facts = drift.extract_facts(self.code(), ["timestamps"]) + self.assertEqual([entry["id"] for entry in facts], ["timestamps:rule-1", "timestamps:rule-2", "timestamps:rule-3"]) + self.assertTrue(facts[0]["statement"].startswith("**Storage**")) + self.assertIn("--timezone", facts[2]["statement"]) + self.assertEqual(facts[0]["evidence"]["file"], "CLAUDE.md") + self.assertEqual(facts[0]["evidence"]["line"], 7) + self.assertNotIn("Unrelated", " ".join(entry["statement"] for entry in facts)) + + def test_ids_are_unique_and_areas_filter(self): + facts = drift.extract_facts(self.code()) + ids = [entry["id"] for entry in facts] + self.assertEqual(len(ids), len(set(ids))) + self.assertEqual({entry["area"] for entry in facts}, set(drift.AREAS)) + for entry in facts: + self.assertEqual(set(entry), {"id", "area", "statement", "evidence", "terms"}) + self.assertEqual(set(entry["evidence"]), {"file", "line", "excerpt"}) + self.assertEqual({entry["area"] for entry in drift.extract_facts(self.code(), ["skills", "env"])}, {"skills", "env"}) + + def test_missing_checkout_is_a_clean_error(self): + with self.assertRaises(drift.DriftError): + drift.extract_facts(self.base / "nope") + + +class SectionTests(TempDirs): + def test_navigation_pages_are_collected_in_order_without_duplicates(self): + config = json.loads((self.docs() / "docs.json").read_text()) + self.assertEqual(drift.navigation_pages(config), ["operate/ncl-cli", "changelog/index", "reference/env", "missing/page"]) + + def test_sections_split_on_headings_and_skip_code_fences_comments_and_tags(self): + sections = {section["id"]: section for section in drift.load_sections(self.docs())} + self.assertEqual(set(sections), { + "operate/ncl-cli#the-ncl-admin-cli", "operate/ncl-cli#agents-can-run-ncl-too", "operate/ncl-cli#working-with-resources", + "operate/ncl-cli#widgets", "operate/ncl-cli#widgets-2", "reference/env#read-from-env", + }) + agents = sections["operate/ncl-cli#agents-can-run-ncl-too"] + self.assertEqual(agents["title"], "The ncl admin CLI") + self.assertEqual(agents["level"], 2) + self.assertIn("## not a heading", agents["text"]) + self.assertNotIn("", agents["text"]) + self.assertNotIn("verified-against", sections["operate/ncl-cli#the-ncl-admin-cli"]["text"]) + self.assertEqual(sections["operate/ncl-cli#widgets"]["level"], 3) + self.assertEqual(agents["line"], 11) + self.assertEqual(sections["operate/ncl-cli#working-with-resources"]["line"], 22) + self.assertEqual(sections["reference/env#read-from-env"]["line"], 5) + self.assertIn("cli_scope", agents["keywords"]) + + def test_changelog_pages_are_excluded_unless_asked(self): + default = {section["page"] for section in drift.load_sections(self.docs())} + self.assertNotIn("changelog/index", default) + included = {section["page"] for section in drift.load_sections(self.docs(), include_changelog=True)} + self.assertIn("changelog/index", included) + + def test_missing_docs_json_is_a_clean_error(self): + with self.assertRaises(drift.DriftError): + drift.load_sections(self.base) + + +class SearchTests(TempDirs): + def test_identifier_matches_rank_the_right_section_first(self): + sections = drift.load_sections(self.docs()) + index = drift.SectionIndex(sections) + scope = {"id": "container-config:cli-scope-values", "area": "container-config", + "statement": "`cli_scope` accepts exactly these values: `disabled`, `group`, `global`.", "terms": ["cli_scope", "cli-scope"]} + hits = index.search(scope, 3) + self.assertEqual(hits[0][0], "operate/ncl-cli#agents-can-run-ncl-too") + self.assertLessEqual(len(hits), 3) + self.assertTrue(all(isinstance(score, float) for _id, score in hits)) + limit = {"id": "env:LIMIT", "area": "env", "statement": "`LIMIT` defaults to `2048`.", "terms": ["LIMIT"]} + self.assertEqual(index.search(limit, 1)[0][0], "reference/env#read-from-env") + widget = {"id": "ncl:widgets-poke-flags", "area": "ncl", "statement": "`ncl widgets poke` accepts `--id`.", "terms": ["ncl", "widgets", "poke", "id"]} + self.assertEqual(index.search(widget, 1)[0][0], "operate/ncl-cli#widgets") + nothing = {"id": "x", "area": "env", "statement": "zzzz qqqq", "terms": ["zzzz"]} + self.assertEqual(index.search(nothing, 3), []) + + +class QuestionTests(unittest.TestCase): + def test_request_shape_matches_the_documented_api(self): + entry = {"id": "env:X", "area": "env", "statement": "`X` defaults to 1.", "evidence": {"file": "src/config.ts", "line": 3, "excerpt": "x"}, "terms": ["X"]} + section = {"id": "p#h", "page": "p", "title": "T", "heading": "h", "level": 2, "line": 1, "text": "y" * (drift.SECTION_LIMIT + 50)} + request = drift.build_request(entry, section, code_commit="abc123") + self.assertEqual(set(request), {"state", "model", "questions"}) + self.assertEqual(request["model"], "jev-latest") + self.assertEqual(set(request["state"]), {"fact", "doc_section"}) + self.assertEqual(request["state"]["fact"]["evidence"]["commit"], "abc123") + self.assertEqual(len(request["state"]["doc_section"]["text"]), drift.SECTION_LIMIT) + self.assertEqual(set(request["questions"]), {"contradicts", "covers", "staleness"}) + self.assertEqual(request["questions"]["contradicts"]["type"], "noul") + self.assertEqual(request["questions"]["covers"]["type"], "noul") + self.assertEqual(request["questions"]["staleness"]["type"], "score") + self.assertEqual(len(request["questions"]["staleness"]["criteria"]), 4) + for question in request["questions"].values(): + self.assertTrue(question["instructions"]) + self.assertNotIn("terms", request["state"]["fact"]) + + +class DecisionTests(unittest.TestCase): + def test_pair_verdicts_follow_the_gates(self): + self.assertEqual(drift.decide_pair(answers(contradicts=0.7), THRESHOLDS)["verdict"], "DRIFT") + self.assertEqual(drift.decide_pair(answers(contradicts=0.3, covers=0.7), THRESHOLDS)["verdict"], "OK") + self.assertEqual(drift.decide_pair(answers(contradicts=0.3, covers=0.3), THRESHOLDS)["verdict"], "UNRELATED") + self.assertEqual(drift.decide_pair(answers(contradicts=0.5, covers=0.9), THRESHOLDS)["verdict"], "UNSURE") + self.assertEqual(drift.decide_pair(answers(contradicts=0.2, covers=0.5), THRESHOLDS)["verdict"], "UNSURE") + missing = drift.decide_pair({}, THRESHOLDS) + self.assertEqual((missing["verdict"], missing["note"], missing["contradicts"]), ("UNSURE", "no answer", None)) + bad = drift.decide_pair({"contradicts": noul("high"), "covers": noul(float("nan"))}, THRESHOLDS) + self.assertEqual(bad["verdict"], "UNSURE") + + def test_staleness_rounds_to_a_level_and_is_gated_on_confidence(self): + decision = drift.decide_pair(answers(staleness=1.4, confidence=0.5), THRESHOLDS) + self.assertEqual((decision["staleness_label"], decision["staleness_gated"]), ("slightly outdated", False)) + decision = drift.decide_pair(answers(staleness=1.5, confidence=0.49), THRESHOLDS) + self.assertEqual((decision["staleness_label"], decision["staleness_gated"]), ("wrong", True)) + self.assertEqual(drift.staleness_label(9), "dangerous if followed") + self.assertEqual(drift.staleness_label(-1), "current") + self.assertIsNone(drift.staleness_label(None)) + + def test_thresholds_are_overridable(self): + strict = dict(THRESHOLDS, contradicts=0.9) + self.assertEqual(drift.decide_pair(answers(contradicts=0.8), strict)["verdict"], "UNSURE") + loose = dict(THRESHOLDS, contradicts=0.6, covers=0.6) + self.assertEqual(drift.decide_pair(answers(contradicts=0.6), loose)["verdict"], "DRIFT") + self.assertEqual(drift.decide_pair(answers(contradicts=0.4, covers=0.6), loose)["verdict"], "OK") + + def test_fact_verdict_folds_pairs_with_drift_winning(self): + entry = {"id": "f", "area": "env", "statement": "s", "evidence": {"file": "a", "line": 1, "excerpt": ""}} + drifty = pair("p#a", drift.decide_pair(answers(contradicts=0.75, covers=0.9, staleness=1.6), THRESHOLDS)) + okay = pair("p#b", drift.decide_pair(answers(contradicts=0.1, covers=0.95), THRESHOLDS)) + unrelated = pair("p#c", drift.decide_pair(answers(contradicts=0.05, covers=0.1), THRESHOLDS)) + unsure = pair("p#d", drift.decide_pair(answers(contradicts=0.5, covers=0.5), THRESHOLDS)) + result = drift.decide_fact(entry, [okay, drifty, unrelated], THRESHOLDS) + self.assertEqual((result["verdict"], result["best"]["section"]["id"]), ("DRIFT", "p#a")) + self.assertEqual(result["documented_anywhere"], 0.95) + result = drift.decide_fact(entry, [unrelated, okay], THRESHOLDS) + self.assertEqual((result["verdict"], result["best"]["section"]["id"]), ("OK", "p#b")) + result = drift.decide_fact(entry, [unrelated, unsure], THRESHOLDS) + self.assertEqual((result["verdict"], result["best"]["section"]["id"]), ("UNSURE", "p#d")) + result = drift.decide_fact(entry, [unrelated], THRESHOLDS) + self.assertEqual(result["verdict"], "MISSING") + self.assertIn("max covers 0.10", result["note"]) + result = drift.decide_fact(entry, [], THRESHOLDS) + self.assertEqual((result["verdict"], result["best"], result["documented_anywhere"]), ("MISSING", None, None)) + self.assertIn("no doc section matched", result["note"]) + result = drift.decide_fact(entry, [], THRESHOLDS, failed=2) + self.assertEqual((result["verdict"], result["failed_pairs"]), ("UNSURE", 2)) + self.assertIn("2 request(s) failed", result["note"]) + result = drift.decide_fact(entry, [unrelated], THRESHOLDS, failed=1) + self.assertEqual(result["verdict"], "UNSURE") + result = drift.decide_fact(entry, [drifty], THRESHOLDS, failed=1) + self.assertEqual(result["verdict"], "DRIFT") + + def test_ranking_puts_drift_first_by_contradiction_then_missing_unsure_ok(self): + entry = {"id": "f", "area": "env", "statement": "s", "evidence": {"file": "a", "line": 1, "excerpt": ""}} + + def result(fact_id, pairs): + return drift.decide_fact(dict(entry, id=fact_id), pairs, THRESHOLDS) + + results = [ + result("ok", [pair("p#1", drift.decide_pair(answers(contradicts=0.1, covers=0.9), THRESHOLDS))]), + result("drift-low", [pair("p#2", drift.decide_pair(answers(contradicts=0.72), THRESHOLDS))]), + result("missing", []), + result("unsure", [pair("p#3", drift.decide_pair(answers(contradicts=0.5, covers=0.5), THRESHOLDS))]), + result("drift-high", [pair("p#4", drift.decide_pair(answers(contradicts=0.95), THRESHOLDS))]), + ] + self.assertEqual([entry["id"] for entry in drift.rank(results)], ["drift-high", "drift-low", "missing", "unsure", "ok"]) + summary = drift.summarize(results, pair_count=4, request_count=4) + self.assertEqual(summary["verdicts"], {"DRIFT": 2, "MISSING": 1, "UNSURE": 1, "OK": 1}) + self.assertEqual(summary["areas"]["env"]["DRIFT"], 2) + self.assertEqual((summary["facts"], summary["pairs"], summary["requests"]), (5, 4, 4)) + + +class TransportTests(unittest.TestCase): + class Response: + def __init__(self, body): + self.body = body + + def read(self): + return json.dumps(self.body).encode() + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def test_http_transport_sends_bearer_key_strips_private_keys_and_retries_on_429(self): + seen = [] + sleeps = [] + + def opener(request, timeout): + seen.append(request) + if len(seen) == 1: + raise urllib.error.HTTPError(request.full_url, 429, "Too Many Requests", {}, io.BytesIO(b"{}")) + return self.Response(response()) + + transport = drift.HttpTransport("secret-key", opener=opener, sleep=sleeps.append) + result = transport({"state": {"x": 1}, "model": "jev-latest", "questions": {}, "_key": "f|s"}) + self.assertEqual(result["model"], "jev-latest") + self.assertEqual(len(seen), 2) + self.assertEqual(sleeps, [1.0]) + request = seen[0] + self.assertEqual(request.full_url, "https://api.typesafe.ai/v1/systemone") + self.assertEqual(request.get_method(), "POST") + self.assertEqual(request.get_header("Authorization"), "Bearer secret-key") + self.assertEqual(json.loads(request.data.decode()), {"state": {"x": 1}, "model": "jev-latest", "questions": {}}) + + def test_http_transport_does_not_retry_client_errors(self): + def opener(request, timeout): + raise urllib.error.HTTPError(request.full_url, 422, "Unprocessable", {}, io.BytesIO(b'{"detail":"bad question"}')) + + transport = drift.HttpTransport("k", opener=opener, sleep=lambda s: self.fail("should not sleep")) + with self.assertRaises(drift.DriftError) as raised: + transport({}) + self.assertIn("422", str(raised.exception)) + self.assertIn("bad question", str(raised.exception)) + + def test_http_transport_gives_up_after_the_last_attempt(self): + def opener(request, timeout): + raise urllib.error.HTTPError(request.full_url, 529, "Overloaded", {}, io.BytesIO(b"")) + + transport = drift.HttpTransport("k", attempts=3, opener=opener, sleep=lambda s: None) + with self.assertRaises(drift.DriftError): + transport({}) + + def test_invalid_json_body_is_a_clean_error(self): + class Broken(self.Response): + def read(self): + return b"not json" + + transport = drift.HttpTransport("k", opener=lambda request, timeout: Broken({}), sleep=lambda s: None) + with self.assertRaises(drift.DriftError) as raised: + transport({}) + self.assertIn("could not be read", str(raised.exception)) + + def test_missing_or_malformed_key_is_rejected_without_echoing_it(self): + with self.assertRaises(drift.DriftError): + drift.HttpTransport("") + secret = "ts-FAKE-KEY-with-newline\n" + with self.assertRaises(drift.DriftError) as raised: + drift.HttpTransport(secret) + self.assertNotIn("ts-FAKE-KEY", str(raised.exception)) + self.assertIn("whitespace", str(raised.exception)) + + def test_error_bodies_that_echo_the_key_are_redacted(self): + secret = "ts-FAKE-KEY-0123456789" + + def opener(request, timeout): + raise urllib.error.HTTPError(request.full_url, 401, "Unauthorized", {}, + io.BytesIO(("bad header Bearer %s" % secret).encode())) + + transport = drift.HttpTransport(secret, opener=opener, sleep=lambda s: None) + with self.assertRaises(drift.DriftError) as raised: + transport({}) + self.assertNotIn(secret, str(raised.exception)) + self.assertIn("[redacted]", str(raised.exception)) + + def test_redaction_happens_before_the_error_body_is_truncated(self): + secret = "ts-FAKE-KEY-0123456789" + body = ("x" * (301 - len(secret)) + secret).encode() + + def opener(request, timeout): + raise urllib.error.HTTPError(request.full_url, 401, "Unauthorized", {}, io.BytesIO(body)) + + transport = drift.HttpTransport(secret, opener=opener, sleep=lambda s: None) + with self.assertRaises(drift.DriftError) as raised: + transport({}) + self.assertNotIn(secret[:-1], str(raised.exception)) + self.assertIn("[redacted]", str(raised.exception)) + + def test_truncated_response_bodies_become_a_drift_error(self): + import http.client + + class Truncated(self.Response): + def read(self): + raise http.client.IncompleteRead(b"partial") + + transport = drift.HttpTransport("ts-FAKE-KEY-0123456789", opener=lambda request, timeout: Truncated({}), sleep=lambda s: None) + with self.assertRaises(drift.DriftError) as raised: + transport({}) + self.assertIn("IncompleteRead", str(raised.exception)) + + def test_default_opener_is_resolved_at_call_time_so_tests_can_block_it(self): + transport = drift.HttpTransport("ts-FAKE-KEY-0123456789", sleep=lambda s: None) + with patch.object(drift.urllib.request, "urlopen", side_effect=urllib.error.URLError("blocked")): + with self.assertRaises(drift.DriftError) as raised: + transport({}) + self.assertIn("blocked", str(raised.exception)) + + def test_fixture_transport_replays_by_pair_key_and_rejects_unknown_pairs(self): + transport = drift.FixtureTransport({"f|s": {"answers": {}}}) + self.assertEqual(transport({"_key": "f|s"}), {"answers": {}}) + with self.assertRaises(drift.DriftError): + transport({"_key": "f|other"}) + + +class RunPairsTests(unittest.TestCase): + def jobs(self, count): + jobs = [] + for number in range(count): + entry = {"id": "env:F%d" % number, "area": "env", "statement": "s", "evidence": {"file": "a", "line": 1, "excerpt": ""}, "terms": []} + section = {"id": "p#s%d" % number, "page": "p", "title": "T", "heading": "s", "level": 2, "line": 1, "text": "t"} + jobs.append((entry, section, 1.5)) + return jobs + + def test_concurrent_runs_keep_job_order_and_sum_usage(self): + lock = threading.Lock() + seen = [] + + def transport(payload): + with lock: + seen.append(payload["_key"]) + number = int(payload["_key"].split("|")[0][5:]) + return response(contradicts=0.9 if number % 2 else 0.1, tokens=number) + + outcomes, usage, errors = drift.run_pairs(self.jobs(6), transport, THRESHOLDS, concurrency=3) + self.assertEqual(errors, []) + self.assertEqual([outcome["key"] for outcome in outcomes], ["env:F%d|p#s%d" % (n, n) for n in range(6)]) + self.assertEqual(usage["input_tokens"], sum(range(6))) + self.assertEqual([outcome["decision"]["verdict"] for outcome in outcomes], ["OK", "DRIFT"] * 3) + self.assertEqual(sorted(seen), sorted("env:F%d|p#s%d" % (n, n) for n in range(6))) + self.assertEqual(outcomes[0]["lexical_score"], 1.5) + + def test_failures_are_collected_without_losing_completed_pairs(self): + def transport(payload): + if payload["_key"].startswith("env:F1|"): + raise drift.DriftError("HTTP 529 from TypeSafe: overloaded") + return response() + + outcomes, _usage, errors = drift.run_pairs(self.jobs(3), transport, THRESHOLDS, concurrency=2) + self.assertEqual(len(errors), 1) + self.assertIn("env:F1|p#s1", errors[0]) + self.assertEqual([outcome is None for outcome in outcomes], [False, True, False]) + sequential, _usage, errors = drift.run_pairs(self.jobs(3), transport, THRESHOLDS, concurrency=1) + self.assertEqual([outcome is None for outcome in sequential], [False, True, False]) + self.assertEqual(len(errors), 1) + + def test_sequential_runs_stop_after_consecutive_failures_and_hide_unexpected_error_text(self): + def transport(payload): + raise RuntimeError("Bearer ts-SECRET-IN-MESSAGE") + + outcomes, _usage, errors = drift.run_pairs(self.jobs(5), transport, THRESHOLDS, concurrency=1) + self.assertEqual(outcomes, [None] * 5) + self.assertEqual(len(errors), drift.MAX_CONSECUTIVE_FAILURES + 1) + self.assertIn("stopped after", errors[-1]) + self.assertNotIn("SECRET", " ".join(errors)) + self.assertIn("unexpected RuntimeError", errors[0]) + concurrent_outcomes, _usage, errors = drift.run_pairs(self.jobs(4), transport, THRESHOLDS, concurrency=2) + self.assertEqual(concurrent_outcomes, [None] * 4) + self.assertEqual(len(errors), 4) + + def test_response_without_answers_is_an_error(self): + _outcomes, _usage, errors = drift.run_pairs(self.jobs(1), lambda payload: {"model": "x"}, THRESHOLDS) + self.assertEqual(len(errors), 1) + self.assertIn("no 'answers' map", errors[0]) + + +class PipelineTests(TempDirs): + def setUp(self): + super().setUp() + self.output = self.base / "out" + self.patcher = patch.object(drift.urllib.request, "urlopen", side_effect=AssertionError("live TypeSafe call attempted")) + self.patcher.start() + self.addCleanup(self.patcher.stop) + + def run_main(self, argv, environ=None): + stdout, stderr = io.StringIO(), io.StringIO() + code = drift.main(argv, environ={} if environ is None else environ, stdout=stdout, stderr=stderr) + return code, stdout.getvalue(), stderr.getvalue() + + def test_facts_only_needs_neither_docs_nor_key(self): + code, out, err = self.run_main(["--code", str(self.code()), "--facts-only"]) + self.assertEqual(code, 0, err) + self.assertIn("ncl:widgets-verbs", out) + self.assertIn("facts from", out) + self.assertFalse(self.output.exists()) + code, out, _ = self.run_main(["--code", str(self.code()), "--facts-only", "--json", "--areas", "skills"]) + self.assertEqual(code, 0) + payload = json.loads(out) + self.assertEqual({entry["area"] for entry in payload["facts"]}, {"skills"}) + + def test_plan_prints_pairs_without_a_key(self): + code, out, err = self.run_main(["--code", str(self.code()), "--docs", str(self.docs()), "--plan", "--areas", "container-config", "--top-k", "2"]) + self.assertEqual(code, 0, err) + self.assertIn("container-config:cli-scope-values", out) + self.assertIn("-> operate/ncl-cli#agents-can-run-ncl-too", out) + self.assertIn("pairs", out) + self.assertNotIn("changelog", out) + + def test_missing_key_exits_with_a_clear_message_and_no_request(self): + code, out, err = self.run_main(["--code", str(self.code()), "--docs", str(self.docs()), "--output-dir", str(self.output)], environ={}) + self.assertEqual(code, 2) + self.assertIn("TYPESAFE_API_KEY is not set", err) + self.assertIn("--fixture", err) + self.assertEqual(out, "") + self.assertFalse(self.output.exists()) + + def test_docs_argument_is_required_for_a_live_run(self): + with self.assertRaises(SystemExit): + with patch("sys.stderr", io.StringIO()): + drift.parse_args(["--code", "."]) + + def test_fixture_replay_runs_the_whole_pipeline_offline(self): + code, out, err = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output)]) + self.assertEqual(code, 0, err) + self.assertIn("fixture mode", out) + self.assertIn("Nothing was written", out) + for token in ("DRIFT", "MISSING", "UNSURE", "OK", "slightly outdated", "reference/ncl-cli#groups", "tokens: input", "wall time"): + self.assertIn(token, out) + saved = sorted(self.output.glob("drift-*.json")) + self.assertEqual(len(saved), 1) + payload = json.loads(saved[0].read_text()) + self.assertEqual(payload["mode"], "fixture") + self.assertEqual(payload["summary"]["verdicts"], {"DRIFT": 1, "MISSING": 2, "UNSURE": 1, "OK": 2}) + self.assertEqual(payload["summary"]["requests"], 6) + self.assertEqual(payload["usage"]["input_tokens"], 11230) + self.assertEqual([result["verdict"] for result in payload["results"]][:1], ["DRIFT"]) + first = payload["results"][0] + self.assertEqual(first["id"], "ncl:groups-config-update-flags") + self.assertEqual(first["best"]["section"]["page"], "reference/ncl-cli") + self.assertEqual(first["best"]["decision"]["staleness_label"], "slightly outdated") + self.assertIn("pairs", first) + self.assertIn("lexical_score", first["pairs"][0]) + no_candidates = next(result for result in payload["results"] if result["id"] == "timestamps:rule-1") + self.assertEqual((no_candidates["verdict"], no_candidates["pairs"]), ("MISSING", [])) + + def test_fixture_pairs_get_the_documented_questions(self): + fixture = json.loads(FIXTURE.read_text()) + self.assertEqual(len(fixture["facts"]), 6) + self.assertEqual(len(fixture["responses"]), 6) + jobs = drift.plan_pairs(fixture["facts"], fixture["sections"], fixture["candidates"], fixture["scores"]) + transport = drift.FixtureTransport(fixture["responses"]) + drift.run_pairs(jobs, transport, THRESHOLDS) + self.assertEqual(len(transport.requests), 6) + for request in transport.requests: + self.assertEqual(request["model"], "jev-latest") + self.assertEqual(set(request["questions"]), {"contradicts", "covers", "staleness"}) + self.assertEqual(request["_key"].split("|")[1], request["state"]["doc_section"]["page"] + "#" + request["_key"].split("#")[1]) + for answer_map in fixture["responses"].values(): + for name, answer in answer_map["answers"].items(): + if answer["type"] == "score": + self.assertAlmostEqual(sum(answer["probabilities"].values()), 1.0, places=3, msg=name) + self.assertTrue(0 <= answer["confidence"] <= 1) + else: + self.assertTrue(0 <= answer["noul"] <= 1) + + def test_flags_override_the_gate_and_areas_filter_the_fixture(self): + code, out, _ = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output), "--json", "--contradicts-threshold", "0.8"]) + self.assertEqual(code, 0) + payload = json.loads(out) + self.assertEqual(payload["summary"]["verdicts"]["DRIFT"], 0) + self.assertEqual(payload["summary"]["verdicts"]["UNSURE"], 2) + code, out, _ = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output), "--json", "--areas", "gateway"]) + self.assertEqual(code, 0) + payload = json.loads(out) + self.assertEqual(payload["summary"]["facts"], 2) + self.assertEqual(set(payload["summary"]["areas"]), {"gateway"}) + + def test_fixture_mode_honours_limit_plan_and_facts_only(self): + code, out, err = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output), "--json", "--limit-facts", "1"]) + self.assertEqual(code, 0, err) + self.assertEqual(json.loads(out)["summary"]["facts"], 1) + self.assertEqual(len(list(self.output.glob("drift-*.json"))), 1) + code, out, _ = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output), "--plan"]) + self.assertEqual(code, 0) + self.assertIn("-> reference/ncl-cli#groups", out) + code, out, _ = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output), "--facts-only"]) + self.assertEqual(code, 0) + self.assertIn("ncl:groups-config-update-flags", out) + self.assertEqual(len(list(self.output.glob("drift-*.json"))), 1) + + def test_invalid_flags_are_rejected(self): + for argv in (["--contradicts-threshold", "0.4"], ["--covers-threshold", "1.5"], ["--top-k", "0"], ["--areas", "bogus"], ["--concurrency", "0"]): + with self.assertRaises(SystemExit, msg=argv): + with patch("sys.stderr", io.StringIO()): + drift.parse_args(["--fixture", "x"] + argv) + + def test_key_never_appears_in_output_or_saved_json(self): + sentinel = "ts-live-FAKE-KEY-0123456789" + code, out, err = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output)], environ={"TYPESAFE_API_KEY": sentinel}) + self.assertEqual(code, 0) + saved = next(self.output.glob("drift-*.json")).read_text() + for text in (out, err, saved): + self.assertNotIn(sentinel, text) + + def test_partial_failure_keeps_completed_pairs_and_exits_nonzero(self): + fixture = json.loads(FIXTURE.read_text()) + original = drift.FixtureTransport(fixture["responses"]) + + def flaky(payload): + if payload["_key"].startswith("env:CONTAINER_PIDS_LIMIT|"): + raise drift.DriftError("HTTP 529 from TypeSafe: overloaded") + return original(payload) + + with patch.object(drift, "FixtureTransport", return_value=flaky): + code, out, err = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output), "--json"]) + self.assertEqual(code, 1) + self.assertIn("keeping 5 completed pair(s)", err) + payload = json.loads(out) + self.assertIn("env:CONTAINER_PIDS_LIMIT|", payload["partial"]) + self.assertEqual(payload["summary"]["requests"], 5) + self.assertEqual(payload["summary"]["verdicts"]["DRIFT"], 1) + pids = next(result for result in payload["results"] if result["id"] == "env:CONTAINER_PIDS_LIMIT") + self.assertEqual((pids["verdict"], pids["pairs"], pids["failed_pairs"]), ("UNSURE", [], 1)) + self.assertIn("request(s) failed", pids["note"]) + self.assertEqual(len(list(self.output.glob("drift-*.json"))), 1) + + def test_output_files_never_overwrite_each_other(self): + with patch.object(drift, "stamp", return_value="fixed"): + first = drift.write_output(self.output, {"a": 1}) + second = drift.write_output(self.output, {"a": 2}) + self.assertNotEqual(first, second) + self.assertEqual(json.loads(first.read_text()), {"a": 1}) + self.assertEqual(json.loads(second.read_text()), {"a": 2}) + + def test_bad_fixture_is_a_clean_error(self): + broken = self.base / "broken.json" + broken.write_text("[]") + code, out, err = self.run_main(["--fixture", str(broken), "--output-dir", str(self.output)]) + self.assertEqual(code, 1) + self.assertIn("error:", err) + self.assertIn("'facts'", err) + + def test_live_mode_with_mocked_transport_records_a_replayable_fixture(self): + def fake(payload): + if payload["_key"].startswith("env:LIMIT|"): + raise drift.DriftError("HTTP 529 from TypeSafe: overloaded") + return response(contradicts=0.9 if "cli-scope" in payload["_key"] else 0.1) + record = self.base / "record.json" + with patch.object(drift, "HttpTransport", return_value=fake): + code, out, err = self.run_main( + ["--code", str(self.code()), "--docs", str(self.docs()), "--output-dir", str(self.output), + "--areas", "container-config,env", "--top-k", "1", "--record", str(record), "--json"], + environ={"TYPESAFE_API_KEY": "ts-FAKE-KEY-0123456789"}) + self.assertEqual(code, 1, err) + payload = json.loads(out) + self.assertEqual(payload["mode"], "live") + self.assertIn("[1/", err) + self.assertIn("env:LIMIT|", payload["partial"]) + limit = next(result for result in payload["results"] if result["id"] == "env:LIMIT") + self.assertEqual((limit["verdict"], limit["failed_pairs"]), ("UNSURE", 1)) + self.assertGreaterEqual(payload["summary"]["verdicts"]["DRIFT"], 1) + self.assertIn(".env.example", " ".join(payload["notes"])) if not payload["notes"] else None + self.assertIn("gateway-seam.md is absent", " ".join(payload["notes"])) + recorded = json.loads(record.read_text()) + self.assertEqual(set(recorded), {"schema_version", "recorded_at", "code_commit", "docs_commit", "facts", "sections", "candidates", "scores", "responses"}) + self.assertEqual(len(recorded["responses"]), payload["summary"]["requests"]) + self.assertEqual(recorded["candidates"]["env:LIMIT"], []) + self.assertEqual(sum(len(ids) for ids in recorded["candidates"].values()), len(recorded["responses"])) + self.assertEqual(set(recorded["scores"]), set(recorded["responses"])) + code, replay_out, err = self.run_main(["--fixture", str(record), "--output-dir", str(self.output), "--json"]) + self.assertEqual(code, 0, err) + replayed = json.loads(replay_out)["summary"]["verdicts"] + self.assertEqual(replayed["DRIFT"], payload["summary"]["verdicts"]["DRIFT"]) + self.assertEqual(replayed["OK"], payload["summary"]["verdicts"]["OK"]) + + +if __name__ == "__main__": + unittest.main() From 876a3e9ea0bb63d0ee68b0dbb7553694a4b8ec95 Mon Sep 17 00:00:00 2001 From: glifocat Date: Thu, 17 Sep 2026 13:55:00 +0200 Subject: [PATCH 2/4] feat(typesafe-docs-drift): benchmark TypeSafe against a prompt-and-parse LLM Add drift-benchmark.py: every contender judges the same 60 fact x section pairs sampled (stratified, seeded) from the recorded live run. Contenders are the recorded TypeSafe answers, TypeSafe live (sequential), and headless `claude -p` with tools disabled, 10 pairs per call, strict JSON out; a read-only `codex exec` fallback is implemented. Reports wall time, per-pair p50/p95 with the measured CLI startup overhead separated, tokens, cost, parse failures and format violations per run, run-to-run flips with their denominator, pairwise agreement, and precision/recall on `contradicts` against a 30-pair gold file labeled by an LLM-assisted reviewer. Only the recorded contender runs by default; network and CLI use is opt-in through --contenders. The TypeSafe key is checked before any paid contender starts, never printed, and removed from every spawned CLI's environment; CLI diagnostics are redacted; a timeout or interrupt kills the CLI's whole process group. Results of the first run are recorded in docs/typesafe-docs-drift.md. --- CHANGELOG.md | 6 + docs/typesafe-docs-drift.md | 82 + skills/typesafe-docs-drift/SKILL.md | 10 + .../fixtures/benchmark-gold.json | 188 + .../fixtures/benchmark-workload.json | 3268 +++++++++++++++++ .../scripts/drift-benchmark.py | 705 ++++ tests/test_drift_benchmark.py | 448 +++ 7 files changed, 4707 insertions(+) create mode 100644 skills/typesafe-docs-drift/fixtures/benchmark-gold.json create mode 100644 skills/typesafe-docs-drift/fixtures/benchmark-workload.json create mode 100644 skills/typesafe-docs-drift/scripts/drift-benchmark.py create mode 100644 tests/test_drift_benchmark.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cf240d9..1370139 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,12 @@ milestone and is not yet released. time, raw answers under a gitignored directory, `--facts-only` and `--plan` for the key-free half, and `--fixture` / `--record` replay. Reads `TYPESAFE_API_KEY` from the environment only and writes to neither repository. +- `drift-benchmark.py` for `typesafe-docs-drift`: runs TypeSafe (recorded and + live) and a prompt-and-parse LLM through headless `claude -p` (tools disabled, + 10 pairs per call) on the same 60 recorded fact x section pairs and reports + wall time, per-pair latency with startup overhead separated, tokens, cost, + parse failures, run-to-run flips, agreement, and precision/recall on + `contradicts` against a 30-pair gold file labeled by an LLM-assisted reviewer. - OpenCode public-wizard E2E for OpenRouter, DeepSeek, and custom/self-hosted API-key endpoints across exe.dev, Proxmox and Windows WSL2. Select the model with `--opencode-model`, the endpoint with `--opencode-base-url`, and its API diff --git a/docs/typesafe-docs-drift.md b/docs/typesafe-docs-drift.md index 0ee29ad..bcaa478 100644 --- a/docs/typesafe-docs-drift.md +++ b/docs/typesafe-docs-drift.md @@ -158,6 +158,88 @@ resume` flags (0.71, staleness confidence below the gate). The raw payload is in the gitignored output directory of the machine that ran it; the numbers above are the only part recorded here. +## Benchmark: TypeSafe versus a prompt-and-parse LLM + +`skills/typesafe-docs-drift/scripts/drift-benchmark.py` runs every contender +on the same work and reports speed, cost, stability and accuracy. + +**Workload.** 60 fact x section pairs sampled from the recorded live run above +(`fixtures/benchmark-workload.json`, seed 20260916): the 11 `DRIFT` pairs of +the 7 `DRIFT` facts, the best pair of the 1 `MISSING` and the 11 `UNSURE` +facts, and 37 seeded-random `OK` pairs. Every contender gets the same fact +statement, evidence excerpt and doc section text, and the same three +questions with the same criteria text. + +**Contenders.** + +- `typesafe-recorded`: the recorded answers; wall time is the sum of the + recorded per-request latencies (the recorded run used 4 workers). +- `typesafe-live`: the same requests sent again one at a time. +- `claude:haiku`, `claude:sonnet`: `claude -p --model --output-format json + --tools "" --strict-mcp-config --no-session-persistence --system-prompt ...`, + 10 pairs per call, strict JSON array out (`contradicts` and `covers` as + booleans, `staleness` 0 to 3). CLI defaults otherwise, so the model's default + thinking is on. `codex` (`codex exec --ephemeral -c 'sandbox_mode="read-only"'`) + is implemented as a fallback and was not needed. + +**Metrics.** Wall time for the first run; p50 seconds per pair (for the CLI +contenders the call time divided by the batch size); `p50 adj` subtracts the +measured startup overhead (one trivial `Reply with the single word OK.` call, +which also contains one minimal API round trip, so the subtraction favors the +LLM); tokens and cost from the TypeSafe `usage` block and the CLI's `usage` and +`total_cost_usd`; parse failures (invalid JSON, a missing or mistyped field, or +a failed call, counted per pair); flip rate (share of pairs whose `contradicts` +verdict differs between two runs); pairwise agreement; precision and recall on +`contradicts` against the gold file. TypeSafe probabilities are binarized at +the skill's 0.7 gate. An unanswered gold pair counts as a predicted no. + +**Gold set.** `fixtures/benchmark-gold.json` holds 30 labels with a one-line +justification each: all `DRIFT` and `MISSING` pairs plus 18 seeded-random +others, 12 positive. **The labels were written by Claude, the LLM-assisted +reviewer that built this tool**, by reading the NanoClaw source and the doc +section without looking at any contender's answer. No human has verified them; +three are marked `borderline`. Two biases to keep in mind: the sample was +stratified by TypeSafe's own verdicts, so most positives are pairs TypeSafe +already flagged, which favors its recall; and the labeler is a Claude model, +which may favor the Claude contenders' reading of borderline cases. + +**Results.** One run on 2026-09-17 10:12 to 10:34 UTC from one macOS machine, +60 pairs, two passes per live contender, `claude` CLI 2.1.273, `jev-latest`: + +| Contender | Pairs | Wall time | p50 per pair | p50 per pair, startup subtracted | Tokens in / out | Cost | Parse failures (run 1 / run 2) | Flip rate on `contradicts` | Precision | Recall | +|---|---|---|---|---|---|---|---|---|---|---| +| typesafe-recorded | 60 | 40.1 s (sum of recorded latencies) | 0.66 s | 0.66 s | 95,720 / 3,180 | not reported | 0 | n/a | 1.00 | 0.92 | +| typesafe-live | 60 | 41.3 s | 0.68 s | 0.68 s | 94,880 / 3,180 | not reported | 0 / 0 | 3.3% (2/60) | 1.00 | 0.92 | +| claude:haiku | 60 | 359.6 s | 5.57 s | 5.09 s | 81,994 / 32,776 | $0.33 | 0 / 0 | 6.7% (4/60) | 0.85 | 0.92 | +| claude:sonnet | 60 | 284.8 s | 3.99 s | 3.66 s | 110,521 / 25,665 | $0.70 | 0 / 0 | 11.7% (7/60) | 0.83 | 0.83 | + +- Measured startup overhead per CLI call: haiku 4.8 s, sonnet 3.3 s. Batch times + ranged from 14.5 s to 114.2 s for 10 pairs. +- Every pair parsed for every contender. Haiku wrapped its array in a code + fence or prose in all 6 calls of both runs and sonnet in 1 of 6; the parser + recovered them, as a real pipeline would. +- Agreement on `contradicts` (run 1): TypeSafe recorded vs live 98.3%, TypeSafe + live vs sonnet 90.0%, vs haiku 85.0%, haiku vs sonnet 88.3%. On `covers`: + 100%, 96.7%, 93.3%, 96.7%. +- The CLI's token counts include its cached system prompt tokens; TypeSafe + does not report a price, so its cost column is empty rather than zero. +- Read precision and recall with the two gold-set biases above in mind. With 12 + positives, one pair moves recall by 8 points. + +Reproduce (the LLM contenders spend tokens on the operator's Claude plan, and +`typesafe-live` needs `TYPESAFE_API_KEY`): + +```bash +python3 skills/typesafe-docs-drift/scripts/drift-benchmark.py # offline: recorded answers vs gold +python3 skills/typesafe-docs-drift/scripts/drift-benchmark.py \ + --contenders typesafe-recorded,typesafe-live,claude:haiku,claude:sonnet --runs 2 +``` + +`--record --results --write-workload ` rebuilds the +sample from another recorded run; `--limit` and `--seed` change it; +`--from-results` re-renders a saved `benchmark-*.json`. Tests: +`tests/test_drift_benchmark.py` (mocked subprocess and transport). + ## Tests `tests/test_typesafe_docs_drift.py` runs in the offline suite: every extractor diff --git a/skills/typesafe-docs-drift/SKILL.md b/skills/typesafe-docs-drift/SKILL.md index fa10391..ee77d1e 100644 --- a/skills/typesafe-docs-drift/SKILL.md +++ b/skills/typesafe-docs-drift/SKILL.md @@ -85,6 +85,16 @@ gateway,timestamps`), `--limit-facts`, `--top-k` (3), `--include-changelog`, `--staleness-threshold` (0.5, score confidence), `--concurrency` (4), `--model`, `--timeout`, `--json`. +## Benchmark + +[drift-benchmark.py](scripts/drift-benchmark.py) compares TypeSafe with a +prompt-and-parse LLM (`claude -p`, tools disabled) on the same recorded pairs: +speed, tokens, cost, parse failures, stability, agreement, and precision/recall +against [benchmark-gold.json](fixtures/benchmark-gold.json), whose labels were +written by an LLM-assisted reviewer. With no flags it is offline and scores the +recorded answers only. Method and results: +[docs/typesafe-docs-drift.md](../../docs/typesafe-docs-drift.md#benchmark-typesafe-versus-a-prompt-and-parse-llm). + ## Read the output One row per fact, ranked `DRIFT` (a section contradicts the fact, highest diff --git a/skills/typesafe-docs-drift/fixtures/benchmark-gold.json b/skills/typesafe-docs-drift/fixtures/benchmark-gold.json new file mode 100644 index 0000000..0eca9b0 --- /dev/null +++ b/skills/typesafe-docs-drift/fixtures/benchmark-gold.json @@ -0,0 +1,188 @@ +{ + "schema_version": 1, + "labeled_at": "2026-09-17", + "labeled_by": "Claude (LLM-assisted reviewer), by reading the NanoClaw source at 6e5008fe and the nanoclaw-docs section text at e3e1546, without looking at any contender's answers for the pair. Not independently verified by a human; spot-check before relying on it.", + "rule": "contradicts is true when the section states a name, value, default, option list or behavior that cannot hold if the fact holds; a usage synopsis that omits a flag the code declares counts, and is marked borderline. Silence is not a contradiction.", + "labels": { + "ncl:approvals-status-enum|reference/ncl-cli#approvals": { + "contradicts": true, + "covers": true, + "borderline": false, + "justification": "Docs list a fifth status `awaiting_reason`; the CLI enum in approvals.ts:47 has four values." + }, + "ncl:approvals-status-enum|reference/db-schema#pending-approvals": { + "contradicts": true, + "covers": true, + "borderline": false, + "justification": "db-schema status column also lists `awaiting_reason`, which the `ncl approvals` enum does not accept as a filter value." + }, + "ncl:dropped-messages-reason-enum|operate/ncl-cli#dropped-messages": { + "contradicts": true, + "covers": true, + "borderline": false, + "justification": "Docs list `unknown_sender_decline_notify` as a reason; dropped-messages.ts:20 enum has only four values." + }, + "ncl:dropped-messages-reason-enum|reference/ncl-cli#dropped-messages": { + "contradicts": true, + "covers": true, + "borderline": false, + "justification": "Same extra `unknown_sender_decline_notify` value in the reference columns list." + }, + "ncl:dropped-messages-reason-enum|operate/troubleshooting#agent-never-replies": { + "contradicts": true, + "covers": true, + "borderline": false, + "justification": "Troubleshooting reason table has an `unknown_sender_decline_notify` row the CLI enum lacks." + }, + "ncl:tasks-update-flags|reference/ncl-cli#tasks": { + "contradicts": true, + "covers": true, + "borderline": false, + "justification": "Docs synopsis for update omits `--dangerously-override-recurrence-limit`, `--group`, `--session`, which tasks.ts declares." + }, + "ncl:tasks-cancel-flags|reference/ncl-cli#tasks": { + "contradicts": true, + "covers": true, + "borderline": false, + "justification": "Docs synopsis `cancel (--id | --all)` omits the declared `--group` and `--session`." + }, + "ncl:tasks-resume-flags|reference/ncl-cli#tasks": { + "contradicts": true, + "covers": true, + "borderline": true, + "justification": "Docs synopsis `resume --id` omits declared `--group`/`--session`; an omission in a synopsis, so borderline." + }, + "env:WEBHOOK_PORT|operate/troubleshooting#webhook-channel-is-silent": { + "contradicts": true, + "covers": true, + "borderline": false, + "justification": "Docs say WEBHOOK_PORT is process-environment only, not `.env`; config.ts getWebhookPort reads process.env then .env." + }, + "env:WEBHOOK_PORT|reference/environment-variables#host-process-environment-only": { + "contradicts": true, + "covers": true, + "borderline": false, + "justification": "Reference page files WEBHOOK_PORT under 'no .env fallback'; the code has a .env fallback." + }, + "skills:add-ollama-provider|extend/providers#ollama-add-ollama-provider": { + "contradicts": true, + "covers": true, + "borderline": true, + "justification": "Docs say the skill patches readContainerConfig and Docker args; the skill description says only env overrides and a model setting. The docs are the accurate side, so borderline as drift." + }, + "timestamps:rule-1|guides/scheduled-tasks#schedule-recurring-work": { + "contradicts": false, + "covers": false, + "borderline": false, + "justification": "Scheduled-tasks guide says nothing about timestamp storage rules; unrelated." + }, + "ncl:tasks-delete-flags|reference/ncl-cli#tasks": { + "contradicts": true, + "covers": true, + "borderline": true, + "justification": "Docs synopsis `delete --id` omits declared `--group`/`--session`; omission in a synopsis, borderline." + }, + "ncl:wirings-create-flags|reference/ncl-cli#wirings": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Every flag named in the help text appears in the wirings reference section." + }, + "skills:debug|reference/skills-catalog#operational-skills": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Catalog row for `/debug` matches the skill description." + }, + "ncl:tasks-create-flags|reference/ncl-cli#tasks": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Docs create synopsis lists the same seven flags the code declares." + }, + "timestamps:rule-2|reference/environment-variables#container-side-variables-agent-runner": { + "contradicts": false, + "covers": false, + "borderline": false, + "justification": "Section only has the container `TZ` row; it does not address display formatting rules." + }, + "skills:update-nanoclaw|operate/upgrading#updating-nanoclaw": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Upgrading page describes `/update-nanoclaw` consistently with the skill description." + }, + "skills:add-mattermost|channels/mattermost#install": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Mattermost install section names `/add-mattermost` and agrees with the description." + }, + "skills:add-wechat|channels/more-channels#wechat": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "WeChat section names `/add-wechat`; nothing conflicts." + }, + "container-config:cli_scope-default|reference/container-config#cli-scope": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Docs say `group` (default), matching migration 015." + }, + "env:NANOCLAW_DEFAULT_MODEL|reference/environment-variables#read-from-env": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Row gives 'provider default' and the intro states process env precedence over .env; consistent with an empty default." + }, + "gateway:selection-unknown|reference/environment-variables#read-from-env": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Row says unknown names fail instead of disabling the gateway; consistent." + }, + "skills:add-github|reference/skills-catalog#channel-installs": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Catalog row for `/add-github` matches the description." + }, + "skills:slack-a2a-rooms|channels/slack#rooms-and-membership": { + "contradicts": false, + "covers": false, + "borderline": false, + "justification": "Section describes A2A room admission via SLACK_A2A_ROOMS consistently but never names the `/slack-a2a-rooms` skill." + }, + "ncl:sessions-history-flags|reference/ncl-cli#sessions": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Docs `history [--limit ]`; positional id equals `--id`, so the flag set matches." + }, + "ncl:destinations-add-flags|reference/ncl-cli#destinations": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Docs add synopsis has the same four flags." + }, + "ncl:roles-role-enum|reference/ncl-cli#roles": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Docs list `owner` | `admin`, same as the enum." + }, + "env:NANOCLAW_RUNTIME_DRIVER|reference/environment-variables#read-from-env": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Row: default docker, process env then .env, unknown names abort startup; consistent." + }, + "env:CONTAINER_MEMORY_LIMIT|reference/environment-variables#read-from-env": { + "contradicts": false, + "covers": true, + "borderline": false, + "justification": "Row: unset (unbounded), .env or process environment; consistent." + } + } +} diff --git a/skills/typesafe-docs-drift/fixtures/benchmark-workload.json b/skills/typesafe-docs-drift/fixtures/benchmark-workload.json new file mode 100644 index 0000000..cb7bb30 --- /dev/null +++ b/skills/typesafe-docs-drift/fixtures/benchmark-workload.json @@ -0,0 +1,3268 @@ +{ + "schema_version": 1, + "source": { + "recorded_at": "2026-09-16T22:29:05.246419+00:00", + "code_commit": "6e5008fe", + "docs_commit": "e3e1546" + }, + "seed": 20260916, + "limit": 60, + "strata": { + "DRIFT": 11, + "MISSING": 1, + "UNSURE": 11, + "OK": 37 + }, + "sections": [ + { + "id": "channels/mattermost#install", + "page": "channels/mattermost", + "title": "Mattermost", + "heading": "Install", + "line": 20, + "text": "Mattermost is offered in the first-run setup wizard, or add it later by running `/add-mattermost` in Claude Code. The flow:\n\n\n\n The skill first looks for a Mattermost you already have: a configured `MATTERMOST_BASE_URL`, a healthy server on `localhost:8065` / `127.0.0.1:8065` (probed with `/api/v4/system/ping`), or a Mattermost Docker container it can offer to start. A found server is only a suggestion: you pick it, enter a different URL, or ask for a **local evaluation server** (Mattermost Team Edition plus PostgreSQL in containers, bound to `127.0.0.1:8065`, with config under `.nanoclaw/mattermost/`). Nothing is installed without showing you what will be created and getting approval.\n\n\n Mattermost Desktop sends its configured server URL as the WebSocket `Origin`, so `ServiceSettings.SiteURL` must match the base URL you chose, and `ServiceSettings.WebsocketURL` must stay blank. The skill sets this via `mmctl` (host-local or inside the container) when it can, or gives you the exact commands; the evaluation server's Compose template sets it for you. It then verifies both values through the public `/api/v4/config/client?format=old` endpoint.\n\n\n The skill copies the vendored adapter and its tests from the `channels` branch, appends `import './mattermost.js';` to `src/channels/index.ts`, and pins the WebSocket dependencies. It also removes the unscoped `chat-adapter-mattermost` npm package if present: nothing in NanoClaw imports it, and it is typosquat-shaped against the scoped `@chat-adapter` family.\n\n\n Two different places in Mattermost: first, as a System Admin, turn on **System Console → Integrations → Bot Accounts → Enable Bot Account Creation** (this only permits creation). Then, back in the workspace, create the bot under **Product menu → Integrations → Bot Accounts → Add Bot Account** and copy its access token. Add the bot to each team and channel it should hear. Mattermost never adds bots to teams or channels automatically. The skill verifies the token against `/api/v4/users/me`.\n\n\n Approval cards require the Mattermost **server** to reach NanoClaw. You provide a callback URL routable from the server (NanoClaw's base URL or the full `/webhook/mattermost` route; the adapter normalizes either). Mattermost does not sign action callbacks, so the skill generates a random shared secret; a callback without it is refused with `401` before anything is dispatched. For a private callback host, add it under **System Console → Environment → Developer → Allow untrusted internal connections**.\n\n\n The skill asks for your Mattermost username (without `@`), resolves it against `/api/v4/users/username/`, and opens the DM between you and the bot, capturing the `mattermost:` platform ID that `/init-first-agent` wires as your first chat.\n\n\n `pnpm run build`, the registration and adapter regression tests, then a service restart so the channel and credentials load. For a first channel, continue with `/init-first-agent`; otherwise `/manage-channels`.\n\n\n\nThe credentials written to `.env`: `MATTERMOST_BASE_URL`, `MATTERMOST_BOT_TOKEN`, `MATTERMOST_CALLBACK_URL`, `MATTERMOST_CALLBACK_SECRET`. The adapter factory returns nothing without the first two, and `MATTERMOST_BASE_URL` is refreshed on every re-run so you can switch servers without touching credentials." + }, + { + "id": "channels/more-channels#wechat", + "page": "channels/more-channels", + "title": "More channels", + "heading": "WeChat", + "line": 85, + "text": "Personal WeChat via Tencent's official iLink Bot API — first-party, free, and works with any personal account, unlike puppet-based libraries (wechaty/PadLocal) that risk bans. Login is a QR scan (the URL is written to `data/wechat/qr.txt` and logged); messages arrive by long-polling with a persisted cursor, so nothing is dropped across restarts and no public URL is needed.\n\n- **Transport:** native (Tencent iLink Bot API, long-poll)\n- **Credentials:** `WECHAT_ENABLED=true`, then scan the QR code; auth is saved under `data/wechat/`\n- **Threads:** no\n- **Install:** `/add-wechat`" + }, + { + "id": "channels/slack#rooms-and-membership", + "page": "channels/slack", + "title": "Slack", + "heading": "Rooms and membership", + "line": 208, + "text": "The agents feature adds room membership handling. When a Slack user invites an agent bot into a room, NanoClaw can adopt the room and create the required messaging group. The current Chat SDK does not dispatch bot-departure events, so removing a bot does not reliably update its NanoClaw wiring automatically. Shared agent rooms admit bot-authored messages only when the room is on the `SLACK_A2A_ROOMS` allowlist; bot messages remain blocked everywhere else." + }, + { + "id": "channels/slack#set-up-the-first-agent", + "page": "channels/slack", + "title": "Slack", + "heading": "Set up the first agent", + "line": 73, + "text": "Run the installer:\n\n ```bash\n bash nanoclaw.sh\n ```\n\n\n Select Slack when the setup wizard asks for a channel. When it asks how to create the app, select **Create it for me**.\n\n\n Sign in with your NanoClaw account if prompted. Approve the Slack workspace connection in your browser, then return to the terminal.\n\n If more than one workspace is connected, select the workspace where the agent should live.\n\n\n NanoClaw creates a Socket Mode app, installs it, and stores the derived bot and app tokens in `.env`. The host never receives the workspace-level manager credential when provisioning runs through the managed service.\n\n If the workspace requires admin approval, managed setup opens the app's install link and waits up to five minutes for approval. When the managed service returns the bot token, setup continues without token pasting.\n\n If approval takes longer, the provisioning module is older, or you use a manager token directly, setup falls back to **Finish installing in Slack**. After approval, install the app, open **OAuth & Permissions**, and paste the `xoxb-…` Bot User OAuth Token at the next prompt. The app-level token is already stored.\n\n\n Setup resolves your Slack member ID, wires the direct message to the first agent group, restarts the service, and sends the welcome message. Reply in Slack to verify the connection.\n\n\n\n\n\n Use this path when you do not want managed provisioning or your workspace requires a self-managed app.\n\n\n\n Go to [api.slack.com/apps](https://api.slack.com/apps), select **Create New App**, then select **From scratch**.\n\n\n Under **OAuth & Permissions**, add these Bot Token Scopes:\n\n ```text\n chat:write\n im:write\n channels:history\n groups:history\n im:history\n channels:read\n groups:read\n mpim:read\n users:read\n reactions:write\n files:read\n files:write\n ```\n\n\n Under **App Home**, enable the Messages Tab. Then enable **Allow users to send Slash commands and messages from the messages tab**.\n\n\n For Socket Mode, create an App-Level Token with `connections:write`, enable Socket Mode, and subscribe to `message.channels`, `message.groups`, `message.im`, and `app_mention`. Socket Mode needs no public URL.\n\n For webhook delivery, keep the Signing Secret and point Event Subscriptions and Interactivity to `https:///webhook/slack`. Your public HTTPS endpoint must reach NanoClaw's webhook port, which defaults to `3000`.\n\n\n Install the app to the workspace. Run `/add-slack` from Claude Code, `$add-slack` from Codex, or load `add-slack` with your harness's skill tool. Choose the same delivery mode and paste the credentials when prompted.\n\n\n\n\n\n\n The manual checklist matches the base `/add-slack` app. A managed app is created with the full scope and event set for group-DM agent rooms, canvases, onboarding, and membership updates. To run the agents feature on a manually created app, add the extra scopes and events listed under \"What you get\" above." + }, + { + "id": "extend/providers#ollama-add-ollama-provider", + "page": "extend/providers", + "title": "Agent providers", + "heading": "Ollama (`/add-ollama-provider`)", + "line": 67, + "text": "The skill remains in the repository, but its instructions target an older configuration path: they patch `readContainerConfig` and Docker argument assembly, then edit a generated group `container.json`. Current NanoClaw materializes that file from database-backed configuration and realizes a validated session spec through a driver. Those legacy edits are not a supported drop-in switch on current main.\n\nBefore using this skill, its integration needs to be ported and tested against the current config and admission paths. Do not treat direct edits to generated `container.json` as persistent configuration. Local-model tools such as [Atomic Chat and Ollama](/extend/tools) are a separate option: they give an existing agent a model to call rather than replacing its provider." + }, + { + "id": "guides/scheduled-tasks#schedule-recurring-work", + "page": "guides/scheduled-tasks", + "title": "Schedule recurring work", + "heading": "Schedule recurring work", + "line": 1, + "text": "v2 has no scheduler service. A scheduled task is a row in an isolated per-task session's `messages_in` table with `kind='task'` and a `process_after` timestamp — the same inbox machinery, swept by the same host sweep, as any chat message. You drive it through the `ncl tasks` CLI: ask the agent in chat and it runs `ncl tasks` on your behalf, or run the commands yourself from the host. This tutorial assumes a working install with a wired agent (see [your first agent](/guides/first-agent)).\n\n\nAgents stamped from a [template](/templates/overview) may arrive with recurring tasks already defined. They are created **paused** — list them with `ncl tasks list --group --status paused` and enable the ones you want with `ncl tasks resume `.\n\n\n\n\n\nAsk your agent, in whatever chat it's wired to:\n\n```text\nScout, every weekday at 9am remind me to post my standup update\n```\n\nMechanically, the agent runs `ncl tasks create` with a `--prompt`, a cron `--recurrence` (`0 9 * * 1-5`), and (for one-shots) a `--process-after` timestamp for the first run. NanoClaw provisions an isolated per-task session for the series and inserts a `messages_in` row there with `kind='task'`, status `pending`, your `process_after`, the cron expression, and `series_id` set to the task's own id. See the [session DB schema](/reference/db-schema#session-databases) for the columns.\n\nThe agent confirms with the generated id. Passing `--name \"standup\"` yields a readable id like `standup-a25c`; without a name, ids are `t-`. Recurrences over 4 fires per day are refused unless the task carries a `--script` gate (or the explicit `--dangerously-override-recurrence-limit` flag) — a high-frequency wake with no gate is almost always a mistake. For a one-shot task — \"remind me tomorrow at 2pm to review the budget\" — it passes `--process-after` and omits `--recurrence`.\n\n\n\n\nThere is no separate per-task scheduler. The [host sweep](/concepts/architecture#what-runs-when) periodically queues active sessions for reconciliation, waiting 60 seconds after the previous tick's work drains. Message writes and container-exit events can trigger earlier checks. Reconciliation counts pending rows with `process_after <= now` and wakes the task session's container if one isn't already running. The agent's poll loop then picks the task up like any inbound message and executes the prompt.\n\nTwo consequences worth internalizing:\n\n- **Timing is approximate.** A task due at 09:00:00 runs when reconciliation finds it due, plus queue work and any container cold start. The roughly one-minute resync is not a strict latency bound; do not schedule anything that needs second-level timing.\n- **Recurring tasks are clones, not loops.** When a recurring occurrence completes, reconciliation computes the next run with cron-parser, inserts a fresh `pending` row carrying the same `series_id` forward, and clears `recurrence` on the completed row so it isn't re-cloned. The series id is the stable handle; individual occurrence ids come and go. Once a series has no live occurrence left and its container has stopped, reconciliation closes its per-task session. History remains until explicitly deleted.\n\n\n\n\nYou can manage tasks by asking the agent in chat:\n\n```text\nScout, list my scheduled tasks\nScout, pause the standup reminder\nScout, move the standup reminder to 9:30\nScout, cancel the standup reminder\n```\n\nBehind those phrases (and available directly from the host) is the `ncl tasks` resource:\n\n| Command | Behavior |\n|---|---|\n| `ncl tasks list` | One row per series — the live `pending` or `paused` occurrence, not the pile of completed firings, each enriched with run count, failures, and next fire. |\n| `ncl tasks get ` | Full detail for one series: schedule, run history, failures, and the recent run-log lines. |\n| `ncl tasks update ` | Edits the live occurrence (matches by id *or* series id). Omitted fields are untouched; an empty `--prompt` is rejected. `--recurrence null` clears the schedule (the task becomes one-shot) and `--script null` removes the gate. |\n| `ncl tasks pause ` / `resume ` | Flip status between `pending` and `paused`. Paused rows are never counted as due, so they don't fire or wake anything. |\n| `ncl tasks run ` | Fire once now without changing the schedule — a safe test that neither consumes a one-shot nor advances a recurring series. |\n| `ncl tasks cancel ` | End the series (`status='cancelled'`, `recurrence` nulled); history rows stay. `--all` is the kill switch across every live task. |\n| `ncl tasks delete ` | For modern isolated tasks, delete the whole task session, mailbox history, attachments, heartbeat, and run log. Refused while its container is running. Legacy shared sessions keep their mailbox and lose only this series and its run log. |\n\nFull flags are in the [ncl tasks reference](/reference/ncl-cli#tasks).\n\n\n\n\nEvery firing that reaches the agent is a full model API call. For tasks that run more than a few times a day, `ncl tasks create` accepts an optional `--script` — a bash script that runs *before* the agent is invoked:\n\n```text\nScout, every hour check for open PRs needing my review. Use a script that\ncurls the GitHub API and only wake yourself if there are any.\n```\n\nThe contract, from `container/agent-runner/src/scheduling/task-script.ts`:\n\n1. When the task fires, the script runs first inside the container — 30-second timeout, 1 MB output cap.\n2. The **last line of stdout** must be JSON: `{ \"wakeAgent\": true|false, \"data\": ... }`.\n3. `wakeAgent: false` marks the occurrence completed without invoking the agent — a normal, quiet run; the sweep clones the next occurrence as usual. A script **error, timeout, or invalid output** is different: the occurrence is marked **failed**, consecutive failures back the series off exponentially (next run = the later of the cron slot and the backoff), and after 8 consecutive script failures the series auto-pauses with a run-log note — fix the script, then `ncl tasks resume `.\n4. `wakeAgent: true` — the agent wakes with `data` injected into the task prompt as `scriptOutput`. The prompt's `` element carries both the scheduled time (`time`, from `process_after`) and a fresh `current_time` stamped when the run reaches the agent — so relative wording like \"today\" stays correct even when a run fires late.\n\nBe honest about what this saves: the container still wakes on every firing (the sweep can't know the script's verdict in advance). The script gates the expensive part — the agent invocation — not the container spawn. Skip scripts entirely for tasks that need the agent's judgment every time, like daily briefings.\n\n\n\n\nWhen you say \"9am\", three layers have to agree on whose 9am. The chain: the host resolves `TIMEZONE` once at startup — `process.env.TZ`, then the `.env` file's `TZ`, then the system locale, falling back to `UTC`, each candidate validated as a real IANA identifier (`src/config.ts`). Every container is spawned with `-e TZ=` (`src/container-runner.ts`), and the agent-runner's `timezone.ts` resolves the same way inside.\n\nThe result: naive timestamps the agent passes (`2026-06-11T09:00:00`, no offset) and **cron expressions** — both at schedule time and when the host sweep computes the next recurrence — are interpreted in the task's timezone: the owning group's override when one is set, otherwise that instance value (`src/modules/scheduling/create.ts`). To pin the install default:\n\n```bash\n# in .env (or the host's environment)\nTZ=Europe/Madrid\n```\n\nRestart the host after changing it; the install default is read once at startup.\n\nA group can override the default without touching `.env`: `ncl groups config update --timezone Europe/Lisbon` (or pass `--timezone` on `ncl groups create`). Scheduled-task times follow the override immediately — no restart — and the group's containers spawn with it as `TZ` (`src/cli/resources/groups.ts`, `src/container-runner.ts:485`). Pass `\"\"` to clear back to the install default.\n\n\n\n\nBecause scheduling is a host CLI resource now, you read it directly:\n\n```bash\nncl tasks list # every live series in scope\nncl tasks list --group # scope to one agent group\nncl tasks get standup-a25c # one series: schedule, run count, failures, recent run log\n```\n\nUnder the hood each series lives as `messages_in` rows with `kind='task'` in the agent group's isolated per-task session (`data/v2-sessions///inbound.db`), not the central DB. A healthy recurring series shows one `pending` row (the next run) plus one `completed` row per past firing, all sharing a `series_id` — but `ncl tasks get` already surfaces that history, so you rarely need to open the database yourself." + }, + { + "id": "operate/credentials#setting-it-up", + "page": "operate/credentials", + "title": "Credentials", + "heading": "Setting it up", + "line": 24, + "text": "The [setup wizard](/installation) handles the vault as one of its steps, with three paths:\n\n- **Fresh install (default)** — runs the OneCLI installer (a Docker Compose stack plus the `onecli` CLI), points the CLI at the new instance, and writes `ONECLI_URL` to `.env`.\n- **Reuse existing** — if setup detects a working `onecli` on the host, it asks before reinstalling. Choosing reuse leaves the running gateway untouched (re-running the installer would rebind the listener and break other apps using it) and just records its URL.\n- **Remote vault** — set `NANOCLAW_ONECLI_API_HOST` before running setup to point at a OneCLI instance on another machine. Setup installs only the CLI, health-checks the remote URL, and writes `ONECLI_URL`. Pass the remote vault's API token via `NANOCLAW_ONECLI_API_TOKEN` so setup can authenticate the CLI and write `ONECLI_API_KEY`.\n\nSetup installs the gateway and `onecli` CLI at the versions pinned in `versions.json` (`onecli-gateway`, `onecli-cli`), never at `latest`. It also probes the gateway's `/v1/health`: `@onecli-sh/sdk` 2.x talks to the gateway's `/v1` API, and a gateway that predates it answers `404` to every vault call — failures that look transient but are permanent. Setup never upgrades the gateway for you; it warns and points to the verbatim migration runbook, [`docs/onecli-upgrades.md`](https://github.com/nanocoai/nanoclaw/blob/main/docs/onecli-upgrades.md). The gateway pin also moves through `/update-nanoclaw` — see [Upgrading](/operate/upgrading).\n\nOn an existing install (for example after `/update-nanoclaw` brings in OneCLI as a breaking change), run the `/init-onecli` skill instead. It installs the vault, migrates any credentials it finds in `.env` (Anthropic keys, `OPENAI_API_KEY`, and similar container-facing secrets) into the vault, and removes the raw values from `.env`. Channel tokens like `TELEGRAM_BOT_TOKEN` stay in `.env` — the host process uses those, not the containers." + }, + { + "id": "operate/ncl-cli#dropped-messages", + "page": "operate/ncl-cli", + "title": "The ncl admin CLI", + "heading": "dropped-messages", + "line": 142, + "text": "Read-only log of messages the router dropped, aggregated per sender and chat with a `reason` (`no_agent_wired`, `no_agent_engaged`, `unknown_sender_strict`, `unknown_sender_request_approval`, `unknown_sender_decline_notify`). Your first stop when a message got no reply:\n\n```bash\nncl dropped-messages list\n```" + }, + { + "id": "operate/ncl-cli#messaging-groups-and-wirings", + "page": "operate/ncl-cli", + "title": "The ncl admin CLI", + "heading": "messaging-groups and wirings", + "line": 89, + "text": "A messaging group is one chat on one platform (unique `channel_type` + `platform_id`); a wiring connects it to an agent group and sets engagement rules. List commands accept column filters and `--limit`:\n\n```bash\nncl messaging-groups list --channel_type telegram\nncl wirings create --messaging-group-id --agent-group-id --engage-mode mention-sticky\nncl wirings update --id --session-mode per-thread\n```\n\nWiring columns: `engage_mode` (`mention` | `mention-sticky` | `pattern`), `engage_pattern`, `sender_scope` (`all` | `known`), `ignored_message_policy` (`drop` | `accumulate`), `session_mode` (`shared` | `per-thread` | `agent-shared`), `threads` (honor platform thread ids for this wiring; unset inherits the channel's default — you can turn threads *off* on a threaded platform, never *on* for one that has none), and `priority` (fanout order when several agents are wired to one chat; higher first). Engagement flags you omit on `create` default from the channel adapter's declaration.\n\n`messaging-groups send` injects a message into a chat as if a sender posted it, waking the wired agent — the host uses it to send a welcome on first wire:\n\n```bash\nncl messaging-groups send --channel-type telegram --platform-id --text \"hello\"\n```" + }, + { + "id": "operate/troubleshooting#agent-never-replies", + "page": "operate/troubleshooting", + "title": "Troubleshooting", + "heading": "Agent never replies", + "line": 53, + "text": "Trace the message through the pipeline, in order:\n\n\n\n ```bash\n ncl dropped-messages list\n ```\n Each row aggregates drops per chat, with a `reason`:\n\n | Reason | Meaning | Fix |\n |---|---|---|\n | `no_agent_wired` | No wiring exists for that chat | Create one: `ncl wirings create --messaging-group-id --agent-group-id ` |\n | `no_agent_engaged` | A wiring exists but its engage rules didn't fire | Check `engage_mode`: `mention` needs an @mention (or a DM); `pattern` matches `engage_pattern` against each message |\n | `unknown_sender_strict` | Sender not recognized, strict policy | Add the sender, or relax the policy — see [Hardening](/operate/hardening#restrict-who-can-talk-to-your-agents) |\n | `unknown_sender_request_approval` | Sender not recognized; an Allow/Deny card was sent to an approver's DM | The approver answers the card in their DM; `ncl dropped-messages list` shows the recorded drop either way |\n | `unknown_sender_decline_notify` | Sender not recognized; in a DM the bot politely declined and notified an owner or admin | Add the sender if they should have access, or change the policy — repeated declines are deduplicated for 24 hours |\n\n\n ```bash\n ncl sessions list # container_status should be \"running\"\n grep 'Spawning session' logs/nanoclaw.log | tail -5\n docker ps --filter label=nanoclaw-install --format '{{.Names}} {{.Status}}'\n ```\n `stopped` is normal between messages — the sweep restarts the container when due messages arrive. If nothing spawns at all, see [Container won't spawn](#container-wont-spawn) below.\n\n\n Inspect the session DBs (`inbound.db` / `outbound.db` under `data/v2-sessions///`). The `/debug` skill walks you through the exact queries. A `Container exited non-zero` warning in the host log carries the exit `code` and a `stderrTail` — the last ~10 lines of container stderr — which is where a boot failure usually shows. A clean or signal-killed exit logs `Container exited` at info level instead." + }, + { + "id": "operate/troubleshooting#webhook-channel-is-silent", + "page": "operate/troubleshooting", + "title": "Troubleshooting", + "heading": "Webhook channel is silent", + "line": 108, + "text": "Webhook-based adapters (Slack, Teams, and similar) register routes on a shared local HTTP server. Confirm it's up and routed:\n\n```bash\ngrep 'Webhook server started' logs/nanoclaw.log # shows port + registered adapters\ncurl -i http://localhost:3000/webhook/slack # 404 \"Unknown adapter\" = not registered\n```\n\nThe server listens on `WEBHOOK_PORT` (default `3000`). If another process already holds that port, the listen fails and the host crashes with `FATAL Uncaught exception` — `WEBHOOK_PORT` is read from the process environment only (`src/webhook-server.ts:113`), so set it in the service definition or your shell, not `.env`, then restart. Channel-specific failures (tokens, tunnel/public URL, platform-side config) are covered on each channel page: [Slack](/channels/slack), [Teams](/channels/teams), [Channels overview](/channels/overview)." + }, + { + "id": "operate/upgrading#updating-nanoclaw", + "page": "operate/upgrading", + "title": "Upgrading", + "heading": "Updating NanoClaw", + "line": 21, + "text": "Run `/update-nanoclaw` in a Claude Code session at your project root. The skill archives the controller scripts from the upstream ref into a temporary directory and runs `prepare` from there, so an update always starts with the upstream controller rather than the one in your checkout. Later commands run from the stage worktree or from your checkout.\n\n\n\n The controller refuses to start unless your working tree is clean and your checkout is on a named branch. It records the current HEAD, creates a backup branch named `backup/pre-update---` and a backup tag named `pre-update---`, then creates a stage worktree on branch `update-nanoclaw/`. Pick one strategy for it to apply there:\n\n - **Merge**, the default.\n - **Rebase**.\n - **Cherry-pick**, with a named commit list.\n\n\n A conflicting merge leaves the transaction in the `conflict` phase and the controller exits `2`. Resolve the conflict inside the stage worktree, never in your live checkout. Then resume from the stage worktree:\n\n ```bash\n pnpm exec tsx scripts/update-nanoclaw.ts resume --id \"$id\"\n ```\n\n\n Validation runs entirely in the stage worktree. It refreshes your installed skills first, commits the refreshed payloads, then runs `pnpm install --frozen-lockfile`, `pnpm run build`, and `pnpm test`. If the staged changes touched `container/agent-runner`, it also runs a Bun install and a `tsc --noEmit` pass over that directory. If Bun is not on your host, that typecheck is deferred to the image build instead. Your live checkout is still untouched at this point.\n\n\n Cutover requires a validated transaction, a clean live tree, and a live HEAD that has not moved since prepare. A moved HEAD stops it with `Live checkout moved after the update was staged`. The controller then stops the service, drains this install's containers by label with a 300 second timeout, snapshots your mutable state, resets your checkout to the staged commit, and reinstalls and rebuilds. If the staged changes touched `container/`, the agent image is refreshed in the same step: `bash container/build.sh pull` on an install with `NANOCLAW_HARDENED_IMAGE=true`, and `bash container/build.sh` otherwise. See [the hardened runtime](/guides/echo-hardened-runtime) for which of those applies to you.\n\n\n Your service stays down until every requirement is acknowledged as succeeded. A pending requirement blocks the finish step, and so does one acknowledged as failed. Requirements are derived for you and come in two kinds. A **breaking change** requirement is raised for each added `[BREAKING]` line in the `CHANGELOG.md` diff. An **external component** requirement is raised when the `onecli-gateway` or `onecli-cli` pin in `versions.json` moved, because those run outside your checkout and git cannot restore them. Follow [`docs/onecli-upgrades.md`](https://github.com/nanocoai/nanoclaw/blob/main/docs/onecli-upgrades.md) for that one. An external component you acknowledge as succeeded must carry a rollback instruction with it.\n\n\n The finish step stamps the upgrade marker, restarts your service through the detected service mode, and health-verifies the result. A failure at any of those steps rolls your install back automatically.\n\n\n Cleanup removes the stage worktree and its branch. It keeps the backup branch, the backup tag, and the snapshot, so you can still roll back afterwards. Pruning takes the id of the transaction to keep, and removes older finished ones and their git refs. Preview it with `--dry-run` first. Pruning always retains the keeper, everything newer, anything still in flight, and anything whose state it cannot read safely.\n\n\n\n\n There is no `ncl` subcommand for upgrading. The admin CLI carries no update resource; the controller scripts are the interface.\n\n\nTo back out before cutover, abandon the transaction. Nothing has moved yet, so there is nothing to restore." + }, + { + "id": "reference/container-config#cli-scope", + "page": "reference/container-config", + "title": "Container configuration", + "heading": "cli_scope", + "line": 68, + "text": "Host-side enforcement of what the agent can do through the in-container `ncl` CLI. One of:\n\n- `disabled` — all CLI requests rejected; the CLI instructions fragment is also dropped from the composed CLAUDE.md (that part applies at next spawn)\n- `group` (default) — only `groups`, `sessions`, `destinations`, `members`, and `tasks` resources plus `wirings get`/`update` for the current conversation's wiring, auto-scoped to the agent's own group; the agent cannot change `cli_scope` itself\n- `global` — full resource access; mutations still require approval\n\nThis is the only field **not** written to `container.json` — the host checks the database on every request. See [ncl CLI reference](/reference/ncl-cli#access-control) for the full enforcement rules." + }, + { + "id": "reference/db-schema#pending-approvals", + "page": "reference/db-schema", + "title": "Database schema", + "heading": "pending_approvals", + "line": 195, + "text": "Host-side records for any approval-requiring request: `install_packages` / `add_mcp_server` (session-bound), and OneCLI credential approvals (`session_id` may be NULL, `action='onecli_credential'`). The OneCLI-specific columns let the host edit the admin card when a request expires and sweep stale rows on startup.\n\n| Column | Type | Default | Meaning |\n|---|---|---|---|\n| `approval_id` | TEXT | — | Primary key |\n| `session_id` | TEXT | `NULL` | FK → `sessions(id)` |\n| `request_id` | TEXT | — | The requesting side's correlation ID |\n| `action` | TEXT | — | What's being approved |\n| `payload` | TEXT | — | JSON request payload |\n| `agent_group_id` | TEXT | `NULL` | FK → `agent_groups(id)` |\n| `channel_type` / `platform_id` / `platform_message_id` | TEXT | `NULL` | Where the admin card lives, for later edits |\n| `instance` | TEXT | `NULL` | Adapter instance that delivered the card, so later edits resolve the same bot identity; `NULL` means the default instance (`channel_type`) (migration 023) |\n| `expires_at` | TEXT | `NULL` | Expiry for credential approvals |\n| `status` | TEXT | `'pending'` | `pending` \\| `approved` \\| `rejected` \\| `expired` \\| `awaiting_reason` (a reject-with-reason hold, see [security](/concepts/security#human-in-the-loop-approvals)) |\n| `title` | TEXT | `''` | Card title (render metadata) |\n| `options_json` | TEXT | `'[]'` | Card options (render metadata) |\n| `approver_user_id` | TEXT | `NULL` | When set — the named approver of an agent-to-agent [message policy](#agent_message_policies) — only that exact user may resolve the card; `NULL` keeps the group/owner authorization path (migration 018) |\n| `question` | TEXT | `''` | The original card body, persisted so terminal card edits (resolved, timed out — even across a host restart) can re-render it; pre-migration rows fall back to the title (migration 021) |\n| `created_at` | TEXT | — | ISO timestamp |" + }, + { + "id": "reference/environment-variables#container-side-variables-agent-runner", + "page": "reference/environment-variables", + "title": "Environment variables", + "heading": "Container-side variables (agent runner)", + "line": 88, + "text": "Read inside the agent container by the agent runner. The host injects runtime, routing, and provider/gateway variables at spawn; these are not a blanket passthrough of the host environment. The rest are not passed through from the host environment; overriding them means baking them into a custom image or a provider container config.\n\n| Variable | Default | Read in | Effect |\n|---|---|---|---|\n| `TZ` | injected from host `TIMEZONE` | `agent-runner/src/timezone.ts:68` | Timezone for timestamps and scheduling inside the container. |\n| `NANOCLAW_CONVERSATIONS_DIR` | `/workspace/agent/conversations` | `agent-runner/src/providers/claude.ts:309` | Where pre-compaction conversation transcripts are archived as Markdown. |\n| `CLAUDE_TRANSCRIPT_ROTATE_BYTES` | `12582912` (12 MiB) | `agent-runner/src/providers/claude.ts:339` | Transcript size past which the session is rotated instead of resumed — cold resumes of huge transcripts would outlive the idle ceiling. |\n| `CLAUDE_TRANSCRIPT_ROTATE_AGE_DAYS` | `14` | `agent-runner/src/providers/claude.ts:347` | Age trigger for transcript rotation, measured from the first entry. Zero or negative disables the age check; size alone governs. |\n| `CLAUDE_CONFIG_DIR` | `~/.claude` | `agent-runner/src/providers/claude.ts:360` | Base directory the runner scans for Claude Code project transcripts. |\n| `CLAUDE_CODE_AUTO_COMPACT_WINDOW` | `165000` | `agent-runner/src/providers/claude.ts:453` | Token window at which Claude Code auto-compacts context. Raise for 1M-context model variants. |\n\nLine numbers reference [`nanocoai/nanoclaw`](https://github.com/nanocoai/nanoclaw) at `639577c3` (v2.1.54) and will drift with future releases." + }, + { + "id": "reference/environment-variables#host-process-environment-only", + "page": "reference/environment-variables", + "title": "Environment variables", + "heading": "Host process environment only", + "line": 38, + "text": "Read from `process.env` at host startup, with no `.env` fallback. Putting these in `.env` has no effect — the generated launchd plist, systemd unit, and nohup wrapper don't source it. Set them in the service definition or export them before `pnpm run dev`.\n\n| Variable | Default | Read in | Effect |\n|---|---|---|---|\n| `NANOCLAW_INSTALL_ID` | first 8 characters of SHA-1 of the project-root path (`process.cwd()` by default) | `src/install-slug.ts` | Explicit install identity for deployments where the checkout path is not stable. Must be 1–32 lowercase letters, digits, `_` or `-`, starting with a letter or digit; invalid values throw. No `.env` fallback. |\n| `CONTAINER_IMAGE` | per-install tag | `src/config.ts:70` | Full agent image override (name and tag). |\n| `CONTAINER_IMAGE_BASE` | per-install name | `src/config.ts:69` | Base image name override. The default embeds an install slug so two checkouts on one host don't clobber each other's image. |\n| `NANOCLAW_TEMPLATES_DIR` | `templates/` at the project root | `src/config.ts:63` | Local directory [templates](/templates/overview) resolve against; `--template ` is relative to it. A local path only, never a URL. Resolved **once at host startup**, so setting it on an `ncl` invocation has no effect — `ncl` is a socket client and the host has already resolved the path. Change it in the service definition and restart the host. |\n| `WEBHOOK_PORT` | `3000` | `src/webhook-server.ts:113` | Port for the local webhook server. |\n| `LOG_LEVEL` | `info` | `src/log.ts:16` | One of `debug`, `info`, `warn`, `error`, `fatal`. |\n\n\n`CONTAINER_TIMEOUT`, `IDLE_TIMEOUT`, `MAX_CONCURRENT_CONTAINERS`, `CONTAINER_MAX_OUTPUT_SIZE`, and `MAX_MESSAGES_PER_PROMPT` were removed from the code and are no longer read anywhere (host or container). Container idle/stuck detection is now handled by the periodic host-sweep loop (`src/host-sweep.ts`), not a per-container timeout env var. Setting any of these five in `.env` or the process environment has no effect." + }, + { + "id": "reference/environment-variables#read-from-env", + "page": "reference/environment-variables", + "title": "Environment variables", + "heading": "Read from .env", + "line": 13, + "text": "NanoClaw's own parser ([`src/env.ts`](https://github.com/nanocoai/nanoclaw/blob/main/src/env.ts)) returns only the keys requested by each caller from `.env` at the project root. The parser itself does not load values into `process.env`. For the core settings below, the process environment takes precedence over `.env` unless noted. `ANTHROPIC_BASE_URL` in the Claude provider is read from `.env` only. The shared parser strips matching outer single or double quotes, ignores blank values, and does not execute shell expressions. These parsing rules apply to callers of `readEnvFile`/`envValue`; some setup steps retain their own readers.\n\n| Variable | Default | Read in | Effect |\n|---|---|---|---|\n| `ASSISTANT_NAME` | `Andy` | `src/config.ts:29` | Assistant name. Also sets the default trigger, `@`. |\n| `ASSISTANT_HAS_OWN_NUMBER` | `false` | `src/config.ts:47` | Set `true` when the assistant runs on its own WhatsApp number instead of yours. |\n| `ONECLI_URL` | — | `src/config.ts:75` | URL of the OneCLI Agent Vault. |\n| `ONECLI_API_KEY` | — | `src/config.ts:76` | API key for the vault. See [Credentials](/operate/credentials). |\n| `TZ` | system timezone, else `UTC` | `src/config.ts:101` | IANA timezone for scheduled tasks and timestamps. Invalid values are skipped, not errored. Injected into every agent container at spawn. |\n| `CONTAINER_CPU_LIMIT` | _unset_ (unbounded) | `src/config.ts:80` | Opt-in CPU cap per agent container, passed through to `docker create --cpus` (e.g. `2`). Empty leaves containers unbounded — the current default. Can be set in `.env` or the process environment. |\n| `CONTAINER_MEMORY_LIMIT` | _unset_ (unbounded) | `src/config.ts:81` | Opt-in memory cap per agent container, passed through to `docker create --memory` (e.g. `8g`). A hard cap only on a swapless host — there the runaway is OOM-killed; with swap present the container can spill past it. NanoClaw doesn't manage swap. Can be set in `.env` or the process environment. |\n| `CONTAINER_PIDS_LIMIT` | `2048` | `src/config.ts:88` | Process/thread cap per agent container, passed through to `docker create --pids-limit` — a fork-bomb backstop, and the one container limit that's on by default. Set `0` (or any non-numeric value) to drop the cap. Resolved with `??` rather than `||`, so an explicitly empty value in the process environment wins and disables the cap instead of falling through to `.env` or the default. Can be set in `.env` or the process environment. |\n| `NANOCLAW_EGRESS_LOCKDOWN` | `false` | `src/config.ts:92` | Set `true` to force all agent traffic through the OneCLI gateway on an internal Docker network. Spawns fail fast if the gateway isn't running. Can be set in `.env` or the process environment. See [Hardening](/operate/hardening). |\n| `NANOCLAW_EGRESS_NETWORK` | `nanoclaw-egress` | `src/config.ts:93` | Name of the locked-down Docker network. Can be set in `.env` or the process environment. |\n| `ONECLI_GATEWAY_CONTAINER` | `onecli` | `src/config.ts:95` | Name of the gateway container attached as the only egress hop. Can be set in `.env` or the process environment. |\n| `DEFAULT_AGENT_PROVIDER` | `claude` | `src/config.ts:36` | Instance-wide default agent provider stamped onto newly created agent groups. Existing groups are never retroactively changed, and per-group `ncl groups config update --provider` overrides it. Can be set in `.env` or the process environment. |\n| `NANOCLAW_DEFAULT_MODEL` | provider default | `src/config.ts` → `configFromDb` | Install-wide model used when the group's model is `NULL`. The group's own model wins. Restart the host to reload the setting, then restart affected agent sessions to apply it. |\n| `NANOCLAW_FAST_MODE` | `false` | `src/config.ts` → Claude provider `settings.fastMode` | Enables the Claude fast serving tier only for `1` or `true` (case-insensitive). Other values stay off. May change provider billing. No per-group fast-mode field; other providers must explicitly support the option. |\n| `NANOCLAW_RUNTIME_DRIVER` | `docker` | `src/drivers/index.ts` | Registered session driver. Process environment, then `.env`; unknown names abort startup. |\n| `NANOCLAW_GATEWAY_PROVIDER` | `onecli` | `src/gateway-providers/index.ts` | Registered gateway provider. Process environment, then `.env`; unknown names fail instead of disabling the gateway. |\n| `NANOCLAW_SESSION_MATERIAL_ROOT` | `data/session-materials` | `src/drivers/index.ts` | Root for materialized session files. Process environment, then `.env`. |\n| `ANTHROPIC_BASE_URL` | — | `src/providers/claude.ts:21` | Custom Anthropic-compatible endpoint, passed into agent containers with a placeholder auth token. Only active when setup has registered the custom-endpoint provider config; the real token stays in the vault. |" + }, + { + "id": "reference/ncl-cli#approvals", + "page": "reference/ncl-cli", + "title": "ncl CLI reference", + "heading": "approvals", + "line": 46, + "text": "In-flight approval cards waiting for an admin response. Read-only; rows are deleted after approve/reject/expiry.\n\n| Operation | Invocation | Access |\n| --- | --- | --- |\n| list | `ncl approvals list [-- ] [--limit N]` | open |\n| get | `ncl approvals get --id ` | open |\n\nColumns (all usable as list filters): `approval_id`, `session_id`, `request_id`, `action` (e.g. `install_packages`, `add_mcp_server`, `onecli_credential`), `payload` (JSON), `created_at`, `agent_group_id`, `channel_type`, `platform_id`, `platform_message_id`, `expires_at`, `status` (`pending` | `approved` | `rejected` | `expired` | `awaiting_reason`), `approver_user_id`, `title`, `options_json`." + }, + { + "id": "reference/ncl-cli#destinations", + "page": "reference/ncl-cli", + "title": "ncl CLI reference", + "heading": "destinations", + "line": 57, + "text": "Per-agent routing entries and ACL — each row lets an agent send to a target (channel or another agent) under a local name. Local names are unique per agent. No `get`, `create`, `update`, or `delete`; mutations go through `add`/`remove`, which also project the change into every active session's `inbound.db` so running containers pick it up without restart.\n\n| Operation | Invocation | Access |\n| --- | --- | --- |\n| list | `ncl destinations list [--agent-group-id ] [...]` | open |\n| add | `ncl destinations add --agent-group-id --local-name --target-type --target-id ` | approval |\n| remove | `ncl destinations remove --agent-group-id --local-name ` | approval |\n\n`--target-id` is a `messaging_groups.id` when `--target-type channel`, an `agent_groups.id` when `agent`." + }, + { + "id": "reference/ncl-cli#dropped-messages", + "page": "reference/ncl-cli", + "title": "ncl CLI reference", + "heading": "dropped-messages", + "line": 69, + "text": "Log of messages dropped by the router or access gate, aggregated by `(channel_type, platform_id)` with a running count. List-only.\n\n| Operation | Invocation | Access |\n| --- | --- | --- |\n| list | `ncl dropped-messages list [--reason ] [...]` | open |\n\nColumns: `channel_type`, `platform_id`, `user_id`, `sender_name`, `reason` (`no_agent_wired` | `no_agent_engaged` | `unknown_sender_strict` | `unknown_sender_request_approval` | `unknown_sender_decline_notify`), `messaging_group_id`, `agent_group_id`, `message_count`, `first_seen`, `last_seen`." + }, + { + "id": "reference/ncl-cli#groups", + "page": "reference/ncl-cli", + "title": "ncl CLI reference", + "heading": "groups", + "line": 79, + "text": "Agent groups — logical agent identities with their own workspace folder, history, and container image.\n\n| Operation | Invocation | Access | Notes |\n| --- | --- | --- | --- |\n| list | `ncl groups list` | open | |\n| get | `ncl groups get --id ` | open | |\n| create | `ncl groups create --folder [--name ] [--timezone ]` or `ncl groups create --template [--name ] [--timezone ] [--id ] [--new] [--yes]` | approval | **Without `--template`:** `--folder` required and immutable, `--name` defaults to the folder, idempotent on `--folder` (creating an existing folder returns the existing row), and new ids are prefixed `ag-`. **With `--template `:** stamps from a local [template](/templates/overview) under `templates/`. Combining it with `--folder` is **rejected** — a bare group *is* its folder, a templated group *is* its plugin. The display name is `--name`, else the manifest's `agentName`, else the template ref's leaf folder; the group folder derives from that name, with a random `-XXXXXXXX` suffix on collision. Always creates a new agent, except when updating one this template already made: a group can only be targeted if its workspace holds that plugin's manifest, so a hand-built group or one stamped from another template is rejected. Convergent rather than folder-idempotent: if a group already carries that plugin, the command returns an in-place [update plan](/templates/updating) instead of creating anything — `--yes` applies it, `--id` picks the target when several groups carry it, and `--new` forces a fresh stamp. `--name` and `--timezone` apply only when a group is created; both are ignored on an in-place update. Reader notices come back as `templateReport` on a create, and as `report` on an update. |\n| update | `ncl groups update --id --name ` | approval | `name` is the only updatable column. |\n| delete | `ncl groups delete --id ` | approval | Custom cascading handler (not the generic single-table delete): removes sessions, destinations, approvals, role grants, memberships, and wirings in one FK-ordered transaction. Does **not** kill running containers or clean up `groups//` and `data/v2-sessions//` on disk. |\n| restart | `ncl groups restart --id [--rebuild] [--message ]` | approval | Host caller: restarts all running containers in the group. Agent caller: `--id` auto-filled, only the calling session restarts. `--rebuild` rebuilds the image first (required after package changes). `--message` queues an on-wake instruction so the fresh container starts and acts on it; without it the container stays stopped until the next user message. |\n| config get | `ncl groups config get --id ` | open | Shows `agent_group_id`, `provider`, `model`, `effort`, `image_tag`, `assistant_name`, `max_messages_per_prompt`, `skills`, `mcp_servers`, `packages_apt`, `packages_npm`, `additional_mounts`, `cli_scope`, `timezone`, `updated_at`. |\n| config update | `ncl groups config update --id [--provider] [--model] [--effort] [--image-tag] [--assistant-name] [--max-messages-per-prompt] [--cli-scope ] [--timezone ]` | approval | At least one flag required. Most changes take effect after `ncl groups restart`; `--timezone` grounds task scheduling immediately (empty string clears it back to the install default). |\n| config add-mcp-server | `ncl groups config add-mcp-server --id --name --command [--args ] [--env ]` or `--url [--headers ]` | approval | Exactly one of `--command`/`--url`. Server name: 1-64 chars of `[A-Za-z0-9_-]`. `--url` is a remote Streamable HTTP endpoint — HTTPS required (plain HTTP only for localhost / `host.docker.internal`), and URLs with embedded credentials, fragments, or secret-looking query params are rejected. Restart required to take effect. |\n| config remove-mcp-server | `ncl groups config remove-mcp-server --id --name ` | approval | Restart required to take effect. |\n| config add-package | `ncl groups config add-package --id --apt \\| --npm ` | approval | At least one of `--apt`/`--npm`. Requires `ncl groups restart --rebuild`. |\n| config remove-package | `ncl groups config remove-package --id --apt \\| --npm ` | approval | Requires `ncl groups restart --rebuild`. |\n| config add-mount | `ncl groups config add-mount --id --host --container [--ro]` | approval | **Host-only** — never runnable from inside a container (mounting host paths is a filesystem-access boundary). Adds a bind mount to `additional_mounts`; `--ro` makes it read-only. Restart required to take effect. |\n| config remove-mount | `ncl groups config remove-mount --id --host --container ` | approval | Host-only. Removes the matching mount. Restart required to take effect. |\n\n`add-mcp-server` and `remove-mcp-server` refuse to touch a server owned by a stamped [template](/templates/overview) — update the template and [restamp](/templates/updating) instead. A plugin-owned server removed by hand reappears on the next restamp." + }, + { + "id": "reference/ncl-cli#members", + "page": "reference/ncl-cli", + "title": "ncl CLI reference", + "heading": "members", + "line": 102, + "text": "Membership rows granting unprivileged users access to an agent group. Admins/owners are implicitly members and need no row. Checked by the router when a wiring's `sender_scope` is `known`.\n\n| Operation | Invocation | Access | Notes |\n| --- | --- | --- | --- |\n| list | `ncl members list [--agent-group-id ] [...]` | open | Columns: `user_id`, `agent_group_id`, `added_by`, `added_at`. |\n| add | `ncl members add --user --group [--added-by ]` | approval | Idempotent (`INSERT OR IGNORE`). `--user` must reference an existing `users.id`. |\n| remove | `ncl members remove --user --group ` | approval | Errors if no matching row. |" + }, + { + "id": "reference/ncl-cli#policies", + "page": "reference/ncl-cli", + "title": "ncl CLI reference", + "heading": "policies", + "line": 136, + "text": "Per-message approval gate on an agent-to-agent connection. A row forces every message from one agent to another to be approved by a human before delivery — without un-wiring the [destination](#destinations). **No row = free flow.** Directed and per-pair: gate both directions with two policies. Operator-only — agents can't manage their own gates, and self-messages (`--from` equal to `--to`) are never gated.\n\n| Operation | Invocation | Access |\n| --- | --- | --- |\n| list | `ncl policies list [-- ]` | open |\n| set | `ncl policies set --from --to --approver ` | approval |\n| remove | `ncl policies remove --from --to ` | approval |\n\n`set` upserts — re-running with a different `--approver` replaces it. Only the named `--approver` (or an owner) can approve a held message. Removing a policy returns the edge to free flow. Columns: `from_agent_group_id`, `to_agent_group_id`, `approver`, `created_at`." + }, + { + "id": "reference/ncl-cli#roles", + "page": "reference/ncl-cli", + "title": "ncl CLI reference", + "heading": "roles", + "line": 148, + "text": "Privilege grants. `owner` is always global; `admin` can be global (`agent_group_id` null) or scoped to one group. Admin at a group implies membership.\n\n| Operation | Invocation | Access | Notes |\n| --- | --- | --- | --- |\n| list | `ncl roles list [--role ] [...]` | open | Columns: `user_id`, `role` (`owner` \\| `admin`), `agent_group_id`, `granted_by`, `granted_at`. |\n| grant | `ncl roles grant --user --role [--group ] [--granted-by ]` | approval | Idempotent. Passing `--group` with `--role owner` is rejected — owner is always global. |\n| revoke | `ncl roles revoke --user --role [--group ]` | approval | Match includes scope: omit `--group` for global grants, pass it for scoped ones. |" + }, + { + "id": "reference/ncl-cli#sessions", + "page": "reference/ncl-cli", + "title": "ncl CLI reference", + "heading": "sessions", + "line": 158, + "text": "The runtime unit — one `(agent_group, messaging_group, thread)` combination mapped to a container. Created automatically by the router; read-only in the CLI.\n\n| Operation | Invocation | Access |\n| --- | --- | --- |\n| list | `ncl sessions list [--agent-group-id ] [--status active] [...]` | open |\n| get | `ncl sessions get --id ` | open |\n| history | `ncl sessions history [--limit ]` | open |\n\nColumns: `id`, `agent_group_id`, `messaging_group_id` (null for agent-shared sessions), `thread_id` (per-thread mode only), `agent_provider` (null = inherit from group), `status` (`active` | `closed`), `container_status` (`running` | `idle` | `stopped` — `idle` is reserved and currently unused), `last_active`, `created_at`.\n\n`history` merges inbound and outbound rows chronologically so an agent can catch up on another session in its own group. Human output is `timestamp|direction|kind|sender|text`, with text capped at 200 characters. `--json` returns the raw rows with full text. The default limit is 50; the command selects the newest rows and prints them oldest-to-newest." + }, + { + "id": "reference/ncl-cli#tasks", + "page": "reference/ncl-cli", + "title": "ncl CLI reference", + "heading": "tasks", + "line": 172, + "text": "Scheduled tasks — the CLI home for what used to be the agent's `schedule_task` MCP tools. Each task is a **series**: the live `pending`/`paused` row is the next run, and completed rows are its run history. Tasks are created from the host or by an agent inside its own task session, and each runs in an isolated per-task session that the host closes once the series is spent (history remains until explicit deletion). Ids are `t-`, or `-` when you pass `--name`.\n\n| Operation | Invocation | Access |\n| --- | --- | --- |\n| list | `ncl tasks list [--status ] [--group ] [--session ] [--all]` | open |\n| get | `ncl tasks get --id [--session ]` | open |\n| create | `ncl tasks create --prompt [--name ] [--process-after ] [--recurrence ] [--script ] [--group ] [--dangerously-override-recurrence-limit]` | open |\n| update | `ncl tasks update --id [--prompt] [--process-after] [--recurrence] [--script]` | open |\n| run | `ncl tasks run --id ` | open |\n| pause / resume | `ncl tasks pause --id ` / `ncl tasks resume --id ` | open |\n| cancel | `ncl tasks cancel (--id \\| --all)` | open |\n| delete | `ncl tasks delete --id ` | open |\n| append-log | `ncl tasks append-log --msg [--id ]` | open |\n\n| Flag | Notes |\n| --- | --- |\n| `--prompt` | Required on create. Must be nonblank when supplied on create or update; omit on update to retain the current prompt. |\n| `--process-after` | First/next run time — ISO 8601, offset, or naive-local in the group timezone (falling back to the install timezone). Required for one-shots; with `--recurrence` the first run is derived from the cron grid. |\n| `--recurrence` | Cron expression, evaluated in the group timezone (falling back to the install timezone). On `update`, `\"null\"`/`\"none\"` clears it (back to one-shot). Recurrences over 4 fires/day are refused unless the task has a `--script` gate or you pass `--dangerously-override-recurrence-limit`. |\n| `--name` | Short label folded into a readable series id (`-`). |\n| `--script` | Pre-task gate bash script — runs first; the agent only wakes if the script's last stdout line is JSON with `\"wakeAgent\": true`. |\n| `--group` | Agent group id (host callers). Auto-filled to the caller's own group inside a container. |\n| `--all` | On `cancel`, the kill switch — cancels every live task in scope. |\n| `--msg` | On `append-log`, the note to add. Inside a task run, `--id` is auto-derived from the caller's session. |\n\n`run` fires a task now without changing its schedule (a safe test that neither consumes a one-shot nor advances a recurring series — unlike `update --process-after now`). `append-log` adds a note to a task's run log (`tasks/.md`); every run already auto-logs its final text, so it's for additive mid-run notes only.\n\n`delete` refuses a running task session. For an isolated task it removes the session mailbox, history, attachments, heartbeat, session directory and row, and run log. For legacy tasks in a shared session it removes only the series rows and run log." + }, + { + "id": "reference/ncl-cli#wirings", + "page": "reference/ncl-cli", + "title": "ncl CLI reference", + "heading": "wirings", + "line": 229, + "text": "Connects a messaging group to an agent group — which agent handles which chat. Many-to-many in both directions. Under `group` CLI scope, agents get self-service access to the wiring of their current conversation only: `wirings get` and `wirings update` (limited to `engage_mode`/`engage_pattern`; the id is force-filled, updates stay approval-gated).\n\n| Operation | Invocation | Access |\n| --- | --- | --- |\n| list / get | `ncl wirings list` / `ncl wirings get --id ` | open |\n| create | `ncl wirings create --messaging-group-id --agent-group-id [flags]` | approval |\n| update | `ncl wirings update --id [flags]` | approval |\n| delete | `ncl wirings delete --id ` | approval |\n\n| Flag | Type | Notes |\n| --- | --- | --- |\n| `--messaging-group-id` | string | On create: pass this, or the natural key instead (`--channel-type` + `--platform-id` [+ `--instance`]). Not updatable. Create is idempotent on the (messaging group, agent group) pair. |\n| `--agent-group-id` | string | On create: pass this or `--agent-group `. References `agent_groups.id`. Not updatable. |\n| `--priority` | number | Updatable, default 0. Fanout order when several agents are wired to one chat — higher first. |\n| `--engage-mode` | enum | Updatable. Omitted engage flags default from the target channel adapter's declaration for the chat context (DM vs group), falling back to `mention` when the channel declares none. `mention` — only when @mentioned or in DMs. `mention-sticky` — after one mention in a thread, responds to all subsequent messages there. `pattern` — matches every message against `engage_pattern`. |\n| `--engage-pattern` | string | Updatable. Regex, required when `engage_mode=pattern`; use `.` for always-on. Ignored in mention modes. |\n| `--sender-scope` | enum | Updatable, default `all`. `all` — any sender (subject to `unknown_sender_policy`). `known` — only users with a role or membership in the agent group. |\n| `--ignored-message-policy` | enum | Updatable, default `drop`. `drop` — agent never sees non-engaging messages. `accumulate` — stored as background context so the agent has prior context when triggered. |\n| `--session-mode` | enum | Updatable, default `shared`. `shared` — one session per (agent, messaging group). `per-thread` — one per thread/topic. `agent-shared` — one across all messaging groups wired to the agent. Threaded adapters in group chats force per-thread regardless, unless `--threads false` disables threads for the wiring. |\n| `--threads` | boolean | Updatable. Per-wiring thread override: `true`/`false` (or `1`/`0`). Omitted (`NULL`) inherits the channel adapter's declared thread default for the context. Hard-ANDed with the adapter's raw capability at router fanout — it can disable threads on a threaded platform, but never enable them on a non-threaded one. Disabling also lifts the per-thread session-mode forcing above. |" + }, + { + "id": "reference/skills-catalog#channel-installs", + "page": "reference/skills-catalog", + "title": "Skills catalog", + "heading": "Channel installs", + "line": 22, + "text": "Each skill wires up one messaging channel. See [Channels](/channels/overview) for the full setup guides.\n\n| Skill | What it does |\n| --- | --- |\n| `/add-dial` | Adds the [Dial](/channels/dial) channel — a real phone number for SMS and AI voice calls through the `dial` CLI; native adapter, email sign-in, 6-digit SMS pairing, owner-only or public line |\n| `/add-dial-number` | Adds another phone number to an installed [Dial](/channels/dial) channel — a second owner-only or public line, wired to the same agent or a different one |\n| `/add-deltachat` | Adds the [Delta Chat](/channels/more-channels#delta-chat) channel via `@deltachat/stdio-rpc-server` — native adapter, email-based messaging with end-to-end encryption |\n| `/add-discord` | Adds the [Discord](/channels/discord) bot channel via Chat SDK |\n| `/add-emacs` | Adds [Emacs](/channels/more-channels#emacs) as a channel — interactive chat buffer and org-mode integration over a local HTTP bridge, no bot token needed |\n| `/add-gchat` | Adds the [Google Chat](/channels/more-channels#google-chat) channel via Chat SDK |\n| `/add-github` | Adds the [GitHub](/channels/more-channels#github) channel via Chat SDK — PR and issue comment threads as conversations |\n| `/add-imessage` | Adds the [iMessage](/channels/imessage) channel — local backend (this Mac's `chat.db`, macOS only) or hosted backend (a native iMessage line via [photon.codes](https://photon.codes)) |\n| `/add-linear` | Adds the [Linear](/channels/more-channels#linear) channel via Chat SDK — issue comment threads as conversations |\n| `/add-matrix` | Adds the [Matrix](/channels/more-channels#matrix) channel via Chat SDK — works with any homeserver |\n| `/add-mattermost` | Adds the [Mattermost](/channels/mattermost) bot channel via Chat SDK. Copies a vendored adapter rather than an npm package, offers to reuse a local server when one is running, and can create an evaluation server when none exists |\n| `/add-resend` | Adds the [Resend](/channels/more-channels#resend-email) email channel via Chat SDK |\n| `/add-signal` | Adds the [Signal](/channels/signal) channel via signal-cli TCP daemon — native adapter, no Chat SDK bridge |\n| `/add-slack` | Adds the base [Slack](/channels/slack) channel via Chat SDK — one bot, direct messages, and channel chat |\n| `/add-teams` | Adds the [Microsoft Teams](/channels/teams) channel via Chat SDK |\n| `/add-telegram` | Adds the [Telegram](/channels/telegram) channel via Chat SDK |\n| `/add-webex` | Adds the [Webex](/channels/more-channels#webex) channel via Chat SDK |\n| `/add-wechat` | Adds the [WeChat](/channels/more-channels#wechat) personal channel via Tencent's official iLink Bot API — long-polling and QR scan, no webhook |\n| `/add-whatsapp` | Adds the [WhatsApp](/channels/whatsapp) channel via native Baileys adapter — QR code or pairing code authentication |\n| `/add-whatsapp-cloud` | Adds the [WhatsApp Business Cloud API](/channels/more-channels#whatsapp-cloud-api) channel via Chat SDK — official Meta API |" + }, + { + "id": "reference/skills-catalog#operational-skills", + "page": "reference/skills-catalog", + "title": "Skills catalog", + "heading": "Operational skills", + "line": 88, + "text": "Setup, maintenance, migration, and day-to-day administration of your install.\n\n| Skill | What it does |\n| --- | --- |\n| `/customize` | Adds new capabilities or modifies NanoClaw behavior — interactive skill that asks questions to understand what you want |\n| `/debug` | Debugs container agent issues — logs, session DBs, mounts, authentication problems, and common failures |\n| `/init-first-agent` | Walks you through creating the first agent for a DM channel — resolves your channel identity, wires the DM group, and triggers a welcome DM |\n| `/init-onecli` | Installs and initializes OneCLI Agent Vault, migrating existing `.env` credentials to the vault |\n| `/learn` | Distills a reusable skill from a source — a directory, URL, pasted notes, or the work just done in the conversation — or refines an existing one, writing a `.claude/skills//SKILL.md` to the project's skill guidelines. Creates or refines skills; it doesn't install them from a registry |\n| `/manage-channels` | Wires channels to agent groups, manages isolation levels, and adds new channel groups |\n| `/manage-mounts` | Configures which host directories agent containers can access — view, add, or remove mount allowlist entries |\n| `/migrate-from-openclaw` | Migrates from OpenClaw to NanoClaw v2 — detects an existing install, extracts identity, credentials, and tasks, then guides interactive migration |\n| `/migrate-from-v1` | Finishes migrating a NanoClaw v1 install into v2 after `migrate-v2.sh` — seeds the owner, reconciles container configs, helps port custom v1 code |\n| `/migrate-memory` | Carries an agent group's memory across a provider switch (e.g. Claude ↔ Codex) — run after `ncl groups config update --provider`; reads the source provider's memory store, distills it into the target's, and restarts the group |\n| `/migrate-slack-agents` | Migrates a classic single-bot Slack install to one provisioned Slack app per agent group, preserving identities, workspaces, memory, and wiring — or records the choice to stay on classic |\n| `/migrate-nanoclaw` | Extracts your customizations from a fork, generates a replayable migration guide, and upgrades to upstream by reapplying them on a clean base |\n| `/setup` | Runs initial NanoClaw setup — install, configure, first-time setup |\n| `/update-nanoclaw` | Brings upstream updates into a customized install with preview, selective cherry-pick, and low token usage |\n| `/update-skills` | Refreshes installed channel/provider payloads and dependency pins from upstream without repeating credential prompts or channel wiring |" + }, + { + "id": "reference/skills-catalog#provider-installs", + "page": "reference/skills-catalog", + "title": "Skills catalog", + "heading": "Provider installs", + "line": 59, + "text": "Each skill swaps the agent runtime for a different provider. See [Providers](/extend/providers).\n\n| Skill | What it does |\n| --- | --- |\n| `/add-codex` | Uses Codex (CLI + AppServer) as the full agent provider in place of the Claude Agent SDK — ChatGPT subscription or `OPENAI_API_KEY`, per-group via `ncl groups config update --provider codex` |\n| `/add-ollama-provider` | Legacy Ollama integration workflow; its config patches need reconciliation with current database-backed group configuration before use |\n| `/add-opencode` | Installs OpenCode as an agent provider, selected per group with `ncl groups config update --provider opencode` — OpenRouter, OpenAI, Google, DeepSeek, and more via OpenCode config |" + }, + { + "id": "reference/skills-catalog#tool-installs", + "page": "reference/skills-catalog", + "title": "Skills catalog", + "heading": "Tool installs", + "line": 69, + "text": "Each skill adds a capability or MCP tool to your agents. See [Tools](/extend/tools).\n\n| Skill | What it does |\n| --- | --- |\n| `/add-anydoc` | Adds local office-document-to-Markdown conversion to agent containers with the pinned Firecrawl AnyDoc CLI — Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and text-based PDF without uploading to a hosted parser |\n| `/add-atomic-chat-tool` | Adds the Atomic Chat MCP server so agents can call local models served by the Atomic Chat desktop app via its OpenAI-compatible API |\n| `/add-clidash` | Adds clidash — a zero-dependency, read-only web dashboard that derives its tabs and tables at runtime from any CLI that lists resources as JSON, pre-wired for NanoClaw's `ncl` CLI |\n| `/add-dashboard` | Adds a monitoring dashboard — installs `@nanoco/nanoclaw-dashboard` and a pusher that sends periodic JSON snapshots |\n| `/add-dial-tool` | Gives chosen agents the [Dial](/channels/dial#the-dial-tool-sms-and-calls-from-inside-the-sandbox) `dial` CLI as a container tool — SMS, AI voice calls, and inbound codes from the sandbox; OneCLI credential injection for `api.getdial.ai`, scoped per agent with block rules |\n| `/add-karpathy-llm-wiki` | Adds a persistent wiki knowledge base to a group, based on Karpathy's LLM Wiki pattern |\n| `/add-macos-statusbar` | Adds a macOS menu bar status indicator with Start, Stop, and Restart controls (macOS only) |\n| `/add-mnemon` | Adds persistent graph-based memory via mnemon — agents recall past context before responding and remember insights after each turn |\n| `/add-ollama-tool` | Adds the Ollama MCP server so agents can call local models and optionally manage the Ollama model library |\n| `/add-rtk` | Installs the rtk token-compression proxy into agent containers — routes Bash tool calls through rtk for 60–90% token savings on dev commands |\n| `/add-tavily-tool` | Adds Tavily Search and Extract as keyless remote MCP tools through the pinned `mcp-remote` bridge — no API key by default; an optional paid-key upgrade goes through the OneCLI vault |\n| `/add-vercel` | Adds Vercel deployment capability — installs the Vercel CLI in agent containers with OneCLI credential injection for `api.vercel.com` |" + } + ], + "pairs": [ + { + "key": "ncl:approvals-status-enum|reference/ncl-cli#approvals", + "stratum": "DRIFT", + "fact": { + "id": "ncl:approvals-status-enum", + "area": "ncl", + "statement": "`ncl approvals` column `status` accepts exactly these values: `pending`, `approved`, `rejected`, `expired`.", + "evidence": { + "file": "src/cli/resources/approvals.ts", + "line": 44, + "excerpt": "name: 'status' enum: ['pending', 'approved', 'rejected', 'expired']" + } + }, + "section_id": "reference/ncl-cli#approvals", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.96 + }, + "covers": { + "type": "noul", + "noul": 0.93 + }, + "staleness": { + "type": "score", + "score": 1.82, + "confidence": 0.81, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.02, + "1": 0.14, + "2": 0.84, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1086, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.807 + }, + { + "key": "ncl:approvals-status-enum|reference/db-schema#pending-approvals", + "stratum": "DRIFT", + "fact": { + "id": "ncl:approvals-status-enum", + "area": "ncl", + "statement": "`ncl approvals` column `status` accepts exactly these values: `pending`, `approved`, `rejected`, `expired`.", + "evidence": { + "file": "src/cli/resources/approvals.ts", + "line": 44, + "excerpt": "name: 'status' enum: ['pending', 'approved', 'rejected', 'expired']" + } + }, + "section_id": "reference/db-schema#pending-approvals", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.94 + }, + "covers": { + "type": "noul", + "noul": 0.89 + }, + "staleness": { + "type": "score", + "score": 1.72, + "confidence": 0.66, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.06, + "1": 0.2, + "2": 0.71, + "3": 0.03 + } + } + }, + "usage": { + "input_tokens": 1410, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.654 + }, + { + "key": "ncl:dropped-messages-reason-enum|operate/ncl-cli#dropped-messages", + "stratum": "DRIFT", + "fact": { + "id": "ncl:dropped-messages-reason-enum", + "area": "ncl", + "statement": "`ncl dropped-messages` column `reason` accepts exactly these values: `no_agent_wired`, `no_agent_engaged`, `unknown_sender_strict`, `unknown_sender_request_approval`.", + "evidence": { + "file": "src/cli/resources/dropped-messages.ts", + "line": 17, + "excerpt": "name: 'reason' enum: ['no_agent_wired', 'no_agent_engaged', 'unknown_sender_strict', 'unknown_sender_request_approval']" + } + }, + "section_id": "operate/ncl-cli#dropped-messages", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.91 + }, + "covers": { + "type": "noul", + "noul": 0.92 + }, + "staleness": { + "type": "score", + "score": 1.7, + "confidence": 0.7, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.06, + "1": 0.19, + "2": 0.75, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 993, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.644 + }, + { + "key": "ncl:dropped-messages-reason-enum|reference/ncl-cli#dropped-messages", + "stratum": "DRIFT", + "fact": { + "id": "ncl:dropped-messages-reason-enum", + "area": "ncl", + "statement": "`ncl dropped-messages` column `reason` accepts exactly these values: `no_agent_wired`, `no_agent_engaged`, `unknown_sender_strict`, `unknown_sender_request_approval`.", + "evidence": { + "file": "src/cli/resources/dropped-messages.ts", + "line": 17, + "excerpt": "name: 'reason' enum: ['no_agent_wired', 'no_agent_engaged', 'unknown_sender_strict', 'unknown_sender_request_approval']" + } + }, + "section_id": "reference/ncl-cli#dropped-messages", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.92 + }, + "covers": { + "type": "noul", + "noul": 0.92 + }, + "staleness": { + "type": "score", + "score": 1.88, + "confidence": 0.87, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.03, + "1": 0.07, + "2": 0.9, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1066, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.605 + }, + { + "key": "ncl:dropped-messages-reason-enum|operate/troubleshooting#agent-never-replies", + "stratum": "DRIFT", + "fact": { + "id": "ncl:dropped-messages-reason-enum", + "area": "ncl", + "statement": "`ncl dropped-messages` column `reason` accepts exactly these values: `no_agent_wired`, `no_agent_engaged`, `unknown_sender_strict`, `unknown_sender_request_approval`.", + "evidence": { + "file": "src/cli/resources/dropped-messages.ts", + "line": 17, + "excerpt": "name: 'reason' enum: ['no_agent_wired', 'no_agent_engaged', 'unknown_sender_strict', 'unknown_sender_request_approval']" + } + }, + "section_id": "operate/troubleshooting#agent-never-replies", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.86 + }, + "covers": { + "type": "noul", + "noul": 0.81 + }, + "staleness": { + "type": "score", + "score": 1.78, + "confidence": 0.77, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.06, + "1": 0.1, + "2": 0.84, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1440, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.642 + }, + { + "key": "ncl:tasks-update-flags|reference/ncl-cli#tasks", + "stratum": "DRIFT", + "fact": { + "id": "ncl:tasks-update-flags", + "area": "ncl", + "statement": "`ncl tasks update` (open) declares exactly these flags (any other flag is rejected): `--id`, `--prompt`, `--process-after`, `--recurrence`, `--dangerously-override-recurrence-limit`, `--script`, `--group`, `--session`.", + "evidence": { + "file": "src/cli/resources/tasks.ts", + "line": 574, + "excerpt": "update: { access: 'open', args: [--id, --prompt, --process-after, --recurrence, --dangerously-override-recurrence-limit, --script, --group, --session] }" + } + }, + "section_id": "reference/ncl-cli#tasks", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.87 + }, + "covers": { + "type": "noul", + "noul": 0.92 + }, + "staleness": { + "type": "score", + "score": 1.78, + "confidence": 0.76, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.02, + "1": 0.19, + "2": 0.78, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 1815, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.911 + }, + { + "key": "ncl:tasks-cancel-flags|reference/ncl-cli#tasks", + "stratum": "DRIFT", + "fact": { + "id": "ncl:tasks-cancel-flags", + "area": "ncl", + "statement": "`ncl tasks cancel` (open) declares exactly these flags (any other flag is rejected): `--id`, `--all`, `--group`, `--session`.", + "evidence": { + "file": "src/cli/resources/tasks.ts", + "line": 598, + "excerpt": "cancel: { access: 'open', args: [--id, --all, --group, --session] }" + } + }, + "section_id": "reference/ncl-cli#tasks", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.88 + }, + "covers": { + "type": "noul", + "noul": 0.9 + }, + "staleness": { + "type": "score", + "score": 1.76, + "confidence": 0.75, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.04, + "1": 0.17, + "2": 0.78, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 1761, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.588 + }, + { + "key": "ncl:tasks-resume-flags|reference/ncl-cli#tasks", + "stratum": "DRIFT", + "fact": { + "id": "ncl:tasks-resume-flags", + "area": "ncl", + "statement": "`ncl tasks resume` (open) declares exactly these flags (any other flag is rejected): `--id`, `--group`, `--session`.", + "evidence": { + "file": "src/cli/resources/tasks.ts", + "line": 642, + "excerpt": "resume: { access: 'open', args: [--id, --group, --session] }" + } + }, + "section_id": "reference/ncl-cli#tasks", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.71 + }, + "covers": { + "type": "noul", + "noul": 0.85 + }, + "staleness": { + "type": "score", + "score": 1.38, + "confidence": 0.37, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.16, + "1": 0.31, + "2": 0.53, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1754, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.685 + }, + { + "key": "env:WEBHOOK_PORT|operate/troubleshooting#webhook-channel-is-silent", + "stratum": "DRIFT", + "fact": { + "id": "env:WEBHOOK_PORT", + "area": "env", + "statement": "`WEBHOOK_PORT` is read from the process environment first, then from `.env`; its default is `3000`.", + "evidence": { + "file": "src/config.ts", + "line": 119, + "excerpt": "const raw = process.env.WEBHOOK_PORT || envConfig.WEBHOOK_PORT || '3000';" + } + }, + "section_id": "operate/troubleshooting#webhook-channel-is-silent", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.93 + }, + "covers": { + "type": "noul", + "noul": 0.63 + }, + "staleness": { + "type": "score", + "score": 1.94, + "confidence": 0.85, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.05, + "1": 0.02, + "2": 0.88, + "3": 0.05 + } + } + }, + "usage": { + "input_tokens": 1113, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.621 + }, + { + "key": "env:WEBHOOK_PORT|reference/environment-variables#host-process-environment-only", + "stratum": "DRIFT", + "fact": { + "id": "env:WEBHOOK_PORT", + "area": "env", + "statement": "`WEBHOOK_PORT` is read from the process environment first, then from `.env`; its default is `3000`.", + "evidence": { + "file": "src/config.ts", + "line": 119, + "excerpt": "const raw = process.env.WEBHOOK_PORT || envConfig.WEBHOOK_PORT || '3000';" + } + }, + "section_id": "reference/environment-variables#host-process-environment-only", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.78 + }, + "covers": { + "type": "noul", + "noul": 0.56 + }, + "staleness": { + "type": "score", + "score": 1.89, + "confidence": 0.82, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.05, + "1": 0.04, + "2": 0.88, + "3": 0.03 + } + } + }, + "usage": { + "input_tokens": 1440, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.613 + }, + { + "key": "skills:add-ollama-provider|extend/providers#ollama-add-ollama-provider", + "stratum": "DRIFT", + "fact": { + "id": "skills:add-ollama-provider", + "area": "skills", + "statement": "A workspace skill `/add-ollama-provider` exists under `.claude/skills/`; it is invoked when: Route a NanoClaw agent group to a local Ollama model instead of the Anthropic API. Ollama speaks the Anthropic API natively (v1/messages), so no provider code changes are needed — just env var overrides and a model setting. Use when the user wants to run their agent locally, cut API costs, or experiment with open-weight models. See docs/ollama.md for background.", + "evidence": { + "file": ".claude/skills/add-ollama-provider/SKILL.md", + "line": 2, + "excerpt": "name: add-ollama-provider / description: Route a NanoClaw agent group to a local Ollama model instead of the Anthropic API. Ollama speaks the Anthropic API natively (v1/messages), so no provider code changes are needed — just env var overrides and a model setting. Use when the user wants to run their agent locally, cut API costs, or experiment with open-weight models. See docs/ollama.md for backg…" + } + }, + "section_id": "extend/providers#ollama-add-ollama-provider", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.76 + }, + "covers": { + "type": "noul", + "noul": 0.93 + }, + "staleness": { + "type": "score", + "score": 1.69, + "confidence": 0.52, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.19, + "1": 0.01, + "2": 0.71, + "3": 0.09 + } + } + }, + "usage": { + "input_tokens": 1202, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.695 + }, + { + "key": "timestamps:rule-1|guides/scheduled-tasks#schedule-recurring-work", + "stratum": "MISSING", + "fact": { + "id": "timestamps:rule-1", + "area": "timestamps", + "statement": "**Storage**: every timestamp written from JS is `new Date().toISOString()` (ISO-8601 UTC with `Z`). Central SQL receives that timestamp as a parameter; never use `datetime('now')` or `strftime(...)` there because they are SQLite-specific and the naive shape is misparsed as local time by `new Date()`. SQLite-only mailbox SQL may use `strftime('%Y-%m-%dT%H:%M:%fZ','now')`. SQLite-only comparisons wrap both sides in `datetime()`.", + "evidence": { + "file": "CLAUDE.md", + "line": 268, + "excerpt": "**Storage**: every timestamp written from JS is `new Date().toISOString()` (ISO-8601 UTC with `Z`). Central SQL receives that timestamp as a parameter; never use `datetime('now')` or `strftime(...)` there because they are SQLite-specific and the naive shape is misparsed as local time by `new Date()`. SQLite-only mailbox SQL may use `strftime('%Y-%m-%dT%H:%M:%fZ','now')`. SQLite-only comparisons w…" + } + }, + "section_id": "guides/scheduled-tasks#schedule-recurring-work", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.06 + }, + "covers": { + "type": "noul", + "noul": 0.08 + }, + "staleness": { + "type": "score", + "score": 0.06, + "confidence": 0.94, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.96, + "1": 0.01, + "2": 0.02, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 2537, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.718 + }, + { + "key": "ncl:groups-config-remove-package-flags|reference/ncl-cli#groups", + "stratum": "UNSURE", + "fact": { + "id": "ncl:groups-config-remove-package-flags", + "area": "ncl", + "statement": "`ncl groups config remove-package` (approval) names these flags in its help text: `--rebuild`, `--id`, `--apt`, `--npm`.", + "evidence": { + "file": "src/cli/resources/groups.ts", + "line": 553, + "excerpt": "config remove-package: { access: 'approval', description: ... --rebuild --id --apt --npm }" + } + }, + "section_id": "reference/ncl-cli#groups", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.34 + }, + "covers": { + "type": "noul", + "noul": 0.86 + }, + "staleness": { + "type": "score", + "score": 0.98, + "confidence": 0.42, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.3, + "1": 0.43, + "2": 0.27, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 2257, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.715 + }, + { + "key": "ncl:sessions-history-flags|reference/ncl-cli#sessions", + "stratum": "UNSURE", + "fact": { + "id": "ncl:sessions-history-flags", + "area": "ncl", + "statement": "`ncl sessions history` (open) declares exactly these flags (any other flag is rejected): `--id`, `--limit`.", + "evidence": { + "file": "src/cli/resources/sessions.ts", + "line": 54, + "excerpt": "history: { access: 'open', args: [--id, --limit] }" + } + }, + "section_id": "reference/ncl-cli#sessions", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.34 + }, + "covers": { + "type": "noul", + "noul": 0.83 + }, + "staleness": { + "type": "score", + "score": 1.27, + "confidence": 0.26, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.23, + "1": 0.28, + "2": 0.49, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1175, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.62 + }, + { + "key": "ncl:tasks-get-flags|reference/ncl-cli#tasks", + "stratum": "UNSURE", + "fact": { + "id": "ncl:tasks-get-flags", + "area": "ncl", + "statement": "`ncl tasks get` (open) declares exactly these flags (any other flag is rejected): `--id`, `--group`, `--session`.", + "evidence": { + "file": "src/cli/resources/tasks.ts", + "line": 470, + "excerpt": "get: { access: 'open', args: [--id, --group, --session] }" + } + }, + "section_id": "reference/ncl-cli#tasks", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.37 + }, + "covers": { + "type": "noul", + "noul": 0.91 + }, + "staleness": { + "type": "score", + "score": 1.37, + "confidence": 0.36, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.11, + "1": 0.42, + "2": 0.47, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1754, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.613 + }, + { + "key": "ncl:tasks-append-log-flags|reference/ncl-cli#tasks", + "stratum": "UNSURE", + "fact": { + "id": "ncl:tasks-append-log-flags", + "area": "ncl", + "statement": "`ncl tasks append-log` (open) declares exactly these flags (any other flag is rejected): `--msg`, `--id`, `--group`.", + "evidence": { + "file": "src/cli/resources/tasks.ts", + "line": 546, + "excerpt": "append-log: { access: 'open', args: [--msg, --id, --group] }" + } + }, + "section_id": "reference/ncl-cli#tasks", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.62 + }, + "covers": { + "type": "noul", + "noul": 0.92 + }, + "staleness": { + "type": "score", + "score": 1.63, + "confidence": 0.62, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.03, + "1": 0.31, + "2": 0.66, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1756, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.6 + }, + { + "key": "ncl:tasks-run-flags|reference/ncl-cli#tasks", + "stratum": "UNSURE", + "fact": { + "id": "ncl:tasks-run-flags", + "area": "ncl", + "statement": "`ncl tasks run` (open) declares exactly these flags (any other flag is rejected): `--id`, `--group`, `--session`.", + "evidence": { + "file": "src/cli/resources/tasks.ts", + "line": 613, + "excerpt": "run: { access: 'open', args: [--id, --group, --session] }" + } + }, + "section_id": "reference/ncl-cli#tasks", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.69 + }, + "covers": { + "type": "noul", + "noul": 0.89 + }, + "staleness": { + "type": "score", + "score": 1.39, + "confidence": 0.38, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.16, + "1": 0.29, + "2": 0.54, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 1754, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.661 + }, + { + "key": "ncl:tasks-pause-flags|reference/ncl-cli#tasks", + "stratum": "UNSURE", + "fact": { + "id": "ncl:tasks-pause-flags", + "area": "ncl", + "statement": "`ncl tasks pause` (open) declares exactly these flags (any other flag is rejected): `--id`, `--group`, `--session`.", + "evidence": { + "file": "src/cli/resources/tasks.ts", + "line": 628, + "excerpt": "pause: { access: 'open', args: [--id, --group, --session] }" + } + }, + "section_id": "reference/ncl-cli#tasks", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.46 + }, + "covers": { + "type": "noul", + "noul": 0.83 + }, + "staleness": { + "type": "score", + "score": 1.4, + "confidence": 0.39, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.12, + "1": 0.37, + "2": 0.51, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1754, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.651 + }, + { + "key": "ncl:tasks-delete-flags|reference/ncl-cli#tasks", + "stratum": "UNSURE", + "fact": { + "id": "ncl:tasks-delete-flags", + "area": "ncl", + "statement": "`ncl tasks delete` (open) declares exactly these flags (any other flag is rejected): `--id`, `--group`, `--session`.", + "evidence": { + "file": "src/cli/resources/tasks.ts", + "line": 656, + "excerpt": "delete: { access: 'open', args: [--id, --group, --session] }" + } + }, + "section_id": "reference/ncl-cli#tasks", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.35 + }, + "covers": { + "type": "noul", + "noul": 0.89 + }, + "staleness": { + "type": "score", + "score": 1.41, + "confidence": 0.4, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.08, + "1": 0.43, + "2": 0.48, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 1754, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.664 + }, + { + "key": "env:NANOCLAW_DEFAULT_MODEL|reference/environment-variables#read-from-env", + "stratum": "UNSURE", + "fact": { + "id": "env:NANOCLAW_DEFAULT_MODEL", + "area": "env", + "statement": "`NANOCLAW_DEFAULT_MODEL` is read from the process environment first, then from `.env`; it is unset (empty) by default.", + "evidence": { + "file": "src/config.ts", + "line": 50, + "excerpt": "export const DEFAULT_MODEL = process.env.NANOCLAW_DEFAULT_MODEL || envConfig.NANOCLAW_DEFAULT_MODEL || '';" + } + }, + "section_id": "reference/environment-variables#read-from-env", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.45 + }, + "covers": { + "type": "noul", + "noul": 0.84 + }, + "staleness": { + "type": "score", + "score": 1.24, + "confidence": 0.23, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.31, + "1": 0.14, + "2": 0.54, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 2155, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.708 + }, + { + "key": "env:NANOCLAW_FAST_MODE|reference/environment-variables#read-from-env", + "stratum": "UNSURE", + "fact": { + "id": "env:NANOCLAW_FAST_MODE", + "area": "env", + "statement": "`NANOCLAW_FAST_MODE` is read from the process environment first, then from `.env`; it is unset (empty) by default.", + "evidence": { + "file": "src/config.ts", + "line": 56, + "excerpt": "(process.env.NANOCLAW_FAST_MODE || envConfig.NANOCLAW_FAST_MODE || '').toLowerCase()," + } + }, + "section_id": "reference/environment-variables#read-from-env", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.44 + }, + "covers": { + "type": "noul", + "noul": 0.9 + }, + "staleness": { + "type": "score", + "score": 0.44, + "confidence": 0.56, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.65, + "1": 0.28, + "2": 0.07, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 2155, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.723 + }, + { + "key": "skills:catalog|reference/skills-catalog#tool-installs", + "stratum": "UNSURE", + "fact": { + "id": "skills:catalog", + "area": "skills", + "statement": "There are exactly 52 workspace skills under `.claude/skills/` with a SKILL.md: `add-anydoc`, `add-atomic-chat-tool`, `add-clidash`, `add-codex`, `add-dashboard`, `add-deltachat`, `add-dial`, `add-dial-number`, `add-dial-tool`, `add-discord`, `add-emacs`, `add-gchat`, `add-github`, `add-imessage`, `add-karpathy-llm-wiki`, `add-linear`, `add-macos-statusbar`, `add-matrix`, `add-mattermost`, `add-mnemon`, `add-ollama-provider`, `add-ollama-tool`, `add-opencode`, `add-resend`, `add-rtk`, `add-signal`, `add-slack`, `add-tavily-tool`, `add-teams`, `add-telegram`, `add-vercel`, `add-webex`, `add-wechat`, `add-whatsapp`, `add-whatsapp-cloud`, `customize`, `debug`, `init-first-agent`, `init-onecli`, `learn`, `manage-channels`, `manage-mounts`, `migrate-from-openclaw`, `migrate-from-v1`, `migrate-memory`, `migrate-nanoclaw`, `migrate-slack-agents`, `setup`, `slack-a2a-rooms`, `slack-agent-flow`, `update-nanoclaw`, `update-skills`.", + "evidence": { + "file": ".claude/skills", + "line": 1, + "excerpt": "directory listing" + } + }, + "section_id": "reference/skills-catalog#tool-installs", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.06 + }, + "covers": { + "type": "noul", + "noul": 0.45 + }, + "staleness": { + "type": "score", + "score": 0.26, + "confidence": 0.74, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.84, + "1": 0.07, + "2": 0.09, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1718, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.591 + }, + { + "key": "timestamps:rule-2|reference/environment-variables#container-side-variables-agent-runner", + "stratum": "UNSURE", + "fact": { + "id": "timestamps:rule-2", + "area": "timestamps", + "statement": "**Display**: anything shown to an agent or a user renders in the install timezone — `formatLocalTime` (prose) or `formatLocalStamp` (log lines) from `src/timezone.ts` / `container/agent-runner/src/timezone.ts`. `--json` output, DB values, and operator logs stay ISO.", + "evidence": { + "file": "CLAUDE.md", + "line": 269, + "excerpt": "**Display**: anything shown to an agent or a user renders in the install timezone — `formatLocalTime` (prose) or `formatLocalStamp` (log lines) from `src/timezone.ts` / `container/agent-runner/src/timezone.ts`. `--json` output, DB values, and operator logs stay ISO." + } + }, + "section_id": "reference/environment-variables#container-side-variables-agent-runner", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.08 + }, + "covers": { + "type": "noul", + "noul": 0.46 + }, + "staleness": { + "type": "score", + "score": 0.25, + "confidence": 0.75, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.81, + "1": 0.13, + "2": 0.06, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1470, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.617 + }, + { + "key": "skills:update-nanoclaw|operate/upgrading#updating-nanoclaw", + "stratum": "OK", + "fact": { + "id": "skills:update-nanoclaw", + "area": "skills", + "statement": "A workspace skill `/update-nanoclaw` exists under `.claude/skills/`; it is invoked when: Transactionally update a customized NanoClaw checkout from official upstream without exposing live mounted source, with fork-safe skill refresh, mutable-state snapshots, migration gates, exact-code upgrade markers, detected service restart, health verification, and automatic local rollback. Use for routine merge, rebase, or selective upstream updates.", + "evidence": { + "file": ".claude/skills/update-nanoclaw/SKILL.md", + "line": 2, + "excerpt": "name: update-nanoclaw / description: Transactionally update a customized NanoClaw checkout from official upstream without exposing live mounted source, with fork-safe skill refresh, mutable-state snapshots, migration gates, exact-code upgrade markers, detected service restart, health verification, and automatic local rollback. Use for routine merge, rebase, or selective upstream updates." + } + }, + "section_id": "operate/upgrading#updating-nanoclaw", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.05 + }, + "covers": { + "type": "noul", + "noul": 0.93 + }, + "staleness": { + "type": "score", + "score": 0.13, + "confidence": 0.87, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.93, + "1": 0.03, + "2": 0.02, + "3": 0.02 + } + } + }, + "usage": { + "input_tokens": 1895, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.662 + }, + { + "key": "gateway:selection-unknown|reference/environment-variables#read-from-env", + "stratum": "OK", + "fact": { + "id": "gateway:selection-unknown", + "area": "gateway", + "statement": "Setting `NANOCLAW_GATEWAY_PROVIDER` to a name with no registered gateway provider throws at startup and lists the installed kinds; other gateways arrive as skills (overlays) and the host never silently falls back to another gateway.", + "evidence": { + "file": "src/gateway-providers/index.ts", + "line": 44, + "excerpt": "`NANOCLAW_GATEWAY_PROVIDER='${kind}' but no gateway provider is registered for '${kind}'; ` +" + } + }, + "section_id": "reference/environment-variables#read-from-env", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.08 + }, + "covers": { + "type": "noul", + "noul": 0.88 + }, + "staleness": { + "type": "score", + "score": 0.29, + "confidence": 0.71, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.79, + "1": 0.14, + "2": 0.07, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 2180, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.714 + }, + { + "key": "skills:learn|reference/skills-catalog#operational-skills", + "stratum": "OK", + "fact": { + "id": "skills:learn", + "area": "skills", + "statement": "A workspace skill `/learn` exists under `.claude/skills/`; it is invoked when: Distill a reusable skill from anything — a directory, a URL, pasted notes, or what you just did together — or refine an existing skill with new learnings. Use when the user says '/learn', 'learn this', 'turn this into a skill', 'capture this workflow', 'make a skill from ', or 'improve/update the skill'. Produces or updates a .claude/skills//SKILL.md authored to NanoClaw's skill guidelines. (This CREATES or REFINES a skill from a source; it does not install existing skills from a registry.)", + "evidence": { + "file": ".claude/skills/learn/SKILL.md", + "line": 2, + "excerpt": "name: learn / description: Distill a reusable skill from anything — a directory, a URL, pasted notes, or what you just did together — or refine an existing skill with new learnings. Use when the user says '/learn', 'learn this', 'turn this into a skill', 'capture this workflow', 'make a skill from ', or 'improve/update the skill'. Produces or updates a .claude/skills//SKILL.m…" + } + }, + "section_id": "reference/skills-catalog#operational-skills", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.93 + }, + "staleness": { + "type": "score", + "score": 0.02, + "confidence": 0.98, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.98, + "1": 0.02, + "2": 0.0, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1703, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.658 + }, + { + "key": "skills:debug|reference/skills-catalog#operational-skills", + "stratum": "OK", + "fact": { + "id": "skills:debug", + "area": "skills", + "statement": "A workspace skill `/debug` exists under `.claude/skills/`; it is invoked when: Debug container agent issues. Use when things aren't working, container fails, authentication problems, or to understand how the container system works. Covers logs, session DBs, mounts, and common issues.", + "evidence": { + "file": ".claude/skills/debug/SKILL.md", + "line": 2, + "excerpt": "name: debug / description: Debug container agent issues. Use when things aren't working, container fails, authentication problems, or to understand how the container system works. Covers logs, session DBs, mounts, and common issues." + } + }, + "section_id": "reference/skills-catalog#operational-skills", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.92 + }, + "staleness": { + "type": "score", + "score": 0.02, + "confidence": 0.98, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.99, + "1": 0.01, + "2": 0.0, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1559, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.647 + }, + { + "key": "ncl:destinations-target_type-enum|reference/ncl-cli#destinations", + "stratum": "OK", + "fact": { + "id": "ncl:destinations-target_type-enum", + "area": "ncl", + "statement": "`ncl destinations` column `target_type` accepts exactly these values: `channel`, `agent`.", + "evidence": { + "file": "src/cli/resources/destinations.ts", + "line": 53, + "excerpt": "name: 'target_type' enum: ['channel', 'agent']" + } + }, + "section_id": "reference/ncl-cli#destinations", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.95 + }, + "staleness": { + "type": "score", + "score": 0.01, + "confidence": 0.99, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.99, + "1": 0.01, + "2": 0.0, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1085, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.714 + }, + { + "key": "skills:add-wechat|channels/more-channels#wechat", + "stratum": "OK", + "fact": { + "id": "skills:add-wechat", + "area": "skills", + "statement": "A workspace skill `/add-wechat` exists under `.claude/skills/`; it is invoked when: Add WeChat (personal) channel integration via Tencent's official iLink Bot API. Uses long-polling and QR scan — no webhook, no ToS risk, no paid token.", + "evidence": { + "file": ".claude/skills/add-wechat/SKILL.md", + "line": 2, + "excerpt": "name: add-wechat / description: Add WeChat (personal) channel integration via Tencent's official iLink Bot API. Uses long-polling and QR scan — no webhook, no ToS risk, no paid token." + } + }, + "section_id": "channels/more-channels#wechat", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.93 + }, + "staleness": { + "type": "score", + "score": 0.2, + "confidence": 0.8, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.9, + "1": 0.04, + "2": 0.02, + "3": 0.04 + } + } + }, + "usage": { + "input_tokens": 1092, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.727 + }, + { + "key": "ncl:wirings-engage_mode-enum|operate/ncl-cli#messaging-groups-and-wirings", + "stratum": "OK", + "fact": { + "id": "ncl:wirings-engage_mode-enum", + "area": "ncl", + "statement": "`ncl wirings` column `engage_mode` accepts exactly these values: `pattern`, `mention`, `mention-sticky`.", + "evidence": { + "file": "src/cli/resources/wirings.ts", + "line": 71, + "excerpt": "name: 'engage_mode' enum: ['pattern', 'mention', 'mention-sticky']" + } + }, + "section_id": "operate/ncl-cli#messaging-groups-and-wirings", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.04 + }, + "covers": { + "type": "noul", + "noul": 0.96 + }, + "staleness": { + "type": "score", + "score": 0.02, + "confidence": 0.98, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.99, + "1": 0.01, + "2": 0.0, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1217, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.591 + }, + { + "key": "skills:add-mattermost|channels/mattermost#install", + "stratum": "OK", + "fact": { + "id": "skills:add-mattermost", + "area": "skills", + "statement": "A workspace skill `/add-mattermost` exists under `.claude/skills/`; it is invoked when: Add a self-hosted or cloud Mattermost bot channel through the Chat SDK bridge, reusing a healthy server when available and linking to official server setup guidance when needed.", + "evidence": { + "file": ".claude/skills/add-mattermost/SKILL.md", + "line": 2, + "excerpt": "name: add-mattermost / description: Add a self-hosted or cloud Mattermost bot channel through the Chat SDK bridge, reusing a healthy server when available and linking to official server setup guidance when needed." + } + }, + "section_id": "channels/mattermost#install", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.04 + }, + "covers": { + "type": "noul", + "noul": 0.91 + }, + "staleness": { + "type": "score", + "score": 0.02, + "confidence": 0.98, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.98, + "1": 0.01, + "2": 0.01, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1750, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.687 + }, + { + "key": "container-config:cli_scope-default|reference/container-config#cli-scope", + "stratum": "OK", + "fact": { + "id": "container-config:cli_scope-default", + "area": "container-config", + "statement": "`container_configs.cli_scope` defaults to 'group'.", + "evidence": { + "file": "src/db/migrations/015-cli-scope.ts", + "line": 9, + "excerpt": "cli_scope NOT NULL DEFAULT 'group'" + } + }, + "section_id": "reference/container-config#cli-scope", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.02 + }, + "covers": { + "type": "noul", + "noul": 0.98 + }, + "staleness": { + "type": "score", + "score": 0.0, + "confidence": 1.0, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 1.0, + "1": 0.0, + "2": 0.0, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1036, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.654 + }, + { + "key": "env:NANOCLAW_RUNTIME_DRIVER|reference/environment-variables#read-from-env", + "stratum": "OK", + "fact": { + "id": "env:NANOCLAW_RUNTIME_DRIVER", + "area": "env", + "statement": "`NANOCLAW_RUNTIME_DRIVER` selects the session runtime driver; it is read from the process environment first, then from `.env`, and defaults to `docker`. A configured name with no registered implementation is an error at startup, not a fallback.", + "evidence": { + "file": "src/drivers/index.ts", + "line": 63, + "excerpt": "DEFAULT_DRIVER_KIND = 'docker'" + } + }, + "section_id": "reference/environment-variables#read-from-env", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.94 + }, + "staleness": { + "type": "score", + "score": 0.02, + "confidence": 0.98, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.98, + "1": 0.01, + "2": 0.01, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 2160, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.761 + }, + { + "key": "skills:migrate-slack-agents|reference/skills-catalog#operational-skills", + "stratum": "OK", + "fact": { + "id": "skills:migrate-slack-agents", + "area": "skills", + "statement": "A workspace skill `/migrate-slack-agents` exists under `.claude/skills/`; it is invoked when: Migrate a classic single-bot Slack install to one provisioned Slack app per existing agent group while preserving agent identities, workspaces, memory, and wiring behavior — or record the operator's choice to stay on classic, which remains supported. Use when /update-nanoclaw surfaces the Slack agents requirement, or standalone any time later.", + "evidence": { + "file": ".claude/skills/migrate-slack-agents/SKILL.md", + "line": 2, + "excerpt": "name: migrate-slack-agents / description: Migrate a classic single-bot Slack install to one provisioned Slack app per existing agent group while preserving agent identities, workspaces, memory, and wiring behavior — or record the operator's choice to stay on classic, which remains supported. Use when /update-nanoclaw surfaces the Slack agents requirement, or standalone any time later." + } + }, + "section_id": "reference/skills-catalog#operational-skills", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.04 + }, + "covers": { + "type": "noul", + "noul": 0.94 + }, + "staleness": { + "type": "score", + "score": 0.33, + "confidence": 0.67, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.69, + "1": 0.29, + "2": 0.02, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1637, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.745 + }, + { + "key": "env:CONTAINER_MEMORY_LIMIT|reference/environment-variables#read-from-env", + "stratum": "OK", + "fact": { + "id": "env:CONTAINER_MEMORY_LIMIT", + "area": "env", + "statement": "`CONTAINER_MEMORY_LIMIT` is read from the process environment first, then from `.env`; it is unset (empty) by default.", + "evidence": { + "file": "src/config.ts", + "line": 100, + "excerpt": "export const CONTAINER_MEMORY_LIMIT = process.env.CONTAINER_MEMORY_LIMIT || envConfig.CONTAINER_MEMORY_LIMIT || '';" + } + }, + "section_id": "reference/environment-variables#read-from-env", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.11 + }, + "covers": { + "type": "noul", + "noul": 0.91 + }, + "staleness": { + "type": "score", + "score": 0.35, + "confidence": 0.65, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.72, + "1": 0.21, + "2": 0.06, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 2160, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.693 + }, + { + "key": "skills:add-whatsapp-cloud|reference/skills-catalog#channel-installs", + "stratum": "OK", + "fact": { + "id": "skills:add-whatsapp-cloud", + "area": "skills", + "statement": "A workspace skill `/add-whatsapp-cloud` exists under `.claude/skills/`; it is invoked when: Add WhatsApp Business Cloud API channel via Chat SDK. Official Meta API.", + "evidence": { + "file": ".claude/skills/add-whatsapp-cloud/SKILL.md", + "line": 2, + "excerpt": "name: add-whatsapp-cloud / description: Add WhatsApp Business Cloud API channel via Chat SDK. Official Meta API." + } + }, + "section_id": "reference/skills-catalog#channel-installs", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.02 + }, + "covers": { + "type": "noul", + "noul": 0.96 + }, + "staleness": { + "type": "score", + "score": 0.0, + "confidence": 1.0, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 1.0, + "1": 0.0, + "2": 0.0, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1716, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.658 + }, + { + "key": "ncl:wirings-create-flags|reference/ncl-cli#wirings", + "stratum": "OK", + "fact": { + "id": "ncl:wirings-create-flags", + "area": "ncl", + "statement": "`ncl wirings create` (approval) names these flags in its help text: `--messaging-group-id`, `--channel-type`, `--platform-id`, `--instance`, `--agent-group-id`, `--agent-group`, `--engage-mode`, `--engage-pattern`, `--session-mode`, `--sender-scope`, `--ignored-message-policy`, `--threads`, `--priority`.", + "evidence": { + "file": "src/cli/resources/wirings.ts", + "line": 168, + "excerpt": "create: { access: 'approval', description: ... --messaging-group-id --channel-type --platform-id --instance --agent-group-id --agent-group --engage-mode --engage-pattern --session-mode --sender-scope --ignored-message-policy --threads --priority }" + } + }, + "section_id": "reference/ncl-cli#wirings", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.08 + }, + "covers": { + "type": "noul", + "noul": 0.89 + }, + "staleness": { + "type": "score", + "score": 0.81, + "confidence": 0.43, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.38, + "1": 0.43, + "2": 0.18, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 1731, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.676 + }, + { + "key": "skills:add-linear|reference/skills-catalog#channel-installs", + "stratum": "OK", + "fact": { + "id": "skills:add-linear", + "area": "skills", + "statement": "A workspace skill `/add-linear` exists under `.claude/skills/`; it is invoked when: Add Linear channel integration via Chat SDK. Issue comment threads as conversations.", + "evidence": { + "file": ".claude/skills/add-linear/SKILL.md", + "line": 2, + "excerpt": "name: add-linear / description: Add Linear channel integration via Chat SDK. Issue comment threads as conversations." + } + }, + "section_id": "reference/skills-catalog#channel-installs", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.02 + }, + "covers": { + "type": "noul", + "noul": 0.95 + }, + "staleness": { + "type": "score", + "score": 0.01, + "confidence": 0.99, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 1.0, + "1": 0.0, + "2": 0.0, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1703, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.6 + }, + { + "key": "ncl:approvals-verbs|reference/ncl-cli#approvals", + "stratum": "OK", + "fact": { + "id": "ncl:approvals-verbs", + "area": "ncl", + "statement": "`ncl approvals` exposes exactly these verbs, with their access level for agent callers: list (open), get (open). `open` runs inline; `approval` sends an approval card when called from inside a container.", + "evidence": { + "file": "src/cli/resources/approvals.ts", + "line": 3, + "excerpt": "registerResource plural 'approvals' operations {'list': 'open', 'get': 'open'} custom []" + } + }, + "section_id": "reference/ncl-cli#approvals", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.15 + }, + "covers": { + "type": "noul", + "noul": 0.85 + }, + "staleness": { + "type": "score", + "score": 0.24, + "confidence": 0.76, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.83, + "1": 0.09, + "2": 0.07, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 1108, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.792 + }, + { + "key": "env:ONECLI_API_KEY|reference/environment-variables#read-from-env", + "stratum": "OK", + "fact": { + "id": "env:ONECLI_API_KEY", + "area": "env", + "statement": "`ONECLI_API_KEY` is read from the process environment first, then from `.env`; it has no default.", + "evidence": { + "file": "src/config.ts", + "line": 95, + "excerpt": "export const ONECLI_API_KEY = process.env.ONECLI_API_KEY || envConfig.ONECLI_API_KEY;" + } + }, + "section_id": "reference/environment-variables#read-from-env", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.22 + }, + "covers": { + "type": "noul", + "noul": 0.85 + }, + "staleness": { + "type": "score", + "score": 0.94, + "confidence": 0.06, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.41, + "1": 0.27, + "2": 0.28, + "3": 0.04 + } + } + }, + "usage": { + "input_tokens": 2146, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.651 + }, + { + "key": "ncl:groups-delete-flags|reference/ncl-cli#groups", + "stratum": "OK", + "fact": { + "id": "ncl:groups-delete-flags", + "area": "ncl", + "statement": "`ncl groups delete` (approval) names these flags in its help text: `--id`.", + "evidence": { + "file": "src/cli/resources/groups.ts", + "line": 225, + "excerpt": "delete: { access: 'approval', description: ... --id }" + } + }, + "section_id": "reference/ncl-cli#groups", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.85 + }, + "staleness": { + "type": "score", + "score": 0.03, + "confidence": 0.97, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.98, + "1": 0.01, + "2": 0.01, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 2233, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.582 + }, + { + "key": "ncl:policies-remove-flags|reference/ncl-cli#policies", + "stratum": "OK", + "fact": { + "id": "ncl:policies-remove-flags", + "area": "ncl", + "statement": "`ncl policies remove` (approval) names these flags in its help text: `--from`, `--to`.", + "evidence": { + "file": "src/cli/resources/policies.ts", + "line": 44, + "excerpt": "remove: { access: 'approval', description: ... --from --to }" + } + }, + "section_id": "reference/ncl-cli#policies", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.04 + }, + "covers": { + "type": "noul", + "noul": 0.82 + }, + "staleness": { + "type": "score", + "score": 0.04, + "confidence": 0.96, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.98, + "1": 0.01, + "2": 0.01, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1137, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.637 + }, + { + "key": "ncl:tasks-create-flags|reference/ncl-cli#tasks", + "stratum": "OK", + "fact": { + "id": "ncl:tasks-create-flags", + "area": "ncl", + "statement": "`ncl tasks create` (open) declares exactly these flags (any other flag is rejected): `--name`, `--prompt`, `--recurrence`, `--dangerously-override-recurrence-limit`, `--process-after`, `--script`, `--group`.", + "evidence": { + "file": "src/cli/resources/tasks.ts", + "line": 484, + "excerpt": "create: { access: 'open', args: [--name, --prompt, --recurrence, --dangerously-override-recurrence-limit, --process-after, --script, --group] }" + } + }, + "section_id": "reference/ncl-cli#tasks", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.08 + }, + "covers": { + "type": "noul", + "noul": 0.93 + }, + "staleness": { + "type": "score", + "score": 0.12, + "confidence": 0.88, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.91, + "1": 0.06, + "2": 0.03, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1808, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.614 + }, + { + "key": "ncl:roles-role-enum|reference/ncl-cli#roles", + "stratum": "OK", + "fact": { + "id": "ncl:roles-role-enum", + "area": "ncl", + "statement": "`ncl roles` column `role` accepts exactly these values: `owner`, `admin`.", + "evidence": { + "file": "src/cli/resources/roles.ts", + "line": 5, + "excerpt": "name: 'role' enum: ['owner', 'admin']" + } + }, + "section_id": "reference/ncl-cli#roles", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.95 + }, + "staleness": { + "type": "score", + "score": 0.01, + "confidence": 0.99, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 1.0, + "1": 0.0, + "2": 0.0, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1096, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.65 + }, + { + "key": "skills:init-first-agent|reference/skills-catalog#operational-skills", + "stratum": "OK", + "fact": { + "id": "skills:init-first-agent", + "area": "skills", + "statement": "A workspace skill `/init-first-agent` exists under `.claude/skills/`; it is invoked when: Walk the operator through wiring the first NanoClaw agent to a DM channel — resolve the operator's channel identity, select or create the agent, and trigger a welcome DM via the normal delivery path. Use after channel credentials are configured and the service is running.", + "evidence": { + "file": ".claude/skills/init-first-agent/SKILL.md", + "line": 2, + "excerpt": "name: init-first-agent / description: Walk the operator through wiring the first NanoClaw agent to a DM channel — resolve the operator's channel identity, select or create the agent, and trigger a welcome DM via the normal delivery path. Use after channel credentials are configured and the service is running." + } + }, + "section_id": "reference/skills-catalog#operational-skills", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.04 + }, + "covers": { + "type": "noul", + "noul": 0.93 + }, + "staleness": { + "type": "score", + "score": 0.29, + "confidence": 0.71, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.74, + "1": 0.24, + "2": 0.02, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1591, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.678 + }, + { + "key": "skills:add-opencode|reference/skills-catalog#provider-installs", + "stratum": "OK", + "fact": { + "id": "skills:add-opencode", + "area": "skills", + "statement": "A workspace skill `/add-opencode` exists under `.claude/skills/`; it is invoked when: Use OpenCode as an agent provider. OpenRouter, OpenAI, Google, DeepSeek, etc. via OpenCode config — not the Anthropic Agent SDK. Per group via `ncl groups config update --provider opencode`; host passes OPENCODE_* and XDG mount when spawning containers.", + "evidence": { + "file": ".claude/skills/add-opencode/SKILL.md", + "line": 2, + "excerpt": "name: add-opencode / description: Use OpenCode as an agent provider. OpenRouter, OpenAI, Google, DeepSeek, etc. via OpenCode config — not the Anthropic Agent SDK. Per group via `ncl groups config update --provider opencode`; host passes OPENCODE_* and XDG mount when spawning containers." + } + }, + "section_id": "reference/skills-catalog#provider-installs", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.06 + }, + "covers": { + "type": "noul", + "noul": 0.92 + }, + "staleness": { + "type": "score", + "score": 0.6, + "confidence": 0.4, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.5, + "1": 0.42, + "2": 0.08, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1184, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.726 + }, + { + "key": "skills:add-whatsapp|reference/skills-catalog#channel-installs", + "stratum": "OK", + "fact": { + "id": "skills:add-whatsapp", + "area": "skills", + "statement": "A workspace skill `/add-whatsapp` exists under `.claude/skills/`; it is invoked when: Add WhatsApp channel via native Baileys adapter. Direct connection — no Chat SDK bridge. Uses QR code or pairing code for authentication.", + "evidence": { + "file": ".claude/skills/add-whatsapp/SKILL.md", + "line": 2, + "excerpt": "name: add-whatsapp / description: Add WhatsApp channel via native Baileys adapter. Direct connection — no Chat SDK bridge. Uses QR code or pairing code for authentication." + } + }, + "section_id": "reference/skills-catalog#channel-installs", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.95 + }, + "staleness": { + "type": "score", + "score": 0.06, + "confidence": 0.94, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.95, + "1": 0.05, + "2": 0.0, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1739, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.692 + }, + { + "key": "ncl:members-add-flags|reference/ncl-cli#members", + "stratum": "OK", + "fact": { + "id": "ncl:members-add-flags", + "area": "ncl", + "statement": "`ncl members add` (approval) names these flags in its help text: `--user`, `--group`.", + "evidence": { + "file": "src/cli/resources/members.ts", + "line": 33, + "excerpt": "add: { access: 'approval', description: ... --user --group }" + } + }, + "section_id": "reference/ncl-cli#members", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.06 + }, + "covers": { + "type": "noul", + "noul": 0.81 + }, + "staleness": { + "type": "score", + "score": 0.15, + "confidence": 0.85, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.89, + "1": 0.07, + "2": 0.04, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1064, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.655 + }, + { + "key": "ncl:groups-config-add-mcp-server-flags|reference/ncl-cli#groups", + "stratum": "OK", + "fact": { + "id": "ncl:groups-config-add-mcp-server-flags", + "area": "ncl", + "statement": "`ncl groups config add-mcp-server` (approval) names these flags in its help text: `--id`, `--name`, `--command`, `--args`, `--env`, `--url`, `--headers`.", + "evidence": { + "file": "src/cli/resources/groups.ts", + "line": 453, + "excerpt": "config add-mcp-server: { access: 'approval', description: ... --id --name --command --args --env --url --headers }" + } + }, + "section_id": "reference/ncl-cli#groups", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.1 + }, + "covers": { + "type": "noul", + "noul": 0.86 + }, + "staleness": { + "type": "score", + "score": 0.57, + "confidence": 0.43, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.57, + "1": 0.3, + "2": 0.12, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 2277, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.639 + }, + { + "key": "skills:add-vercel|reference/skills-catalog#tool-installs", + "stratum": "OK", + "fact": { + "id": "skills:add-vercel", + "area": "skills", + "statement": "A workspace skill `/add-vercel` exists under `.claude/skills/`; it is invoked when: Add Vercel deployment capability to NanoClaw agents. Installs the Vercel CLI in agent containers and sets up OneCLI credential injection for api.vercel.com. Use when the user wants agents to deploy web applications to Vercel.", + "evidence": { + "file": ".claude/skills/add-vercel/SKILL.md", + "line": 2, + "excerpt": "name: add-vercel / description: Add Vercel deployment capability to NanoClaw agents. Installs the Vercel CLI in agent containers and sets up OneCLI credential injection for api.vercel.com. Use when the user wants agents to deploy web applications to Vercel." + } + }, + "section_id": "reference/skills-catalog#tool-installs", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.93 + }, + "staleness": { + "type": "score", + "score": 0.13, + "confidence": 0.87, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.89, + "1": 0.1, + "2": 0.01, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1526, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.613 + }, + { + "key": "skills:slack-a2a-rooms|channels/slack#rooms-and-membership", + "stratum": "OK", + "fact": { + "id": "skills:slack-a2a-rooms", + "area": "skills", + "statement": "A workspace skill `/slack-a2a-rooms` exists under `.claude/skills/`; it is invoked when: Agent-to-agent Slack rooms — a group DM (MPIM) holding a human plus two or more NanoClaw sibling bots, where each bot hears the room over its own Socket Mode connection. Registers the admission policy on the Slack channel's bot-inbound guard so bot-authored inbound is admitted only for rooms allowlisted in SLACK_A2A_ROOMS (re-attributed as slack:bot:, hop-limited via SLACK_A2A_MAX_HOPS), plus scripts/open-a2a-room.ts to open a room and register it.", + "evidence": { + "file": ".claude/skills/slack-a2a-rooms/SKILL.md", + "line": 2, + "excerpt": "name: slack-a2a-rooms / description: Agent-to-agent Slack rooms — a group DM (MPIM) holding a human plus two or more NanoClaw sibling bots, where each bot hears the room over its own Socket Mode connection. Registers the admission policy on the Slack channel's bot-inbound guard so bot-authored inbound is admitted only for rooms allowlisted in SLACK_A2A_ROOMS (re-attributed as slack:bot:, …" + } + }, + "section_id": "channels/slack#rooms-and-membership", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.08 + }, + "covers": { + "type": "noul", + "noul": 0.86 + }, + "staleness": { + "type": "score", + "score": 0.48, + "confidence": 0.52, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.64, + "1": 0.26, + "2": 0.09, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 1182, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.703 + }, + { + "key": "ncl:policies-set-flags|reference/ncl-cli#policies", + "stratum": "OK", + "fact": { + "id": "ncl:policies-set-flags", + "area": "ncl", + "statement": "`ncl policies set` (approval) names these flags in its help text: `--from`, `--to`, `--approver`.", + "evidence": { + "file": "src/cli/resources/policies.ts", + "line": 25, + "excerpt": "set: { access: 'approval', description: ... --from --to --approver }" + } + }, + "section_id": "reference/ncl-cli#policies", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.04 + }, + "covers": { + "type": "noul", + "noul": 0.9 + }, + "staleness": { + "type": "score", + "score": 0.02, + "confidence": 0.98, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.99, + "1": 0.01, + "2": 0.0, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1147, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.658 + }, + { + "key": "skills:add-slack|channels/slack#set-up-the-first-agent", + "stratum": "OK", + "fact": { + "id": "skills:add-slack", + "area": "skills", + "statement": "A workspace skill `/add-slack` exists under `.claude/skills/`; it is invoked when: Add Slack channel integration via Chat SDK.", + "evidence": { + "file": ".claude/skills/add-slack/SKILL.md", + "line": 2, + "excerpt": "name: add-slack / description: Add Slack channel integration via Chat SDK." + } + }, + "section_id": "channels/slack#set-up-the-first-agent", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.92 + }, + "staleness": { + "type": "score", + "score": 0.04, + "confidence": 0.96, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.97, + "1": 0.02, + "2": 0.01, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1579, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.653 + }, + { + "key": "ncl:wirings-verbs|reference/ncl-cli#wirings", + "stratum": "OK", + "fact": { + "id": "ncl:wirings-verbs", + "area": "ncl", + "statement": "`ncl wirings` exposes exactly these verbs, with their access level for agent callers: list (open), get (open), update (approval), delete (approval), create (approval). `open` runs inline; `approval` sends an approval card when called from inside a container.", + "evidence": { + "file": "src/cli/resources/wirings.ts", + "line": 48, + "excerpt": "registerResource plural 'wirings' operations {'list': 'open', 'get': 'open', 'update': 'approval', 'delete': 'approval'} custom ['create']" + } + }, + "section_id": "reference/ncl-cli#wirings", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.1 + }, + "covers": { + "type": "noul", + "noul": 0.92 + }, + "staleness": { + "type": "score", + "score": 0.26, + "confidence": 0.74, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.79, + "1": 0.16, + "2": 0.05, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1686, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.631 + }, + { + "key": "env:CONTAINER_PIDS_LIMIT|reference/environment-variables#read-from-env", + "stratum": "OK", + "fact": { + "id": "env:CONTAINER_PIDS_LIMIT", + "area": "env", + "statement": "`CONTAINER_PIDS_LIMIT` is read from the process environment first, then from `.env`; its default is `2048`. It is resolved with `??`, so an explicitly empty value wins over the default.", + "evidence": { + "file": "src/config.ts", + "line": 107, + "excerpt": "export const CONTAINER_PIDS_LIMIT = process.env.CONTAINER_PIDS_LIMIT ?? envConfig.CONTAINER_PIDS_LIMIT ?? '2048';" + } + }, + "section_id": "reference/environment-variables#read-from-env", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.25 + }, + "covers": { + "type": "noul", + "noul": 0.92 + }, + "staleness": { + "type": "score", + "score": 0.87, + "confidence": 0.13, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.51, + "1": 0.18, + "2": 0.26, + "3": 0.05 + } + } + }, + "usage": { + "input_tokens": 2186, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.651 + }, + { + "key": "ncl:destinations-add-flags|reference/ncl-cli#destinations", + "stratum": "OK", + "fact": { + "id": "ncl:destinations-add-flags", + "area": "ncl", + "statement": "`ncl destinations add` (approval) names these flags in its help text: `--agent-group-id`, `--local-name`, `--target-type`, `--target-id`.", + "evidence": { + "file": "src/cli/resources/destinations.ts", + "line": 96, + "excerpt": "add: { access: 'approval', description: ... --agent-group-id --local-name --target-type --target-id }" + } + }, + "section_id": "reference/ncl-cli#destinations", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.04 + }, + "covers": { + "type": "noul", + "noul": 0.88 + }, + "staleness": { + "type": "score", + "score": 0.05, + "confidence": 0.95, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.96, + "1": 0.02, + "2": 0.02, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1113, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.672 + }, + { + "key": "ncl:destinations-remove-flags|reference/ncl-cli#destinations", + "stratum": "OK", + "fact": { + "id": "ncl:destinations-remove-flags", + "area": "ncl", + "statement": "`ncl destinations remove` (approval) names these flags in its help text: `--agent-group-id`, `--local-name`.", + "evidence": { + "file": "src/cli/resources/destinations.ts", + "line": 123, + "excerpt": "remove: { access: 'approval', description: ... --agent-group-id --local-name }" + } + }, + "section_id": "reference/ncl-cli#destinations", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.04 + }, + "covers": { + "type": "noul", + "noul": 0.88 + }, + "staleness": { + "type": "score", + "score": 0.04, + "confidence": 0.96, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.98, + "1": 0.01, + "2": 0.01, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1098, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.691 + }, + { + "key": "skills:add-signal|reference/skills-catalog#channel-installs", + "stratum": "OK", + "fact": { + "id": "skills:add-signal", + "area": "skills", + "statement": "A workspace skill `/add-signal` exists under `.claude/skills/`; it is invoked when: Add Signal channel integration via signal-cli device-link. Native adapter — no Chat SDK bridge.", + "evidence": { + "file": ".claude/skills/add-signal/SKILL.md", + "line": 2, + "excerpt": "name: add-signal / description: Add Signal channel integration via signal-cli device-link. Native adapter — no Chat SDK bridge." + } + }, + "section_id": "reference/skills-catalog#channel-installs", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.19 + }, + "covers": { + "type": "noul", + "noul": 0.95 + }, + "staleness": { + "type": "score", + "score": 0.58, + "confidence": 0.42, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.54, + "1": 0.35, + "2": 0.11, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1717, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.723 + }, + { + "key": "skills:add-github|reference/skills-catalog#channel-installs", + "stratum": "OK", + "fact": { + "id": "skills:add-github", + "area": "skills", + "statement": "A workspace skill `/add-github` exists under `.claude/skills/`; it is invoked when: Add GitHub channel integration via Chat SDK. PR and issue comment threads as conversations.", + "evidence": { + "file": ".claude/skills/add-github/SKILL.md", + "line": 2, + "excerpt": "name: add-github / description: Add GitHub channel integration via Chat SDK. PR and issue comment threads as conversations." + } + }, + "section_id": "reference/skills-catalog#channel-installs", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.02 + }, + "covers": { + "type": "noul", + "noul": 0.95 + }, + "staleness": { + "type": "score", + "score": 0.0, + "confidence": 1.0, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 1.0, + "1": 0.0, + "2": 0.0, + "3": 0.0 + } + } + }, + "usage": { + "input_tokens": 1715, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.662 + }, + { + "key": "skills:init-onecli|operate/credentials#setting-it-up", + "stratum": "OK", + "fact": { + "id": "skills:init-onecli", + "area": "skills", + "statement": "A workspace skill `/init-onecli` exists under `.claude/skills/`; it is invoked when: Install and initialize OneCLI Agent Vault. Migrates existing .env credentials to the vault. Use after /update-nanoclaw brings in OneCLI as a breaking change, or for first-time OneCLI setup.", + "evidence": { + "file": ".claude/skills/init-onecli/SKILL.md", + "line": 2, + "excerpt": "name: init-onecli / description: Install and initialize OneCLI Agent Vault. Migrates existing .env credentials to the vault. Use after /update-nanoclaw brings in OneCLI as a breaking change, or for first-time OneCLI setup." + } + }, + "section_id": "operate/credentials#setting-it-up", + "recorded_response": { + "model": "jev-1.13.0", + "answers": { + "contradicts": { + "type": "noul", + "noul": 0.03 + }, + "covers": { + "type": "noul", + "noul": 0.93 + }, + "staleness": { + "type": "score", + "score": 0.04, + "confidence": 0.96, + "legend": { + "0": "Current: the section states the fact correctly, or does not touch the point the fact makes at all.", + "1": "Slightly outdated: a name, default value, flag spelling or minor detail differs from the fact, but a reader following the section would still succeed.", + "2": "Wrong: the section states something the code no longer does or never did on this point; a reader following it would be misled, get an error, or miss a required step.", + "3": "Dangerous if followed: following the section on this point could lose data, expose a credential, weaken a security boundary, or leave the install broken." + }, + "probabilities": { + "0": 0.97, + "1": 0.01, + "2": 0.01, + "3": 0.01 + } + } + }, + "usage": { + "input_tokens": 1445, + "output_tokens": 53 + } + }, + "recorded_elapsed_seconds": 0.632 + } + ] +} diff --git a/skills/typesafe-docs-drift/scripts/drift-benchmark.py b/skills/typesafe-docs-drift/scripts/drift-benchmark.py new file mode 100644 index 0000000..116035e --- /dev/null +++ b/skills/typesafe-docs-drift/scripts/drift-benchmark.py @@ -0,0 +1,705 @@ +#!/usr/bin/env python3 +"""Benchmark: TypeSafe System One versus a prompt-and-parse LLM on the same docs-drift pairs. + +Every contender judges the same fact x section pairs (same evidence text, same +doc section text) taken from a recorded ``typesafe-docs-drift.py --record`` run: + +* ``typesafe-recorded`` the recorded answers, no network. +* ``typesafe-live`` the same requests sent again, sequentially, for timing. +* ``claude:`` headless ``claude -p`` with tools disabled, 10 pairs per call, + strict JSON out (contradicts, covers, staleness). +* ``codex`` ``codex exec`` in a read-only sandbox, same prompt (fallback). + +Only ``typesafe-recorded`` runs by default; every contender that uses the +network or spawns a CLI must be named in ``--contenders``. The script reports +wall time, per-pair latency, tokens, cost, parse failures, run-to-run flips, +pairwise agreement, and precision/recall on ``contradicts`` against a +hand-labeled gold file. ``TYPESAFE_API_KEY`` is read from the environment only +for ``typesafe-live``, is never printed, and is removed from the environment of +every spawned CLI. +""" + +import argparse +import importlib.util +import json +import math +import os +from pathlib import Path +import random +import re +import signal +import subprocess +import sys +import time + + +SCRIPT_DIR = Path(__file__).resolve().parent +SKILL_DIR = SCRIPT_DIR.parent +DEFAULT_WORKLOAD = SKILL_DIR / "fixtures" / "benchmark-workload.json" +DEFAULT_GOLD = SKILL_DIR / "fixtures" / "benchmark-gold.json" +DEFAULT_OUTPUT_DIR = SKILL_DIR / "output" +DEFAULT_LIMIT = 60 +DEFAULT_SEED = 20260916 +DEFAULT_BATCH = 10 +SYSTEM_PROMPT = ( + "You are a documentation drift checker. You compare one fact derived from source code with one " + "documentation section. You answer with JSON only: no prose, no code fences." +) +OVERHEAD_PROMPT = "Reply with the single word OK." + + +def load_drift(): + spec = importlib.util.spec_from_file_location("typesafe_docs_drift", SCRIPT_DIR / "typesafe-docs-drift.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +drift = load_drift() +BenchmarkError = drift.DriftError + + +# --------------------------------------------------------------------------- +# Workload +# --------------------------------------------------------------------------- + +def build_workload(record, results=None, limit=DEFAULT_LIMIT, seed=DEFAULT_SEED, thresholds=None): + """Stratified sample of recorded pairs: DRIFT pairs of DRIFT facts, the best pair of every + MISSING and UNSURE fact, then a seeded random fill with the best pair of OK facts.""" + thresholds = thresholds or {"contradicts": 0.7, "covers": 0.7, "staleness": 0.5} + sections = {section["id"]: section for section in record["sections"]} + elapsed = {} + for result in (results or {}).get("results", []): + for pair in result.get("pairs", []): + elapsed[pair["key"]] = pair.get("elapsed_seconds") + strata = {"DRIFT": [], "MISSING": [], "UNSURE": [], "OK": []} + for entry in record["facts"]: + pairs = [] + for section_id in record["candidates"].get(entry["id"], []): + key = drift.pair_key(entry["id"], section_id) + response = record["responses"].get(key) + if response is None or section_id not in sections: + continue + pairs.append({"key": key, "section": {"id": section_id}, "response": response, + "decision": drift.decide_pair(response.get("answers") or {}, thresholds)}) + verdict = drift.decide_fact(entry, pairs, thresholds) + if not pairs: + continue + if verdict["verdict"] == "DRIFT": + chosen = [pair for pair in pairs if pair["decision"]["verdict"] == "DRIFT"] + else: + chosen = [pair for pair in pairs if pair["section"]["id"] == verdict["best"]["section"]["id"]][:1] + for pair in chosen: + strata[verdict["verdict"]].append({ + "key": pair["key"], + "stratum": verdict["verdict"], + "fact": {name: entry[name] for name in ("id", "area", "statement", "evidence")}, + "section_id": pair["section"]["id"], + "recorded_response": pair["response"], + "recorded_elapsed_seconds": elapsed.get(pair["key"]), + }) + picked = strata["DRIFT"] + strata["MISSING"] + strata["UNSURE"] + if limit and len(picked) > limit: + picked = picked[:limit] + room = max(0, (limit or len(picked) + len(strata["OK"])) - len(picked)) + fill = sorted(strata["OK"], key=lambda pair: pair["key"]) + random.Random(seed).shuffle(fill) + picked = picked + fill[:room] + used = sorted({pair["section_id"] for pair in picked}) + return { + "schema_version": 1, + "source": {"recorded_at": record.get("recorded_at"), "code_commit": record.get("code_commit"), + "docs_commit": record.get("docs_commit")}, + "seed": seed, + "limit": limit, + "strata": {name: sum(1 for pair in picked if pair["stratum"] == name) for name in strata}, + "sections": [{name: sections[section_id][name] for name in ("id", "page", "title", "heading", "line", "text")} + for section_id in used], + "pairs": picked, + } + + +def load_json(path, what): + try: + return json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError) as error: + raise BenchmarkError("could not read %s %s: %s" % (what, path, error)) + + +def load_workload(path, limit=0): + data = load_json(path, "workload") + if not isinstance(data, dict) or "pairs" not in data or "sections" not in data: + raise BenchmarkError("workload %s must be an object with 'pairs' and 'sections'" % path) + if limit and limit < len(data["pairs"]): + data = dict(data, pairs=data["pairs"][:limit]) + return data + + +def pair_state(pair, sections): + section = sections[pair["section_id"]] + return drift.build_state(dict(pair["fact"], terms=[]), section) + + +# --------------------------------------------------------------------------- +# Contenders +# --------------------------------------------------------------------------- + +def verdict_from_answers(answers, gate): + """Binarize TypeSafe answers the way the skill's gate does: yes means probability >= gate.""" + def real(value, low, high): + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + return None + return float(value) if low <= value <= high else None + + # Same bar as the LLM parser: every field present, correctly typed and in range, or the pair is a parse failure. + def field(question, name): + answer = answers.get(question) if isinstance(answers, dict) else None + return answer.get(name) if isinstance(answer, dict) else None + + contradicts = real(field("contradicts", "noul"), 0, 1) + covers = real(field("covers", "noul"), 0, 1) + staleness = real(field("staleness", "score"), 0, len(drift.STALENESS_LEVELS) - 1) + if contradicts is None or covers is None or staleness is None: + return None + return {"contradicts": contradicts >= gate, "covers": covers >= gate, + "staleness": int(math.floor(staleness + 0.5)), + "p_contradicts": round(contradicts, 3), "p_covers": round(covers, 3)} + + +def count(value): + """A token count from an API payload; anything that is not a plain non-negative number counts as zero.""" + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + return 0 + return int(value) + + +def empty_run(name): + return {"contender": name, "verdicts": {}, "latencies": [], "adjusted_latencies": [], "wall_seconds": 0.0, + "input_tokens": 0, "output_tokens": 0, "cost_usd": None, "parse_failures": 0, "format_violations": 0, + "calls": 0, "failed_calls": 0, "overhead_seconds": None, "errors": []} + + +def run_typesafe_recorded(workload, gate): + run = empty_run("typesafe-recorded") + for pair in workload["pairs"]: + response = pair["recorded_response"] + response = response if isinstance(response, dict) else {} + verdict = verdict_from_answers(response.get("answers"), gate) + if verdict is None: + run["parse_failures"] += 1 + else: + run["verdicts"][pair["key"]] = verdict + if pair.get("recorded_elapsed_seconds") is not None: + run["latencies"].append(pair["recorded_elapsed_seconds"]) + usage = response.get("usage") if isinstance(response.get("usage"), dict) else {} + run["input_tokens"] += count(usage.get("input_tokens")) + run["output_tokens"] += count(usage.get("output_tokens")) + run["calls"] += 1 + # The recorded run was concurrent; the sum of request latencies is the sequential-equivalent wall time. + if len(run["latencies"]) == len(workload["pairs"]): + run["wall_seconds"] = round(sum(run["latencies"]), 3) + else: # no --results file, or only some latencies: a partial sum would pose as the whole workload + run["wall_seconds"] = None + run["latencies"] = [] + run["adjusted_latencies"] = list(run["latencies"]) + return run + + +def run_typesafe_live(workload, gate, transport, model=None, log=None): + run = empty_run("typesafe-live") + sections = {section["id"]: section for section in workload["sections"]} + started = time.monotonic() + for index, pair in enumerate(workload["pairs"], 1): + payload = {"state": pair_state(pair, sections), "model": model or drift.DEFAULT_MODEL, + "questions": drift.build_questions(), "_key": pair["key"]} + begun = time.monotonic() + run["calls"] += 1 + try: + response = transport(payload) + except BenchmarkError as error: + run["latencies"].append(round(time.monotonic() - begun, 3)) + run["errors"].append("%s: %s" % (pair["key"], error)) + run["parse_failures"] += 1 + run["failed_calls"] += 1 + continue + latency = time.monotonic() - begun + run["latencies"].append(round(latency, 3)) + response = response if isinstance(response, dict) else {} + verdict = verdict_from_answers(response.get("answers"), gate) + if verdict is None: + run["parse_failures"] += 1 + else: + run["verdicts"][pair["key"]] = verdict + usage = response.get("usage") if isinstance(response.get("usage"), dict) else {} + run["input_tokens"] += count(usage.get("input_tokens")) + run["output_tokens"] += count(usage.get("output_tokens")) + if log: + log("[typesafe-live %d/%d] %.1fs" % (index, len(workload["pairs"]), latency)) + run["wall_seconds"] = round(time.monotonic() - started, 3) + run["adjusted_latencies"] = list(run["latencies"]) + return run + + +def build_llm_prompt(batch, sections): + questions = drift.build_questions() + items = [] + for number, pair in enumerate(batch, 1): + state = pair_state(pair, sections) + items.append({"id": "p%02d" % number, "fact": state["fact"], "doc_section": state["doc_section"]}) + instructions = { + "task": "For every item, judge `doc_section` against `fact`. The three judgments are independent.", + "contradicts": {"question": questions["contradicts"]["instructions"], "true": questions["contradicts"]["criteria"]["true"], + "false": questions["contradicts"]["criteria"]["false"]}, + "covers": {"question": questions["covers"]["instructions"], "true": questions["covers"]["criteria"]["true"], + "false": questions["covers"]["criteria"]["false"]}, + "staleness": {"question": questions["staleness"]["instructions"], + "levels": {str(index): text for index, text in enumerate(questions["staleness"]["criteria"])}}, + "output": ("Return one JSON array and nothing else, with exactly one object per item, in order: " + '{"id": "", "contradicts": true|false, "covers": true|false, "staleness": 0|1|2|3}'), + } + return json.dumps({"instructions": instructions, "items": items}, indent=1, ensure_ascii=False) + + +def is_strict_json_array(text): + """True when the whole reply is one JSON array, as the prompt demands (no prose, no code fence).""" + try: + return isinstance(json.loads((text or "").strip()), list) + except json.JSONDecodeError: + return False + + +def parse_llm_answer(text, expected): + """Return {item id: verdict} for the valid objects in the model's reply; anything else is a parse failure. + + Recovery is what a real pipeline would do (strip a fence, take the outermost array); whether the reply + obeyed the strict-JSON contract is reported separately by ``is_strict_json_array``. An id answered twice + is ambiguous and dropped.""" + text = (text or "").strip() + text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text) + start, end = text.find("["), text.rfind("]") + if start < 0 or end <= start: + return {} + try: + data = json.loads(text[start:end + 1]) + except json.JSONDecodeError: + return {} + verdicts = {} + seen = set() + for item in data if isinstance(data, list) else []: + if not isinstance(item, dict) or not isinstance(item.get("id"), str) or item["id"] not in expected: + continue + if item["id"] in seen: + verdicts.pop(item["id"], None) + continue + seen.add(item["id"]) + contradicts, covers, staleness = item.get("contradicts"), item.get("covers"), item.get("staleness") + if not isinstance(contradicts, bool) or not isinstance(covers, bool): + continue + if isinstance(staleness, bool) or not isinstance(staleness, int) or not 0 <= staleness <= 3: + continue + verdicts[item["id"]] = {"contradicts": contradicts, "covers": covers, "staleness": staleness} + return verdicts + + +def child_environment(environ): + return {name: value for name, value in environ.items() if name != drift.KEY_ENV} + + +_SECRET_NAME = re.compile(r"KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL", re.I) + + +def redact_secrets(text, environ): + """Blank any secret-looking environment value a CLI echoed into its diagnostics.""" + for name, value in environ.items(): + if value and len(value) >= 8 and _SECRET_NAME.search(name): + text = text.replace(value, "[redacted]") + return text + + +class CliError(BenchmarkError): + """A failed CLI call; ``usage`` holds whatever the CLI still reported, so failures are not free.""" + + def __init__(self, message, usage=None): + super().__init__(message) + self.usage = usage + + +class Finished: + def __init__(self, returncode, stdout, stderr): + self.returncode, self.stdout, self.stderr = returncode, stdout, stderr + + +def spawn(command, input=None, capture_output=True, text=True, timeout=None, env=None): + """Like subprocess.run, but in its own process group so a timeout also stops the CLI's children.""" + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=text, env=env, start_new_session=True) + try: + stdout, stderr = process.communicate(input, timeout=timeout) + except BaseException: # timeout, Ctrl+C, anything: the detached group must not keep spending tokens + try: + os.killpg(process.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + process.kill() + try: + process.communicate(timeout=5) + except Exception: + pass + raise + return Finished(process.returncode, stdout, stderr) + + +def cli_command(contender): + kind, _, model = contender.partition(":") + if kind == "claude": + return ["claude", "-p", "--model", model or "haiku", "--output-format", "json", "--tools", "", + "--strict-mcp-config", "--no-session-persistence", "--system-prompt", SYSTEM_PROMPT] + if kind == "codex": + command = ["codex", "exec", "--ephemeral", "-c", 'sandbox_mode="read-only"'] + if model: + command += ["-m", model] + return command + ["-"] + raise BenchmarkError("unknown contender %s" % contender) + + +def envelope_usage(envelope): + raw = envelope.get("usage") if isinstance(envelope.get("usage"), dict) else {} + cost = envelope.get("total_cost_usd") + return {"input_tokens": sum(count(raw.get(name)) for name in ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens")), + "output_tokens": count(raw.get("output_tokens")), + "cost_usd": float(cost) if isinstance(cost, (int, float)) and not isinstance(cost, bool) else None} + + +def claude_usage(stdout): + """Usage from whatever envelope a failed ``claude -p`` still printed, or None.""" + try: + envelope = json.loads(stdout or "") + except json.JSONDecodeError: + return None + if isinstance(envelope, list): + envelope = next((event for event in reversed(envelope) if isinstance(event, dict) and event.get("type") == "result"), None) + return envelope_usage(envelope) if isinstance(envelope, dict) else None + + +def call_cli(contender, prompt, run, environ, timeout): + """Spawn the CLI once; return (reply text, usage dict).""" + command = cli_command(contender) + try: + result = run(command, input=prompt, capture_output=True, text=True, timeout=timeout, env=child_environment(environ)) + except FileNotFoundError: + raise BenchmarkError("%s is not on PATH" % command[0]) + except subprocess.TimeoutExpired: + raise BenchmarkError("%s timed out after %ss" % (command[0], timeout)) + if result.returncode != 0: + detail = (result.stderr or result.stdout or "").strip().splitlines() + raise CliError("%s exited %d: %s" % ( + command[0], result.returncode, redact_secrets(detail[-1], environ)[:200] if detail else "no output"), + claude_usage(result.stdout) if contender.startswith("claude") else None) + if contender.startswith("claude"): + try: + envelope = json.loads(result.stdout) + except json.JSONDecodeError: + raise BenchmarkError("claude -p did not return JSON") + if isinstance(envelope, list): # some versions emit the event list; the result event is last + envelope = next((event for event in reversed(envelope) if isinstance(event, dict) and event.get("type") == "result"), {}) + if not isinstance(envelope, dict): + raise BenchmarkError("claude -p returned JSON that is not a result object") + usage = envelope_usage(envelope) + if envelope.get("is_error"): + raise CliError("claude -p reported an error: %s" % redact_secrets(str(envelope.get("result")), environ)[:200], usage) + reply = envelope.get("result") + return reply if isinstance(reply, str) else "", usage + # codex exec prints plain text; its token use is not machine-readable here, so it is unknown rather than zero. + return result.stdout, {"input_tokens": None, "output_tokens": None, "cost_usd": None} + + +def add_usage(outcome, usage): + if not usage: + return + for name in ("input_tokens", "output_tokens"): + if usage.get(name) is None: + outcome[name] = None # unknown for this CLI; never report it as zero + elif outcome[name] is not None: + outcome[name] += usage[name] + if usage.get("cost_usd") is not None: + outcome["cost_usd"] = round((outcome["cost_usd"] or 0) + usage["cost_usd"], 6) + + +def run_llm(contender, workload, run=spawn, environ=None, batch_size=DEFAULT_BATCH, timeout=600, + overhead=None, log=None): + environ = os.environ if environ is None else environ + outcome = empty_run(contender) + outcome["overhead_seconds"] = overhead + sections = {section["id"]: section for section in workload["sections"]} + pairs = workload["pairs"] + started = time.monotonic() + for offset in range(0, len(pairs), batch_size): + batch = pairs[offset: offset + batch_size] + ids = {"p%02d" % number: pair["key"] for number, pair in enumerate(batch, 1)} + begun = time.monotonic() + outcome["calls"] += 1 + failure = None + try: + reply, usage = call_cli(contender, build_llm_prompt(batch, sections), run, environ, timeout) + except BenchmarkError as error: + failure = error + wall = time.monotonic() - begun + outcome["latencies"].extend([round(wall / len(batch), 3)] * len(batch)) + outcome["adjusted_latencies"].extend([round(max(0.0, wall - (overhead or 0.0)) / len(batch), 3)] * len(batch)) + if failure is not None: + outcome["errors"].append("batch %d: %s" % (offset // batch_size + 1, failure)) + outcome["parse_failures"] += len(batch) + outcome["failed_calls"] += 1 + add_usage(outcome, getattr(failure, "usage", None)) + continue + if not is_strict_json_array(reply): + outcome["format_violations"] += 1 + add_usage(outcome, usage) + verdicts = parse_llm_answer(reply, ids) + outcome["parse_failures"] += len(batch) - len(verdicts) + for item_id, verdict in verdicts.items(): + outcome["verdicts"][ids[item_id]] = verdict + if log: + log("[%s batch %d] %d pairs in %.1fs, %d parsed" % (contender, offset // batch_size + 1, len(batch), wall, len(verdicts))) + outcome["wall_seconds"] = round(time.monotonic() - started, 3) + return outcome + + +def measure_overhead(contender, run=spawn, environ=None, timeout=300): + """Wall time of one trivial call: process start, auth, and one minimal API round trip.""" + environ = os.environ if environ is None else environ + begun = time.monotonic() + call_cli(contender, OVERHEAD_PROMPT, run, environ, timeout) + return round(time.monotonic() - begun, 3) + + +# --------------------------------------------------------------------------- +# Metrics +# --------------------------------------------------------------------------- + +def percentile(values, fraction): + if not values: + return None + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + low, high = int(math.floor(position)), int(math.ceil(position)) + return round(ordered[low] + (ordered[high] - ordered[low]) * (position - low), 3) + + +def flip_rate(first, second, field="contradicts"): + """Share of pairs answered in both runs whose verdict differs, with that denominator.""" + shared = [key for key in first["verdicts"] if key in second["verdicts"]] + if not shared: + return None + flips = sum(1 for key in shared if first["verdicts"][key][field] != second["verdicts"][key][field]) + return {"pairs": len(shared), "flips": flips, "rate": round(flips / len(shared), 4)} + + +def agreement(first, second, field): + shared = [key for key in first["verdicts"] if key in second["verdicts"]] + if not shared: + return None + same = sum(1 for key in shared if first["verdicts"][key][field] == second["verdicts"][key][field]) + return {"pairs": len(shared), "rate": round(same / len(shared), 4)} + + +def precision_recall(run, gold, keys, field="contradicts"): + """Gold pairs in the workload only. A pair the contender failed to answer counts as a predicted no.""" + tp = fp = fn = tn = 0 + for key in keys: + label = gold.get(key) + if label is None: + continue + predicted = bool((run["verdicts"].get(key) or {}).get(field)) + actual = bool(label[field]) + if predicted and actual: + tp += 1 + elif predicted: + fp += 1 + elif actual: + fn += 1 + else: + tn += 1 + return {"tp": tp, "fp": fp, "fn": fn, "tn": tn, + "precision": round(tp / (tp + fp), 4) if tp + fp else None, + "recall": round(tp / (tp + fn), 4) if tp + fn else None} + + +def summarize(runs_by_contender, gold, keys): + rows = [] + for name, runs in runs_by_contender.items(): + first = runs[0] + rows.append({ + "contender": name, + "pairs": len(keys), + "answered": len(first["verdicts"]), + "wall_seconds": first["wall_seconds"], + "p50_seconds": percentile(first["latencies"], 0.5), + "p95_seconds": percentile(first["latencies"], 0.95), + "p50_adjusted_seconds": percentile(first["adjusted_latencies"], 0.5), + "overhead_seconds": first["overhead_seconds"], + "input_tokens": first["input_tokens"], + "output_tokens": first["output_tokens"], + "cost_usd": first["cost_usd"], + "parse_failures": first["parse_failures"], + "parse_failures_by_run": [entry["parse_failures"] for entry in runs], + "format_violations_by_run": [entry.get("format_violations", 0) for entry in runs], + "failed_calls_by_run": [entry.get("failed_calls", 0) for entry in runs], + "flip": flip_rate(first, runs[1]) if len(runs) > 1 else None, + "gold": precision_recall(first, gold, keys), + "gold_covers": precision_recall(first, gold, keys, "covers"), + "errors": first["errors"][:5], + }) + names = list(runs_by_contender) + pairwise = [] + for index, left in enumerate(names): + for right in names[index + 1:]: + pairwise.append({"a": left, "b": right, + "contradicts": agreement(runs_by_contender[left][0], runs_by_contender[right][0], "contradicts"), + "covers": agreement(runs_by_contender[left][0], runs_by_contender[right][0], "covers")}) + return {"rows": rows, "agreement": pairwise, "gold_pairs": sum(1 for key in keys if key in gold)} + + +def fmt(value, pattern="%s", missing="-"): + return missing if value is None else pattern % value + + +def render(summary): + columns = ("Contender", "Pairs", "Wall s", "p50 s/pair", "p50 adj", "Tokens in/out", "Cost $", "Parse fail", "Flip %", "Precision", "Recall") + rows = [] + for row in summary["rows"]: + rows.append(( + row["contender"], str(row["pairs"]), fmt(row["wall_seconds"], "%.1f"), fmt(row["p50_seconds"], "%.2f"), + fmt(row["p50_adjusted_seconds"], "%.2f"), "%s / %s" % (fmt(row["input_tokens"], "%d"), fmt(row["output_tokens"], "%d")), + fmt(row["cost_usd"], "%.4f"), "/".join(str(count) for count in row["parse_failures_by_run"]), + "-" if row["flip"] is None else "%.1f (%d/%d)" % (row["flip"]["rate"] * 100, row["flip"]["flips"], row["flip"]["pairs"]), + fmt(row["gold"]["precision"], "%.2f"), fmt(row["gold"]["recall"], "%.2f"), + )) + lines = [drift.table(columns, rows), "", + "Parse fail is per run (pairs). Flip % is over the pairs answered in both runs (flips/pairs).", + "Precision and recall are for `contradicts` on %d hand-labeled gold pairs." % summary["gold_pairs"]] + for row in summary["rows"]: + if any(row["format_violations_by_run"]) or any(row["failed_calls_by_run"]): + lines.append("%s: replies that were not a bare JSON array per run %s; failed calls per run %s" % ( + row["contender"], row["format_violations_by_run"], row["failed_calls_by_run"])) + for entry in summary["agreement"]: + lines.append("agreement %s vs %s: contradicts %s, covers %s" % ( + entry["a"], entry["b"], + fmt(entry["contradicts"] and entry["contradicts"]["rate"] * 100, "%.1f%%"), + fmt(entry["covers"] and entry["covers"]["rate"] * 100, "%.1f%%"))) + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--workload", default=str(DEFAULT_WORKLOAD), help="sampled pairs to judge (default: the shipped fixture)") + parser.add_argument("--record", help="build the workload from this typesafe-docs-drift --record file instead") + parser.add_argument("--results", help="with --record: the run's saved drift-*.json, for recorded per-request latencies") + parser.add_argument("--write-workload", help="with --record: save the sampled workload here and exit") + parser.add_argument("--limit", type=int, default=DEFAULT_LIMIT, help="pairs in the sample (default %d)" % DEFAULT_LIMIT) + parser.add_argument("--seed", type=int, default=DEFAULT_SEED, help="seed for the OK fill (default %d)" % DEFAULT_SEED) + parser.add_argument("--gold", help="hand-labeled gold file (default: the shipped fixture; with --from-results: the labels saved in that file)") + parser.add_argument("--contenders", default="typesafe-recorded", + help="comma-separated: typesafe-recorded (offline), typesafe-live, claude:, codex[:]") + parser.add_argument("--runs", type=int, default=2, help="runs per live contender, for the flip rate (default 2)") + parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH, help="pairs per LLM call (default %d)" % DEFAULT_BATCH) + parser.add_argument("--gate", type=float, default=0.7, help="TypeSafe yes threshold on the noul probabilities (default 0.7)") + parser.add_argument("--timeout", type=float, default=600.0, help="seconds per CLI call or HTTP request") + parser.add_argument("--from-results", help="re-render a saved benchmark-*.json; runs nothing") + parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR)) + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + if args.runs < 1 or args.batch_size < 1 or args.limit < 0: + parser.error("--runs and --batch-size must be at least 1 and --limit at least 0") + if not 0.5 <= args.gate <= 1: + parser.error("--gate must be between 0.5 and 1") + args.contender_list = [name.strip() for name in args.contenders.split(",") if name.strip()] + for name in args.contender_list: + if name not in ("typesafe-recorded", "typesafe-live") and name.partition(":")[0] not in ("claude", "codex"): + parser.error("unknown contender %s" % name) + return args + + +def main(argv=None, environ=None, stdout=None, stderr=None, run=spawn): + environ = os.environ if environ is None else environ + stdout = sys.stdout if stdout is None else stdout + stderr = sys.stderr if stderr is None else stderr + args = parse_args(argv) + log = lambda line: print(line, file=stderr) + try: + if args.from_results: + payload = load_json(args.from_results, "benchmark results") + if args.gold: + gold_file = load_json(args.gold, "gold") + payload["gold"] = {"labeled_by": gold_file.get("labeled_by"), "labels": gold_file.get("labels", {})} + gold = (payload.get("gold") or {}).get("labels", {}) + payload["summary"] = summarize(payload["runs"], gold, payload["keys"]) + payload["gold"]["pairs"] = payload["summary"]["gold_pairs"] + print(json.dumps(payload, indent=2) if args.json else render(payload["summary"]), file=stdout) + return 0 + if args.record: + workload = build_workload(load_json(args.record, "record"), load_json(args.results, "results") if args.results else None, + args.limit, args.seed) + if args.write_workload: + Path(args.write_workload).write_text(json.dumps(workload, indent=1, ensure_ascii=False) + "\n") + print("wrote %d pairs (%s) to %s" % (len(workload["pairs"]), workload["strata"], args.write_workload), file=stdout) + return 0 + else: + workload = load_workload(args.workload, args.limit) + gold_path = args.gold or str(DEFAULT_GOLD) + gold_file = load_json(gold_path, "gold") if Path(gold_path).exists() else {"labels": {}} + gold = gold_file.get("labels", {}) + keys = [pair["key"] for pair in workload["pairs"]] + runs_by_contender = {} + if "typesafe-live" in args.contender_list and not environ.get(drift.KEY_ENV): + # Checked before anything runs, so a paid contender earlier in the list is never wasted. + print("%s is not set; typesafe-live needs it exported in this shell. It is never read from a file." % drift.KEY_ENV, file=stderr) + return 2 + for name in args.contender_list: + if name == "typesafe-recorded": + runs_by_contender[name] = [run_typesafe_recorded(workload, args.gate)] + elif name == "typesafe-live": + transport = drift.HttpTransport(environ.get(drift.KEY_ENV), timeout=args.timeout) + runs_by_contender[name] = [run_typesafe_live(workload, args.gate, transport, log=log) for _ in range(args.runs)] + else: + try: + overhead = measure_overhead(name, run, environ, args.timeout) + log("[%s] startup overhead %.1fs" % (name, overhead)) + except BenchmarkError as error: + overhead = None + log("[%s] startup overhead could not be measured (%s); adjusted latency equals raw" % (name, error)) + runs_by_contender[name] = [run_llm(name, workload, run, environ, args.batch_size, args.timeout, overhead, log) + for _ in range(args.runs)] + except BenchmarkError as error: + print("error: %s" % error, file=stderr) + return 1 + except Exception as error: # never let a traceback carry request details to the terminal + print("error: unexpected %s: %s" % (type(error).__name__, redact_secrets(drift.redact_env(str(error), environ), environ)), file=stderr) + return 1 + summary = summarize(runs_by_contender, gold, keys) + payload = { + "schema_version": 1, + "generated_at": drift.utc_now().isoformat(), + "workload": {"source": workload.get("source"), "seed": workload.get("seed"), "pairs": len(keys), "strata": workload.get("strata")}, + "gold": {"pairs": summary["gold_pairs"], "labeled_by": gold_file.get("labeled_by"), + "labels": {key: gold[key] for key in keys if key in gold}}, + "gate": args.gate, + "batch_size": args.batch_size, + "keys": keys, + "summary": summary, + "runs": runs_by_contender, + } + output_path = drift.write_output(args.output_dir, payload, prefix="benchmark") + if args.json: + print(json.dumps(payload, indent=2), file=stdout) + else: + print(render(summary), file=stdout) + print("\nraw runs: %s" % output_path, file=stdout) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_drift_benchmark.py b/tests/test_drift_benchmark.py new file mode 100644 index 0000000..1462040 --- /dev/null +++ b/tests/test_drift_benchmark.py @@ -0,0 +1,448 @@ +"""Docs-drift benchmark: workload sampling, LLM prompt-and-parse, metrics and CLI, all offline.""" + +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "skills/typesafe-docs-drift/scripts/drift-benchmark.py" +WORKLOAD = ROOT / "skills/typesafe-docs-drift/fixtures/benchmark-workload.json" +GOLD = ROOT / "skills/typesafe-docs-drift/fixtures/benchmark-gold.json" + + +def load(): + spec = importlib.util.spec_from_file_location("drift_benchmark_test", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +bench = load() + + +def response(contradicts, covers, staleness=0.2, tokens=100): + return {"model": "jev-latest", "answers": {"contradicts": {"type": "noul", "noul": contradicts}, + "covers": {"type": "noul", "noul": covers}, + "staleness": {"type": "score", "score": staleness, "confidence": 0.9}}, + "usage": {"input_tokens": tokens, "output_tokens": 10}} + + +def record(ok_facts=8): + facts, sections, candidates, responses = [], [], {}, {} + + def add(fact_id, answers): + facts.append({"id": fact_id, "area": fact_id.split(":")[0], "statement": "`%s` fact." % fact_id, + "evidence": {"file": "src/x.ts", "line": 1, "excerpt": "x"}, "terms": []}) + candidates[fact_id] = [] + for number, (contradicts, covers) in enumerate(answers, 1): + section_id = "page#%s-%d" % (fact_id.replace(":", "-"), number) + sections.append({"id": section_id, "page": "page", "title": "T", "heading": "h", "level": 2, "line": number, "text": "doc text"}) + candidates[fact_id].append(section_id) + responses["%s|%s" % (fact_id, section_id)] = response(contradicts, covers) + + add("ncl:drift", [(0.9, 0.9), (0.8, 0.8), (0.1, 0.1)]) + add("env:missing", [(0.05, 0.1), (0.05, 0.2)]) + add("env:unsure", [(0.5, 0.6)]) + for number in range(ok_facts): + add("skills:ok%d" % number, [(0.05, 0.95), (0.05, 0.1)]) + return {"recorded_at": "t", "code_commit": "c", "docs_commit": "d", "facts": facts, "sections": sections, + "candidates": candidates, "scores": {}, "responses": responses} + + +def small_workload(pairs=3): + data = bench.build_workload(record(), limit=0) + data["pairs"] = data["pairs"][:pairs] + return data + + +class Completed: + def __init__(self, stdout="", returncode=0, stderr=""): + self.stdout, self.returncode, self.stderr = stdout, returncode, stderr + + +def claude_envelope(items, tokens=(1000, 50), cost=0.01): + return json.dumps({"type": "result", "is_error": False, "result": json.dumps(items), "total_cost_usd": cost, + "usage": {"input_tokens": tokens[0], "cache_creation_input_tokens": 5, "cache_read_input_tokens": 7, + "output_tokens": tokens[1]}}) + + +class WorkloadTests(unittest.TestCase): + def test_sample_takes_every_drift_missing_and_unsure_row_then_a_seeded_ok_fill(self): + workload = bench.build_workload(record(), limit=6, seed=1) + self.assertEqual(workload["strata"], {"DRIFT": 2, "MISSING": 1, "UNSURE": 1, "OK": 2}) + keys = [pair["key"] for pair in workload["pairs"]] + self.assertEqual(keys[:2], ["ncl:drift|page#ncl-drift-1", "ncl:drift|page#ncl-drift-2"]) + self.assertIn("env:missing|page#env-missing-2", keys) + self.assertEqual(keys, [pair["key"] for pair in bench.build_workload(record(), limit=6, seed=1)["pairs"]]) + other = [pair["key"] for pair in bench.build_workload(record(), limit=6, seed=2)["pairs"]] + self.assertEqual(keys[:4], other[:4]) + self.assertNotEqual(keys[4:], other[4:]) + self.assertEqual({section["id"] for section in workload["sections"]}, {pair["section_id"] for pair in workload["pairs"]}) + for pair in workload["pairs"]: + self.assertNotIn("terms", pair["fact"]) + self.assertIn("recorded_response", pair) + + def test_recorded_latencies_come_from_the_results_file(self): + results = {"results": [{"pairs": [{"key": "env:unsure|page#env-unsure-1", "elapsed_seconds": 0.42}]}]} + workload = bench.build_workload(record(), results, limit=0) + unsure = next(pair for pair in workload["pairs"] if pair["stratum"] == "UNSURE") + self.assertEqual(unsure["recorded_elapsed_seconds"], 0.42) + + def test_shipped_workload_and_gold_are_consistent(self): + workload = json.loads(WORKLOAD.read_text()) + gold = json.loads(GOLD.read_text()) + self.assertEqual(len(workload["pairs"]), 60) + self.assertEqual(sum(workload["strata"].values()), 60) + keys = {pair["key"] for pair in workload["pairs"]} + self.assertEqual(len(gold["labels"]), 30) + self.assertTrue(set(gold["labels"]) <= keys) + self.assertIn("LLM-assisted", gold["labeled_by"]) + must = {pair["key"] for pair in workload["pairs"] if pair["stratum"] in ("DRIFT", "MISSING")} + self.assertTrue(must <= set(gold["labels"])) + for label in gold["labels"].values(): + self.assertIsInstance(label["contradicts"], bool) + self.assertIsInstance(label["covers"], bool) + self.assertTrue(len(label["justification"]) > 20) + + +class PromptTests(unittest.TestCase): + def test_prompt_carries_the_same_questions_and_state_as_the_typesafe_request(self): + workload = small_workload(2) + sections = {section["id"]: section for section in workload["sections"]} + prompt = json.loads(bench.build_llm_prompt(workload["pairs"], sections)) + questions = bench.drift.build_questions() + self.assertEqual(prompt["instructions"]["contradicts"]["question"], questions["contradicts"]["instructions"]) + self.assertEqual(prompt["instructions"]["covers"]["true"], questions["covers"]["criteria"]["true"]) + self.assertEqual(len(prompt["instructions"]["staleness"]["levels"]), 4) + self.assertEqual([item["id"] for item in prompt["items"]], ["p01", "p02"]) + self.assertEqual(prompt["items"][0], dict(id="p01", **bench.pair_state(workload["pairs"][0], sections))) + + def test_parse_accepts_fenced_json_and_rejects_bad_items(self): + expected = {"p01": "a", "p02": "b", "p03": "c", "p04": "d"} + text = '```json\n[{"id":"p01","contradicts":true,"covers":false,"staleness":2},' \ + '{"id":"p02","contradicts":"yes","covers":true,"staleness":1},' \ + '{"id":"p03","contradicts":false,"covers":true,"staleness":7},' \ + '{"id":"p09","contradicts":false,"covers":true,"staleness":0},' \ + '{"id":"p01","contradicts":false,"covers":true,"staleness":0}]\n```' + self.assertEqual(bench.parse_llm_answer(text, expected), {}, "p01 answered twice is ambiguous and dropped") + self.assertEqual(bench.parse_llm_answer(text.replace('{"id":"p01","contradicts":false,"covers":true,"staleness":0}', '{"id":[]}'), expected), + {"p01": {"contradicts": True, "covers": False, "staleness": 2}}) + self.assertFalse(bench.is_strict_json_array(text)) + self.assertFalse(bench.is_strict_json_array("Here: []")) + self.assertTrue(bench.is_strict_json_array(' [{"id": "p01"}]\n')) + self.assertEqual(bench.parse_llm_answer("Sure! Here you go: [{\"id\":\"p04\",\"contradicts\":false,\"covers\":true,\"staleness\":0}] Hope it helps", expected), + {"p04": {"contradicts": False, "covers": True, "staleness": 0}}) + for bad in ("", "no json", "[not json]", '{"id":"p01"}', '[{"id":"p01","contradicts":true,"covers":true,"staleness":true}]'): + self.assertEqual(bench.parse_llm_answer(bad, expected), {}, bad) + + def test_cli_commands_disable_tools_and_sandbox_codex(self): + command = bench.cli_command("claude:haiku") + self.assertEqual(command[:6], ["claude", "-p", "--model", "haiku", "--output-format", "json"]) + self.assertEqual(command[command.index("--tools") + 1], "") + self.assertIn("--strict-mcp-config", command) + self.assertIn("--no-session-persistence", command) + codex = bench.cli_command("codex") + self.assertEqual(codex[:3], ["codex", "exec", "--ephemeral"]) + self.assertIn('sandbox_mode="read-only"', codex) + self.assertNotIn("--dangerously-bypass-approvals-and-sandbox", codex) + with self.assertRaises(bench.BenchmarkError): + bench.cli_command("gpt:4") + + +class LlmRunTests(unittest.TestCase): + def test_batches_sum_usage_and_subtract_the_measured_overhead(self): + workload = bench.build_workload(record(), limit=0) + workload["pairs"] = workload["pairs"][:5] + calls = [] + + def run(command, input, capture_output, text, timeout, env): + calls.append((command, input, env)) + items = [{"id": item["id"], "contradicts": True, "covers": True, "staleness": 1} for item in json.loads(input)["items"]] + if len(calls) == 2: + items = items[:1] + [{"id": "p02", "contradicts": "maybe"}] + return Completed(claude_envelope(items)) + + clock = iter([0, 0, 4.0, 4.0, 10.0, 10.0, 12.0, 12.0]) + with patch.object(bench.time, "monotonic", side_effect=lambda: next(clock)): + outcome = bench.run_llm("claude:haiku", workload, run=run, environ={"TYPESAFE_API_KEY": "ts-FAKE-KEY-0123456789", "PATH": "/bin"}, + batch_size=2, overhead=2.0) + self.assertEqual(outcome["calls"], 3) + self.assertEqual([len(json.loads(prompt)["items"]) for _c, prompt, _e in calls], [2, 2, 1]) + self.assertEqual(outcome["input_tokens"], 3 * 1012) + self.assertEqual(outcome["output_tokens"], 150) + self.assertEqual(outcome["cost_usd"], 0.03) + self.assertEqual(outcome["parse_failures"], 1) + self.assertEqual(outcome["format_violations"], 0) + self.assertEqual(len(outcome["verdicts"]), 4) + self.assertEqual(outcome["latencies"], [2.0, 2.0, 3.0, 3.0, 2.0]) + self.assertEqual(outcome["adjusted_latencies"], [1.0, 1.0, 2.0, 2.0, 0.0]) + for _command, _prompt, env in calls: + self.assertNotIn("TYPESAFE_API_KEY", env) + self.assertEqual(env["PATH"], "/bin") + + def test_cli_failures_count_every_pair_in_the_batch_as_a_parse_failure(self): + workload = small_workload(3) + replies = iter([Completed("", 1, "boom\nlast line"), Completed("not json"), + Completed(json.dumps({"is_error": True, "result": "rate limited"}))]) + outcome = bench.run_llm("claude:sonnet", workload, run=lambda *a, **k: next(replies), environ={}, batch_size=1) + self.assertEqual(outcome["parse_failures"], 3) + self.assertEqual((outcome["calls"], outcome["failed_calls"], len(outcome["latencies"])), (3, 3, 3)) + self.assertEqual(outcome["verdicts"], {}) + self.assertIn("exited 1: last line", outcome["errors"][0]) + self.assertIn("did not return JSON", outcome["errors"][1]) + self.assertIn("rate limited", outcome["errors"][2]) + + def test_failed_calls_still_cost_what_the_cli_reported_and_odd_envelopes_do_not_crash(self): + failed = json.dumps({"is_error": True, "result": "overloaded", "total_cost_usd": 0.5, + "usage": {"input_tokens": 900, "output_tokens": "many"}}) + replies = iter([Completed(failed), Completed("null"), Completed(json.dumps({"result": None, "usage": None})), + Completed(failed, returncode=1, stderr="boom")]) + outcome = bench.run_llm("claude:haiku", small_workload(4), run=lambda *a, **k: next(replies), environ={}, batch_size=1) + self.assertEqual((outcome["cost_usd"], outcome["input_tokens"], outcome["output_tokens"]), (1.0, 1800, 0)) + self.assertEqual((outcome["failed_calls"], outcome["parse_failures"]), (3, 4)) + self.assertIsNone(bench.verdict_from_answers({"contradicts": True}, 0.7)) + self.assertIsNone(bench.verdict_from_answers([], 0.7)) + + def test_cli_diagnostics_never_carry_secret_environment_values(self): + environ = {"ANTHROPIC_AUTH_TOKEN": "sk-SENTINEL-0123456789", "PATH": "/bin"} + replies = iter([Completed("", 1, "auth failed for sk-SENTINEL-0123456789"), + Completed(json.dumps({"is_error": True, "result": "bad token sk-SENTINEL-0123456789"}))]) + outcome = bench.run_llm("claude:haiku", small_workload(2), run=lambda *a, **k: next(replies), environ=environ, batch_size=1) + self.assertEqual(len(outcome["errors"]), 2) + self.assertNotIn("SENTINEL", json.dumps(outcome)) + self.assertIn("[redacted]", outcome["errors"][0]) + + def test_timeout_kills_the_whole_process_group(self): + import sys as _sys + script = "import subprocess,sys,time; subprocess.Popen([sys.executable,'-c','import time; time.sleep(30)']); time.sleep(30)" + with self.assertRaises(subprocess.TimeoutExpired): + bench.spawn([_sys.executable, "-c", script], input="", timeout=1) + finished = bench.spawn([_sys.executable, "-c", "import sys; print(sys.stdin.read().upper())"], input="ok", timeout=20) + self.assertEqual((finished.returncode, finished.stdout.strip()), (0, "OK")) + + def test_missing_cli_and_timeouts_are_clean_errors(self): + def missing(*args, **kwargs): + raise FileNotFoundError("claude") + + def slow(*args, **kwargs): + raise subprocess.TimeoutExpired("claude", 1) + + with self.assertRaises(bench.BenchmarkError) as raised: + bench.measure_overhead("claude:haiku", run=missing, environ={}) + self.assertIn("not on PATH", str(raised.exception)) + with self.assertRaises(bench.BenchmarkError): + bench.measure_overhead("claude:haiku", run=slow, environ={}) + + def test_event_list_output_and_codex_plain_text_are_understood(self): + workload = small_workload(1) + items = [{"id": "p01", "contradicts": False, "covers": True, "staleness": 0}] + events = json.dumps([{"type": "system"}, json.loads(claude_envelope(items))]) + outcome = bench.run_llm("claude:haiku", workload, run=lambda *a, **k: Completed(events), environ={}) + self.assertEqual(len(outcome["verdicts"]), 1) + outcome = bench.run_llm("codex", workload, run=lambda *a, **k: Completed("thinking...\n" + json.dumps(items)), environ={}) + self.assertEqual(len(outcome["verdicts"]), 1) + self.assertIsNone(outcome["cost_usd"]) + self.assertIsNone(outcome["input_tokens"], "codex token use is unknown, not zero") + summary = bench.summarize({"codex": [outcome]}, {}, [workload["pairs"][0]["key"]]) + self.assertIn("- / -", bench.render(summary)) + + +class TypeSafeRunTests(unittest.TestCase): + def test_recorded_run_binarizes_at_the_gate_and_sums_recorded_usage(self): + workload = bench.build_workload(record(), limit=0) + run = bench.run_typesafe_recorded(workload, 0.7) + self.assertTrue(run["verdicts"]["ncl:drift|page#ncl-drift-1"]["contradicts"]) + self.assertFalse(run["verdicts"]["env:unsure|page#env-unsure-1"]["contradicts"]) + self.assertTrue(bench.run_typesafe_recorded(workload, 0.5)["verdicts"]["env:unsure|page#env-unsure-1"]["contradicts"]) + self.assertEqual(run["input_tokens"], 100 * len(workload["pairs"])) + self.assertEqual(run["parse_failures"], 0) + self.assertIsNone(run["wall_seconds"], "no recorded latencies means unknown, not zero") + for bad in ({"contradicts": {"noul": True}, "covers": {"noul": "0.9"}, "staleness": {"score": 1}}, + {"contradicts": {"noul": 0.9}, "covers": {"noul": 0.9}}, + {"contradicts": {"noul": 1.4}, "covers": {"noul": 0.9}, "staleness": {"score": 1}}, + {"contradicts": {"noul": 0.4}, "covers": {"noul": 0.9}, "staleness": {"score": 9}}, None): + self.assertIsNone(bench.verdict_from_answers(bad, 0.7), bad) + + def test_live_run_sends_the_skill_request_sequentially_and_survives_failures(self): + workload = small_workload(3) + seen = [] + + def transport(payload): + seen.append(payload) + if len(seen) == 2: + raise bench.BenchmarkError("HTTP 529 from TypeSafe: overloaded") + return response(0.9, 0.9, tokens=7) + + run = bench.run_typesafe_live(workload, 0.7, transport) + self.assertEqual(len(seen), 3) + self.assertEqual(set(seen[0]["questions"]), {"contradicts", "covers", "staleness"}) + self.assertEqual(seen[0]["model"], "jev-latest") + self.assertEqual(set(seen[0]["state"]), {"fact", "doc_section"}) + self.assertEqual((len(run["verdicts"]), run["parse_failures"], run["input_tokens"]), (2, 1, 14)) + self.assertEqual((len(run["errors"]), run["calls"], run["failed_calls"], len(run["latencies"])), (1, 3, 1, 3)) + + +class MetricTests(unittest.TestCase): + def run_with(self, verdicts): + run = bench.empty_run("x") + run["verdicts"] = {key: {"contradicts": c, "covers": v, "staleness": 0} for key, (c, v) in verdicts.items()} + return run + + def test_percentiles_flip_rate_and_agreement(self): + self.assertEqual(bench.percentile([1, 2, 3, 4], 0.5), 2.5) + self.assertEqual(bench.percentile([1, 2, 3, 4, 100], 0.95), 80.8) + self.assertIsNone(bench.percentile([], 0.5)) + first = self.run_with({"a": (True, True), "b": (False, True), "c": (True, False), "d": (False, False)}) + second = self.run_with({"a": (True, True), "b": (True, True), "c": (True, True)}) + self.assertEqual(bench.flip_rate(first, second), {"pairs": 3, "flips": 1, "rate": round(1 / 3, 4)}) + self.assertEqual(bench.agreement(first, second, "covers"), {"pairs": 3, "rate": round(2 / 3, 4)}) + self.assertIsNone(bench.flip_rate(first, self.run_with({}))) + + def test_precision_and_recall_count_unanswered_gold_pairs_as_a_no(self): + gold = {"a": {"contradicts": True, "covers": True}, "b": {"contradicts": True, "covers": True}, + "c": {"contradicts": False, "covers": True}, "z": {"contradicts": True, "covers": True}} + run = self.run_with({"a": (True, True), "c": (True, True)}) + scores = bench.precision_recall(run, gold, ["a", "b", "c", "d"]) + self.assertEqual((scores["tp"], scores["fp"], scores["fn"], scores["tn"]), (1, 1, 1, 0)) + self.assertEqual((scores["precision"], scores["recall"]), (0.5, 0.5)) + empty = bench.precision_recall(self.run_with({}), {"c": {"contradicts": False, "covers": False}}, ["c"]) + self.assertEqual((empty["precision"], empty["recall"]), (None, None)) + + def test_summary_rows_and_rendered_table_have_the_requested_columns(self): + first = self.run_with({"a": (True, True)}) + first.update(wall_seconds=3.0, latencies=[1.0], adjusted_latencies=[0.5], input_tokens=9, output_tokens=1, cost_usd=0.002) + second = self.run_with({"a": (False, True)}) + second["parse_failures"] = 4 + summary = bench.summarize({"claude:haiku": [first, second], "typesafe-recorded": [self.run_with({"a": (True, True)})]}, + {"a": {"contradicts": True, "covers": True}}, ["a"]) + row = summary["rows"][0] + self.assertEqual((row["flip"]["rate"], row["flip"]["pairs"], row["gold"]["precision"], row["gold"]["recall"]), (1.0, 1, 1.0, 1.0)) + self.assertEqual(row["parse_failures_by_run"], [0, 4]) + self.assertIsNone(summary["rows"][1]["flip"]) + self.assertEqual(summary["agreement"][0]["contradicts"], {"pairs": 1, "rate": 1.0}) + text = bench.render(summary) + for column in ("Contender", "Pairs", "Wall s", "p50 s/pair", "Tokens in/out", "Cost $", "Parse fail", "Flip %", "Precision", "Recall"): + self.assertIn(column, text) + self.assertIn("agreement claude:haiku vs typesafe-recorded: contradicts 100.0%", text) + self.assertIn("0/4", text) + self.assertIn("100.0 (1/1)", text) + + +class CliTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="drift-benchmark-") + self.addCleanup(self.temporary.cleanup) + self.output = Path(self.temporary.name) / "out" + self.patcher = patch.object(bench.drift.urllib.request, "urlopen", side_effect=AssertionError("live TypeSafe call attempted")) + self.patcher.start() + self.addCleanup(self.patcher.stop) + + def run_main(self, argv, environ=None, run=None): + stdout, stderr = io.StringIO(), io.StringIO() + runner = run or (lambda *a, **k: self.fail("no CLI may be spawned")) + code = bench.main(argv + ["--output-dir", str(self.output)], environ={} if environ is None else environ, + stdout=stdout, stderr=stderr, run=runner) + return code, stdout.getvalue(), stderr.getvalue() + + def test_default_run_is_offline_and_scores_the_recorded_answers_against_the_gold(self): + code, out, err = self.run_main(["--json"]) + self.assertEqual(code, 0, err) + payload = json.loads(out) + row = payload["summary"]["rows"][0] + self.assertEqual((row["contender"], row["pairs"], row["parse_failures"]), ("typesafe-recorded", 60, 0)) + self.assertEqual(payload["gold"]["pairs"], 30) + self.assertIn("LLM-assisted", payload["gold"]["labeled_by"]) + self.assertGreater(row["input_tokens"], 0) + self.assertIsNotNone(row["gold"]["precision"]) + self.assertEqual(len(list(self.output.glob("benchmark-*.json"))), 1) + + def test_llm_contender_runs_twice_after_one_overhead_call_and_limit_applies(self): + prompts = [] + + def run(command, input, capture_output, text, timeout, env): + prompts.append(input) + if input == bench.OVERHEAD_PROMPT: + return Completed(claude_envelope([])) + items = [{"id": item["id"], "contradicts": len(prompts) % 2 == 0, "covers": True, "staleness": 0} for item in json.loads(input)["items"]] + return Completed(claude_envelope(items)) + + code, out, err = self.run_main(["--contenders", "typesafe-recorded,claude:haiku", "--limit", "12", "--batch-size", "10", "--json"], + environ={"TYPESAFE_API_KEY": "ts-FAKE-KEY-0123456789"}, run=run) + self.assertEqual(code, 0, err) + self.assertEqual(len(prompts), 1 + 2 * 2) + payload = json.loads(out) + rows = {row["contender"]: row for row in payload["summary"]["rows"]} + self.assertEqual(rows["claude:haiku"]["pairs"], 12) + self.assertEqual(rows["claude:haiku"]["answered"], 12) + self.assertEqual(rows["claude:haiku"]["flip"]["pairs"], 12) + self.assertIsNotNone(rows["claude:haiku"]["overhead_seconds"]) + self.assertEqual(len(payload["summary"]["agreement"]), 1) + for text in (out, err, next(self.output.glob("benchmark-*.json")).read_text()): + self.assertNotIn("ts-FAKE-KEY", text) + + def test_a_failed_overhead_probe_does_not_lose_the_run(self): + def run(command, input, capture_output, text, timeout, env): + if input == bench.OVERHEAD_PROMPT: + return Completed("", 1, "rate limited") + items = [{"id": item["id"], "contradicts": False, "covers": True, "staleness": 0} for item in json.loads(input)["items"]] + return Completed(claude_envelope(items)) + + code, out, err = self.run_main(["--contenders", "claude:haiku", "--limit", "3", "--runs", "1", "--json"], run=run) + self.assertEqual(code, 0, err) + self.assertIn("could not be measured", err) + row = json.loads(out)["summary"]["rows"][0] + self.assertEqual((row["answered"], row["overhead_seconds"]), (3, None)) + + def test_typesafe_live_needs_the_key_and_uses_the_mocked_transport(self): + code, out, err = self.run_main(["--contenders", "claude:haiku,typesafe-live"], environ={}) + self.assertEqual(code, 2, "the key is checked before any paid contender is spawned") + self.assertIn("TYPESAFE_API_KEY is not set", err) + self.assertEqual(out, "") + with patch.object(bench.drift, "HttpTransport", return_value=lambda payload: response(0.9, 0.9, tokens=3)): + code, out, err = self.run_main(["--contenders", "typesafe-live", "--limit", "4", "--json"], + environ={"TYPESAFE_API_KEY": "ts-FAKE-KEY-0123456789"}) + self.assertEqual(code, 0, err) + payload = json.loads(out) + self.assertEqual(len(payload["runs"]["typesafe-live"]), 2) + self.assertEqual(payload["summary"]["rows"][0]["flip"]["rate"], 0.0) + self.assertEqual(payload["summary"]["rows"][0]["input_tokens"], 12) + + def test_saved_results_can_be_rendered_again_with_the_gold_they_were_scored_on(self): + custom = Path(self.temporary.name) / "gold.json" + first_key = json.loads(WORKLOAD.read_text())["pairs"][0]["key"] + custom.write_text(json.dumps({"labeled_by": "someone else", "labels": {first_key: {"contradicts": False, "covers": True}}})) + code, _out, _err = self.run_main(["--gold", str(custom)]) + self.assertEqual(code, 0) + saved = next(self.output.glob("benchmark-*.json")) + code, out, _err = self.run_main(["--from-results", str(saved), "--json"]) + self.assertEqual(code, 0) + payload = json.loads(out) + self.assertEqual((payload["gold"]["pairs"], payload["gold"]["labeled_by"]), (1, "someone else")) + self.assertEqual(payload["summary"]["rows"][0]["gold"]["fp"], 1) + code, out, _err = self.run_main(["--from-results", str(saved)]) + self.assertIn("typesafe-recorded", out) + self.assertEqual(len(list(self.output.glob("benchmark-*.json"))), 1) + + def test_record_mode_writes_a_workload_and_bad_input_is_a_clean_error(self): + source = Path(self.temporary.name) / "record.json" + source.write_text(json.dumps(record())) + target = Path(self.temporary.name) / "workload.json" + code, out, err = self.run_main(["--record", str(source), "--write-workload", str(target), "--limit", "6"]) + self.assertEqual(code, 0, err) + self.assertEqual(len(json.loads(target.read_text())["pairs"]), 6) + code, _out, err = self.run_main(["--workload", str(source)]) + self.assertEqual(code, 1) + self.assertIn("'pairs'", err) + with self.assertRaises(SystemExit): + with patch("sys.stderr", io.StringIO()): + bench.parse_args(["--contenders", "gpt:4"]) + + +if __name__ == "__main__": + unittest.main() From 41ef4d2c2073070a39af6dc08e232322e50716a8 Mon Sep 17 00:00:00 2001 From: glifocat Date: Wed, 23 Sep 2026 18:11:11 +0200 Subject: [PATCH 3/4] typesafe-docs-drift: exit 0 on partial runs unless --strict; collapse HTML error bodies; note the gold set's enum caveat --- docs/typesafe-docs-drift.md | 16 +++++++++--- skills/typesafe-docs-drift/SKILL.md | 3 ++- .../scripts/typesafe-docs-drift.py | 22 +++++++++++++--- tests/test_typesafe_docs_drift.py | 26 ++++++++++++++++--- 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/docs/typesafe-docs-drift.md b/docs/typesafe-docs-drift.md index bcaa478..9ef3732 100644 --- a/docs/typesafe-docs-drift.md +++ b/docs/typesafe-docs-drift.md @@ -104,8 +104,11 @@ python3 skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py \ Requests run with `--concurrency` 4 workers by default; 429 and 529 are retried with backoff. A pair that fails after retries is reported in `partial` and counted in its fact's `failed_pairs`; the fact is judged on the pairs that -completed and is never called `MISSING` on that basis, and the exit status -is 1. With `--concurrency 1` the run stops after three consecutive failures. +completed and is never called `MISSING` on that basis. The summary ends with a +`partial:` line naming the failed pairs (an HTML error body, such as a proxy or +WAF page, is collapsed to its text), and the exit status stays 0 so a mostly +complete run is still usable; pass `--strict` to exit 1 on any failed pair. +With `--concurrency 1` the run stops after three consecutive failures. `--record` writes facts, sections, candidates, lexical scores and raw responses in the fixture format, dropping pairs that failed so the record replays cleanly. Raw answers for every run are saved as @@ -201,7 +204,14 @@ section without looking at any contender's answer. No human has verified them; three are marked `borderline`. Two biases to keep in mind: the sample was stratified by TypeSafe's own verdicts, so most positives are pairs TypeSafe already flagged, which favors its recall; and the labeler is a Claude model, -which may favor the Claude contenders' reading of borderline cases. +which may favor the Claude contenders' reading of borderline cases. A third +caveat: the gold answers the benchmark's question (does the section contradict +the extracted fact?), not "is the doc wrong?". Both `enum` positives in the set +(`ncl approvals` `status`, `ncl dropped-messages` `reason`) are cases where the +doc lists a value the database really uses and the CLI column `enum` is the +stale side, so a contender scores a hit for flagging a code bug as doc drift. +A human spot check of 10 seeded pairs on 2026-09-23 agreed with all 10 +`contradicts` and `covers` labels and disagreed with one `borderline` flag. **Results.** One run on 2026-09-17 10:12 to 10:34 UTC from one macOS machine, 60 pairs, two passes per live contender, `claude` CLI 2.1.273, `jev-latest`: diff --git a/skills/typesafe-docs-drift/SKILL.md b/skills/typesafe-docs-drift/SKILL.md index ee77d1e..53374f5 100644 --- a/skills/typesafe-docs-drift/SKILL.md +++ b/skills/typesafe-docs-drift/SKILL.md @@ -83,7 +83,8 @@ Flags: `--areas` (comma-separated subset of `ncl,env,container-config,skills, gateway,timestamps`), `--limit-facts`, `--top-k` (3), `--include-changelog`, `--contradicts-threshold` (0.7), `--covers-threshold` (0.7), `--staleness-threshold` (0.5, score confidence), `--concurrency` (4), -`--model`, `--timeout`, `--json`. +`--model`, `--timeout`, `--json`, `--strict` (exit 1 when any pair failed after +retries; by default failures are listed under `partial` and the exit status is 0). ## Benchmark diff --git a/skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py b/skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py index 00db5c3..c5d66a9 100644 --- a/skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py +++ b/skills/typesafe-docs-drift/scripts/typesafe-docs-drift.py @@ -786,6 +786,17 @@ def pair_key(fact_id, section_id): # Transports # --------------------------------------------------------------------------- + +def summarize_error_body(body): + """Keep an error body readable: an HTML page (a proxy or WAF answer) is collapsed to its text.""" + text = body.strip() + if text[:1] == "<" or "]*>.*?", " ", text, flags=re.S | re.I) + text = re.sub(r"<[^>]+>", " ", text) + text = "HTML error page: " + " ".join(text.split()) + return text[:300] + + class HttpTransport: """POST to the TypeSafe API with exponential backoff on 429/529.""" @@ -828,7 +839,7 @@ def __call__(self, payload): detail = "" try: # Redact before truncating: a cut through the key would defeat the replacement. - detail = self.redact(error.read().decode("utf-8", "replace"))[:300] + detail = summarize_error_body(self.redact(error.read().decode("utf-8", "replace"))) except Exception: # pragma: no cover - best effort detail = "" finally: @@ -1145,7 +1156,7 @@ def render_table(results): return table(("Verdict", "Area", "Fact", "Doc section", "Contra", "Covers", "Stale", "Evidence"), rows) -def render_summary(summary, usage, wall_seconds, thresholds, output_path, notes=()): +def render_summary(summary, usage, wall_seconds, thresholds, output_path, notes=(), partial=None): lines = ["", "Summary"] verdicts = summary["verdicts"] lines.append(" facts: %d, pairs: %d, requests: %d" % (summary["facts"], summary["pairs"], summary["requests"])) @@ -1158,6 +1169,8 @@ def render_summary(summary, usage, wall_seconds, thresholds, output_path, notes= lines.append(" wall time: %.1fs" % wall_seconds) for note in notes: lines.append(" note: %s" % note) + if partial: + lines.append(" partial: %d request(s) failed after retries: %s" % (summary["pairs"] - summary["requests"], partial)) if output_path: lines.append(" raw answers: %s" % output_path) return "\n".join(lines) @@ -1241,6 +1254,7 @@ def parse_args(argv): parser.add_argument("--covers-threshold", type=float, default=0.7, help="yes probability needed to call a section covering (default 0.7)") parser.add_argument("--staleness-threshold", type=float, default=0.5, help="score confidence below which the staleness label is marked '?' (default 0.5)") parser.add_argument("--json", action="store_true", help="print the full result JSON to stdout instead of the table") + parser.add_argument("--strict", action="store_true", help="exit 1 when any pair failed after retries (default: exit 0 and report the failures under `partial`)") args = parser.parse_args(argv) for name in ("contradicts_threshold", "covers_threshold"): if not 0.5 <= getattr(args, name) <= 1: @@ -1385,8 +1399,8 @@ def main(argv=None, environ=None, stdout=None, stderr=None): mode, len(facts), len(jobs)), file=stdout) print("", file=stdout) print(render_table(results), file=stdout) - print(render_summary(summary, usage, wall, thresholds, output_path, notes), file=stdout) - return 1 if error_text else 0 + print(render_summary(summary, usage, wall, thresholds, output_path, notes, error_text), file=stdout) + return 1 if error_text and args.strict else 0 def redact_env(text, environ): diff --git a/tests/test_typesafe_docs_drift.py b/tests/test_typesafe_docs_drift.py index 6d2ded6..6a89a22 100644 --- a/tests/test_typesafe_docs_drift.py +++ b/tests/test_typesafe_docs_drift.py @@ -815,7 +815,7 @@ def test_key_never_appears_in_output_or_saved_json(self): for text in (out, err, saved): self.assertNotIn(sentinel, text) - def test_partial_failure_keeps_completed_pairs_and_exits_nonzero(self): + def test_partial_failure_keeps_completed_pairs_and_exits_zero(self): fixture = json.loads(FIXTURE.read_text()) original = drift.FixtureTransport(fixture["responses"]) @@ -826,7 +826,7 @@ def flaky(payload): with patch.object(drift, "FixtureTransport", return_value=flaky): code, out, err = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output), "--json"]) - self.assertEqual(code, 1) + self.assertEqual(code, 0) self.assertIn("keeping 5 completed pair(s)", err) payload = json.loads(out) self.assertIn("env:CONTAINER_PIDS_LIMIT|", payload["partial"]) @@ -837,6 +837,26 @@ def flaky(payload): self.assertIn("request(s) failed", pids["note"]) self.assertEqual(len(list(self.output.glob("drift-*.json"))), 1) + def test_strict_makes_a_partial_run_exit_nonzero_and_the_table_names_it(self): + fixture = json.loads(FIXTURE.read_text()) + original = drift.FixtureTransport(fixture["responses"]) + + def flaky(payload): + if payload["_key"].startswith("env:CONTAINER_PIDS_LIMIT|"): + raise drift.DriftError("HTTP 403 from TypeSafe: HTML error page: Access denied") + return original(payload) + + with patch.object(drift, "FixtureTransport", return_value=flaky): + code, out, err = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output), "--strict"]) + self.assertEqual(code, 1) + self.assertIn("partial: 1 request(s) failed after retries: env:CONTAINER_PIDS_LIMIT|", out) + self.assertIn("HTML error page: Access denied", out) + + def test_html_error_bodies_are_collapsed_to_text(self): + body = "403

Access denied

Request blocked.

" + self.assertEqual(drift.summarize_error_body(body), "HTML error page: 403 Access denied Request blocked.") + self.assertEqual(drift.summarize_error_body('{"error": "bad"}'), '{"error": "bad"}') + def test_output_files_never_overwrite_each_other(self): with patch.object(drift, "stamp", return_value="fixed"): first = drift.write_output(self.output, {"a": 1}) @@ -864,7 +884,7 @@ def fake(payload): ["--code", str(self.code()), "--docs", str(self.docs()), "--output-dir", str(self.output), "--areas", "container-config,env", "--top-k", "1", "--record", str(record), "--json"], environ={"TYPESAFE_API_KEY": "ts-FAKE-KEY-0123456789"}) - self.assertEqual(code, 1, err) + self.assertEqual(code, 0, err) payload = json.loads(out) self.assertEqual(payload["mode"], "live") self.assertIn("[1/", err) From d421333bfa152bee875b0610ac8536ae0a96aeaa Mon Sep 17 00:00:00 2001 From: glifocat Date: Wed, 23 Sep 2026 19:17:48 +0200 Subject: [PATCH 4/4] test(shared-terminal): wait for the state file to parse, not just exist The server creates its private state file with O_EXCL and writes the body afterwards, so setUp could read an empty file and fail with a JSON decode error (ubuntu CI on this branch and on main's #25 merge run). Poll until the file parses, within the same 8 s deadline. --- tests/test_shared_terminal.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_shared_terminal.py b/tests/test_shared_terminal.py index c0639c4..183101a 100644 --- a/tests/test_shared_terminal.py +++ b/tests/test_shared_terminal.py @@ -35,10 +35,16 @@ def setUp(self): [sys.executable, str(SERVER), '--cwd', str(self.root), '--state-file', str(self.state_path)], stdout=self.output, stderr=subprocess.STDOUT, start_new_session=True, ) + # The server creates the state file (O_EXCL) before its body is flushed, so wait for + # parseable JSON rather than for the path to exist, or a fast reader sees an empty file. deadline = time.monotonic() + 8 - while not self.state_path.exists() and self.process.poll() is None and time.monotonic() < deadline: - time.sleep(0.025) - if not self.state_path.exists(): + self.state = None + while self.state is None and self.process.poll() is None and time.monotonic() < deadline: + try: + self.state = json.loads(self.state_path.read_text()) + except (FileNotFoundError, json.JSONDecodeError): + time.sleep(0.025) + if self.state is None: startup_status = self.process.poll() self.process.terminate() try: @@ -52,7 +58,6 @@ def setUp(self): self.temp.cleanup() self.fail(f'Shared terminal did not create its private state file ' f'(startup exit={startup_status}): {diagnostic}') - self.state = json.loads(self.state_path.read_text()) self.url = self.state['url'] self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) self.wait_text('shared-terminal$')