From fb282b175a2b44b059d51c5ed0450cfd8b978fc7 Mon Sep 17 00:00:00 2001 From: glifocat Date: Wed, 16 Sep 2026 23:29:08 +0200 Subject: [PATCH 1/3] feat(typesafe-triage): add dry-run label triage with fixture replay Add a typesafe-triage skill that fetches recently updated open NanoClaw issues and PRs read-only through gh, sends one TypeSafe System One fan-out request per item (area, kind, priority, needs_repro or pr_ready), gates on confidence with triage/unresolved below the gate, and prints proposals next to existing labels with AGREE/DISAGREE/NEW verdicts. Raw answers are saved under a gitignored directory; --fixture replays recorded responses offline and --record captures a live run in the same format. The key is read from TYPESAFE_API_KEY only, rejected if it contains whitespace, and redacted from error text. Nothing is written to GitHub. --- .claude-plugin/plugin.json | 2 +- .gitignore | 3 + CHANGELOG.md | 11 +- README.md | 1 + docs/skills-catalog.md | 33 +- docs/typesafe-triage.md | 92 ++ skills/typesafe-triage/SKILL.md | 93 ++ .../fixtures/nanoclaw-sample.json | 472 ++++++++++ .../scripts/typesafe-triage.py | 848 ++++++++++++++++++ tests/test_typesafe_triage.py | 555 ++++++++++++ 10 files changed, 2103 insertions(+), 7 deletions(-) create mode 100644 docs/typesafe-triage.md create mode 100644 skills/typesafe-triage/SKILL.md create mode 100644 skills/typesafe-triage/fixtures/nanoclaw-sample.json create mode 100644 skills/typesafe-triage/scripts/typesafe-triage.py create mode 100644 tests/test_typesafe_triage.py diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index a05a44f..41e45b3 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "nanoclaw-e2e", "description": "Install and verify NanoClaw on exe.dev, Proxmox, macOS or Windows WSL2, then triage failures against upstream issues and PRs with form-aware reporting.", - "version": "0.12.0", + "version": "0.13.0", "author": { "name": "Nano Co", "url": "https://github.com/nanocoai" diff --git a/.gitignore b/.gitignore index 9ffdc46..b8315e3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ __pycache__/ # typesafe-docs-drift raw answers (doc excerpts and code snippets) and this clone's worktrees. /skills/typesafe-docs-drift/output/ /.worktrees/ + +# typesafe-triage raw answers (may contain issue bodies). +/skills/typesafe-triage/output/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 1370139..ad255f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,8 @@ instead of being backfilled as releases. ## Unreleased -Development milestone (plugin 0.12.0); supersedes the unreleased 0.11.0 -milestone and is not yet released. +Development milestone (plugin 0.13.0); supersedes the unreleased 0.11.0 and +0.12.0 milestones and is not yet released. ### Added @@ -63,6 +63,13 @@ milestone and is not yet released. wall time, per-pair latency with startup overhead separated, tokens, cost, parse failures, run-to-run flips, agreement, and precision/recall on `contradicts` against a 30-pair gold file labeled by an LLM-assisted reviewer. +- `typesafe-triage` skill: a dry-run label triage of open nanocoai/nanoclaw + issues and PRs through the TypeSafe System One API. One fan-out request per + item (area, kind, priority, needs-repro or PR readiness), a confidence gate + that falls back to `triage/unresolved`, AGREE/DISAGREE/NEW against existing + labels, a summary with agreement rates, token usage and wall time, raw answers + saved under a gitignored directory, and `--fixture` replay for offline runs. + Reads `TYPESAFE_API_KEY` from the environment only and never writes to GitHub. - OpenCode public-wizard E2E for OpenRouter, DeepSeek, and custom/self-hosted API-key endpoints across exe.dev, Proxmox and Windows WSL2. Select the model with `--opencode-model`, the endpoint with `--opencode-base-url`, and its API diff --git a/README.md b/README.md index 192a5c3..e2bf87b 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ selected provider's authentication choices from that exact Git revision. | [e2e-windows](skills/e2e-windows/SKILL.md) | Qualify Windows WSL2 with the local Docker Desktop engine, then run the public wizard with validated reply, service and sanitized evidence. | | [e2e-triage](skills/e2e-triage/SKILL.md) | Research unexpected E2E failures against upstream issues and PRs, recommend next actions, and prepare drafts using the target repository’s current issue forms. | | [typesafe-docs-drift](skills/typesafe-docs-drift/SKILL.md) | Rank where the docs portal disagrees with the NanoClaw code using the TypeSafe decision API: deterministic fact extraction, lexical section search, contradicts / covers / staleness per pair, DRIFT / MISSING / UNSURE / OK report; offline fixture replay; nothing written. | +| [typesafe-triage](skills/typesafe-triage/SKILL.md) | Dry-run label triage of NanoClaw issues and PRs with the TypeSafe decision API: confidence-gated area, kind, priority and triage proposals compared with existing labels; offline fixture replay; nothing written to GitHub. | | [shared-terminal](skills/shared-terminal/SKILL.md) | One local browser terminal shared by the human and agent for supervised installers and interactive handoffs on macOS or Linux. | Each skill follows the [Agent Skills format](https://agentskills.io). Use it with diff --git a/docs/skills-catalog.md b/docs/skills-catalog.md index fad55cf..e3a1bc1 100644 --- a/docs/skills-catalog.md +++ b/docs/skills-catalog.md @@ -1,13 +1,14 @@ # NanoClaw OSS Dev Tools: skill catalog -Verified: **2026-09-17**. This checkout contains **8 skills**, with plugin -manifest version **0.12.0** (development, unreleased). Three drive headless setup steps; `e2e-wizard` and +Verified: **2026-09-24**. This checkout contains **9 skills**, with plugin +manifest version **0.13.0** (development, unreleased). Three drive headless setup steps; `e2e-wizard` and `e2e-windows` drive the public interactive wizard. The shared `e2e-triage` skill researches unexpected failures and prepares reporting recommendations. The workflows and their qualifications are listed separately below. `shared-terminal` provides a local terminal for human and agent handoffs. `typesafe-docs-drift` ranks where the docs portal disagrees with the code through the TypeSafe -decision API and writes nothing. +decision API and writes nothing. `typesafe-triage` previews confidence-gated +issue and PR labels from the TypeSafe decision API without writing to GitHub. Installed skill copies must be updated separately, and the manifest version does not imply a tagged GitHub release. @@ -34,6 +35,7 @@ attempt exposed a harness tool-path bug and remains a retained failure. See the | [e2e-windows](../skills/e2e-windows/SKILL.md) | Fresh Windows WSL2 distribution using the local Docker Desktop Linux engine. | Public wizard passed again on 2026-09-14 at current NanoClaw `main`; terminal-close and post-Windows-reboot inference also passed with locally validated sanitized evidence. | WSL-only restart still left the service/socket unavailable, and Docker Desktop did not start automatically after Windows reboot. | | [e2e-triage](../skills/e2e-triage/SKILL.md) | Agent workflow on the operator machine, using retained evidence and current upstream trackers. | Replayed retained Windows CA failure against live issues/PRs and the current NanoClaw bug form on 2026-09-12. | Instruction-driven; direct shell runs do not invoke it. No public submission was performed during validation. | | [typesafe-docs-drift](../skills/typesafe-docs-drift/SKILL.md) | Ranks disagreements between a NanoClaw checkout and the nanoclaw-docs portal through the TypeSafe System One API: deterministic facts, lexical candidate sections, one fan-out request per pair, gated DRIFT / MISSING / UNSURE / OK verdicts. | Offline fixture replay and mocked-transport tests; one live run on 2026-09-16 (142 facts, 426 requests, 71.5 s: DRIFT 7, MISSING 1, UNSURE 11, OK 123; see [typesafe-docs-drift.md](typesafe-docs-drift.md)). | Detection only; candidate recall is lexical top-3; fixture answers are hand-written. | +| [typesafe-triage](../skills/typesafe-triage/SKILL.md) | Dry-run label triage of open nanocoai/nanoclaw issues and PRs through the TypeSafe System One API, with a confidence gate and comparison against existing labels. | Offline fixture replay and mocked-transport tests only; no live TypeSafe call has been recorded yet. | Live agreement rates are unmeasured; the fixture answers are hand-written. It proposes labels only and never applies them. | | [shared-terminal](../skills/shared-terminal/SKILL.md) | One local PTY with browser and agent control, for supervised interactive work. | macOS browser typing, interactive prompts, agent/human handoff and shutdown passed; real HTTP/PTY tests passed. | Native Windows, WSL and browser forwarding are unqualified. | The shared baseline for the three headless skills tested NanoClaw @@ -312,7 +314,7 @@ run inside the dedicated WSL2 distribution from the NanoClaw checkout. **Entry point:** [SKILL.md](../skills/e2e-triage/SKILL.md), invoked by the testing agent after an unexpected failure or directly against a retained run. Install it -alongside the E2E skill; the plugin includes all eight. This workflow adds no +alongside the E2E skill; the plugin includes all nine. This workflow adds no runtime or GitHub dependency to the shell/Python test drivers. - Captures distinct failures without rewriting installation/recovery outcomes. @@ -367,6 +369,29 @@ Validation: fixture replay through the full pipeline and the mocked tests in `tests/test_typesafe_docs_drift.py`, plus one live run on 2026-09-16 whose sanitized summary is recorded on the detail page. +## typesafe-triage + +**Entry point:** [typesafe-triage.py](../skills/typesafe-triage/scripts/typesafe-triage.py), +run from any directory; it needs no NanoClaw checkout. + +- Needs Python 3.10+ and, for live mode, an authenticated `gh` plus + `TYPESAFE_API_KEY` exported in the shell. Standard library only; no SDK. +- Fetches the 30 most recently updated open issues and 20 open PRs read-only + (`gh api`), sends one fan-out request per item (area, kind, priority, and + `needs_repro` for issues or `pr_ready` for PRs), and prints proposed labels + with confidence, the existing labels and an AGREE / DISAGREE / NEW verdict. +- Gates on confidence (area and kind 0.6, priority 0.8, yes/no 0.7, all + overridable) and proposes `triage/unresolved` below the gate instead of guessing. +- `--fixture` replays recorded items and responses offline; the shipped fixture + holds five real items fetched on 2026-09-16 with hand-written answers. + `--record` saves a live run in the same format. Raw answers go to a gitignored + output directory. +- Writes nothing to GitHub. Details: [typesafe-triage.md](typesafe-triage.md). + +Validation is offline only: fixture replay through the full pipeline and the +mocked-transport tests in `tests/test_typesafe_triage.py`. No live TypeSafe +request has been made from this repository yet. + ## Installation and maintenance Use the [README installation instructions](../README.md#1-install-the-skill). diff --git a/docs/typesafe-triage.md b/docs/typesafe-triage.md new file mode 100644 index 0000000..43b2b47 --- /dev/null +++ b/docs/typesafe-triage.md @@ -0,0 +1,92 @@ +# TypeSafe triage dry run + +`skills/typesafe-triage/scripts/typesafe-triage.py` previews how TypeSafe's +System One model (`jev-latest`) would label open NanoClaw issues and pull +requests. It never writes to GitHub. This page records the questions, the +confidence gate, where the key comes from, and how to run and replay it. + +## What one request contains + +Every item gets exactly one `POST https://api.typesafe.ai/v1/systemone` with a +structured `state` and four questions (speculative fan-out: all questions are +answered in parallel and code decides which answers matter). + +State: `repository`, `item_type`, `number`, `title`, `author_association`, +`body` (first 12,000 characters). Pull requests add `draft`, `changed_files` +(first 80 paths), `changed_file_count` and `template` (whether the +`nanoclaw-pr-template:v2` marker is present and which of its eight sections are +`filled`, `empty` or `missing`). Existing labels are deliberately not sent. + +| Question | Type | Options and outcome | +|---|---|---| +| `area` | choice | The 16 `area/*` labels; one line of rubric each, derived from NanoClaw's CLAUDE.md, docs and `.github/labeler.yml` path mapping. | +| `kind` | choice | `kind/bug`, `kind/feature`, `kind/documentation`, `kind/question`, `kind/security`, `kind/hardening`, `kind/cleanup`, using the label descriptions. | +| `priority` | score | Four ordered levels with concrete descriptions; the score rounds to `priority/low`, `medium`, `high` or `critical`. | +| `needs_repro` (issues) | noul | Yes means the report lacks reproduction steps or a concrete failing case, proposing `triage/needs-repro`. Skipped when the top `kind` is not `kind/bug` or `kind/security`. | +| `pr_ready` (PRs) | noul | Yes means the template is filled, tests or a reason for none are stated and the diff matches the title, proposing `Status: Needs Review`; a confident no proposes `triage/needs-author`. | + +## Confidence gate + +Choice and score answers carry `confidence` (0 to 1) derived from their +probability distribution. Noul answers are a probability of yes. + +| Question | Default gate | Flag | +|---|---|---| +| `area`, `kind` | confidence >= 0.6 | `--area-threshold`, `--kind-threshold` | +| `priority` | confidence >= 0.8 | `--priority-threshold` | +| `needs_repro`, `pr_ready` | yes if p >= 0.7, no if p <= 0.3, otherwise uncertain | `--noul-threshold` | + +Below the gate the script proposes `triage/unresolved` and shows the model's top +answer in parentheses instead of guessing. Priority is maintainer-set in NanoClaw, +which is why its gate is the strictest. + +Each proposal is compared with the item's existing labels: `AGREE` when the label +is present, `DISAGREE` when a different label of the same `area/`, `kind/` or +`priority/` family is present, `NEW` when the family is absent. The summary gives +the agreement rate per question over compared items, the count of items with a +gated question, token usage and wall time. + +## The key + +Live mode reads `TYPESAFE_API_KEY` from the environment when the first request +is built. If it is unset the script exits with status 2 and a message; it does +not look for a file. Export it for the shell session only, for example from a +password manager. Never commit it, never put it in `.env` or a fixture, and never +paste it into an issue. The script does not log request headers, and the saved +JSON contains only items, responses and decisions. + +## Run + +Offline, using the shipped fixture (five real nanocoai/nanoclaw items fetched +read-only on 2026-09-16 with hand-written answers in the documented shapes): + +```bash +python3 skills/typesafe-triage/scripts/typesafe-triage.py \ + --fixture skills/typesafe-triage/fixtures/nanoclaw-sample.json +``` + +Live dry run (30 most recently updated open issues, 20 open PRs): + +```bash +export TYPESAFE_API_KEY=... +python3 skills/typesafe-triage/scripts/typesafe-triage.py --record /tmp/triage-record.json +``` + +`--record` writes items plus raw responses in the fixture format so the run can +be replayed with `--fixture` without spending tokens. Raw answers for every run +are saved as `triage-.json` under `skills/typesafe-triage/output/` +(gitignored) or `--output-dir`; `--json` prints the same payload to stdout. +`--repo`, `--issues` and `--prs` change the target and sample size. The issues +endpoint interleaves pull requests, so the fetch pages until it has enough real +issues. Requests run one at a time; if one fails after retries (429 and 529 are +retried with backoff), the completed items are still saved and printed, the +failing item is named in `partial`, and the exit status is 1. + +## Tests + +`tests/test_typesafe_triage.py` runs in the offline suite: question and request +shape, template-section detection, gate and verdict logic, `gh` output parsing +with a mocked runner, HTTP transport with a mocked opener (bearer header, retry +on 429/529, no retry on 422), fixture replay end to end, and a check that a key +in the environment never reaches stdout, stderr or the saved JSON. No test +contacts the network. diff --git a/skills/typesafe-triage/SKILL.md b/skills/typesafe-triage/SKILL.md new file mode 100644 index 0000000..a8be9ae --- /dev/null +++ b/skills/typesafe-triage/SKILL.md @@ -0,0 +1,93 @@ +--- +name: typesafe-triage +description: Dry-run label triage for nanocoai/nanoclaw issues and pull requests using the TypeSafe System One decision API. Proposes area, kind, priority and triage labels with confidence, compares them with the existing labels, and writes nothing to GitHub. Use to preview or evaluate automated triage, or to replay the offline fixture without a key. +license: MIT +--- + +# TypeSafe triage dry run + +Fetch the most recently updated open issues and pull requests, ask TypeSafe one +fan-out request per item, gate every answer on confidence, and print proposed +labels next to the labels the item already carries. The script is read-only +towards GitHub. Applying labels is a separate, human decision. + +Entry point: [typesafe-triage.py](scripts/typesafe-triage.py) (Python 3.10+, +standard library only). Offline fixture: +[nanoclaw-sample.json](fixtures/nanoclaw-sample.json). Details of the questions, +gate and output: [docs/typesafe-triage.md](../../docs/typesafe-triage.md). + +## Sources this skill drives + +- NanoClaw label taxonomy: `gh label list -R nanocoai/nanoclaw` (the `area/*`, + `kind/*`, `priority/*` and `triage/*` families and their descriptions). +- NanoClaw `CONTRIBUTING.md` (issue forms, "exactly one `area/*`", maintainer-only + priority, `triage/needs-repro` meaning) and `.github/labeler.yml` (path to area + mapping used for the area rubrics). +- NanoClaw `.github/PULL_REQUEST_TEMPLATE.md`, marker `nanoclaw-pr-template:v2`, + whose section headings the script checks for content. +- TypeSafe HTTP API: `POST https://api.typesafe.ai/v1/systemone`, model + `jev-latest`, question types `noul`, `choice`, `score` + ([API reference](https://docs.typesafe.ai/api.md), + [confidence](https://docs.typesafe.ai/confidence.md), + [speculative fan-out](https://docs.typesafe.ai/patterns/fan-out.md)). + +Re-check the rubrics when the label set or the labeler mapping changes. + +## Prerequisites + +- `gh` authenticated with read access to the target repository (live mode only). +- `TYPESAFE_API_KEY` exported in the shell (live mode only). The script reads it + from the environment at request time, never from a file, and never prints it. + Do not put the key in `.env`, a fixture, or any file inside a repository. + +## Run + +Offline demonstration, no key and no network: + +```bash +python3 skills/typesafe-triage/scripts/typesafe-triage.py \ + --fixture skills/typesafe-triage/fixtures/nanoclaw-sample.json +``` + +Live dry run against nanocoai/nanoclaw (30 issues, 20 PRs; nothing is written): + +```bash +export TYPESAFE_API_KEY=... # from your password manager, this shell only +python3 skills/typesafe-triage/scripts/typesafe-triage.py --record /tmp/triage-record.json +``` + +`--record` saves the fetched items and raw responses so the same run can be +replayed later with `--fixture`. Raw answers for every run land in +`skills/typesafe-triage/output/` (gitignored) or `--output-dir`. Item bodies are +included there; keep the directory private. + +Flags: `--repo`, `--issues`, `--prs`, `--area-threshold` (0.6), +`--kind-threshold` (0.6), `--priority-threshold` (0.8), `--noul-threshold` (0.7), +`--model`, `--timeout`, `--json`. + +## Read the output + +One row per proposal. `Verdict` compares the proposal with the existing labels: +`AGREE` (already present), `DISAGREE` (a different label from the same +`area/`, `kind/` or `priority/` family is present), `NEW` (that family is +absent), `SKIP` (not applicable, for example `needs_repro` on a feature +request). Below the confidence gate the proposal is `triage/unresolved` with the +model's top answer in parentheses, so a maintainer sees what it leaned towards +without the script guessing. + +The summary reports agreement per question over compared items only +(agree / (agree + disagree)), the number of items with at least one gated +question, token usage and wall time. + +## Limits + +- Dry run only. There is no flag that writes labels, comments or anything else. +- The fixture's answers are hand-written in the documented response shapes to + exercise the pipeline; they are not evidence of model accuracy. Evaluate + accuracy with a live `--record` run and the agreement summary. +- `priority/*` is maintainer-set; the default 0.8 gate keeps most priority + proposals as `triage/unresolved` on purpose. +- Existing labels are never sent to the model, so agreement is a fair comparison. +- Requests are sequential; 50 items take roughly a minute. A request that fails + after retries stops the run, but completed items are still saved and printed + (exit status 1, `partial` names the failing item). diff --git a/skills/typesafe-triage/fixtures/nanoclaw-sample.json b/skills/typesafe-triage/fixtures/nanoclaw-sample.json new file mode 100644 index 0000000..db1436c --- /dev/null +++ b/skills/typesafe-triage/fixtures/nanoclaw-sample.json @@ -0,0 +1,472 @@ +{ + "schema_version": 1, + "repo": "nanocoai/nanoclaw", + "recorded_at": "2026-09-16T20:45:00Z", + "note": "Items are real open nanocoai/nanoclaw issues/PRs fetched read-only on 2026-09-16 (bodies truncated). Responses are hand-written in the documented TypeSafe answer shapes, not recorded from the API.", + "items": [ + { + "type": "issue", + "number": 3839, + "title": "[bug] registry-skills: add-opencode reapply pass hangs in bun test until the 6-hour cancel", + "body": "### Affected NanoClaw version or commit\n\nCore `19e2ed30` (#3817 head) with the `providers` registry branch at the SHA `registry-skills.yml` pinned for that run. The reapply pass that hung is unchanged on current `main`.\n\n### Host platform\n\nLinux (GitHub `ubuntu-latest` runner, Bun 1.4.0, Node 22)\n\n### What happened?\n\nIn `registry-skills.yml`, the `test (add-opencode, true, true)` matrix job hung and was cancelled by GitHub at the 6-hour default. `provider-promotion` and `registry gate` then failed closed for the whole run.\n\n`scripts/test-registry-skills.ts` runs the provider round trip: apply \u2192 remove \u2192 verify contracts \u2192 apply again. The `add-opencode` SKILL.md test step is:\n\n```\ncd container/agent-runner && bun test --isolate src/providers/opencode*.test.ts src/providers/mcp-to-opencode.test.ts\n```\n\nThe first apply pass ran that command and finished in 13 s. The second apply pass (the reapply, `scripts/test-registry-skills.ts:368`) ran the same command in the same checkout and never returned. No test output after the hang survives; the job log was lost with the cancellation.\n\n### What did you expect?\n\nThe reapply pass finishes in about the same time as the first pass, and the job completes in around 3 minutes like every other run of it.\n\n### How can we reproduce it?\n\nNot reproduced on demand. Evidence so far:\n\n- #3818, which carries 3817's commit, ran the same job twice and passed both times at about 3 min.\n- Three local passes of `pnpm exec tsx scripts/test-registry-skills.ts add-opencode` are green.\n- #3836 puts a 20-minute `timeout-minutes` on the job so a recurrence fails fast instead of taking 6 h. That bounds the symptom; this issue tracks the cause.\n\nThe shape (first pass fine, second run of the same `bun test --isolate` set in the same tree blocks exit) points at a handle, listener, or child process left open by one of the ten opencode test files on the `providers` branch:\n\n```\ncontainer/agent-runner/src/providers/mcp-to-opencode.test.ts\ncontainer/agent-runner/src/providers/opencode-registration.test.ts\ncontainer/agent-runner/src/providers/opencode.attachments.test.ts\ncontainer/agent-runner/src/providers/opencode.compaction.test.ts\ncontainer/agent-runner/src/providers/opencode.config.test.ts\ncontainer/agent-runner/src/providers/opencode.empty-resume.test.ts\ncontainer/agent-runner/src/providers/opencode.factory.test.ts\ncontainer/agent-runner/src/providers/opencode.memory.test.ts\ncontainer/agent-runner/src/providers/opencode.question.test.ts\n```\n\nSuggested first steps:\n\n1. Loop the reapply pass on a runner-sized Linux box with `bun test --isolate --timeout 30000` to turn a hang into a named failing test.\n2. Check each file for a spawned `opencode` process, HTTP server, or timer that is not closed in `afterAll`.\n3. Once the culprit is known, add a per-test timeout to the SKILL.md test step so a future leak fails the job with a test name instead of a silent stall.\n\n### OS version and CPU architecture\n\nGitHub-hosted `ubuntu-latest`, x86_64.\n\n### Install or update path\n\nCI only: `scripts/test-registry-skills.ts` in a temporary shared clone. No install involved.\n\n### Docker version\n\nNot applicable; the job does not start containers.\n\n### Channel or interface\n\nNone.\n\n### Agent provider\n\nOpenCode (test suite only, no live provider).\n\n### Agent model\n\nNot applicable.\n\n### Relevant redacted logs\n\nThe hung job's log did not survive the 6-hour cancellation. The one retained fact is the first-pass duration of 13 s for the same command.\n\n### Last version that worked\n\nUnknown. The same job at the same commit passed on #3818, so this is intermittent rather than a regression at a known commit.\n", + "author_association": "MEMBER", + "labels": [ + "area/providers", + "area/repository-maintenance", + "kind/bug", + "triage/needs-repro", + "triage/unresolved" + ], + "url": "https://github.com/nanocoai/nanoclaw/issues/3839", + "updated_at": "2026-09-16T09:54:37Z" + }, + { + "type": "issue", + "number": 3811, + "title": "Central DB has no busy_timeout, so lock contention throws as if it were corruption", + "body": "## Summary\n\nThe central DB (`src/db/compose.ts`) \u2014 task/schedule state, `container_configs`, `agent_groups` \u2014 is opened in WAL mode with no `busy_timeout` set. Any momentary lock contention (two processes touching it at once) throws immediately instead of waiting briefly and retrying, which is indistinguishable from real corruption to a caller seeing `ncl tasks list` / `ncl tasks run` fail.\n\nThe session DBs (`inbound.db`/`outbound.db`) already carry this protection \u2014 see `container/agent-runner/src/cli/ncl.ts`'s `busy_timeout=5000` on both connections. The central DB never did.\n\n## Fix\n\n```ts\n// src/db/compose.ts\nif (!options.readonly) raw.pragma('busy_timeout = 5000');\n```\n\nSame value already used for the session DBs, so behavior stays consistent across all three.\n\n## Scope note\n\nThis does not fix an ungraceful-shutdown-mid-write scenario (the other candidate cause of intermittent task-DB failures on a real install) \u2014 only the contention half. Not the same issue as #3708 (closed, unmerged) or #2640/#2750/#2196/#2188, which are all about `outbound.db`'s readonly/journal handling \u2014 this is the central DB specifically, a different file with no existing coverage.\n\nPR follows.", + "author_association": "NONE", + "labels": [], + "url": "https://github.com/nanocoai/nanoclaw/issues/3811", + "updated_at": "2026-09-14T19:54:04Z" + }, + { + "type": "issue", + "number": 3814, + "title": "Raw process/turn-error text can be delivered to a public channel, not just the owner", + "body": "## Summary\n\n`deliverErrorResult` (`container/agent-runner/src/poll-loop.ts`) posts the SDK's raw error text verbatim back to whatever channel triggered the current turn \u2014 with no check for whether that destination is public. If a container's `claude` subprocess dies mid-turn while answering a real message in a public Discord/Slack channel (not a task run, not a DM), the community sees the raw internal error text.\n\n## Confirmed live\n\nObserved in practice: `Error: Claude Code process exited with code 137` (SIGKILL \u2014 the SDK's own `ProcessTransport.getProcessExitError`, confirmed by reading the published `@anthropic-ai/claude-agent-sdk` package source) posted to an owner DM. The code path that delivered it has no destination check at all, so the only reason it landed in a DM this time is that a DM happened to be the context of that particular turn \u2014 the same path would post identically to a public channel if that's what the turn was answering.\n\n## Confirmed via source\n\n`poll-loop.ts` ~line 661:\n```ts\nif (resultBlocks === 0 && event.isError === true && !routing.taskRun) {\n await deliverErrorResult(event.text, routing);\n ...\n}\n```\n`routing = extractRouting(messages)` (`formatter.ts`) is derived purely from whichever inbound message started the turn \u2014 it carries no public/private distinction. The one existing safeguard is `!routing.taskRun`: a scheduled task's error goes to its run log instead, never to any channel. A live conversational turn has no equivalent protection.\n\nThe host already has what's needed to add one: `messaging_groups.is_group` is used elsewhere (`src/channels/channel-defaults.ts`) to distinguish a DM from a group/public channel. That data isn't threaded down to where `deliverErrorResult` runs, but the distinction the fix needs already exists in the data model.\n\n## Ask\n\nTwo related but separable things, either src/messaging_groups.is_group is not currently available at delivery time:\n1. **Never deliver raw process/turn-level error text to a non-owner or public destination.** For those, substitute a short generic line (\"something went wrong, retrying\" or similar) instead of the SDK/provider's internal message.\n2. Full detail (the real error text) still reaches the owner \u2014 either because the turn actually was in the owner's DM, or via a separate host-side notice to the owner's own channel when it wasn't.\n\n## Related, but distinct\n\n`#3576` (rate-limited turns flood the channel with duplicate error notices) shares this same call site and root cause class (unconditional host-side delivery, no policy gate) but is about *repetition* of a notice already reaching its destination. This issue is about the notice reaching the *wrong* destination in the first place, even a single time. Worth fixing together since they're adjacent, but independently valid and separately fixable.", + "author_association": "NONE", + "labels": [], + "url": "https://github.com/nanocoai/nanoclaw/issues/3814", + "updated_at": "2026-09-14T20:49:32Z" + }, + { + "type": "pr", + "number": 3844, + "title": "fix(setup): replace broken sudo retry with user-owned npm prefix fallback", + "body": "## Problem\n\n`setup.sh`'s pnpm-install fallback fails permanently on Linux when Node is installed from a distro package (e.g. Fedora's `dnf install nodejs`, Debian/Ubuntu's `apt install nodejs`) instead of nvm or Homebrew.\n\nRepro on Fedora 43 (`nodejs-22.22.2` RPM):\n\n```\nnpm error code EACCES\nnpm error syscall mkdir\nnpm error path /usr/local/lib/node_modules\nnpm error Error: EACCES: permission denied, mkdir '/usr/local/lib/node_modules'\n...\nsudo: timed out reading password\nsudo: a password is required\n[bootstrap] pnpm not on PATH after corepack + npm fallback\nSTATUS: deps_failed\n```\n\n## Root cause\n\n- Distro-packaged Node sets npm's global prefix to `/usr/local`, which is root-owned. `npm install -g pnpm` fails with `EACCES`.\n- The existing fallback retries with `sudo npm install -g pnpm`. This can't work: `setup.sh` redirects the whole `install_deps` flow's stdout/stderr to a log file (`>> \"$LOG_FILE\" 2>&1`), so `sudo` has no TTY to show its password prompt on. It just blocks until `sudo: timed out reading password`, then the failure is swallowed by `|| true`.\n- Bootstrap reports `deps_failed` with no actionable message \u2014 the user has no indication a password was ever needed, or what to do about it.\n\nThis doesn't happen on macOS (Homebrew's prefix is user-writable) or with nvm-installed Node, which is presumably why it went unnoticed.\n\n## Fix\n\nReplace the dead sudo retry with a retry against a user-owned npm prefix (`~/.npm-global`), the same pattern documented in npm's own docs for avoiding sudo entirely:\n\n```bash\nnpm install -g \"pnpm@${pinned}\" --prefix \"$HOME/.npm-global\"\n```\n\nNo root needed. On success, `$HOME/.npm-global/bin` is prepended to `PATH` for the rest of the script, and the outcome (success or failure) is always logged \u2014 no more silent swallow.\n\n## Testing\n\nReproduced the original failure on a Fedora 43 box (RPM-packaged Node, root-owned `/usr/local/lib/node_modules`, no `corepack`, no `pnpm`) and confirmed:\n\n- **Before**: `npm install -g pnpm` \u2192 `EACCES` \u2192 `sudo` retry times out \u2192 `STATUS: deps_failed`.\n- **After**: `npm install -g pnpm` \u2192 `EACCES` \u2192 retries with `--prefix ~/.npm-global` \u2192 succeeds \u2192 `pnpm install --frozen-lockfile` and the `better-sqlite3` native-module check both pass \u2192 `STATUS: success`.\n\n\ud83e\udd16 Generated with [Claude Code](https://claude.com/claude-code)", + "author_association": "FIRST_TIME_CONTRIBUTOR", + "labels": [ + "area/setup-installation" + ], + "url": "https://github.com/nanocoai/nanoclaw/pull/3844", + "updated_at": "2026-09-16T15:31:52Z", + "draft": false, + "files": [ + "setup.sh" + ], + "file_count": 1, + "template": { + "marker": false, + "headings": [ + "Problem", + "Root cause", + "Fix", + "Testing" + ], + "sections": { + "Summary": "missing", + "Related work": "missing", + "Change kind": "missing", + "Validation": "missing", + "User and release impact": "missing", + "Security and trust boundaries": "missing", + "Skill delivery": "missing", + "AI assistance": "missing" + }, + "filled": 0, + "total": 8 + } + }, + { + "type": "pr", + "number": 3841, + "title": "fix(opencode): run the memory hook with async spawn so bun test cannot wedge", + "body": "\n\n## Summary\n\nStops the add-opencode `bun test --isolate` run from wedging for six hours in CI.\n- **Problem**: Bun 1.4.0's `spawnSync` can lose the child's exit and spin forever at 100% CPU (oven-sh/bun#34069). A synchronous spin is immune to every `bun test` timeout, matching the hung job exactly: one `bun`, no children, silent until the 6-hour cancel.\n- **Trigger**: the OpenCode memory session hook ran `spawnSync` on every turn, ~33 real `sh -c` spawns per suite run next to sqlite, SSE sockets and timers, the churn the Bun bug needs. Incidence scales with spawn volume and distilled repros do not fire, which is why sibling runs and 30 local runs stayed green.\n- **Fix**: the hook runs through async `spawn` with the same 10 s deadline, stdin payload and exit-status semantics, and settles at the deadline even if a SIGTERM-ignoring hook or a background child keeps stdout open; `prepare` is awaited and an abort or pump failure during preparation is re-checked before the prompt is sent. No `spawnSync` remains in the OpenCode runtime path, which also removes the wedge from production containers where it would freeze the turn watchdog.\n- **Guard**: `scripts/test-registry-skills.ts` runs every skill step in its own process group, SIGKILLs the group after 15 min (the shell alone would orphan a wedged bun), relays Ctrl-C to it, and fails a step on any error or signal. Complements the job-level timeout from #3836.\n- **Not the cause**: the leaked servers and timers in the test files. `bun test` 1.4.0 exits with open handles (verified).\n- **Out of scope**: `container/agent-runner/src/upload-trace.ts` still uses `spawnSync('curl')` on a rare path.\n\n## Related work\n\nCloses #3839\n\n## Change kind\n\n- [x] `kind/bug`\n- [ ] `kind/feature`\n- [ ] `kind/documentation`\n- [ ] `kind/cleanup`\n- [ ] `kind/hardening`\n\n## Validation\n\n- `cd container/agent-runner && bun run typecheck` -> clean\n- `bun test --isolate src/providers/opencode*.test.ts src/providers/mcp-to-opencode.test.ts` x3 -> 125 pass, 0 fail, ~13 s each\n- `pnpm exec vitest run scripts/test-registry-skills` -> 7 pass; `prettier --check` on changed files -> clean; host `tsc --noEmit` -> clean\n- `pnpm exec tsx scripts/test-registry-skills.ts add-opencode` (install -> tests -> image build -> remove -> contract verify -> reapply -> tests) -> `PASS: add-opencode`, run after each commit\n- `opencode.memory.test.ts` exercises the async hook: payload on stdin, verbatim output, exit-code and missing-command failures, undeclared source skip, file rendering, and (new) the deadline path with a background child holding stdout.\n- The wedge itself is not reproducible on demand (Bun issue and 30 local `--smol` runs agree); the guard is the CI-observable regression check.\n- Codex adversarial review: 3 findings (hook waited on `close` past its deadline; timeout killed only the shell; status 0 + ETIMEDOUT read as success), all reproduced by experiment and fixed in the second commit.\n\n- [x] Tests cover the changed behavior (or Validation says why not)\n\n## User and release impact\n\n- [x] No user-visible behavior change\n- [ ] User-visible change \u2014 release note below\n- [ ] Breaking change \u2014 release note below covers detect, why, fix/migration, rollback\n\n```release-note\nSkip it and a maintainer will write one.\n```\n\n## Security and trust boundaries\n\nNone. Same hook command, same shell, same 10 s deadline, same stdin payload; stderr was already discarded. No new inputs, permissions or credentials.\n\n## Skill delivery\n\n- [ ] Not a skill\n- [x] Skill: apply/remove footprint and fresh-clone verification are described above\n\nPayload-only change inside `.claude/skills/add-opencode/payload/` (four files, all already in the `nc:copy` list). The full apply -> remove -> reapply round trip in Validation is the fresh-clone check.\n\n## AI assistance\n\n- [x] AI tools or agents helped produce this change\n\n- [ ] A human has reviewed this PR and stands behind every change\n", + "author_association": "MEMBER", + "labels": [ + "PR: Fix", + "PR: Skill", + "area/providers", + "area/skills", + "core-team", + "delivery/skill", + "follows-guidelines", + "kind/bug" + ], + "url": "https://github.com/nanocoai/nanoclaw/pull/3841", + "updated_at": "2026-09-16T12:38:56Z", + "draft": false, + "files": [ + ".claude/skills/add-opencode/payload/container/agent-runner/src/providers/opencode-memory.ts", + ".claude/skills/add-opencode/payload/container/agent-runner/src/providers/opencode-turn.ts", + ".claude/skills/add-opencode/payload/container/agent-runner/src/providers/opencode.memory.test.ts", + ".claude/skills/add-opencode/payload/container/agent-runner/src/providers/opencode.ts", + "scripts/test-registry-skills.ts" + ], + "file_count": 5, + "template": { + "marker": true, + "headings": [ + "Summary", + "Related work", + "Change kind", + "Validation", + "User and release impact", + "Security and trust boundaries", + "Skill delivery", + "AI assistance" + ], + "sections": { + "Summary": "filled", + "Related work": "filled", + "Change kind": "filled", + "Validation": "filled", + "User and release impact": "filled", + "Security and trust boundaries": "filled", + "Skill delivery": "filled", + "AI assistance": "filled" + }, + "filled": 8, + "total": 8 + } + } + ], + "responses": { + "issue:3839": { + "model": "jev-latest", + "answers": { + "area": { + "type": "choice", + "choice": "area/providers", + "probabilities": { + "area/agent-memory": 0.0069, + "area/agent-runner": 0.0069, + "area/channels": 0.0069, + "area/configuration": 0.0069, + "area/containers": 0.0069, + "area/core": 0.0069, + "area/credentials": 0.0069, + "area/ncl-cli": 0.0069, + "area/providers": 0.5503, + "area/repository-maintenance": 0.28, + "area/scheduled-tasks": 0.0069, + "area/security": 0.0069, + "area/sessions": 0.0069, + "area/setup-installation": 0.0069, + "area/skills": 0.08, + "area/tools": 0.0069 + }, + "confidence": 0.62 + }, + "kind": { + "type": "choice", + "choice": "kind/bug", + "probabilities": { + "kind/bug": 0.94, + "kind/feature": 0.01, + "kind/documentation": 0.01, + "kind/question": 0.01, + "kind/security": 0.01, + "kind/hardening": 0.01, + "kind/cleanup": 0.01 + }, + "confidence": 0.93 + }, + "priority": { + "type": "score", + "score": 1.34, + "legend": { + "0": "Low: cosmetic, rare, or easily worked around; affects an optional path or one unusual setup; no data loss and no security impact.", + "1": "Medium: a real defect or gap on a supported configuration that has a workaround, or a well-scoped improvement most operators would notice; can wait for a normal release.", + "2": "High: breaks a core flow (install, message routing, container spawn, delivery, scheduled runs) for many users with no clean workaround, or a hardening gap that is actively being discussed.", + "3": "Critical: data loss, credential or secret exposure, an exploitable trust-boundary vulnerability, or the service cannot start or run for everyone; must be fixed before the next release." + }, + "probabilities": { + "0": 0.12, + "1": 0.48, + "2": 0.34, + "3": 0.06 + }, + "confidence": 0.55 + }, + "needs_repro": { + "type": "noul", + "noul": 0.78 + } + }, + "usage": { + "input_tokens": 2140, + "output_tokens": 62 + } + }, + "issue:3811": { + "model": "jev-latest", + "answers": { + "area": { + "type": "choice", + "choice": "area/core", + "probabilities": { + "area/agent-memory": 0.0064, + "area/agent-runner": 0.0064, + "area/channels": 0.0064, + "area/configuration": 0.0064, + "area/containers": 0.0064, + "area/core": 0.8604, + "area/credentials": 0.0064, + "area/ncl-cli": 0.05, + "area/providers": 0.0064, + "area/repository-maintenance": 0.0064, + "area/scheduled-tasks": 0.0064, + "area/security": 0.0064, + "area/sessions": 0.0064, + "area/setup-installation": 0.0064, + "area/skills": 0.0064, + "area/tools": 0.0064 + }, + "confidence": 0.84 + }, + "kind": { + "type": "choice", + "choice": "kind/bug", + "probabilities": { + "kind/bug": 0.89, + "kind/feature": 0.01, + "kind/documentation": 0.01, + "kind/question": 0.01, + "kind/security": 0.01, + "kind/hardening": 0.06, + "kind/cleanup": 0.01 + }, + "confidence": 0.88 + }, + "priority": { + "type": "score", + "score": 1.15, + "legend": { + "0": "Low: cosmetic, rare, or easily worked around; affects an optional path or one unusual setup; no data loss and no security impact.", + "1": "Medium: a real defect or gap on a supported configuration that has a workaround, or a well-scoped improvement most operators would notice; can wait for a normal release.", + "2": "High: breaks a core flow (install, message routing, container spawn, delivery, scheduled runs) for many users with no clean workaround, or a hardening gap that is actively being discussed.", + "3": "Critical: data loss, credential or secret exposure, an exploitable trust-boundary vulnerability, or the service cannot start or run for everyone; must be fixed before the next release." + }, + "probabilities": { + "0": 0.1, + "1": 0.68, + "2": 0.19, + "3": 0.03 + }, + "confidence": 0.83 + }, + "needs_repro": { + "type": "noul", + "noul": 0.55 + } + }, + "usage": { + "input_tokens": 1310, + "output_tokens": 62 + } + }, + "issue:3814": { + "model": "jev-latest", + "answers": { + "area": { + "type": "choice", + "choice": "area/core", + "probabilities": { + "area/agent-memory": 0.0062, + "area/agent-runner": 0.0062, + "area/channels": 0.09, + "area/configuration": 0.0062, + "area/containers": 0.0062, + "area/core": 0.4494, + "area/credentials": 0.0062, + "area/ncl-cli": 0.0062, + "area/providers": 0.0062, + "area/repository-maintenance": 0.0062, + "area/scheduled-tasks": 0.0062, + "area/security": 0.38, + "area/sessions": 0.0062, + "area/setup-installation": 0.0062, + "area/skills": 0.0062, + "area/tools": 0.0062 + }, + "confidence": 0.48 + }, + "kind": { + "type": "choice", + "choice": "kind/hardening", + "probabilities": { + "kind/bug": 0.1, + "kind/feature": 0.0075, + "kind/documentation": 0.0075, + "kind/question": 0.0075, + "kind/security": 0.21, + "kind/hardening": 0.66, + "kind/cleanup": 0.0075 + }, + "confidence": 0.66 + }, + "priority": { + "type": "score", + "score": 1.92, + "legend": { + "0": "Low: cosmetic, rare, or easily worked around; affects an optional path or one unusual setup; no data loss and no security impact.", + "1": "Medium: a real defect or gap on a supported configuration that has a workaround, or a well-scoped improvement most operators would notice; can wait for a normal release.", + "2": "High: breaks a core flow (install, message routing, container spawn, delivery, scheduled runs) for many users with no clean workaround, or a hardening gap that is actively being discussed.", + "3": "Critical: data loss, credential or secret exposure, an exploitable trust-boundary vulnerability, or the service cannot start or run for everyone; must be fixed before the next release." + }, + "probabilities": { + "0": 0.03, + "1": 0.15, + "2": 0.69, + "3": 0.13 + }, + "confidence": 0.81 + }, + "needs_repro": { + "type": "noul", + "noul": 0.31 + } + }, + "usage": { + "input_tokens": 1980, + "output_tokens": 62 + } + }, + "pr:3844": { + "model": "jev-latest", + "answers": { + "area": { + "type": "choice", + "choice": "area/setup-installation", + "probabilities": { + "area/agent-memory": 0.002, + "area/agent-runner": 0.002, + "area/channels": 0.002, + "area/configuration": 0.002, + "area/containers": 0.002, + "area/core": 0.002, + "area/credentials": 0.002, + "area/ncl-cli": 0.002, + "area/providers": 0.002, + "area/repository-maintenance": 0.002, + "area/scheduled-tasks": 0.002, + "area/security": 0.002, + "area/sessions": 0.002, + "area/setup-installation": 0.97, + "area/skills": 0.002, + "area/tools": 0.002 + }, + "confidence": 0.97 + }, + "kind": { + "type": "choice", + "choice": "kind/bug", + "probabilities": { + "kind/bug": 0.91, + "kind/feature": 0.01, + "kind/documentation": 0.01, + "kind/question": 0.01, + "kind/security": 0.01, + "kind/hardening": 0.04, + "kind/cleanup": 0.01 + }, + "confidence": 0.9 + }, + "priority": { + "type": "score", + "score": 1.66, + "legend": { + "0": "Low: cosmetic, rare, or easily worked around; affects an optional path or one unusual setup; no data loss and no security impact.", + "1": "Medium: a real defect or gap on a supported configuration that has a workaround, or a well-scoped improvement most operators would notice; can wait for a normal release.", + "2": "High: breaks a core flow (install, message routing, container spawn, delivery, scheduled runs) for many users with no clean workaround, or a hardening gap that is actively being discussed.", + "3": "Critical: data loss, credential or secret exposure, an exploitable trust-boundary vulnerability, or the service cannot start or run for everyone; must be fixed before the next release." + }, + "probabilities": { + "0": 0.05, + "1": 0.32, + "2": 0.55, + "3": 0.08 + }, + "confidence": 0.62 + }, + "pr_ready": { + "type": "noul", + "noul": 0.22 + } + }, + "usage": { + "input_tokens": 2410, + "output_tokens": 62 + } + }, + "pr:3841": { + "model": "jev-latest", + "answers": { + "area": { + "type": "choice", + "choice": "area/providers", + "probabilities": { + "area/agent-memory": 0.0038, + "area/agent-runner": 0.06, + "area/channels": 0.0038, + "area/configuration": 0.0038, + "area/containers": 0.0038, + "area/core": 0.0038, + "area/credentials": 0.0038, + "area/ncl-cli": 0.0038, + "area/providers": 0.7206, + "area/repository-maintenance": 0.0038, + "area/scheduled-tasks": 0.0038, + "area/security": 0.0038, + "area/sessions": 0.0038, + "area/setup-installation": 0.0038, + "area/skills": 0.17, + "area/tools": 0.0038 + }, + "confidence": 0.71 + }, + "kind": { + "type": "choice", + "choice": "kind/bug", + "probabilities": { + "kind/bug": 0.87, + "kind/feature": 0.012, + "kind/documentation": 0.012, + "kind/question": 0.012, + "kind/security": 0.012, + "kind/hardening": 0.012, + "kind/cleanup": 0.07 + }, + "confidence": 0.86 + }, + "priority": { + "type": "score", + "score": 1.5, + "legend": { + "0": "Low: cosmetic, rare, or easily worked around; affects an optional path or one unusual setup; no data loss and no security impact.", + "1": "Medium: a real defect or gap on a supported configuration that has a workaround, or a well-scoped improvement most operators would notice; can wait for a normal release.", + "2": "High: breaks a core flow (install, message routing, container spawn, delivery, scheduled runs) for many users with no clean workaround, or a hardening gap that is actively being discussed.", + "3": "Critical: data loss, credential or secret exposure, an exploitable trust-boundary vulnerability, or the service cannot start or run for everyone; must be fixed before the next release." + }, + "probabilities": { + "0": 0.06, + "1": 0.45, + "2": 0.42, + "3": 0.07 + }, + "confidence": 0.58 + }, + "pr_ready": { + "type": "noul", + "noul": 0.9 + } + }, + "usage": { + "input_tokens": 3120, + "output_tokens": 62 + } + } + } +} diff --git a/skills/typesafe-triage/scripts/typesafe-triage.py b/skills/typesafe-triage/scripts/typesafe-triage.py new file mode 100644 index 0000000..fb74dcf --- /dev/null +++ b/skills/typesafe-triage/scripts/typesafe-triage.py @@ -0,0 +1,848 @@ +#!/usr/bin/env python3 +"""Dry-run issue and PR triage for nanocoai/nanoclaw using the TypeSafe decision API. + +Fetches recently updated open issues and pull requests with ``gh``, sends one +fan-out request per item to TypeSafe System One (area, kind, priority, and a +yes/no question), applies a confidence gate, and prints the proposed labels +next to the existing ones. Nothing is written to GitHub. Raw answers are saved +as JSON under a gitignored output directory. + +``--fixture `` replays recorded items and API responses so the whole +pipeline runs offline. ``TYPESAFE_API_KEY`` is read from the environment only +when the live API is used; it is never read from a file and never printed. +""" + +import argparse +import datetime +import json +import math +import os +from pathlib import Path +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request + + +API_URL = "https://api.typesafe.ai/v1/systemone" +DEFAULT_MODEL = "jev-latest" +DEFAULT_REPO = "nanocoai/nanoclaw" +KEY_ENV = "TYPESAFE_API_KEY" +SKILL_DIR = Path(__file__).resolve().parents[1] +DEFAULT_OUTPUT_DIR = SKILL_DIR / "output" +BODY_LIMIT = 12000 +FILE_LIMIT = 80 +UNRESOLVED = "triage/unresolved" +NEEDS_REPRO = "triage/needs-repro" +NEEDS_AUTHOR = "triage/needs-author" +NEEDS_REVIEW = "Status: Needs Review" +PR_TEMPLATE_MARKER = "nanoclaw-pr-template:v2" +PR_TEMPLATE_SECTIONS = ( + "Summary", + "Related work", + "Change kind", + "Validation", + "User and release impact", + "Security and trust boundaries", + "Skill delivery", + "AI assistance", +) + +# Rubrics come from the NanoClaw label descriptions (`gh label list`), CLAUDE.md, +# docs/ and the path -> area mapping in .github/labeler.yml. One line per option. +AREA_CRITERIA = { + "area/agent-memory": ( + "Persistent per-agent-group memory: the Markdown memory tree, memory migration, " + "memory hooks and memory add-ons (container/agent-runner/src/memory, docs/memory.md, " + "migrate-memory, add-mnemon, add-karpathy-llm-wiki)." + ), + "area/agent-runner": ( + "The Bun agent-runner inside the container: poll loop, prompt formatting, provider " + "abstraction, media handling, status management (container/agent-runner/**, " + "docs/agent-runner-details.md, docs/SDK_DEEP_DIVE.md)." + ), + "area/channels": ( + "Messaging platform adapters and channel install skills: Slack, Discord, Telegram, " + "WhatsApp, Signal, iMessage, Teams, Matrix, GitHub, Linear, email, interactive and " + "typing modules, channel pairing/auth setup (src/channels, setup/channels, add-)." + ), + "area/configuration": ( + "Host and container configuration: env parsing, container_configs, group folder and " + "persona scaffolding, project-doc composition, timezone, .env.example, config examples, " + "the customize skill (src/config*, src/container-config*, src/project-doc-compose*)." + ), + "area/containers": ( + "Docker/container runtime: Dockerfile, image build and pull, container-runner spawn and " + "restart, mounts, drivers, egress lockdown, self-mod apply, image workflows " + "(container/Dockerfile, container/build.sh, src/container-runner*, src/drivers, manage-mounts)." + ), + "area/core": ( + "Host orchestration core: entry point, inbound router, outbound delivery and delivery guard, " + "host sweep, lifecycle, circuit breaker, webhook server, mailbox and DB layer, agent-to-agent, " + "cross-session context (src/index, src/router, src/delivery*, src/host-sweep, src/db, src/mailbox)." + ), + "area/credentials": ( + "Credential gateway seam and gateway providers: OneCLI/Iron Proxy integration, vault, " + "credential helper, registry login, gateway approvals bridge (src/gateway-providers, " + "setup/onecli*, container/skills/onecli-gateway, docs/onecli-upgrades.md)." + ), + "area/ncl-cli": ( + "The ncl admin CLI and its socket/session transport, command gate, resource definitions, " + "query helper script (src/cli, src/command-gate*, container/agent-runner/src/cli, bin/ncl, scripts/q*)." + ), + "area/providers": ( + "Agent providers and their selection/auth: Claude, OpenCode, Codex, Ollama provider " + "config, provider migration (src/providers, container/agent-runner/src/providers, " + "setup/providers, setup/provider-auth*, add-codex, add-opencode, add-ollama-provider)." + ), + "area/repository-maintenance": ( + "Repository plumbing rather than product behavior: GitHub workflows and templates, husky, " + "lint/tsconfig/vitest config, package.json and lockfiles, READMEs, CHANGELOG, CONTRIBUTING, " + "release scripts, versions.json, CLAUDE.md/AGENTS.md (.github/**, package.json, README*, scripts/release*)." + ), + "area/scheduled-tasks": ( + "Scheduled and recurring tasks: cron recurrence, task runs, ncl tasks resource, " + "scheduling modules (src/modules/scheduling, container/agent-runner/src/scheduling, docs/scheduled-tasks.md)." + ), + "area/security": ( + "Privileged-action guard, approvals, permissions and access resolution, mount security, " + "attachment and inbox safety, harness tag stripping, SECURITY.md " + "(src/guard, src/modules/approvals, src/modules/permissions, src/modules/mount-security)." + ), + "area/sessions": ( + "Session resolution and lifecycle: session manager, sessions table, per-session inbound/outbound " + "DB files, session cleanup (src/session-manager*, src/db/sessions*, docs/db-session.md)." + ), + "area/setup-installation": ( + "Setup wizard and installation lifecycle: nanoclaw.sh, setup/ steps, service (launchd/systemd), " + "bootstrap, install slug, upgrade state, update-nanoclaw, v1/OpenClaw migration, init-first-agent, " + "debug skill (setup/**, nanoclaw.sh, migrate-v2.sh, scripts/update*, launchd/**)." + ), + "area/skills": ( + "The skills system itself and skill packaging: skill directives engine, skill apply/lint, " + "registry skills workflow, group skills, container skills directory, skill guidelines docs, " + "or a new/changed operator skill with no more specific area (.claude/skills/**, scripts/skill-*, docs/skills-model.md)." + ), + "area/tools": ( + "Tools the agent calls from inside the container: MCP tool server, plugin MCP, agent-browser, " + "frontend-engineer, and tool add-on skills such as add-vercel, add-rtk, add-tavily-tool, " + "add-ollama-tool, add-dashboard, add-clidash (container/agent-runner/src/mcp-tools, .mcp.json)." + ), +} + +KIND_CRITERIA = { + "kind/bug": "Something is not working as expected: a defect, crash, hang, wrong output or regression.", + "kind/feature": "New capability or improvement: adds behavior, an option, a channel, provider or skill.", + "kind/documentation": "Documentation is wrong, missing or unclear; the change is to docs, comments or guidance text.", + "kind/question": "Usage or design question; asks how something works or should work rather than reporting or changing it.", + "kind/security": "Exploitable vulnerability crossing a trust boundary on a correctly configured install.", + "kind/hardening": "Defense-in-depth improvement that is not exploitable across a trust boundary; reduces blast radius or tightens defaults.", + "kind/cleanup": "Refactor, tidy-up, test-only or tooling change with no user-visible behavior change.", +} + +PRIORITY_LABELS = ("priority/low", "priority/medium", "priority/high", "priority/critical") +PRIORITY_CRITERIA = [ + "Low: cosmetic, rare, or easily worked around; affects an optional path or one unusual setup; " + "no data loss and no security impact.", + "Medium: a real defect or gap on a supported configuration that has a workaround, or a well-scoped " + "improvement most operators would notice; can wait for a normal release.", + "High: breaks a core flow (install, message routing, container spawn, delivery, scheduled runs) " + "for many users with no clean workaround, or a hardening gap that is actively being discussed.", + "Critical: data loss, credential or secret exposure, an exploitable trust-boundary vulnerability, " + "or the service cannot start or run for everyone; must be fixed before the next release.", +] + +CHOICE_QUESTIONS = ("area", "kind") +LABEL_FAMILIES = ("area/", "kind/", "priority/") + + +class TriageError(RuntimeError): + """A failure the operator can act on; printed without a traceback.""" + + +class PartialFailure(TriageError): + """A request failed after earlier items succeeded; carries the completed results.""" + + def __init__(self, message, failed_key, results, usage): + super().__init__(message) + self.failed_key = failed_key + self.results = results + self.usage = usage + + +def utc_now(): + return datetime.datetime.now(datetime.timezone.utc) + + +def stamp(): + return utc_now().strftime("%Y%m%dT%H%M%SZ") + + +# --------------------------------------------------------------------------- +# GitHub fetch (read-only, via gh api) +# --------------------------------------------------------------------------- + +def gh_json(path, run=subprocess.run): + """Run ``gh api `` and return the parsed JSON body.""" + result = run(["gh", "api", path], capture_output=True, text=True) + if result.returncode != 0: + detail = (result.stderr or result.stdout or "").strip().splitlines() + raise TriageError("gh api failed for %s: %s" % (path, detail[-1] if detail else "no output")) + try: + return json.loads(result.stdout) + except json.JSONDecodeError as error: + raise TriageError("gh api returned invalid JSON for %s: %s" % (path, error)) + + +def gh_pages(path, run=subprocess.run, per_page=100, max_pages=10, enough=None): + """Follow ``page=N`` until a short page, ``max_pages`` or ``enough(items)``.""" + collected = [] + joiner = "&" if "?" in path else "?" + for page in range(1, max_pages + 1): + batch = gh_json("%s%sper_page=%d&page=%d" % (path, joiner, per_page, page), run) + if not isinstance(batch, list): + raise TriageError("gh api returned a non-list body for %s" % path) + collected.extend(batch) + if len(batch) < per_page or (enough and enough(collected)): + break + return collected + + +def template_status(body): + """Report which nanoclaw PR template sections carry content.""" + text = body or "" + marker = PR_TEMPLATE_MARKER in text + without_comments = re.sub(r"", "", text, flags=re.S) + headings = re.findall(r"(?m)^#{1,6}\s+(.+?)\s*$", without_comments) + # Template sections are level-2 headings; deeper headings stay inside their section. + chunks = re.split(r"(?m)^##\s+", without_comments) + content = {} + for chunk in chunks[1:]: + heading, _, rest = chunk.partition("\n") + content[heading.strip()] = rest + sections = {} + for name in PR_TEMPLATE_SECTIONS: + rest = content.get(name) + if rest is None: + sections[name] = "missing" + continue + lines = [] + for line in rest.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("- [ ]") or re.match(r"^#{3,6}\s", stripped): + continue + if stripped in ("```release-note", "```"): + continue + if stripped.startswith("Optional: one user-facing line"): + continue + if stripped == "Closes #": + continue + lines.append(stripped) + sections[name] = "filled" if lines else "empty" + filled = sum(1 for value in sections.values() if value == "filled") + return { + "marker": marker, + "headings": headings[:40], + "sections": sections, + "filled": filled, + "total": len(PR_TEMPLATE_SECTIONS), + } + + +def normalize_issue(raw): + return { + "type": "issue", + "number": raw["number"], + "title": raw.get("title") or "", + "body": raw.get("body") or "", + "author_association": raw.get("author_association") or "NONE", + "labels": sorted(label["name"] for label in raw.get("labels") or []), + "url": raw.get("html_url") or "", + "updated_at": raw.get("updated_at") or "", + } + + +def normalize_pull(raw, files): + item = normalize_issue(raw) + item["type"] = "pr" + item["draft"] = bool(raw.get("draft")) + item["files"] = [entry["filename"] for entry in files][:FILE_LIMIT] + item["file_count"] = len(files) + item["template"] = template_status(item["body"]) + return item + + +def fetch_items(repo, issue_limit, pull_limit, run=subprocess.run): + """Return the most recently updated open issues and PRs, newest first.""" + items = [] + if issue_limit > 0: + # The issues endpoint interleaves pull requests, so keep paging until enough real issues arrive. + def enough_issues(collected): + return sum(1 for entry in collected if "pull_request" not in entry) >= issue_limit + + raw_issues = gh_pages("repos/%s/issues?state=open&sort=updated&direction=desc" % repo, run, + enough=enough_issues) + issues = [entry for entry in raw_issues if "pull_request" not in entry] + items.extend(normalize_issue(entry) for entry in issues[:issue_limit]) + if pull_limit > 0: + raw_pulls = gh_pages("repos/%s/pulls?state=open&sort=updated&direction=desc" % repo, run, + enough=lambda collected: len(collected) >= pull_limit) + for entry in raw_pulls[:pull_limit]: + files = gh_pages("repos/%s/pulls/%d/files" % (repo, entry["number"]), run, max_pages=5) + items.append(normalize_pull(entry, files)) + return items + + +# --------------------------------------------------------------------------- +# Questions and state +# --------------------------------------------------------------------------- + +def build_state(repo, item): + state = { + "repository": repo, + "item_type": "pull_request" if item["type"] == "pr" else "issue", + "number": item["number"], + "title": item["title"], + "author_association": item["author_association"], + "body": (item.get("body") or "")[:BODY_LIMIT], + } + if item["type"] == "pr": + state["draft"] = item.get("draft", False) + state["changed_files"] = item.get("files", []) + state["changed_file_count"] = item.get("file_count", len(item.get("files", []))) + template = item.get("template") or template_status(item.get("body")) + state["template"] = { + "nanoclaw_template_marker_present": template["marker"], + "sections": template["sections"], + "headings_in_body": template["headings"], + } + return state + + +def build_questions(item_type): + is_pr = item_type == "pr" + subject = "`title`, `body` and `changed_files`" if is_pr else "`title` and `body`" + questions = { + "area": { + "type": "choice", + "instructions": ( + "Which single NanoClaw subsystem does this %s primarily belong to, judging from %s? " + "Pick the one area a maintainer would file it under." + % ("pull request" if is_pr else "issue", subject) + ), + "criteria": dict(AREA_CRITERIA), + }, + "kind": { + "type": "choice", + "instructions": ( + "What kind of change or report is this %s, judging from %s?" + % ("pull request" if is_pr else "issue", subject) + ), + "criteria": dict(KIND_CRITERIA), + }, + "priority": { + "type": "score", + "instructions": ( + "How urgent is this %s for NanoClaw maintainers, judging from the impact described in %s?" + % ("pull request" if is_pr else "issue", subject) + ), + "criteria": list(PRIORITY_CRITERIA), + }, + } + if is_pr: + questions["pr_ready"] = { + "type": "noul", + "instructions": ( + "Is this pull request ready for maintainer review: the template sections in " + "`template.sections` carry real content (not placeholders), `body` describes tests " + "or validation or gives a reason none is needed, and `changed_files` match the scope " + "stated in `title`?" + ), + "criteria": { + "true": "Template filled, validation or an explicit reason for none, and the change matches its title.", + "false": "Template missing or left as placeholders, no validation evidence and no reason given, or the diff is broader or narrower than the title.", + }, + } + else: + questions["needs_repro"] = { + "type": "noul", + "instructions": ( + "Does this issue, judging from `body`, lack reproduction steps or a concrete failing case: " + "no command sequence or configuration that reproduces it, no exact error output, " + "failing test, or log excerpt?" + ), + "criteria": { + "true": "The report describes the problem abstractly; a maintainer could not reproduce it from what is written.", + "false": "The report includes concrete steps, a failing command, exact error text, a log excerpt, or a pointer to a failing run.", + }, + } + return questions + + +def build_request(repo, item, model=DEFAULT_MODEL): + return { + "state": build_state(repo, item), + "model": model, + "questions": build_questions(item["type"]), + } + + +def item_key(item): + return "%s:%d" % (item["type"], item["number"]) + + +# --------------------------------------------------------------------------- +# Transports +# --------------------------------------------------------------------------- + +class HttpTransport: + """POST to the TypeSafe API with exponential backoff on 429/529.""" + + RETRY_STATUSES = (429, 529) + + def __init__(self, api_key, url=API_URL, timeout=60, attempts=4, sleep=time.sleep, opener=None): + if not api_key: + raise TriageError("%s is not set" % KEY_ENV) + if any(character.isspace() for character in api_key) or not api_key.isprintable(): + # Never echo the value: a newline inside a header would otherwise surface in a ValueError. + raise TriageError("%s contains whitespace or non-printable characters; export the bare key" % KEY_ENV) + self._api_key = api_key + self.url = url + self.timeout = timeout + self.attempts = attempts + self.sleep = sleep + self.opener = opener + + def __call__(self, payload): + body = json.dumps(payload).encode("utf-8") + delay = 1.0 + last_error = None + for attempt in range(1, self.attempts + 1): + request = urllib.request.Request( + self.url, + data=body, + method="POST", + headers={ + "Authorization": "Bearer " + self._api_key, + "Content-Type": "application/json", + "Accept": "application/json", + "User-Agent": "nanoclaw-oss-dev-tools typesafe-triage", + }, + ) + opener = self.opener or urllib.request.urlopen + try: + with opener(request, timeout=self.timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + detail = "" + try: + detail = error.read().decode("utf-8", "replace")[:300] + except Exception: # pragma: no cover - best effort + detail = "" + finally: + error.close() + last_error = "HTTP %s from TypeSafe: %s" % (error.code, self.redact(detail.strip() or str(error.reason))) + if error.code not in self.RETRY_STATUSES or attempt == self.attempts: + raise TriageError(last_error) + except urllib.error.URLError as error: + last_error = "could not reach TypeSafe: %s" % self.redact(str(error.reason)) + if attempt == self.attempts: + raise TriageError(last_error) + self.sleep(delay) + delay *= 2 + raise TriageError(last_error or "TypeSafe request failed") + + def redact(self, text): + # Real keys are long; skip tiny values so redaction cannot mangle ordinary words. + return text.replace(self._api_key, "[redacted]") if len(self._api_key) >= 8 else text + + +class FixtureTransport: + """Replay recorded responses keyed by ``:``.""" + + def __init__(self, responses): + self.responses = responses + self.requests = [] + + def __call__(self, payload): + self.requests.append(payload) + state = payload["state"] + key = "%s:%d" % ("pr" if state["item_type"] == "pull_request" else "issue", state["number"]) + try: + return self.responses[key] + except KeyError: + raise TriageError("fixture has no recorded response for %s" % key) + + +def load_fixture(path): + try: + data = json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError) as error: + raise TriageError("could not read fixture %s: %s" % (path, error)) + if not isinstance(data, dict) or "items" not in data or "responses" not in data: + raise TriageError("fixture %s must be an object with 'items' and 'responses'" % path) + return data + + +# --------------------------------------------------------------------------- +# Decisions +# --------------------------------------------------------------------------- + +def label_family(label): + for prefix in LABEL_FAMILIES: + if label.startswith(prefix): + return prefix + return label + + +def verdict(label, existing): + if label in existing: + return "AGREE" + family = label_family(label) + if family in LABEL_FAMILIES and any(label_family(name) == family for name in existing): + return "DISAGREE" + return "NEW" + + +def priority_label(score): + """Round the score half-up to the nearest level (1.5 -> priority/high).""" + index = int(math.floor(float(score) + 0.5)) + index = max(0, min(len(PRIORITY_LABELS) - 1, index)) + return PRIORITY_LABELS[index] + + +def decide(item, answers, thresholds): + """Turn one item's answers into label proposals with confidence and verdicts.""" + existing = set(item.get("labels") or []) + proposals = [] + + def add(question, label, confidence, gated, note=""): + proposals.append({ + "question": question, + "label": label, + "confidence": None if confidence is None else round(float(confidence), 3), + "gated": gated, + "verdict": verdict(label, existing), + "existing": sorted(name for name in existing if label_family(name) == label_family(label)), + "note": note, + }) + + for question in CHOICE_QUESTIONS: + answer = answers.get(question) or {} + choice = answer.get("choice") + confidence = answer.get("confidence") + criteria = AREA_CRITERIA if question == "area" else KIND_CRITERIA + threshold = thresholds[question] + if choice in criteria and confidence is not None and confidence >= threshold: + add(question, choice, confidence, False) + else: + note = "top %s" % choice if choice else "no answer" + add(question, UNRESOLVED, confidence, True, note) + + answer = answers.get("priority") or {} + score = answer.get("score") + confidence = answer.get("confidence") + if score is not None and confidence is not None and confidence >= thresholds["priority"]: + add("priority", priority_label(score), confidence, False, "score %.2f" % float(score)) + else: + note = "score %.2f" % float(score) if score is not None else "no answer" + add("priority", UNRESOLVED, confidence, True, note) + + noul_threshold = thresholds["noul"] + if item["type"] == "pr": + answer = answers.get("pr_ready") or {} + value = answer.get("noul") + if value is None: + add("pr_ready", UNRESOLVED, None, True, "no answer") + elif value >= noul_threshold: + add("pr_ready", NEEDS_REVIEW, value, False) + elif value <= 1 - noul_threshold: + add("pr_ready", NEEDS_AUTHOR, 1 - value, False, "not ready") + else: + add("pr_ready", UNRESOLVED, value, True, "uncertain") + else: + answer = answers.get("needs_repro") or {} + value = answer.get("noul") + top_kind = (answers.get("kind") or {}).get("choice") + if top_kind not in (None, "kind/bug", "kind/security"): + proposals.append({ + "question": "needs_repro", "label": None, "confidence": None if value is None else round(float(value), 3), + "gated": False, "verdict": "SKIP", "existing": [], "note": "not a bug report (%s)" % top_kind, + }) + elif value is None: + add("needs_repro", UNRESOLVED, None, True, "no answer") + elif value >= noul_threshold: + add("needs_repro", NEEDS_REPRO, value, False) + elif value <= 1 - noul_threshold: + proposals.append({ + "question": "needs_repro", "label": None, "confidence": round(float(value), 3), + "gated": False, "verdict": "AGREE" if NEEDS_REPRO not in existing else "DISAGREE", + "existing": [NEEDS_REPRO] if NEEDS_REPRO in existing else [], "note": "has repro", + }) + else: + add("needs_repro", UNRESOLVED, value, True, "uncertain") + return proposals + + +def summarize(results): + questions = {} + below_gate_items = 0 + for result in results: + item_gated = False + for proposal in result["proposals"]: + entry = questions.setdefault(proposal["question"], {"agree": 0, "disagree": 0, "new": 0, "skip": 0, "below_gate": 0}) + if proposal["gated"]: + entry["below_gate"] += 1 + item_gated = True + continue + if proposal["verdict"] == "AGREE": + entry["agree"] += 1 + elif proposal["verdict"] == "DISAGREE": + entry["disagree"] += 1 + elif proposal["verdict"] == "NEW": + entry["new"] += 1 + else: + entry["skip"] += 1 + if item_gated: + below_gate_items += 1 + for entry in questions.values(): + compared = entry["agree"] + entry["disagree"] + entry["agreement_rate"] = round(entry["agree"] / compared, 3) if compared else None + return {"questions": questions, "items_below_gate": below_gate_items, "items": len(results)} + + +# --------------------------------------------------------------------------- +# Pipeline +# --------------------------------------------------------------------------- + +def evaluate(repo, items, transport, thresholds, model=DEFAULT_MODEL, log=None): + results = [] + usage = {"input_tokens": 0, "output_tokens": 0} + for index, item in enumerate(items, 1): + payload = build_request(repo, item, model) + started = time.monotonic() + try: + response = transport(payload) + except TriageError as error: + raise PartialFailure(str(error), item_key(item), results, usage) + elapsed = time.monotonic() - started + answers = response.get("answers") if isinstance(response, dict) else None + if not isinstance(answers, dict): + raise TriageError("TypeSafe response for %s has no 'answers' map" % item_key(item)) + for name, value in (response.get("usage") or {}).items(): + if name in usage and isinstance(value, (int, float)): + usage[name] += int(value) + results.append({ + "key": item_key(item), + "item": {k: item[k] for k in ("type", "number", "title", "url", "labels", "author_association") if k in item}, + "response": response, + "proposals": decide(item, answers, thresholds), + "elapsed_seconds": round(elapsed, 3), + }) + if log: + log("[%d/%d] %s (%.1fs)" % (index, len(items), item_key(item), elapsed)) + return results, usage + + +def truncate(text, width): + text = " ".join((text or "").split()) + if len(text) <= width: + return text + return text[: max(0, width - 1)] + "…" + + +def render_table(results): + columns = ("#", "Title", "Question", "Proposed", "Conf", "Existing", "Verdict") + rows = [] + for result in results: + item = result["item"] + first = True + for proposal in result["proposals"]: + label = proposal["label"] or "-" + if proposal["gated"] and proposal["note"]: + label = "%s (%s)" % (label, proposal["note"]) + elif proposal["note"] and proposal["label"] not in (None, UNRESOLVED): + label = "%s (%s)" % (label, proposal["note"]) + elif proposal["note"] and proposal["label"] is None: + label = "- (%s)" % proposal["note"] + confidence = "-" if proposal["confidence"] is None else "%.2f" % proposal["confidence"] + rows.append(( + ("%s%d" % ("PR " if item["type"] == "pr" else "#", item["number"])) if first else "", + truncate(item["title"], 44) if first else "", + proposal["question"], + truncate(label, 46), + confidence, + truncate(", ".join(proposal["existing"]) or "-", 34), + proposal["verdict"], + )) + first = False + widths = [len(name) for name in columns] + for row in rows: + for index, cell in enumerate(row): + widths[index] = max(widths[index], len(cell)) + fmt = " ".join("{:<%d}" % width for width in widths) + lines = [fmt.format(*columns).rstrip(), fmt.format(*["-" * width for width in widths]).rstrip()] + lines.extend(fmt.format(*row).rstrip() for row in rows) + return "\n".join(lines) + + +def render_summary(summary, usage, wall_seconds, thresholds, output_path): + lines = ["", "Summary"] + lines.append(" items: %d, items with at least one question below the gate: %d" % (summary["items"], summary["items_below_gate"])) + lines.append(" gate: area >= %.2f, kind >= %.2f, priority >= %.2f, yes/no >= %.2f" % ( + thresholds["area"], thresholds["kind"], thresholds["priority"], thresholds["noul"])) + for question, entry in summary["questions"].items(): + rate = "n/a" if entry["agreement_rate"] is None else "%.0f%%" % (entry["agreement_rate"] * 100) + lines.append(" %-11s agreement %s (agree %d, disagree %d, new %d, skip %d, below gate %d)" % ( + question, rate, entry["agree"], entry["disagree"], entry["new"], entry["skip"], entry["below_gate"])) + lines.append(" tokens: input %d, output %d" % (usage["input_tokens"], usage["output_tokens"])) + lines.append(" wall time: %.1fs" % wall_seconds) + if output_path: + lines.append(" raw answers: %s" % output_path) + return "\n".join(lines) + + +def write_output(directory, payload): + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + base = "triage-%s" % stamp() + for suffix in [""] + ["-%d" % n for n in range(2, 1000)]: + path = directory / (base + suffix + ".json") + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + continue + with os.fdopen(fd, "w") as output: + output.write(json.dumps(payload, indent=2) + "\n") + return path + raise TriageError("could not find a free output name under %s" % directory) + + +def write_record(path, repo, items, results): + record = { + "schema_version": 1, + "repo": repo, + "recorded_at": utc_now().isoformat(), + "items": items, + "responses": {result["key"]: result["response"] for result in results}, + } + Path(path).write_text(json.dumps(record, indent=2) + "\n") + + +def parse_args(argv): + parser = argparse.ArgumentParser( + prog="typesafe-triage.py", + description="Dry-run TypeSafe label triage for NanoClaw issues and PRs. Writes nothing to GitHub.", + ) + parser.add_argument("--repo", default=DEFAULT_REPO, help="owner/name (default %(default)s)") + parser.add_argument("--issues", type=int, default=30, help="open issues to fetch, most recently updated first (default %(default)s)") + parser.add_argument("--prs", type=int, default=20, help="open pull requests to fetch (default %(default)s)") + parser.add_argument("--fixture", help="replay recorded items and responses from this JSON file instead of gh and the API") + parser.add_argument("--record", help="write the fetched items and raw responses to this JSON file for later --fixture replay") + parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR), help="where raw answers are saved (default: skill-local output/, gitignored)") + parser.add_argument("--model", default=DEFAULT_MODEL) + parser.add_argument("--timeout", type=float, default=60.0, help="HTTP timeout per request in seconds") + parser.add_argument("--area-threshold", type=float, default=0.6) + parser.add_argument("--kind-threshold", type=float, default=0.6) + parser.add_argument("--priority-threshold", type=float, default=0.8) + parser.add_argument("--noul-threshold", type=float, default=0.7, help="yes/no probability needed to propose a triage label") + parser.add_argument("--json", action="store_true", help="print the full result JSON to stdout instead of the table") + args = parser.parse_args(argv) + for name in ("area_threshold", "kind_threshold", "priority_threshold"): + value = getattr(args, name) + if not 0 <= value <= 1: + parser.error("--%s must be between 0 and 1" % name.replace("_", "-")) + if not 0.5 <= args.noul_threshold <= 1: + parser.error("--noul-threshold must be between 0.5 and 1") + return args + + +def main(argv=None, environ=None, stdout=None, stderr=None): + environ = os.environ if environ is None else environ + stdout = sys.stdout if stdout is None else stdout + stderr = sys.stderr if stderr is None else stderr + args = parse_args(argv) + thresholds = { + "area": args.area_threshold, + "kind": args.kind_threshold, + "priority": args.priority_threshold, + "noul": args.noul_threshold, + } + started = time.monotonic() + try: + if args.fixture: + fixture = load_fixture(args.fixture) + repo = fixture.get("repo") or args.repo + items = fixture["items"] + transport = FixtureTransport(fixture["responses"]) + mode = "fixture" + else: + api_key = environ.get(KEY_ENV) + if not api_key: + print( + "%s is not set. Export it in this shell (for example `export %s=...` from your " + "password manager) and rerun. The key is read from the environment only; never put it " + "in a file inside the repository. Use --fixture to run offline." % (KEY_ENV, KEY_ENV), + file=stderr, + ) + return 2 + repo = args.repo + transport = HttpTransport(api_key, timeout=args.timeout) + mode = "live" + items = fetch_items(repo, args.issues, args.prs) + if not items: + print("no open items to triage", file=stderr) + return 1 + error_text = None + try: + results, usage = evaluate(repo, items, transport, thresholds, args.model, + log=lambda line: print(line, file=stderr) if mode == "live" else None) + except PartialFailure as partial: + results, usage = partial.results, partial.usage + error_text = "%s failed: %s" % (partial.failed_key, partial) + print("error: %s; keeping %d completed item(s)" % (error_text, len(results)), file=stderr) + if not results: + return 1 + except TriageError as error: + print("error: %s" % error, file=stderr) + return 1 + except Exception as error: # never let a traceback carry request details to the terminal + print("error: unexpected %s: %s" % (type(error).__name__, redact_env(str(error), environ)), file=stderr) + return 1 + wall = time.monotonic() - started + summary = summarize(results) + payload = { + "schema_version": 1, + "generated_at": utc_now().isoformat(), + "mode": mode, + "repo": repo, + "model": args.model, + "thresholds": thresholds, + "usage": usage, + "wall_seconds": round(wall, 3), + "partial": error_text, + "summary": summary, + "results": results, + } + output_path = write_output(args.output_dir, payload) + if args.record and mode == "live": + write_record(args.record, repo, items, results) + if args.json: + print(json.dumps(payload, indent=2), file=stdout) + else: + print("Dry run (%s mode) for %s: %d items. No labels were written." % (mode, repo, len(items)), file=stdout) + print("", file=stdout) + print(render_table(results), file=stdout) + print(render_summary(summary, usage, wall, thresholds, output_path), file=stdout) + return 1 if error_text else 0 + + +def redact_env(text, environ): + key = environ.get(KEY_ENV) or "" + return text.replace(key, "[redacted]") if len(key) >= 8 else text + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_typesafe_triage.py b/tests/test_typesafe_triage.py new file mode 100644 index 0000000..55c0f44 --- /dev/null +++ b/tests/test_typesafe_triage.py @@ -0,0 +1,555 @@ +"""TypeSafe triage dry run: questions, gate, fixture replay and transport, all offline.""" + +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import urllib.error + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "skills/typesafe-triage/scripts/typesafe-triage.py" +FIXTURE = ROOT / "skills/typesafe-triage/fixtures/nanoclaw-sample.json" + + +def load(): + spec = importlib.util.spec_from_file_location("typesafe_triage_test", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +triage = load() +THRESHOLDS = {"area": 0.6, "kind": 0.6, "priority": 0.8, "noul": 0.7} + + +def choice(option, confidence, options): + rest = (1 - 0.9) / (len(options) - 1) + probabilities = {name: (0.9 if name == option else rest) for name in options} + return {"type": "choice", "choice": option, "probabilities": probabilities, "confidence": confidence} + + +def score(value, confidence): + return {"type": "score", "score": value, "legend": {}, "probabilities": {}, "confidence": confidence} + + +def answers(area="area/core", area_conf=0.9, kind="kind/bug", kind_conf=0.9, priority=1.0, priority_conf=0.9, noul=0.5, noul_name="needs_repro"): + return { + "area": choice(area, area_conf, list(triage.AREA_CRITERIA)), + "kind": choice(kind, kind_conf, list(triage.KIND_CRITERIA)), + "priority": score(priority, priority_conf), + noul_name: {"type": "noul", "noul": noul}, + } + + +def issue(number=1, labels=(), body="body"): + return {"type": "issue", "number": number, "title": "Issue %d" % number, "body": body, + "author_association": "NONE", "labels": sorted(labels), "url": "", "updated_at": ""} + + +def pull(number=2, labels=(), body="body", files=("src/router.ts",)): + item = issue(number, labels, body) + item.update({"type": "pr", "draft": False, "files": list(files), "file_count": len(files), + "template": triage.template_status(body)}) + return item + + +class QuestionTests(unittest.TestCase): + def test_area_and_kind_criteria_cover_the_nanoclaw_label_taxonomy(self): + questions = triage.build_questions("issue") + expected_areas = { + "area/agent-memory", "area/agent-runner", "area/channels", "area/configuration", "area/containers", + "area/core", "area/credentials", "area/ncl-cli", "area/providers", "area/repository-maintenance", + "area/scheduled-tasks", "area/security", "area/sessions", "area/setup-installation", "area/skills", + "area/tools", + } + self.assertEqual(set(questions["area"]["criteria"]), expected_areas) + self.assertEqual(set(questions["kind"]["criteria"]), { + "kind/bug", "kind/feature", "kind/documentation", "kind/question", "kind/security", + "kind/hardening", "kind/cleanup", + }) + for criteria in (questions["area"]["criteria"], questions["kind"]["criteria"]): + for option, rubric in criteria.items(): + self.assertTrue(rubric and "\n" not in rubric, option) + self.assertEqual(questions["priority"]["type"], "score") + self.assertEqual(len(questions["priority"]["criteria"]), 4) + self.assertEqual(triage.PRIORITY_LABELS, ("priority/low", "priority/medium", "priority/high", "priority/critical")) + + def test_issues_ask_needs_repro_and_prs_ask_pr_ready(self): + self.assertIn("needs_repro", triage.build_questions("issue")) + self.assertNotIn("pr_ready", triage.build_questions("issue")) + self.assertIn("pr_ready", triage.build_questions("pr")) + self.assertNotIn("needs_repro", triage.build_questions("pr")) + for item_type in ("issue", "pr"): + for question in triage.build_questions(item_type).values(): + self.assertIn(question["type"], ("noul", "choice", "score")) + self.assertTrue(question["instructions"]) + + def test_request_shape_matches_the_documented_api(self): + request = triage.build_request("nanocoai/nanoclaw", pull(body="x" * 20000)) + self.assertEqual(set(request), {"state", "model", "questions"}) + self.assertEqual(request["model"], "jev-latest") + state = request["state"] + self.assertEqual(state["item_type"], "pull_request") + self.assertEqual(len(state["body"]), triage.BODY_LIMIT) + self.assertEqual(state["changed_files"], ["src/router.ts"]) + self.assertIn("sections", state["template"]) + self.assertNotIn("labels", state) + self.assertNotIn("existing_labels", state) + + +class TemplateTests(unittest.TestCase): + TEMPLATE = ( + "\n\n## Summary\n\n\n\n" + "## Related work\n\nCloses #\n\n## Change kind\n\n- [ ] `kind/bug`\n- [ ] `kind/feature`\n\n" + "## Validation\n\n- [ ] Tests cover the changed behavior\n\n## User and release impact\n\n" + "- [ ] No user-visible behavior change\n\n```release-note\nOptional: one user-facing line for the changelog.\n```\n\n" + "## Security and trust boundaries\n\n\n\n## Skill delivery\n\n- [ ] Not a skill\n\n" + "## AI assistance\n\n- [ ] AI tools or agents helped produce this change\n" + ) + + def test_untouched_template_counts_every_section_as_empty(self): + status = triage.template_status(self.TEMPLATE) + self.assertTrue(status["marker"]) + self.assertEqual(status["filled"], 0) + self.assertEqual(set(status["sections"].values()), {"empty"}) + + def test_filled_template_counts_content_and_checked_boxes(self): + body = (self.TEMPLATE + .replace("## Summary\n\n\n", "## Summary\n\nStops the sweep killing turns.\n- **Fix**: poll tick.\n") + .replace("- [ ] `kind/bug`", "- [x] `kind/bug`") + .replace("- [ ] Tests cover the changed behavior", "- [x] Tests cover the changed behavior\n- `pnpm test` -> 12 passed")) + status = triage.template_status(body) + self.assertEqual(status["sections"]["Summary"], "filled") + self.assertEqual(status["sections"]["Change kind"], "filled") + self.assertEqual(status["sections"]["Validation"], "filled") + self.assertEqual(status["sections"]["Related work"], "empty") + self.assertEqual(status["filled"], 3) + + def test_nested_headings_stay_inside_their_section(self): + body = self.TEMPLATE.replace("## Validation\n\n", "## Validation\n\n### Tests\n\nAll tests passed.\n\n") + status = triage.template_status(body) + self.assertEqual(status["sections"]["Validation"], "filled") + self.assertEqual(status["sections"]["User and release impact"], "empty") + self.assertIn("Tests", status["headings"]) + + def test_body_without_the_template_reports_missing_sections_and_its_own_headings(self): + status = triage.template_status("## Problem\n\nIt breaks.\n\n## Testing\n\nRan it.\n") + self.assertFalse(status["marker"]) + self.assertEqual(status["filled"], 0) + self.assertEqual(set(status["sections"].values()), {"missing"}) + self.assertEqual(status["headings"], ["Problem", "Testing"]) + self.assertEqual(triage.template_status(None)["headings"], []) + + +class DecisionTests(unittest.TestCase): + def proposals(self, item, answer_map): + return {p["question"]: p for p in triage.decide(item, answer_map, THRESHOLDS)} + + def test_confident_answers_agree_with_matching_existing_labels(self): + item = issue(labels=["area/core", "kind/bug", "priority/medium"]) + result = self.proposals(item, answers(priority=1.2)) + self.assertEqual((result["area"]["label"], result["area"]["verdict"]), ("area/core", "AGREE")) + self.assertEqual((result["kind"]["label"], result["kind"]["verdict"]), ("kind/bug", "AGREE")) + self.assertEqual((result["priority"]["label"], result["priority"]["verdict"]), ("priority/medium", "AGREE")) + + def test_different_label_in_the_same_family_is_a_disagreement_and_no_label_is_new(self): + item = issue(labels=["area/channels"]) + result = self.proposals(item, answers()) + self.assertEqual(result["area"]["verdict"], "DISAGREE") + self.assertEqual(result["area"]["existing"], ["area/channels"]) + self.assertEqual(result["kind"]["verdict"], "NEW") + + def test_low_confidence_proposes_unresolved_instead_of_guessing(self): + result = self.proposals(issue(), answers(area_conf=0.59, kind_conf=0.6, priority_conf=0.79)) + self.assertEqual(result["area"]["label"], "triage/unresolved") + self.assertTrue(result["area"]["gated"]) + self.assertIn("area/core", result["area"]["note"]) + self.assertEqual(result["kind"]["label"], "kind/bug") + self.assertFalse(result["kind"]["gated"]) + self.assertEqual(result["priority"]["label"], "triage/unresolved") + self.assertTrue(result["priority"]["gated"]) + + def test_thresholds_are_overridable(self): + thresholds = dict(THRESHOLDS, area=0.3, priority=0.5) + result = {p["question"]: p for p in triage.decide(issue(), answers(area_conf=0.35, priority_conf=0.55), thresholds)} + self.assertEqual(result["area"]["label"], "area/core") + self.assertEqual(result["priority"]["label"], "priority/medium") + + def test_score_rounds_to_the_nearest_priority_level(self): + self.assertEqual(triage.priority_label(0.4), "priority/low") + self.assertEqual(triage.priority_label(1.4), "priority/medium") + self.assertEqual(triage.priority_label(1.5), "priority/high") + self.assertEqual(triage.priority_label(2.4), "priority/high") + self.assertEqual(triage.priority_label(2.6), "priority/critical") + self.assertEqual(triage.priority_label(9), "priority/critical") + self.assertEqual(triage.priority_label(-1), "priority/low") + + def test_unknown_choice_from_the_api_is_gated(self): + result = self.proposals(issue(), answers(area="area/does-not-exist", area_conf=0.99)) + self.assertEqual(result["area"]["label"], "triage/unresolved") + self.assertTrue(result["area"]["gated"]) + + def test_missing_answers_are_gated_rather_than_crashing(self): + result = self.proposals(issue(), {}) + for question in ("area", "kind", "priority", "needs_repro"): + self.assertEqual(result[question]["label"], "triage/unresolved", question) + self.assertTrue(result[question]["gated"]) + + def test_needs_repro_thresholds(self): + yes = self.proposals(issue(), answers(noul=0.7))["needs_repro"] + self.assertEqual((yes["label"], yes["verdict"], yes["gated"]), ("triage/needs-repro", "NEW", False)) + agreed = self.proposals(issue(labels=["triage/needs-repro"]), answers(noul=0.9))["needs_repro"] + self.assertEqual(agreed["verdict"], "AGREE") + no = self.proposals(issue(), answers(noul=0.3))["needs_repro"] + self.assertEqual((no["label"], no["verdict"], no["gated"]), (None, "AGREE", False)) + stale = self.proposals(issue(labels=["triage/needs-repro"]), answers(noul=0.1))["needs_repro"] + self.assertEqual(stale["verdict"], "DISAGREE") + unsure = self.proposals(issue(), answers(noul=0.5))["needs_repro"] + self.assertEqual((unsure["label"], unsure["gated"]), ("triage/unresolved", True)) + + def test_needs_repro_is_skipped_when_the_item_is_not_a_bug(self): + result = self.proposals(issue(), answers(kind="kind/feature", noul=0.95))["needs_repro"] + self.assertEqual((result["label"], result["verdict"]), (None, "SKIP")) + result = self.proposals(issue(), answers(kind="kind/security", noul=0.95))["needs_repro"] + self.assertEqual(result["label"], "triage/needs-repro") + + def test_pr_ready_maps_to_review_or_author_labels(self): + ready = self.proposals(pull(), answers(noul=0.85, noul_name="pr_ready"))["pr_ready"] + self.assertEqual((ready["label"], ready["verdict"]), ("Status: Needs Review", "NEW")) + stalled = self.proposals(pull(), answers(noul=0.2, noul_name="pr_ready"))["pr_ready"] + self.assertEqual(stalled["label"], "triage/needs-author") + self.assertAlmostEqual(stalled["confidence"], 0.8) + unsure = self.proposals(pull(), answers(noul=0.6, noul_name="pr_ready"))["pr_ready"] + self.assertEqual((unsure["label"], unsure["gated"]), ("triage/unresolved", True)) + self.assertNotIn("needs_repro", self.proposals(pull(), answers(noul=0.6, noul_name="pr_ready"))) + + def test_summary_reports_agreement_per_question_and_gated_items(self): + results = [ + {"proposals": triage.decide(issue(1, ["area/core", "kind/feature"]), answers(), THRESHOLDS)}, + {"proposals": triage.decide(issue(2, ["area/core"]), answers(area_conf=0.2), THRESHOLDS)}, + {"proposals": triage.decide(issue(3), answers(kind="kind/feature"), THRESHOLDS)}, + ] + summary = triage.summarize(results) + self.assertEqual(summary["items"], 3) + # needs_repro at 0.5 is uncertain on the two bug items; the feature item skips it + self.assertEqual(summary["items_below_gate"], 2) + area = summary["questions"]["area"] + self.assertEqual((area["agree"], area["disagree"], area["new"], area["below_gate"]), (1, 0, 1, 1)) + self.assertEqual(area["agreement_rate"], 1.0) + kind = summary["questions"]["kind"] + self.assertEqual((kind["agree"], kind["disagree"], kind["new"]), (0, 1, 2)) + self.assertEqual(kind["agreement_rate"], 0.0) + self.assertIsNone(summary["questions"]["priority"]["agreement_rate"]) + + +class FetchTests(unittest.TestCase): + def test_fetch_items_uses_gh_api_read_only_and_drops_prs_from_the_issues_endpoint(self): + calls = [] + + class Result: + def __init__(self, stdout): + self.returncode = 0 + self.stdout = stdout + self.stderr = "" + + def run(command, capture_output, text): + calls.append(command) + self.assertEqual(command[:2], ["gh", "api"]) + path = command[2] + self.assertIn("per_page=100&page=", path) + if path.startswith("repos/o/r/issues?"): + return Result(json.dumps([ + {"number": 5, "title": "Real issue", "body": "steps", "author_association": "MEMBER", + "labels": [{"name": "kind/bug"}], "html_url": "u5", "updated_at": "t"}, + {"number": 6, "title": "PR in disguise", "body": "", "author_association": "NONE", + "labels": [], "html_url": "u6", "updated_at": "t", "pull_request": {}}, + {"number": 4, "title": "Older issue", "body": None, "author_association": "NONE", + "labels": [], "html_url": "u4", "updated_at": "t"}, + ])) + if path.startswith("repos/o/r/pulls?"): + return Result(json.dumps([ + {"number": 7, "title": "feat: x", "body": "\n## Summary\n\nDoes x.\n", + "author_association": "CONTRIBUTOR", "labels": [{"name": "area/core"}], "html_url": "u7", + "updated_at": "t", "draft": True}, + ])) + if path == "repos/o/r/pulls/7/files?per_page=100&page=1": + return Result(json.dumps([{"filename": "src/router.ts"}, {"filename": "docs/x.md"}])) + self.fail("unexpected gh call %s" % path) + + items = triage.fetch_items("o/r", 1, 1, run=run) + self.assertEqual([(item["type"], item["number"]) for item in items], [("issue", 5), ("pr", 7)]) + self.assertEqual(items[0]["labels"], ["kind/bug"]) + self.assertEqual(items[1]["files"], ["src/router.ts", "docs/x.md"]) + self.assertTrue(items[1]["draft"]) + self.assertTrue(items[1]["template"]["marker"]) + self.assertEqual(items[1]["template"]["sections"]["Summary"], "filled") + for command in calls: + self.assertNotIn("-X", command) + self.assertNotIn("--method", command) + self.assertNotIn("-f", command) + + def test_issue_fetch_keeps_paging_past_pull_requests(self): + pages = { + 1: [{"number": 100 - n, "title": "pr", "body": "", "author_association": "NONE", "labels": [], + "html_url": "", "updated_at": "", "pull_request": {}} for n in range(100)], + 2: [{"number": 3, "title": "issue", "body": "", "author_association": "NONE", "labels": [], + "html_url": "", "updated_at": ""}] + [ + {"number": 2 - n, "title": "pr", "body": "", "author_association": "NONE", "labels": [], + "html_url": "", "updated_at": "", "pull_request": {}} for n in range(2)], + } + + class Result: + returncode = 0 + stderr = "" + + def __init__(self, stdout): + self.stdout = stdout + + def run(command, capture_output, text): + page = int(command[2].rsplit("page=", 1)[1]) + return Result(json.dumps(pages[page])) + + items = triage.fetch_items("o/r", 5, 0, run=run) + self.assertEqual([item["number"] for item in items], [3]) + + def test_gh_failure_is_reported_as_a_triage_error(self): + class Failed: + returncode = 1 + stdout = "" + stderr = "gh: Not Found (HTTP 404)\n" + + with self.assertRaises(triage.TriageError) as raised: + triage.gh_json("repos/o/r/issues", run=lambda *a, **k: Failed()) + self.assertIn("HTTP 404", str(raised.exception)) + + +class TransportTests(unittest.TestCase): + class Response: + def __init__(self, body): + self.body = body + + def read(self): + return json.dumps(self.body).encode() + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def test_http_transport_sends_bearer_key_and_retries_on_429(self): + seen = [] + sleeps = [] + + def opener(request, timeout): + seen.append(request) + if len(seen) == 1: + raise urllib.error.HTTPError(request.full_url, 429, "Too Many Requests", {}, io.BytesIO(b"{}")) + return self.Response({"model": "jev-latest", "answers": {}, "usage": {"input_tokens": 1, "output_tokens": 1}}) + + transport = triage.HttpTransport("secret-key", opener=opener, sleep=sleeps.append) + result = transport({"state": {"x": 1}, "model": "jev-latest", "questions": {}}) + self.assertEqual(result["model"], "jev-latest") + self.assertEqual(len(seen), 2) + self.assertEqual(sleeps, [1.0]) + request = seen[0] + self.assertEqual(request.full_url, "https://api.typesafe.ai/v1/systemone") + self.assertEqual(request.get_method(), "POST") + self.assertEqual(request.get_header("Authorization"), "Bearer secret-key") + self.assertEqual(json.loads(request.data.decode()), {"state": {"x": 1}, "model": "jev-latest", "questions": {}}) + + def test_http_transport_does_not_retry_client_errors(self): + def opener(request, timeout): + raise urllib.error.HTTPError(request.full_url, 422, "Unprocessable", {}, io.BytesIO(b'{"detail":"bad question"}')) + + transport = triage.HttpTransport("k", opener=opener, sleep=lambda s: self.fail("should not sleep")) + with self.assertRaises(triage.TriageError) as raised: + transport({}) + self.assertIn("422", str(raised.exception)) + self.assertIn("bad question", str(raised.exception)) + + def test_http_transport_gives_up_after_the_last_attempt(self): + def opener(request, timeout): + raise urllib.error.HTTPError(request.full_url, 529, "Overloaded", {}, io.BytesIO(b"")) + + transport = triage.HttpTransport("k", attempts=3, opener=opener, sleep=lambda s: None) + with self.assertRaises(triage.TriageError): + transport({}) + + def test_missing_key_is_rejected_before_any_request(self): + with self.assertRaises(triage.TriageError): + triage.HttpTransport("") + + def test_key_with_whitespace_is_rejected_without_echoing_it(self): + secret = "ts-FAKE-KEY-with-newline\n" + with self.assertRaises(triage.TriageError) as raised: + triage.HttpTransport(secret) + self.assertNotIn("ts-FAKE-KEY", str(raised.exception)) + self.assertIn("whitespace", str(raised.exception)) + + def test_error_bodies_that_echo_the_key_are_redacted(self): + secret = "ts-FAKE-KEY-0123456789" + + def opener(request, timeout): + raise urllib.error.HTTPError(request.full_url, 401, "Unauthorized", {}, + io.BytesIO(("bad header Bearer %s" % secret).encode())) + + transport = triage.HttpTransport(secret, opener=opener, sleep=lambda s: None) + with self.assertRaises(triage.TriageError) as raised: + transport({}) + self.assertNotIn(secret, str(raised.exception)) + self.assertIn("[redacted]", str(raised.exception)) + + def test_default_opener_is_resolved_at_call_time_so_tests_can_block_it(self): + transport = triage.HttpTransport("ts-FAKE-KEY-0123456789", sleep=lambda s: None) + with patch.object(triage.urllib.request, "urlopen", side_effect=urllib.error.URLError("blocked")): + with self.assertRaises(triage.TriageError) as raised: + transport({}) + self.assertIn("blocked", str(raised.exception)) + + def test_fixture_transport_replays_by_item_key_and_rejects_unknown_items(self): + transport = triage.FixtureTransport({"issue:1": {"answers": {}}}) + payload = triage.build_request("o/r", issue(1)) + self.assertEqual(transport(payload), {"answers": {}}) + with self.assertRaises(triage.TriageError): + transport(triage.build_request("o/r", pull(1))) + + +class PipelineTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="typesafe-triage-", dir="/tmp") + self.addCleanup(self.temporary.cleanup) + self.output = Path(self.temporary.name) / "out" + self.patcher = patch.object(triage.urllib.request, "urlopen", + side_effect=AssertionError("live TypeSafe call attempted")) + self.patcher.start() + self.addCleanup(self.patcher.stop) + + def run_main(self, argv, environ=None): + stdout, stderr = io.StringIO(), io.StringIO() + code = triage.main(argv, environ={} if environ is None else environ, stdout=stdout, stderr=stderr) + return code, stdout.getvalue(), stderr.getvalue() + + def test_missing_key_exits_with_a_clear_message_and_no_fetch(self): + with patch.object(triage, "fetch_items", side_effect=AssertionError("gh should not run")): + code, out, err = self.run_main(["--output-dir", str(self.output)], environ={}) + self.assertEqual(code, 2) + self.assertIn("TYPESAFE_API_KEY is not set", err) + self.assertIn("--fixture", err) + self.assertEqual(out, "") + self.assertFalse(self.output.exists()) + + def test_fixture_replay_runs_the_whole_pipeline_offline(self): + code, out, err = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output)]) + self.assertEqual(code, 0, err) + self.assertIn("fixture mode", out) + self.assertIn("No labels were written", out) + for token in ("AGREE", "NEW", "triage/unresolved", "triage/needs-repro", "Status: Needs Review", + "agreement", "tokens: input", "wall time"): + self.assertIn(token, out) + saved = sorted(self.output.glob("triage-*.json")) + self.assertEqual(len(saved), 1) + payload = json.loads(saved[0].read_text()) + self.assertEqual(payload["mode"], "fixture") + self.assertEqual(payload["repo"], "nanocoai/nanoclaw") + self.assertEqual(len(payload["results"]), 5) + self.assertEqual(payload["usage"]["input_tokens"], 10960) + self.assertEqual(set(payload["thresholds"]), {"area", "kind", "priority", "noul"}) + for result in payload["results"]: + self.assertIn("answers", result["response"]) + self.assertTrue(result["proposals"]) + + def test_fixture_items_get_the_documented_questions(self): + fixture = json.loads(FIXTURE.read_text()) + self.assertEqual(len(fixture["items"]), 5) + transport = triage.FixtureTransport(fixture["responses"]) + triage.evaluate(fixture["repo"], fixture["items"], transport, THRESHOLDS) + for request in transport.requests: + self.assertEqual(request["model"], "jev-latest") + expected = "pr_ready" if request["state"]["item_type"] == "pull_request" else "needs_repro" + self.assertEqual(set(request["questions"]), {"area", "kind", "priority", expected}) + for response in fixture["responses"].values(): + for name, answer in response["answers"].items(): + if answer["type"] in ("choice", "score"): + self.assertAlmostEqual(sum(answer["probabilities"].values()), 1.0, places=3, msg=name) + self.assertTrue(0 <= answer["confidence"] <= 1) + else: + self.assertTrue(0 <= answer["noul"] <= 1) + + def test_fixture_summary_shows_agreement_with_existing_labels(self): + code, out, _ = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output), "--json"]) + self.assertEqual(code, 0) + payload = json.loads(out) + summary = payload["summary"]["questions"] + self.assertEqual(summary["area"]["agree"], 3) + self.assertEqual(summary["area"]["disagree"], 0) + self.assertEqual(summary["area"]["below_gate"], 1) + self.assertEqual(summary["needs_repro"]["agree"], 1) + self.assertEqual(summary["priority"]["below_gate"], 3) + self.assertEqual(payload["summary"]["items_below_gate"], 5) + + def test_flags_override_the_gate(self): + code, out, _ = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output), + "--priority-threshold", "0.5", "--area-threshold", "0.4", "--json"]) + self.assertEqual(code, 0) + summary = json.loads(out)["summary"]["questions"] + self.assertEqual(summary["priority"]["below_gate"], 0) + self.assertEqual(summary["area"]["below_gate"], 0) + + def test_invalid_threshold_is_rejected(self): + with self.assertRaises(SystemExit): + with patch("sys.stderr", io.StringIO()): + triage.parse_args(["--noul-threshold", "0.4"]) + + def test_key_never_appears_in_output_or_saved_json(self): + sentinel = "ts-live-FAKE-KEY-0123456789" + code, out, err = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output)], + environ={"TYPESAFE_API_KEY": sentinel}) + self.assertEqual(code, 0) + saved = next(self.output.glob("triage-*.json")).read_text() + for text in (out, err, saved): + self.assertNotIn(sentinel, text) + + def test_partial_failure_keeps_completed_results_and_exits_nonzero(self): + fixture = json.loads(FIXTURE.read_text()) + responses = dict(fixture["responses"]) + failing = triage.FixtureTransport(responses) + original = failing.__call__ + + def flaky(payload): + if payload["state"]["number"] == 3814: + raise triage.TriageError("HTTP 529 from TypeSafe: overloaded") + return original(payload) + + with patch.object(triage, "FixtureTransport", return_value=flaky): + code, out, err = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output), "--json"]) + self.assertEqual(code, 1) + self.assertIn("keeping 2 completed item(s)", err) + payload = json.loads(out) + self.assertEqual(len(payload["results"]), 2) + self.assertIn("issue:3814 failed", payload["partial"]) + self.assertEqual(len(list(self.output.glob("triage-*.json"))), 1) + + def test_output_files_never_overwrite_each_other(self): + with patch.object(triage, "stamp", return_value="fixed"): + first = triage.write_output(self.output, {"a": 1}) + second = triage.write_output(self.output, {"a": 2}) + self.assertNotEqual(first, second) + self.assertEqual(json.loads(first.read_text()), {"a": 1}) + self.assertEqual(json.loads(second.read_text()), {"a": 2}) + + def test_bad_fixture_is_a_clean_error(self): + broken = Path(self.temporary.name) / "broken.json" + broken.write_text("[]") + code, out, err = self.run_main(["--fixture", str(broken), "--output-dir", str(self.output)]) + self.assertEqual(code, 1) + self.assertIn("error:", err) + self.assertIn("'items'", err) + + +if __name__ == "__main__": + unittest.main() From 29e0d3330dcecd7a1bb39f4f7d1b61dcc90a333b Mon Sep 17 00:00:00 2001 From: glifocat Date: Wed, 16 Sep 2026 23:32:28 +0200 Subject: [PATCH 2/3] fix(typesafe-triage): lower the default priority gate to 0.6 from the first live run --- docs/typesafe-triage.md | 2 +- skills/typesafe-triage/SKILL.md | 8 +++++--- skills/typesafe-triage/scripts/typesafe-triage.py | 2 +- tests/test_typesafe_triage.py | 6 +++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/typesafe-triage.md b/docs/typesafe-triage.md index 43b2b47..dee7ff8 100644 --- a/docs/typesafe-triage.md +++ b/docs/typesafe-triage.md @@ -33,7 +33,7 @@ probability distribution. Noul answers are a probability of yes. | Question | Default gate | Flag | |---|---|---| | `area`, `kind` | confidence >= 0.6 | `--area-threshold`, `--kind-threshold` | -| `priority` | confidence >= 0.8 | `--priority-threshold` | +| `priority` | confidence >= 0.6 | `--priority-threshold` | | `needs_repro`, `pr_ready` | yes if p >= 0.7, no if p <= 0.3, otherwise uncertain | `--noul-threshold` | Below the gate the script proposes `triage/unresolved` and shows the model's top diff --git a/skills/typesafe-triage/SKILL.md b/skills/typesafe-triage/SKILL.md index a8be9ae..e720b82 100644 --- a/skills/typesafe-triage/SKILL.md +++ b/skills/typesafe-triage/SKILL.md @@ -62,7 +62,7 @@ replayed later with `--fixture`. Raw answers for every run land in included there; keep the directory private. Flags: `--repo`, `--issues`, `--prs`, `--area-threshold` (0.6), -`--kind-threshold` (0.6), `--priority-threshold` (0.8), `--noul-threshold` (0.7), +`--kind-threshold` (0.6), `--priority-threshold` (0.6), `--noul-threshold` (0.7), `--model`, `--timeout`, `--json`. ## Read the output @@ -85,8 +85,10 @@ question, token usage and wall time. - The fixture's answers are hand-written in the documented response shapes to exercise the pipeline; they are not evidence of model accuracy. Evaluate accuracy with a live `--record` run and the agreement summary. -- `priority/*` is maintainer-set; the default 0.8 gate keeps most priority - proposals as `triage/unresolved` on purpose. +- `priority/*` is maintainer-set. The default 0.6 gate comes from the first live + run on nanoclaw (50 items): at 0.8 it withheld 34 of 48 priority proposals, at + 0.6 it withheld 13 with every fired proposal agreeing with the existing label, + and at 0.5 disagreements appeared. - Existing labels are never sent to the model, so agreement is a fair comparison. - Requests are sequential; 50 items take roughly a minute. A request that fails after retries stops the run, but completed items are still saved and printed diff --git a/skills/typesafe-triage/scripts/typesafe-triage.py b/skills/typesafe-triage/scripts/typesafe-triage.py index fb74dcf..85c9d1b 100644 --- a/skills/typesafe-triage/scripts/typesafe-triage.py +++ b/skills/typesafe-triage/scripts/typesafe-triage.py @@ -746,7 +746,7 @@ def parse_args(argv): parser.add_argument("--timeout", type=float, default=60.0, help="HTTP timeout per request in seconds") parser.add_argument("--area-threshold", type=float, default=0.6) parser.add_argument("--kind-threshold", type=float, default=0.6) - parser.add_argument("--priority-threshold", type=float, default=0.8) + parser.add_argument("--priority-threshold", type=float, default=0.6) parser.add_argument("--noul-threshold", type=float, default=0.7, help="yes/no probability needed to propose a triage label") parser.add_argument("--json", action="store_true", help="print the full result JSON to stdout instead of the table") args = parser.parse_args(argv) diff --git a/tests/test_typesafe_triage.py b/tests/test_typesafe_triage.py index 55c0f44..df42d6c 100644 --- a/tests/test_typesafe_triage.py +++ b/tests/test_typesafe_triage.py @@ -23,7 +23,7 @@ def load(): triage = load() -THRESHOLDS = {"area": 0.6, "kind": 0.6, "priority": 0.8, "noul": 0.7} +THRESHOLDS = {"area": 0.6, "kind": 0.6, "priority": 0.6, "noul": 0.7} def choice(option, confidence, options): @@ -164,7 +164,7 @@ def test_different_label_in_the_same_family_is_a_disagreement_and_no_label_is_ne self.assertEqual(result["kind"]["verdict"], "NEW") def test_low_confidence_proposes_unresolved_instead_of_guessing(self): - result = self.proposals(issue(), answers(area_conf=0.59, kind_conf=0.6, priority_conf=0.79)) + result = self.proposals(issue(), answers(area_conf=0.59, kind_conf=0.6, priority_conf=0.59)) self.assertEqual(result["area"]["label"], "triage/unresolved") self.assertTrue(result["area"]["gated"]) self.assertIn("area/core", result["area"]["note"]) @@ -489,7 +489,7 @@ def test_fixture_summary_shows_agreement_with_existing_labels(self): self.assertEqual(summary["area"]["disagree"], 0) self.assertEqual(summary["area"]["below_gate"], 1) self.assertEqual(summary["needs_repro"]["agree"], 1) - self.assertEqual(summary["priority"]["below_gate"], 3) + self.assertEqual(summary["priority"]["below_gate"], 2) self.assertEqual(payload["summary"]["items_below_gate"], 5) def test_flags_override_the_gate(self): From bb7f1727a5a3418f8399ea917d199ead90ceaa5e Mon Sep 17 00:00:00 2001 From: glifocat Date: Wed, 16 Sep 2026 23:32:45 +0200 Subject: [PATCH 3/3] test(typesafe-triage): update fixture expectations for the 0.6 priority gate --- tests/test_typesafe_triage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_typesafe_triage.py b/tests/test_typesafe_triage.py index df42d6c..384bd8a 100644 --- a/tests/test_typesafe_triage.py +++ b/tests/test_typesafe_triage.py @@ -490,7 +490,7 @@ def test_fixture_summary_shows_agreement_with_existing_labels(self): self.assertEqual(summary["area"]["below_gate"], 1) self.assertEqual(summary["needs_repro"]["agree"], 1) self.assertEqual(summary["priority"]["below_gate"], 2) - self.assertEqual(payload["summary"]["items_below_gate"], 5) + self.assertEqual(payload["summary"]["items_below_gate"], 4) def test_flags_override_the_gate(self): code, out, _ = self.run_main(["--fixture", str(FIXTURE), "--output-dir", str(self.output),