From 69211646a36682a2fe02705e07560edba1850e5d Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 4 Sep 2026 22:23:04 +0800 Subject: [PATCH] docs: refresh verification record Fixes #55 --- docs/self-evolution.md | 61 +++++++++++++++++++++++++++++++++++---- scripts/check_docs.py | 65 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 6 deletions(-) diff --git a/docs/self-evolution.md b/docs/self-evolution.md index 38dd440..6387467 100644 --- a/docs/self-evolution.md +++ b/docs/self-evolution.md @@ -2,8 +2,14 @@ This is the operator guide for the merged self-evolution and multi-party memory features. It records behavior that is callable today, not a promise about -future learning. The verification below was run on `main` at `51d0bce` on -2026-09-03. +future learning. The verification record below is explicitly a historical +snapshot, not a claim about the repository's current `HEAD`. + +Historical verification snapshot: `51be33361422e55e1f2f00c33a0e0f8c56132a91` +(the post-#54 `main` revision, captured before this #55 documentation-only +update). Snapshot date: 2026-09-04. + +Current repository test count at this snapshot: **130 unittest cases**. ## Verified surface @@ -192,21 +198,64 @@ completion is non-regressing and a paired secondary improvement is credible; OpenKyrozen does not emit a competitor-superiority claim from a tool success or an unpaired run. +The historical snapshot's isolated `make benchmark` output had five cases and +five verified successes for both runners. Both runners used provider +`deterministic-fixture` and model `openkyrozen-memory-policy-v1`; each case had +`fixture_verified` evidence. The observed summary was: + +| Runner | Cases | Verified successes | Tool calls | Tokens | Evidence status | +| --- | ---: | ---: | ---: | ---: | --- | +| clean | 5 | 5 | 22 | 973 | `fixture_verified` | +| evolved | 5 | 5 | 22 | 973 | `fixture_verified` | + +The run reported `completion_non_regressing: true`, +`paired_evidence_status: insufficient`, and +`public_superiority_claim_supported: false`. The fixture is deterministic +and proves the scoped memory behavior; it does not provide the independently +paired product evidence required for a public superiority claim. Latency is +diagnostic only and is not treated as a release claim. + +## Verification snapshot commands + +The post-#54 snapshot ran the repository's current checks and smoke coverage: + +```bash +make check +make docs-check +make shell-check +make lint +make test +make benchmark +git diff --check +``` + +The `make test` suite includes the API health/scoping smoke and the CLI +command-loop smoke. `make benchmark` uses the five-case +`benchmarks/multi_party_memory.jsonl` fixture, temporary SQLite databases, the +`openkyrozen-learning-benchmark-v1` protocol, and no provider credentials. +The documented commit is retained only as a historical verification snapshot; +later commits must run these checks again before making current-release claims. + ## Verification and troubleshooting Run the same local gates used for the merged implementation: ```bash make check +make docs-check +make shell-check +make lint tmpdir=$(mktemp -d /tmp/openkyrozen-check.XXXXXX) KYROZEN_DB_PATH="$tmpdir/state.sqlite3" make test +make benchmark git diff --check ``` -The last verification on `main` passed 58 tests, the API health/scoping smoke, -the CLI command-loop smoke, and the five-case multi-party benchmark with four -ablations (`make check` reports the 29-entry base registry). A new artifact is -not immediate: wait for an eligible run, at least +The historical post-#54 verification passed the 130 discovered tests, the API +health/scoping smoke, the CLI command-loop smoke, and the five-case +clean/evolved benchmark described above. `make check` reports the live 31-tool +runtime inventory, including 14 `git_` tools. A new artifact is not immediate: +wait for an eligible run, at least 60 seconds of idle time, reviewer evidence, and then the two-success plus replay gate. A model or environment mismatch intentionally downgrades an artifact to canary. If ChromaDB is unavailable, SQLite remains the durable diff --git a/scripts/check_docs.py b/scripts/check_docs.py index bcea9a9..df2ad7b 100755 --- a/scripts/check_docs.py +++ b/scripts/check_docs.py @@ -6,17 +6,43 @@ import re import shlex import sys +import unittest from pathlib import Path from generate_tool_inventory import ROOT, render_inventory, runtime_routes README_FILES = sorted(ROOT.glob("README*.md")) +VERIFICATION_DOC = ROOT / "docs" / "self-evolution.md" STALE_TOOL_PATTERNS = ( re.compile(r"\b26\b[^\n]*(?:tool|工具|ツール|도구)", re.IGNORECASE), re.compile(r"(?:Git|git)[^\n]{0,24}\b15\b[^\n]*(?:tool|工具|ツール|도구)", re.IGNORECASE), re.compile(r"\b15\b[^\n]*(?:Git|git)[^\n]*(?:tool|工具|ツール|도구)", re.IGNORECASE), ) +STALE_VERIFICATION_CLAIMS = ( + "51d0bce", + "58 tests", + "119 tests", + "359378b", +) +VERIFICATION_COMMANDS = ( + "make check", + "make docs-check", + "make shell-check", + "make lint", + "make test", + "make benchmark", + "git diff --check", +) +BENCHMARK_METADATA = ( + "benchmarks/multi_party_memory.jsonl", + "openkyrozen-learning-benchmark-v1", + "deterministic-fixture", + "openkyrozen-memory-policy-v1", + "fixture_verified", + "paired_evidence_status: insufficient", + "public_superiority_claim_supported: false", +) def _command_blocks(text: str) -> list[str]: @@ -78,6 +104,44 @@ def _check_commands(path: Path, text: str, routes: set[str], targets: set[str]) return errors +def _discovered_test_count() -> int: + suite = unittest.TestLoader().discover( + str(ROOT / "tests"), pattern="test_*.py" + ) + return suite.countTestCases() + + +def _check_verification_record() -> list[str]: + if not VERIFICATION_DOC.exists(): + return ["docs/self-evolution.md is missing"] + text = VERIFICATION_DOC.read_text(encoding="utf-8") + errors: list[str] = [] + for claim in STALE_VERIFICATION_CLAIMS: + if claim in text: + errors.append(f"self-evolution.md contains obsolete verification claim '{claim}'") + + if "historical verification snapshot" not in text.lower(): + errors.append("self-evolution.md must label its verification as a historical snapshot") + if not re.search(r"Historical verification snapshot:\s*`[0-9a-f]{40}`", text): + errors.append("self-evolution.md must identify a full historical verification commit") + + test_count = _discovered_test_count() + expected_test_marker = f"Current repository test count at this snapshot: **{test_count} unittest cases**" + if expected_test_marker not in text: + errors.append(f"self-evolution.md must record the discovered test count ({test_count})") + + for command in VERIFICATION_COMMANDS: + if command not in text: + errors.append(f"self-evolution.md is missing verification command '{command}'") + for marker in BENCHMARK_METADATA: + if marker not in text: + errors.append(f"self-evolution.md is missing benchmark metadata '{marker}'") + for marker in ("API health/scoping smoke", "CLI command-loop smoke"): + if marker not in text: + errors.append(f"self-evolution.md is missing verification evidence '{marker}'") + return errors + + def main() -> int: expected_inventory = render_inventory() import main as agent_main @@ -92,6 +156,7 @@ def main() -> int: routes = runtime_routes(__import__("server")) route_paths = {path for _, path in routes} targets = _make_targets() + errors.extend(_check_verification_record()) for readme in README_FILES: text = readme.read_text(encoding="utf-8") for pattern in STALE_TOOL_PATTERNS: