diff --git a/.codex/config.toml b/.codex/config.toml index 86e5e199..02249f70 100644 --- a/.codex/config.toml +++ b/.codex/config.toml @@ -1,3 +1,15 @@ +# Dev Plane code-intelligence policy: +# - codebase-memory-mcp is the primary structural/impact-analysis backend. +# - GitNexus remains configured but disabled during the migration bake-off. +# - Use rg/Zoekt for exact text, ast-grep for structural rewrites, and LSP/SCIP +# for definition/reference/type truth before escalating to graph analysis. + +[mcp_servers.codebase-memory-mcp] +command = "codebase-memory-mcp" +args = [] +enabled = true + [mcp_servers.gitnexus] command = "npx" args = ["-y", "gitnexus@1.6.12", "mcp"] +enabled = false diff --git a/.mcp.json b/.mcp.json index afe364b8..b03271a9 100644 --- a/.mcp.json +++ b/.mcp.json @@ -1,8 +1,8 @@ { "mcpServers": { - "gitnexus": { - "command": "npx", - "args": ["-y", "gitnexus@1.6.12", "mcp"] + "codebase-memory-mcp": { + "command": "codebase-memory-mcp", + "args": [] } } } diff --git a/.woodpecker/code-intelligence-bakeoff.yaml b/.woodpecker/code-intelligence-bakeoff.yaml new file mode 100644 index 00000000..d06db5de --- /dev/null +++ b/.woodpecker/code-intelligence-bakeoff.yaml @@ -0,0 +1,55 @@ +labels: + platform: linux/amd64 + +when: + - event: manual + +steps: + - name: live-bakeoff + image: node:24-bookworm + environment: + CODEINTEL_GITHUB_TOKEN: + from_secret: codeintel_github_token + CI: "1" + GITNEXUS_NO_UPDATE_NOTIFIER: "1" + GITNEXUS_SKIP_OPTIONAL_GRAMMARS: "1" + CODEINTEL_GITNEXUS_MCP_CMD: "gitnexus mcp" + CODEINTEL_GITNEXUS_ANALYZE_CMD: "gitnexus analyze --index-only" + CODEINTEL_GITNEXUS_VERSION_CMD: "gitnexus --version" + CODEINTEL_CODEBASE_MEMORY_CMD: "codebase-memory-mcp" + CODEINTEL_CODEBASE_MEMORY_VERSION_CMD: "codebase-memory-mcp --version" + commands: + - apt-get update && apt-get install -y --no-install-recommends ca-certificates curl git python3 && rm -rf /var/lib/apt/lists/* + - | + curl -fsSLo /tmp/go.tgz "https://go.dev/dl/go1.26.8.linux-amd64.tar.gz" + rm -rf /usr/local/go + tar -C /usr/local -xzf /tmp/go.tgz + export PATH="/usr/local/go/bin:$${PATH}" + go version + - npm install --global codebase-memory-mcp@0.11.0 gitnexus@1.6.12 + - codebase-memory-mcp --version + - gitnexus --version + - | + export PATH="/usr/local/go/bin:$${PATH}" + export CBM_CACHE_DIR="$${PWD}/.codeintel-cbm" + export GITNEXUS_STORAGE_ROOT="$${PWD}/.codeintel-gitnexus" + export CODEINTEL_REPOS_ROOT="$${PWD}/.codeintel-repos" + mkdir -p "$${CBM_CACHE_DIR}" "$${GITNEXUS_STORAGE_ROOT}" "$${CODEINTEL_REPOS_ROOT}" + chmod 700 "$${CBM_CACHE_DIR}" "$${GITNEXUS_STORAGE_ROOT}" + + git config --global credential.helper store + printf 'https://x-access-token:%s@github.com\n' "$${CODEINTEL_GITHUB_TOKEN}" > "$${HOME}/.git-credentials" + chmod 600 "$${HOME}/.git-credentials" + + git clone --depth 1 https://github.com/dporkka/adacavo.git "$${CODEINTEL_REPOS_ROOT}/adacavo" + git clone --depth 1 https://github.com/nulang-org/nulang.git "$${CODEINTEL_REPOS_ROOT}/nulang" + git clone --depth 1 https://github.com/dporkka/nulang-cloud.git "$${CODEINTEL_REPOS_ROOT}/nulang-cloud" + git clone --depth 1 https://github.com/dporkka/dev-plane.git "$${CODEINTEL_REPOS_ROOT}/dev-plane" + + rm -f "$${HOME}/.git-credentials" + git config --global --unset credential.helper || true + unset CODEINTEL_GITHUB_TOKEN + + export CODEINTEL_RUN_LIVE=1 + bash scripts/verify-codeintel.sh + - cat data/codeintel-bakeoff/suite.json diff --git a/.woodpecker/code-intelligence.yaml b/.woodpecker/code-intelligence.yaml new file mode 100644 index 00000000..9272b386 --- /dev/null +++ b/.woodpecker/code-intelligence.yaml @@ -0,0 +1,53 @@ +labels: + platform: linux/amd64 + +when: + - event: pull_request + path: + include: + - packages/repo-intel/** + - scripts/codeintel_bakeoff.py + - scripts/test_codeintel_bakeoff.py + - scripts/test_codeintel_corpus.py + - scripts/codeintel_provenance.py + - scripts/test_codeintel_provenance.py + - scripts/verify-codeintel.sh + - benchmarks/code-intelligence/** + - docs/code-intelligence.md + - docs/code-intelligence-bakeoff-runbook.md + - tools/codeintel-benchmark/** + - docker-compose.code-intelligence-benchmark.yml + - .woodpecker/code-intelligence.yaml + - .woodpecker/code-intelligence-bakeoff.yaml + - event: push + branch: main + path: + include: + - packages/repo-intel/** + - scripts/codeintel_bakeoff.py + - scripts/test_codeintel_bakeoff.py + - scripts/test_codeintel_corpus.py + - scripts/codeintel_provenance.py + - scripts/test_codeintel_provenance.py + - scripts/verify-codeintel.sh + - benchmarks/code-intelligence/** + - docs/code-intelligence.md + - docs/code-intelligence-bakeoff-runbook.md + - tools/codeintel-benchmark/** + - docker-compose.code-intelligence-benchmark.yml + - .woodpecker/code-intelligence.yaml + - .woodpecker/code-intelligence-bakeoff.yaml + +steps: + - name: harness-tests + image: python:3.13-slim + commands: + - python3 scripts/test_codeintel_bakeoff.py + - python3 scripts/test_codeintel_corpus.py + - python3 scripts/test_codeintel_provenance.py + + - name: repo-intel-tests + image: golang:1.26.8 + commands: + - cd packages/repo-intel + - go test ./... diff --git a/AGENTS.md b/AGENTS.md index b42a859b..20ea0bfe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,44 +1,24 @@ - -# GitNexus — Code Intelligence +# Code Intelligence -This project is indexed by GitNexus as **dev-plane** (5801 symbols, 16255 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +Use `codebase-memory-mcp` as the primary structural code-intelligence backend for Dev Plane. On a fresh checkout, index the repository before relying on graph results. Prefer the cheapest precise tool for each question instead of routing every task through the graph. -> Index stale? Run `node .gitnexus/run.cjs analyze` from the project root — it auto-selects an available runner. No `.gitnexus/run.cjs` yet? `npx gitnexus analyze` (npm 11 crash → `npm i -g gitnexus`; #1939). +## Tool routing -## Always Do +- Exact identifier, string, filename, or regex lookup: use `rg` or Zoekt when available. +- Structural syntax pattern or repeatable codemod: use `ast-grep`. +- Definition, reference, implementation, rename, or type-resolution questions: prefer the language server / SCIP-quality semantic tooling. +- Architecture, dependency, call-path, cross-file impact, or blast-radius questions: use `codebase-memory-mcp` (`get_architecture`, `search_graph`, `trace_path`, `detect_changes`, and `query_graph` as appropriate). +- Conceptual search when names are unknown: use semantic graph search only after cheaper lexical/structural search is insufficient. +- Security/data-flow analysis requiring source-to-sink reasoning: use Joern when available; do not use it for routine navigation. +- Historical intent or ownership: use Git history. +- Correctness: verify with the repository's tests, type checks, linters, and CI; static code intelligence is not proof that a change works. -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. For regression review, compare against the default branch: `detect_changes({scope: "compare", base_ref: "main"})`. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use `query({search_query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`. -- For security review, `explain({target: "fileOrSymbol"})` lists taint findings (source→sink flows; needs `analyze --pdg`). +## Change safety -## Never Do +Before changing a high-fan-in symbol, public API, persistence contract, scheduler path, capability boundary, or execution protocol, inspect upstream/downstream impact with the graph and semantic tooling. Before committing a non-trivial change, run `detect_changes` against the working tree or `main` and confirm the affected scope matches the intended change. -- NEVER edit a function, class, or method without first running `impact` on it. -- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace — use `rename` which understands the call graph. -- NEVER commit changes without running `detect_changes()` to check affected scope. +If graph results conflict with compiler/LSP output or executable tests, treat compiler/LSP and runtime evidence as authoritative and refresh/re-index the graph. -## Resources +## Migration note -| Resource | Use for | -|----------|---------| -| `gitnexus://repo/dev-plane/context` | Codebase overview, check index freshness | -| `gitnexus://repo/dev-plane/clusters` | All functional areas | -| `gitnexus://repo/dev-plane/processes` | All execution flows | -| `gitnexus://repo/dev-plane/process/{name}` | Step-by-step execution trace | - -## CLI - -| Task | Read this skill file | -|------|---------------------| -| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | -| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | -| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | -| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | -| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | -| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | - - +GitNexus remains a temporary fallback during the bake-off but is not the default code-intelligence path. Do not add new GitNexus-specific workflow dependencies unless a measured gap in codebase-memory requires it. diff --git a/CLAUDE.md b/CLAUDE.md index b42a859b..20ea0bfe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,44 +1,24 @@ - -# GitNexus — Code Intelligence +# Code Intelligence -This project is indexed by GitNexus as **dev-plane** (5801 symbols, 16255 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +Use `codebase-memory-mcp` as the primary structural code-intelligence backend for Dev Plane. On a fresh checkout, index the repository before relying on graph results. Prefer the cheapest precise tool for each question instead of routing every task through the graph. -> Index stale? Run `node .gitnexus/run.cjs analyze` from the project root — it auto-selects an available runner. No `.gitnexus/run.cjs` yet? `npx gitnexus analyze` (npm 11 crash → `npm i -g gitnexus`; #1939). +## Tool routing -## Always Do +- Exact identifier, string, filename, or regex lookup: use `rg` or Zoekt when available. +- Structural syntax pattern or repeatable codemod: use `ast-grep`. +- Definition, reference, implementation, rename, or type-resolution questions: prefer the language server / SCIP-quality semantic tooling. +- Architecture, dependency, call-path, cross-file impact, or blast-radius questions: use `codebase-memory-mcp` (`get_architecture`, `search_graph`, `trace_path`, `detect_changes`, and `query_graph` as appropriate). +- Conceptual search when names are unknown: use semantic graph search only after cheaper lexical/structural search is insufficient. +- Security/data-flow analysis requiring source-to-sink reasoning: use Joern when available; do not use it for routine navigation. +- Historical intent or ownership: use Git history. +- Correctness: verify with the repository's tests, type checks, linters, and CI; static code intelligence is not proof that a change works. -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. For regression review, compare against the default branch: `detect_changes({scope: "compare", base_ref: "main"})`. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use `query({search_query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`. -- For security review, `explain({target: "fileOrSymbol"})` lists taint findings (source→sink flows; needs `analyze --pdg`). +## Change safety -## Never Do +Before changing a high-fan-in symbol, public API, persistence contract, scheduler path, capability boundary, or execution protocol, inspect upstream/downstream impact with the graph and semantic tooling. Before committing a non-trivial change, run `detect_changes` against the working tree or `main` and confirm the affected scope matches the intended change. -- NEVER edit a function, class, or method without first running `impact` on it. -- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace — use `rename` which understands the call graph. -- NEVER commit changes without running `detect_changes()` to check affected scope. +If graph results conflict with compiler/LSP output or executable tests, treat compiler/LSP and runtime evidence as authoritative and refresh/re-index the graph. -## Resources +## Migration note -| Resource | Use for | -|----------|---------| -| `gitnexus://repo/dev-plane/context` | Codebase overview, check index freshness | -| `gitnexus://repo/dev-plane/clusters` | All functional areas | -| `gitnexus://repo/dev-plane/processes` | All execution flows | -| `gitnexus://repo/dev-plane/process/{name}` | Step-by-step execution trace | - -## CLI - -| Task | Read this skill file | -|------|---------------------| -| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | -| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | -| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | -| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | -| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | -| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | - - +GitNexus remains a temporary fallback during the bake-off but is not the default code-intelligence path. Do not add new GitNexus-specific workflow dependencies unless a measured gap in codebase-memory requires it. diff --git a/benchmarks/code-intelligence/scenarios.json b/benchmarks/code-intelligence/scenarios.json new file mode 100644 index 00000000..6bcbb53c --- /dev/null +++ b/benchmarks/code-intelligence/scenarios.json @@ -0,0 +1,146 @@ +{ + "version": 9, + "scenarios": [ + { + "id": "adacavo-proposal-flow", + "repository": "dporkka/adacavo", + "intent": "impact", + "prompt": "Trace ProposalService through proposal entry points and the durable proposal-booking workflow. Return the production files that must be reviewed before changing proposal status-transition semantics.", + "expected": [ + "apps/api/services/proposal-service.ts", + "apps/api/services/proposal-service-legacy.ts", + "apps/api/proposals/actions.ts", + "apps/api/proposals/booking.ts", + "apps/api/proposals.ts", + "temporal/workflows/proposal-booking-workflow.ts", + "temporal/activities/proposal-booking-activities.ts" + ], + "backends": { + "codebase-memory": [ + { + "tool": "search_graph", + "arguments": { + "project": "adacavo", + "query": "ProposalService transitionLifecycle proposal status transition booking workflow durable Temporal", + "limit": 30 + } + } + ], + "gitnexus": [ + { + "tool": "query", + "arguments": { + "search_query": "ProposalService transitionLifecycle proposal status transition booking workflow durable Temporal", + "maxTokens": 8000 + } + } + ] + } + }, + { + "id": "nulang-mir-codegen-callers", + "repository": "nulang-org/nulang", + "intent": "impact", + "prompt": "Find the core production entry points that call mir_codegen::compile_mir and should be reviewed before changing MIR-to-bytecode code generation semantics.", + "expected": [ + "src/mir_codegen.rs", + "src/main.rs", + "src/repl.rs", + "src/dap/mod.rs", + "src/aot/mod.rs", + "src/ffi/c_api.rs" + ], + "backends": { + "codebase-memory": [ + { + "tool": "trace_path", + "arguments": { + "project": "nulang", + "function_name": "compile_mir", + "direction": "inbound" + } + } + ], + "gitnexus": [ + { + "tool": "impact", + "arguments": { + "target": "compile_mir", + "direction": "upstream", + "summaryOnly": false, + "maxTokens": 8000 + } + } + ] + } + }, + { + "id": "nulang-cloud-runtime-boundary", + "repository": "dporkka/nulang-cloud", + "intent": "understand", + "prompt": "Locate the core executable runtime boundary between local Wasmtime execution and Firecracker-backed host-agent execution. Return the production implementation files that define those boundaries.", + "expected": [ + "nlc-dev-server/src/runtime.rs", + "host-agent/src/main.rs", + "host-agent/src/vm_factory.rs" + ], + "backends": { + "codebase-memory": [ + { + "tool": "search_graph", + "arguments": { + "project": "nulang-cloud", + "query": "local Wasmtime execution Firecracker host-agent VmFactory runtime boundary", + "limit": 30 + } + } + ], + "gitnexus": [ + { + "tool": "query", + "arguments": { + "search_query": "local Wasmtime execution Firecracker host-agent VmFactory runtime boundary", + "maxTokens": 8000 + } + } + ] + } + }, + { + "id": "dev-plane-run-admission", + "repository": "dporkka/dev-plane", + "intent": "impact", + "prompt": "Trace run admission from scheduler claim through task readiness, budget checks, agent-runner delegation, and worker wiring. Return the production files that define or consume admission semantics.", + "expected": [ + "apps/worker/internal/handlers/scheduler_admission.go", + "apps/worker/internal/handlers/run_handlers.go", + "apps/worker/cmd/worker/main.go", + "apps/api/pkg/agentexecutor/executor.go", + "apps/api/internal/agentrunner/runner_budget.go", + "apps/api/internal/budget/engine.go", + "packages/readiness/task.go" + ], + "backends": { + "codebase-memory": [ + { + "tool": "search_graph", + "arguments": { + "project": "dev-plane", + "query": "run admission SchedulerAdmission task-readiness-v1 CheckRunStart agentexecutor runner budget worker wiring", + "limit": 30 + } + } + ], + "gitnexus": [ + { + "tool": "query", + "arguments": { + "search_query": "run admission SchedulerAdmission task-readiness-v1 CheckRunStart agentexecutor runner budget worker wiring", + "maxTokens": 8000 + } + } + ] + } + } + ] +} diff --git a/docker-compose.code-intelligence-benchmark.yml b/docker-compose.code-intelligence-benchmark.yml new file mode 100644 index 00000000..c773ec5d --- /dev/null +++ b/docker-compose.code-intelligence-benchmark.yml @@ -0,0 +1,33 @@ +services: + benchmark: + build: + context: . + dockerfile: tools/codeintel-benchmark/Dockerfile + args: + CODEBASE_MEMORY_VERSION: "0.11.0" + GITNEXUS_VERSION: "1.6.12" + working_dir: /workspace/dev-plane + environment: + CI: "1" + CODEINTEL_RUN_LIVE: "1" + CODEINTEL_REPOS_ROOT: /repos + HOME: /var/lib/codeintel + CBM_CACHE_DIR: /var/lib/codeintel/cbm + GITNEXUS_STORAGE_ROOT: /var/lib/codeintel/gitnexus + GITNEXUS_NO_UPDATE_NOTIFIER: "1" + GITNEXUS_SKIP_OPTIONAL_GRAMMARS: "1" + CODEINTEL_CODEBASE_MEMORY_CMD: "codebase-memory-mcp" + CODEINTEL_CODEBASE_MEMORY_VERSION_CMD: "codebase-memory-mcp --version" + CODEINTEL_GITNEXUS_MCP_CMD: "gitnexus mcp" + CODEINTEL_GITNEXUS_ANALYZE_CMD: "gitnexus analyze --index-only" + CODEINTEL_GITNEXUS_VERSION_CMD: "gitnexus --version" + volumes: + - .:/workspace/dev-plane:ro + - ${CODEINTEL_REPOS_ROOT:?set CODEINTEL_REPOS_ROOT to the directory containing adacavo, nulang, nulang-cloud, and dev-plane clones}:/repos:ro + - ./data/codeintel-bakeoff:/workspace/dev-plane/data/codeintel-bakeoff + - codeintel-benchmark-cache:/var/lib/codeintel + command: ["bash", "scripts/verify-codeintel.sh"] + +volumes: + codeintel-benchmark-cache: + driver: local diff --git a/docker-compose.code-intelligence.yml b/docker-compose.code-intelligence.yml new file mode 100644 index 00000000..9d9766c0 --- /dev/null +++ b/docker-compose.code-intelligence.yml @@ -0,0 +1,22 @@ +services: + zoekt: + image: ghcr.io/sourcegraph/zoekt:sha-b0de0bb + command: ["zoekt-webserver", "-index", "/data/index", "-listen", ":6070", "-rpc"] + ports: + - "6070:6070" + volumes: + - zoekt-index:/data/index + restart: unless-stopped + + zoekt-sync: + image: ghcr.io/sourcegraph/zoekt:sha-b0de0bb + profiles: ["index"] + entrypoint: ["/bin/sh", "-lc"] + command: ["zoekt-local-sync -index /data/index -f /repos"] + volumes: + - ${CODE_REPOS_ROOT:?set CODE_REPOS_ROOT to a directory containing local git repositories}:/repos:ro + - zoekt-index:/data/index + +volumes: + zoekt-index: + driver: local diff --git a/docs/code-intelligence-bakeoff-runbook.md b/docs/code-intelligence-bakeoff-runbook.md new file mode 100644 index 00000000..b018590c --- /dev/null +++ b/docs/code-intelligence-bakeoff-runbook.md @@ -0,0 +1,189 @@ +# Code-intelligence bake-off runbook + +This runbook is the operational path for deciding whether `codebase-memory-mcp` can replace GitNexus in Dev Plane and, later, Adacavo. + +## Decision rule + +Do not remove GitNexus because a replacement is faster, cheaper, or easier to operate. The checked-in corpus is a curated set of **required core production paths**, not an exhaustive list of every file that may be relevant to a change. The safety question is therefore whether a backend finds every required path, not whether it returns the narrowest answer. + +The gate is implemented by `packages/repo-intel/cmd/codeintel-promote`: + +- every candidate run must complete successfully; +- candidate recall must be `1.00` on every scenario — every curated core path must be found; +- candidate recall must not regress against a successful GitNexus result on any scenario; +- precision and F1 are diagnostic/noise signals only because additional genuinely useful files are not necessarily present in the non-exhaustive gold set; +- latency and output-token cost are considered only after required-core recall is complete. + +`promote: false` is a failed migration gate. + +## Pinned benchmark toolchain + +The reproducible worker pins: + +- Node 24 (`node:24-bookworm` base image); +- Go 1.26.8; +- Python 3 from Debian Bookworm; +- `codebase-memory-mcp@0.11.0`; +- `gitnexus@1.6.12`. + +Broad codebase-memory discovery scenarios use deterministic BM25 `search_graph(query=...)`; symbol/caller scenarios use graph traversal such as `trace_path`. Every codebase-memory query is explicitly scoped to the indexed project so one repository cannot leak results into another repository's score. + +Benchmark output is enriched with: + +- the actual backend/runtime versions; +- the exact Git SHA for every repository used in the run; +- the benchmark corpus version and SHA-256 of its exact bytes; +- the exact prompt, intent, expected core paths, and backend MCP calls for each scenario. + +This means a stored `suite.json` is sufficient to identify what source and what benchmark definition produced the verdict. + +## Local/container run + +Keep canonical clones in one directory using these checkout names: + +```text +/adacavo +/nulang +/nulang-cloud +/dev-plane +``` + +Then run: + +```bash +export CODEINTEL_REPOS_ROOT="$HOME/src" +docker compose -f docker-compose.code-intelligence-benchmark.yml build benchmark +docker compose -f docker-compose.code-intelligence-benchmark.yml run --rm benchmark +``` + +The source repositories are mounted read-only. The benchmark image invokes the globally installed pinned binaries directly rather than resolving them again through `npx`. Mutable backend state is isolated under: + +```text +/var/lib/codeintel/cbm +/var/lib/codeintel/gitnexus +``` + +and persisted in the `codeintel-benchmark-cache` volume. Results are written to: + +```text +data/codeintel-bakeoff/ +``` + +Important outputs: + +- `suite.json` — run metadata, exact corpus provenance, repository revisions, backend observations, and tool versions; +- `.json` — scorer input for an individual scenario; +- `raw//.txt` — raw backend output used to extract repository-tracked file matches. + +To discard persistent backend indexes and force a completely cold run: + +```bash +docker compose -f docker-compose.code-intelligence-benchmark.yml down -v +``` + +## Benchmark corpus + +`benchmarks/code-intelligence/scenarios.json` currently covers four production-oriented paths: + +- Adacavo proposal lifecycle and durable proposal-booking flow, including the legacy service implementation and transactional booking route; +- Nulang MIR-to-bytecode code generation and its core production callers in CLI, REPL, DAP, AOT, and the C embedding API; +- Nulang Cloud's Dev Plane runtime-provider boundary, local Wasmtime runtime, and Firecracker host-agent/VM factory; +- Dev Plane run admission through scheduler admission, task-readiness policy, agent-executor/runner budget delegation, budget engine, and worker wiring. + +`python3 scripts/test_codeintel_corpus.py` protects the corpus contract: unique scenario IDs and core paths, exactly the two benchmark backends, explicit codebase-memory project scoping, and the pinned MCP parameter shapes. + +Do not change an expected path merely because a backend fails to retrieve it. First verify the current production architecture and change the corpus only when source evidence shows that the ground truth changed. + +## Woodpecker setup + +Dev Plane must first be activated in Woodpecker so the forge webhook exists. + +Create a repository secret named: + +```text +codeintel_github_token +``` + +Use a fine-grained GitHub token with the minimum permissions required to clone the private benchmark repositories. It needs read access to repository contents for: + +- `dporkka/adacavo`; +- `dporkka/nulang-cloud`. + +`nulang-org/nulang` and `dporkka/dev-plane` are public and do not require private-repository access. + +The live workflow is manual-only: + +```text +.woodpecker/code-intelligence-bakeoff.yaml +``` + +It is pinned to the existing `linux/amd64` Woodpecker agent pool. The workflow installs the pinned toolchain, shallow-clones the four canonical `main` branches, removes the temporary Git credential and unsets the token before indexing/querying, uses isolated codebase-memory/GitNexus cache roots, runs the live bake-off, captures source/tool/corpus provenance, applies the promotion gate, and prints `suite.json` to the pipeline log. + +The normal PR verification workflow is separate: + +```text +.woodpecker/code-intelligence.yaml +``` + +That workflow never needs the private repository token and runs: + +- the Python harness tests; +- the corpus-contract tests; +- the provenance tests; +- the `packages/repo-intel` Go tests under Go 1.26.8. + +## Running the manual Woodpecker bake-off + +After repository activation and secret creation: + +1. Trigger the `code-intelligence-bakeoff` workflow manually in Woodpecker. +2. Confirm all four repositories cloned successfully. +3. Confirm the log reports the pinned versions before benchmarking. +4. Inspect the per-scenario backend lines for indexing/query errors. +5. Read the final promotion verdict. +6. Preserve the final `suite.json` from the log or configured log storage with the Woodpecker run identifier. + +A failed backend, missing repository, broken/stale index, required-core-path miss, provenance failure, or `promote: false` means **do not remove GitNexus**. + +## Migration sequence after a passing result + +A single passing run is necessary but not sufficient for Adacavo because GitNexus currently participates in safety/freshness workflows there. + +Recommended sequence: + +1. Obtain a clean `promote: true` run on the pinned worker. +2. Repeat the run once from a cold benchmark cache to rule out accidental index-state dependence. +3. Remove the disabled GitNexus fallback from Dev Plane only. +4. Run Dev Plane repo-intel tests and its normal verification profile. +5. Use codebase-memory as the only graph backend in Dev Plane for normal agent work and observe freshness/reliability. +6. Run the benchmark again after that bake-in period. +7. Only then migrate Adacavo's GitNexus-specific safety/freshness scripts and instructions. +8. Keep the GitNexus removal in Adacavo as a separate PR so rollback is trivial. + +Nulang and Nulang Cloud do not need a GitNexus-removal step from this migration branch; their new project configuration already points at codebase-memory. + +## Failure triage + +### Candidate misses a required core file + +Do not lower the benchmark expectation merely to make the candidate pass. First determine whether the expected file is genuinely part of the current production impact surface. If yes, treat the miss as a backend/query-quality issue. Improve the backend-specific query only if the new query describes the intent rather than directly enumerating the answer. + +### Candidate returns many additional files + +Do not treat additional files as an automatic failure. The corpus is intentionally non-exhaustive. Review precision/F1 and the raw output as noise indicators, but the migration safety gate is required-core recall. + +### Both backends miss a required core file + +Validate the corpus against source/semantic truth. If the expectation is correct, both backends are incomplete for that scenario. If the expectation is obsolete, update the corpus in a reviewed change with source evidence. + +### Indexing succeeds but queries return another project + +Confirm the project selector used by codebase-memory and inspect the raw response. Every codebase-memory scenario is explicitly project-scoped; cross-project output is a backend/query failure, not a passing result. + +### Woodpecker cannot clone private repositories + +Verify that `codeintel_github_token` exists as a Dev Plane repository secret, is permitted for manual events, and has read access to both private repositories. Do not broaden it to write/admin permissions. + +### GitHub Actions shows zero-step failures + +Do not use that status as source-validation evidence. The code-intelligence checks are deliberately portable to the local/container and Woodpecker paths above. diff --git a/docs/code-intelligence.md b/docs/code-intelligence.md new file mode 100644 index 00000000..5e297232 --- /dev/null +++ b/docs/code-intelligence.md @@ -0,0 +1,195 @@ +# Code Intelligence Stack + +Dev Plane uses a layered code-intelligence model so agents use the cheapest precise signal first and reserve graph analysis for questions that need it. + +## Ownership + +Shared services: + +- `codebase-memory-mcp`: persistent structural graph, architecture, call paths, impact analysis, cross-repository relationships, and conceptual fallback. +- Zoekt: fast lexical and regex search over the canonical local repository set. + +Per-agent/worktree tools: + +- Native language server (`rust-analyzer`, `gopls`, TypeScript language service, Pyright, etc.) for branch-local semantic truth. +- `ast-grep` for structural search and repeatable codemods. +- `rg`/`fd` for cheap local lookup. +- compiler, tests, linters, and CI for executable verification. + +On-demand only: + +- Joern for deep source-to-sink/data-flow/security analysis. +- SCIP should not be operated as a separate service until a measured cross-repository semantic-indexing gap justifies the extra index lifecycle. + +## Vendor-neutral routing contract + +`packages/repo-intel` owns the stable routing vocabulary. Product code and agents should request an intent instead of depending directly on a vendor's MCP tool names. + +Supported intents are: + +- `locate` — exact or conceptual discovery; +- `understand` — symbol or architecture understanding; +- `impact` — change blast radius; +- `refactor` — structural transformations; +- `security` — deep data-flow/source-to-sink analysis; +- `history` — Git history and ownership; +- `verify` — executable correctness checks. + +The deterministic router maps those requests to the current preferred backend. Exact worktree lookup routes to ripgrep, exact fleet lookup to Zoekt, symbol semantics to LSP, architecture/impact to codebase-memory, structural refactors to ast-grep, deep data flow to Joern, and verification to tests. Mutating/high-risk plans include executable verification rather than treating static analysis as proof. + +This indirection is intentional: codebase-memory, GitNexus, or another graph engine can be replaced without changing the operation contract used by the rest of Dev Plane. + +## Install codebase-memory-mcp + +Install the binary and ensure `codebase-memory-mcp` is on `PATH`. Repository MCP configuration expects that command name directly. + +On the first session for a repository, index it before depending on graph answers. Keep the graph fresh after meaningful branch changes and re-index if graph output disagrees with compiler/LSP evidence. + +For large multi-agent installations, prefer explicit canonical-repository indexing and keep automatic worktree watchers disabled unless every ephemeral worktree should be registered. The bake-off harness indexes canonical repositories explicitly. + +## Run shared Zoekt + +Zoekt indexes existing local clones; it does not need GitHub credentials in this setup. + +```bash +export CODE_REPOS_ROOT="$HOME/src" +docker compose -f docker-compose.code-intelligence.yml --profile index run --rm zoekt-sync +docker compose -f docker-compose.code-intelligence.yml up -d zoekt +``` + +The web/API endpoint is then available on `http://localhost:6070`. + +Re-run `zoekt-sync` after adding/removing repositories or when canonical clones materially change. Agents working in isolated worktrees should use branch-local `rg`, LSP, and `ast-grep` for uncommitted delta context rather than expecting the shared Zoekt index to represent every agent branch. + +## Routing policy + +1. Exact string, identifier, filename, regex: `rg` locally; Zoekt for fleet-wide search. +2. Structural syntax pattern or mass rewrite: `ast-grep`. +3. Definition/reference/implementation/type resolution: language server. +4. Architecture, dependency, call path, blast radius: `codebase-memory-mcp`. +5. Conceptual discovery when names are unknown: codebase-memory search after cheaper lexical/structural search is insufficient. +6. Security source-to-sink/data-flow question: Joern when installed. +7. Historical reason or ownership: Git. +8. Correctness: tests/typecheck/lint/runtime/CI. + +## Multi-agent model + +Shared indexes track canonical repository clones, normally `main`. Each coding agent works in an isolated Git worktree with its own language-server state and local structural tools. Treat an agent's effective context as: + +```text +canonical shared indexes + branch/worktree delta + executable verification +``` + +Do not rebuild fleet-wide indexes for every agent worktree. + +## Backend bake-off + +The benchmark corpus is checked in at `benchmarks/code-intelligence/scenarios.json`. It uses representative production paths from Adacavo, Nulang, Nulang Cloud, and Dev Plane rather than toy symbol lookups. The `expected` arrays are **curated required core paths**, not exhaustive sets of every relevant file. + +The current scenarios cover: + +- Adacavo proposal lifecycle + durable proposal booking; +- Nulang MIR-to-bytecode code generation + core production callers; +- Nulang Cloud's Dev Plane/local-Wasmtime/Firecracker runtime boundaries; +- Dev Plane scheduler/readiness/budget/executor admission semantics. + +Each scenario contains backend-specific MCP calls. Broad codebase-memory discovery uses deterministic BM25 `search_graph(query=...)`, graph caller analysis uses `trace_path`, and every codebase-memory call is explicitly scoped to its indexed project. + +The harness: + +- optionally refreshes each backend index before measurement; +- uses GitNexus `analyze --index-only` so benchmarking does not rewrite agent instruction/skill files; +- calls both backends through MCP for the query phase; +- records repository-tracked files actually mentioned by each backend response; +- excludes MCP startup and indexing time from query latency while recording `index_ms` separately; +- saves raw backend responses under ignored `data/` for auditability; +- estimates output tokens from response length as a relative cost signal; +- binds final results to exact repository SHAs, backend/tool versions, corpus version, corpus SHA-256, prompts, intents, expected paths, and backend calls. + +The corpus contract itself is tested by `scripts/test_codeintel_corpus.py` so stale or unscoped benchmark definitions fail before a live run. + +Run the non-live verification lane with: + +```bash +bash scripts/verify-codeintel.sh +``` + +That executes the harness, corpus, and provenance tests plus `packages/repo-intel` Go tests without requiring either graph backend. + +To execute the live bake-off using local checkouts: + +```bash +export CODEINTEL_RUN_LIVE=1 +export CODEINTEL_REPOS_ROOT="$HOME/src" +bash scripts/verify-codeintel.sh +``` + +For a pinned containerized run: + +```bash +export CODEINTEL_REPOS_ROOT="$HOME/src" +docker compose -f docker-compose.code-intelligence-benchmark.yml build benchmark +docker compose -f docker-compose.code-intelligence-benchmark.yml run --rm benchmark +``` + +The pinned worker installs `codebase-memory-mcp@0.11.0` and `gitnexus@1.6.12`, invokes the installed binaries directly, and isolates persistent state under `/var/lib/codeintel/cbm` and `/var/lib/codeintel/gitnexus`. + +Each scenario produces a scorer input in `data/codeintel-bakeoff/.json`. Score one with: + +```bash +cd packages/repo-intel +go run ./cmd/codeintel-bakeoff ../../data/codeintel-bakeoff/nulang-mir-codegen-callers.json +``` + +The suite-level promotion verdict is: + +```bash +cd packages/repo-intel +go run ./cmd/codeintel-promote ../../data/codeintel-bakeoff/suite.json +``` + +### Promotion semantics + +Required-core recall is the safety metric: + +- the candidate must succeed on every scenario; +- the candidate must achieve `1.00` recall on every scenario — every curated core path must be found; +- the candidate must have no per-scenario recall regression versus a successful GitNexus result; +- precision/F1 are retained as diagnostic noise/relevance signals but do not block promotion because the expected set is intentionally non-exhaustive; +- latency and output-token cost are considered only after required-core recall is complete. + +`codeintel-bakeoff` therefore ranks successful runs by recall first, then F1, latency, and output tokens. `codeintel-promote` returns a non-zero exit code when promotion is unsafe. + +## Woodpecker verification + +`.woodpecker/code-intelligence.yaml` provides the normal provider-independent verification lane. It is pinned to the `linux/amd64` agent pool and runs on relevant pull requests and pushes to `main`: + +- Python 3.13 harness tests; +- corpus-contract tests; +- provenance tests; +- `packages/repo-intel` Go tests using Go 1.26.8. + +The live cross-repository comparison is separate and manual-only in `.woodpecker/code-intelligence-bakeoff.yaml`. It requires the repository to be activated in Woodpecker and a read-only `codeintel_github_token` repository secret for the private Adacavo and Nulang Cloud clones. The token is removed/unset before indexing and backend queries. + +See `docs/code-intelligence-bakeoff-runbook.md` for the operational setup and migration sequence. + +## CI admission + +The code-intelligence lane is intentionally runnable outside GitHub Actions. A hosted Actions run that creates jobs but executes zero steps is an admission/infrastructure failure and must not be treated as evidence that source validation failed. Run the portable or Woodpecker lanes until hosted Actions admission is healthy again. + +Do not weaken or remove repository verification because a hosted CI provider is unavailable; move the same checks to a functioning runner. + +## Migration from GitNexus + +GitNexus remains a temporary fallback where existing repository policy depends on it. Do not delete GitNexus-specific scripts or safety gates until the pinned live corpus returns `promote: true` and a cold-cache repeat confirms the result. + +Recommended cutover order: + +1. pass the live pinned bake-off; +2. repeat from a cold backend cache; +3. remove the disabled GitNexus fallback from Dev Plane only; +4. bake in codebase-memory as Dev Plane's only graph backend and observe freshness/reliability; +5. rerun the corpus; +6. migrate Adacavo's GitNexus-specific safety/freshness integration in a separate rollback-friendly PR. + +Nulang and Nulang Cloud already point at codebase-memory and do not require a GitNexus-removal step from this migration branch. diff --git a/packages/repo-intel/bakeoff.go b/packages/repo-intel/bakeoff.go new file mode 100644 index 00000000..de0dfc4e --- /dev/null +++ b/packages/repo-intel/bakeoff.go @@ -0,0 +1,116 @@ +package repointel + +import ( + "sort" + "strings" + "time" +) + +// BackendRun is one observed backend result for a benchmark scenario. +type BackendRun struct { + Backend Backend + Results []string + Latency time.Duration + OutputTokens int + Error string +} + +// BenchmarkScore captures required-result coverage plus precision/cost signals. +// The benchmark corpus is a set of core required paths rather than an exhaustive +// list of every relevant file, so recall is the primary safety signal. +type BenchmarkScore struct { + Backend Backend + Precision float64 + Recall float64 + F1 float64 + Latency time.Duration + OutputTokens int + Failed bool + Error string +} + +// ScoreRun compares a backend's normalized unique result keys to the expected +// core result keys for a scenario. +func ScoreRun(expected []string, run BackendRun) BenchmarkScore { + expectedSet := normalizedSet(expected) + actualSet := normalizedSet(run.Results) + + matches := 0 + for item := range actualSet { + if _, ok := expectedSet[item]; ok { + matches++ + } + } + + precision := ratio(matches, len(actualSet)) + recall := ratio(matches, len(expectedSet)) + f1 := 0.0 + if precision+recall > 0 { + f1 = 2 * precision * recall / (precision + recall) + } + if len(expectedSet) == 0 && len(actualSet) == 0 { + precision, recall, f1 = 1, 1, 1 + } + + return BenchmarkScore{ + Backend: run.Backend, + Precision: precision, + Recall: recall, + F1: f1, + Latency: run.Latency, + OutputTokens: run.OutputTokens, + Failed: run.Error != "", + Error: run.Error, + } +} + +// RankRuns ranks successful runs by required-result recall first. F1 is a +// secondary relevance/noise signal, followed by latency and output-token cost. +// Failed runs always sort after successful runs. +func RankRuns(expected []string, runs []BackendRun) []BenchmarkScore { + scores := make([]BenchmarkScore, 0, len(runs)) + for _, run := range runs { + scores = append(scores, ScoreRun(expected, run)) + } + + sort.SliceStable(scores, func(i, j int) bool { + a, b := scores[i], scores[j] + if a.Failed != b.Failed { + return !a.Failed + } + if a.Recall != b.Recall { + return a.Recall > b.Recall + } + if a.F1 != b.F1 { + return a.F1 > b.F1 + } + if a.Latency != b.Latency { + return a.Latency < b.Latency + } + if a.OutputTokens != b.OutputTokens { + return a.OutputTokens < b.OutputTokens + } + return a.Backend < b.Backend + }) + + return scores +} + +func normalizedSet(items []string) map[string]struct{} { + set := make(map[string]struct{}, len(items)) + for _, item := range items { + item = strings.ToLower(strings.TrimSpace(item)) + if item == "" { + continue + } + set[item] = struct{}{} + } + return set +} + +func ratio(numerator, denominator int) float64 { + if denominator == 0 { + return 0 + } + return float64(numerator) / float64(denominator) +} diff --git a/packages/repo-intel/bakeoff_test.go b/packages/repo-intel/bakeoff_test.go new file mode 100644 index 00000000..98b488a4 --- /dev/null +++ b/packages/repo-intel/bakeoff_test.go @@ -0,0 +1,103 @@ +package repointel + +import ( + "testing" + "time" +) + +func TestScoreRunComputesRetrievalQuality(t *testing.T) { + score := ScoreRun([]string{"a", "b", "c"}, BackendRun{ + Backend: BackendCodebaseMemory, + Results: []string{"a", "b", "x"}, + Latency: 120 * time.Millisecond, + OutputTokens: 900, + }) + + assertFloatNear(t, score.Precision, 2.0/3.0) + assertFloatNear(t, score.Recall, 2.0/3.0) + assertFloatNear(t, score.F1, 2.0/3.0) + if score.Latency != 120*time.Millisecond { + t.Fatalf("latency = %s", score.Latency) + } + if score.OutputTokens != 900 { + t.Fatalf("tokens = %d", score.OutputTokens) + } +} + +func TestScoreRunNormalizesAndDeduplicatesResults(t *testing.T) { + score := ScoreRun([]string{" Src/Foo.go ", "Bar"}, BackendRun{ + Backend: BackendGitNexus, + Results: []string{"src/foo.go", "SRC/FOO.GO", "bar"}, + }) + + assertFloatNear(t, score.Precision, 1) + assertFloatNear(t, score.Recall, 1) + assertFloatNear(t, score.F1, 1) +} + +func TestRankRunsPrefersRequiredRecallBeforePrecision(t *testing.T) { + expected := []string{"a", "b", "c", "d"} + ranked := RankRuns(expected, []BackendRun{ + { + Backend: BackendGitNexus, + Results: []string{"a", "b", "c"}, + Latency: 10 * time.Millisecond, + }, + { + Backend: BackendCodebaseMemory, + Results: []string{"a", "b", "c", "d", "extra-1", "extra-2", "extra-3", "extra-4"}, + Latency: 200 * time.Millisecond, + }, + }) + + if ranked[0].Backend != BackendCodebaseMemory { + t.Fatalf("winner = %q, want backend with complete required-file recall", ranked[0].Backend) + } + if ranked[0].Recall != 1 { + t.Fatalf("winner recall = %f, want 1", ranked[0].Recall) + } + if ranked[0].F1 >= ranked[1].F1 { + t.Fatalf("test setup requires complete-recall backend to have lower F1: %#v", ranked) + } +} + +func TestRankRunsUsesF1ThenLatencyThenTokensAfterRecallTie(t *testing.T) { + expected := []string{"a", "b"} + ranked := RankRuns(expected, []BackendRun{ + {Backend: BackendGitNexus, Results: []string{"a", "b"}, Latency: 250 * time.Millisecond, OutputTokens: 500}, + {Backend: BackendCodebaseMemory, Results: []string{"a", "b"}, Latency: 150 * time.Millisecond, OutputTokens: 900}, + {Backend: BackendZoekt, Results: []string{"a"}, Latency: 10 * time.Millisecond, OutputTokens: 100}, + }) + + if len(ranked) != 3 { + t.Fatalf("ranked len = %d", len(ranked)) + } + if ranked[0].Backend != BackendCodebaseMemory { + t.Fatalf("winner = %q, want %q", ranked[0].Backend, BackendCodebaseMemory) + } + if ranked[2].Backend != BackendZoekt { + t.Fatalf("last = %q, want lower-recall backend %q", ranked[2].Backend, BackendZoekt) + } +} + +func TestRankRunsPlacesErrorsLast(t *testing.T) { + ranked := RankRuns([]string{"a"}, []BackendRun{ + {Backend: BackendCodebaseMemory, Results: []string{"a"}, Latency: time.Second, OutputTokens: 1000}, + {Backend: BackendGitNexus, Results: []string{"a"}, Latency: time.Millisecond, OutputTokens: 1, Error: "stale index"}, + }) + + if ranked[0].Backend != BackendCodebaseMemory { + t.Fatalf("winner = %q, expected successful backend", ranked[0].Backend) + } + if !ranked[1].Failed { + t.Fatalf("error run should be marked failed: %#v", ranked[1]) + } +} + +func assertFloatNear(t *testing.T, got, want float64) { + t.Helper() + const epsilon = 1e-9 + if got < want-epsilon || got > want+epsilon { + t.Fatalf("got %.12f, want %.12f", got, want) + } +} diff --git a/packages/repo-intel/cmd/codeintel-bakeoff/main.go b/packages/repo-intel/cmd/codeintel-bakeoff/main.go new file mode 100644 index 00000000..b2ab81c7 --- /dev/null +++ b/packages/repo-intel/cmd/codeintel-bakeoff/main.go @@ -0,0 +1,109 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "time" + + repointel "github.com/ai-dev-control-plane/repo-intel" +) + +type inputRun struct { + Backend repointel.Backend `json:"backend"` + Results []string `json:"results"` + LatencyMS int64 `json:"latency_ms"` + OutputTokens int `json:"output_tokens"` + Error string `json:"error,omitempty"` +} + +type inputDocument struct { + Scenario string `json:"scenario"` + Expected []string `json:"expected"` + Runs []inputRun `json:"runs"` +} + +type outputScore struct { + Backend repointel.Backend `json:"backend"` + Precision float64 `json:"precision"` + Recall float64 `json:"recall"` + F1 float64 `json:"f1"` + LatencyMS int64 `json:"latency_ms"` + OutputTokens int `json:"output_tokens"` + Failed bool `json:"failed"` + Error string `json:"error,omitempty"` +} + +type outputDocument struct { + Scenario string `json:"scenario"` + Scores []outputScore `json:"scores"` +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: codeintel-bakeoff ") + os.Exit(2) + } + + data, err := os.ReadFile(os.Args[1]) + if err != nil { + fatalf("read input: %v", err) + } + + var input inputDocument + if err := json.Unmarshal(data, &input); err != nil { + fatalf("decode input: %v", err) + } + if input.Scenario == "" { + fatalf("decode input: scenario is required") + } + if len(input.Expected) == 0 { + fatalf("decode input: expected must contain at least one result key") + } + if len(input.Runs) == 0 { + fatalf("decode input: runs must contain at least one backend observation") + } + + runs := make([]repointel.BackendRun, 0, len(input.Runs)) + for _, run := range input.Runs { + runs = append(runs, repointel.BackendRun{ + Backend: run.Backend, + Results: run.Results, + Latency: time.Duration(run.LatencyMS) * time.Millisecond, + OutputTokens: run.OutputTokens, + Error: run.Error, + }) + } + + output := outputDocument{ + Scenario: input.Scenario, + Scores: formatScores(repointel.RankRuns(input.Expected, runs)), + } + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(output); err != nil { + fatalf("encode output: %v", err) + } +} + +func formatScores(scores []repointel.BenchmarkScore) []outputScore { + formatted := make([]outputScore, 0, len(scores)) + for _, score := range scores { + formatted = append(formatted, outputScore{ + Backend: score.Backend, + Precision: score.Precision, + Recall: score.Recall, + F1: score.F1, + LatencyMS: score.Latency.Milliseconds(), + OutputTokens: score.OutputTokens, + Failed: score.Failed, + Error: score.Error, + }) + } + return formatted +} + +func fatalf(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +} diff --git a/packages/repo-intel/cmd/codeintel-bakeoff/main_test.go b/packages/repo-intel/cmd/codeintel-bakeoff/main_test.go new file mode 100644 index 00000000..771782b1 --- /dev/null +++ b/packages/repo-intel/cmd/codeintel-bakeoff/main_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "testing" + "time" + + repointel "github.com/ai-dev-control-plane/repo-intel" +) + +func TestFormatScoresUsesMillisecondsAndStableJSONShape(t *testing.T) { + formatted := formatScores([]repointel.BenchmarkScore{{ + Backend: repointel.BackendCodebaseMemory, + Precision: 1, + Recall: 0.5, + F1: 2.0 / 3.0, + Latency: 150 * time.Millisecond, + OutputTokens: 321, + }}) + + if len(formatted) != 1 { + t.Fatalf("formatted len = %d, want 1", len(formatted)) + } + if formatted[0].LatencyMS != 150 { + t.Fatalf("latency_ms = %d, want 150", formatted[0].LatencyMS) + } + if formatted[0].Backend != repointel.BackendCodebaseMemory { + t.Fatalf("backend = %q", formatted[0].Backend) + } + if formatted[0].OutputTokens != 321 { + t.Fatalf("output_tokens = %d", formatted[0].OutputTokens) + } +} diff --git a/packages/repo-intel/cmd/codeintel-promote/main.go b/packages/repo-intel/cmd/codeintel-promote/main.go new file mode 100644 index 00000000..cfc77789 --- /dev/null +++ b/packages/repo-intel/cmd/codeintel-promote/main.go @@ -0,0 +1,193 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + repointel "github.com/ai-dev-control-plane/repo-intel" +) + +type inputRun struct { + Backend repointel.Backend `json:"backend"` + Results []string `json:"results"` + LatencyMS int64 `json:"latency_ms"` + OutputTokens int `json:"output_tokens"` + Error string `json:"error,omitempty"` +} + +type inputScenario struct { + ID string `json:"id"` + Repository string `json:"repository"` + Revision string `json:"revision"` + Expected []string `json:"expected"` + Runs []inputRun `json:"runs"` +} + +type inputRepository struct { + Path string `json:"path"` + Revision string `json:"revision"` +} + +type inputCorpus struct { + Version int `json:"version"` + SHA256 string `json:"sha256"` + Path string `json:"path,omitempty"` +} + +type inputSuite struct { + ProvenanceVersion int `json:"provenance_version"` + Corpus inputCorpus `json:"corpus"` + Tools map[string]string `json:"tools"` + Repositories map[string]inputRepository `json:"repositories"` + Scenarios []inputScenario `json:"scenarios"` +} + +type outputDecision struct { + Candidate repointel.Backend `json:"candidate"` + Baseline repointel.Backend `json:"baseline"` + Promote bool `json:"promote"` + Reasons []string `json:"reasons"` + CandidateAvgF1 float64 `json:"candidate_avg_f1"` + BaselineAvgF1 float64 `json:"baseline_avg_f1"` +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: codeintel-promote ") + os.Exit(2) + } + + data, err := os.ReadFile(os.Args[1]) + if err != nil { + fatalf("read input: %v", err) + } + + var input inputSuite + if err := json.Unmarshal(data, &input); err != nil { + fatalf("decode input: %v", err) + } + if len(input.Scenarios) == 0 { + fatalf("decode input: scenarios must not be empty") + } + if err := validatePromotionProvenance(input); err != nil { + fatalf("provenance validation: %v", err) + } + + scenarios := make([]repointel.ScenarioBenchmark, 0, len(input.Scenarios)) + for _, scenario := range input.Scenarios { + if scenario.ID == "" { + fatalf("decode input: scenario id is required") + } + if len(scenario.Expected) == 0 { + fatalf("decode input: scenario %s has no expected results", scenario.ID) + } + runs := make([]repointel.BackendRun, 0, len(scenario.Runs)) + for _, run := range scenario.Runs { + runs = append(runs, repointel.BackendRun{ + Backend: run.Backend, + Results: run.Results, + Latency: time.Duration(run.LatencyMS) * time.Millisecond, + OutputTokens: run.OutputTokens, + Error: run.Error, + }) + } + scenarios = append(scenarios, repointel.ScenarioBenchmark{ + ID: scenario.ID, + Scores: repointel.RankRuns(scenario.Expected, runs), + }) + } + + decision := repointel.EvaluatePromotion( + repointel.BackendCodebaseMemory, + repointel.BackendGitNexus, + scenarios, + repointel.DefaultPromotionPolicy(), + ) + output := outputDecision{ + Candidate: decision.Candidate, + Baseline: decision.Baseline, + Promote: decision.Promote, + Reasons: decision.Reasons, + CandidateAvgF1: decision.CandidateAvgF1, + BaselineAvgF1: decision.BaselineAvgF1, + } + + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(output); err != nil { + fatalf("encode output: %v", err) + } + os.Exit(decisionExitCode(decision.Promote)) +} + +func validatePromotionProvenance(input inputSuite) error { + if input.ProvenanceVersion < 2 { + return fmt.Errorf("provenance_version %d is too old; require >= 2", input.ProvenanceVersion) + } + if input.Corpus.Version <= 0 { + return fmt.Errorf("corpus version is required") + } + if !isSHA256(input.Corpus.SHA256) { + return fmt.Errorf("corpus sha256 is invalid") + } + + for _, backend := range []string{string(repointel.BackendCodebaseMemory), string(repointel.BackendGitNexus)} { + version, ok := input.Tools[backend] + if !ok || invalidProvenanceValue(version) { + return fmt.Errorf("%s tool version is missing or invalid", backend) + } + } + + for _, scenario := range input.Scenarios { + if scenario.Repository == "" { + return fmt.Errorf("scenario %q repository provenance is missing", scenario.ID) + } + if invalidProvenanceValue(scenario.Revision) { + return fmt.Errorf("scenario %q revision is missing or invalid", scenario.ID) + } + repository, ok := input.Repositories[scenario.Repository] + if !ok { + return fmt.Errorf("scenario %q repository provenance for %q is missing", scenario.ID, scenario.Repository) + } + if invalidProvenanceValue(repository.Revision) { + return fmt.Errorf("repository provenance for %q has invalid revision", scenario.Repository) + } + if scenario.Revision != repository.Revision { + return fmt.Errorf("scenario %q revision %q does not match repository provenance %q", scenario.ID, scenario.Revision, repository.Revision) + } + } + return nil +} + +func invalidProvenanceValue(value string) bool { + value = strings.TrimSpace(value) + if value == "" || strings.EqualFold(value, "unknown") { + return true + } + return strings.HasPrefix(strings.ToLower(value), "error:") +} + +func isSHA256(value string) bool { + value = strings.TrimSpace(value) + if len(value) != 64 { + return false + } + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == 32 +} + +func decisionExitCode(promote bool) int { + if promote { + return 0 + } + return 3 +} + +func fatalf(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +} diff --git a/packages/repo-intel/cmd/codeintel-promote/main_test.go b/packages/repo-intel/cmd/codeintel-promote/main_test.go new file mode 100644 index 00000000..fac48a05 --- /dev/null +++ b/packages/repo-intel/cmd/codeintel-promote/main_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "strings" + "testing" +) + +func TestDecisionExitCode(t *testing.T) { + if got := decisionExitCode(true); got != 0 { + t.Fatalf("promote=true exit code = %d, want 0", got) + } + if got := decisionExitCode(false); got == 0 { + t.Fatal("promote=false must return a non-zero exit code") + } +} + +func TestValidatePromotionProvenanceAcceptsBoundSuite(t *testing.T) { + input := validProvenancedSuite() + if err := validatePromotionProvenance(input); err != nil { + t.Fatalf("validatePromotionProvenance() error = %v", err) + } +} + +func TestValidatePromotionProvenanceRejectsMissingProvenanceVersion(t *testing.T) { + input := validProvenancedSuite() + input.ProvenanceVersion = 0 + assertValidationErrorContains(t, input, "provenance_version") +} + +func TestValidatePromotionProvenanceRejectsInvalidCorpusHash(t *testing.T) { + input := validProvenancedSuite() + input.Corpus.SHA256 = "not-a-sha256" + assertValidationErrorContains(t, input, "corpus sha256") +} + +func TestValidatePromotionProvenanceRejectsBackendVersionError(t *testing.T) { + input := validProvenancedSuite() + input.Tools["codebase-memory"] = "error: version unavailable" + assertValidationErrorContains(t, input, "codebase-memory") +} + +func TestValidatePromotionProvenanceRejectsScenarioRevisionMismatch(t *testing.T) { + input := validProvenancedSuite() + input.Scenarios[0].Revision = "different" + assertValidationErrorContains(t, input, "revision") +} + +func TestValidatePromotionProvenanceRejectsMissingRepositoryBinding(t *testing.T) { + input := validProvenancedSuite() + delete(input.Repositories, "owner/repo") + assertValidationErrorContains(t, input, "repository provenance") +} + +func validProvenancedSuite() inputSuite { + const corpusSHA = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + const revision = "0123456789abcdef0123456789abcdef01234567" + return inputSuite{ + ProvenanceVersion: 2, + Corpus: inputCorpus{ + Version: 9, + SHA256: corpusSHA, + }, + Tools: map[string]string{ + "codebase-memory": "codebase-memory-mcp 0.11.0", + "gitnexus": "gitnexus 1.6.12", + }, + Repositories: map[string]inputRepository{ + "owner/repo": {Path: "/repos/repo", Revision: revision}, + }, + Scenarios: []inputScenario{ + { + ID: "s1", + Repository: "owner/repo", + Revision: revision, + Expected: []string{"src/a.go"}, + Runs: []inputRun{ + {Backend: "codebase-memory", Results: []string{"src/a.go"}}, + {Backend: "gitnexus", Results: []string{"src/a.go"}}, + }, + }, + }, + } +} + +func assertValidationErrorContains(t *testing.T, input inputSuite, want string) { + t.Helper() + err := validatePromotionProvenance(input) + if err == nil { + t.Fatalf("expected validation error containing %q", want) + } + if !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(want)) { + t.Fatalf("error %q does not contain %q", err, want) + } +} diff --git a/packages/repo-intel/go.mod b/packages/repo-intel/go.mod index d34be89c..fbeb5a0d 100644 --- a/packages/repo-intel/go.mod +++ b/packages/repo-intel/go.mod @@ -1,6 +1,6 @@ // Package repointel provides repository intelligence capabilities including -// git worktree management and code indexing (stubbed for Phase 1, full -// tree-sitter integration in Phase 2). +// git worktree management, lightweight local indexing, vendor-neutral +// code-intelligence routing, and backend bake-off scoring. module github.com/ai-dev-control-plane/repo-intel go 1.25.11 diff --git a/packages/repo-intel/promotion.go b/packages/repo-intel/promotion.go new file mode 100644 index 00000000..a5f4aa06 --- /dev/null +++ b/packages/repo-intel/promotion.go @@ -0,0 +1,136 @@ +package repointel + +import "fmt" + +// ScenarioBenchmark is the scored backend comparison for one benchmark scenario. +type ScenarioBenchmark struct { + ID string + Scores []BenchmarkScore +} + +// PromotionPolicy defines the required-core-path recall floor for replacing a +// baseline backend. The corpus intentionally is not an exhaustive list of every +// relevant file, so precision/F1 remain diagnostic rather than safety gates. +type PromotionPolicy struct { + MinRecall float64 + Epsilon float64 +} + +// DefaultPromotionPolicy requires every curated core path on every scenario. +func DefaultPromotionPolicy() PromotionPolicy { + return PromotionPolicy{ + MinRecall: 1.0, + Epsilon: 1e-9, + } +} + +// PromotionDecision explains whether a candidate backend is safe to promote. +type PromotionDecision struct { + Candidate Backend + Baseline Backend + Promote bool + Reasons []string + CandidateAvgF1 float64 + BaselineAvgF1 float64 +} + +// EvaluatePromotion enforces a required-recall-first replacement gate. Both +// candidate and baseline must successfully complete every scenario so the +// comparison itself is valid. The candidate must then clear the absolute recall +// floor and avoid any per-scenario recall regression against the baseline. +// F1 is retained for observability but does not block promotion because +// additional useful files can lower precision against a deliberately +// non-exhaustive core-path corpus. +func EvaluatePromotion( + candidate Backend, + baseline Backend, + scenarios []ScenarioBenchmark, + policy PromotionPolicy, +) PromotionDecision { + decision := PromotionDecision{ + Candidate: candidate, + Baseline: baseline, + Promote: false, + Reasons: []string{}, + } + if len(scenarios) == 0 { + decision.Reasons = append(decision.Reasons, "no benchmark scenarios were provided") + return decision + } + + candidateF1Total := 0.0 + baselineF1Total := 0.0 + candidateCount := 0 + baselineCount := 0 + + for _, scenario := range scenarios { + candidateScore, candidateOK := scoreForBackend(scenario.Scores, candidate) + baselineScore, baselineOK := scoreForBackend(scenario.Scores, baseline) + + if !candidateOK { + decision.Reasons = append(decision.Reasons, + fmt.Sprintf("%s: missing candidate %s result", scenario.ID, candidate)) + } + if !baselineOK { + decision.Reasons = append(decision.Reasons, + fmt.Sprintf("%s: missing baseline %s result", scenario.ID, baseline)) + } + if !candidateOK || !baselineOK { + continue + } + + candidateF1Total += candidateScore.F1 + candidateCount++ + baselineF1Total += baselineScore.F1 + baselineCount++ + + if candidateScore.Failed { + detail := candidateScore.Error + if detail == "" { + detail = "unspecified error" + } + decision.Reasons = append(decision.Reasons, + fmt.Sprintf("%s: candidate failed: %s", scenario.ID, detail)) + } + if baselineScore.Failed { + detail := baselineScore.Error + if detail == "" { + detail = "unspecified error" + } + decision.Reasons = append(decision.Reasons, + fmt.Sprintf("%s: baseline failed: %s", scenario.ID, detail)) + } + if candidateScore.Failed || baselineScore.Failed { + continue + } + + if candidateScore.Recall+policy.Epsilon < policy.MinRecall { + decision.Reasons = append(decision.Reasons, + fmt.Sprintf("%s: candidate recall %.3f is below minimum recall %.3f", + scenario.ID, candidateScore.Recall, policy.MinRecall)) + } + if candidateScore.Recall+policy.Epsilon < baselineScore.Recall { + decision.Reasons = append(decision.Reasons, + fmt.Sprintf("%s: recall regression %.3f < baseline %.3f", + scenario.ID, candidateScore.Recall, baselineScore.Recall)) + } + } + + if candidateCount > 0 { + decision.CandidateAvgF1 = candidateF1Total / float64(candidateCount) + } + if baselineCount > 0 { + decision.BaselineAvgF1 = baselineF1Total / float64(baselineCount) + } + decision.Promote = len(decision.Reasons) == 0 + return decision +} + +func scoreForBackend(scores []BenchmarkScore, backend Backend) (BenchmarkScore, bool) { + for _, score := range scores { + if score.Backend == backend { + return score, true + } + } + return BenchmarkScore{}, false +} diff --git a/packages/repo-intel/promotion_test.go b/packages/repo-intel/promotion_test.go new file mode 100644 index 00000000..224d2dd2 --- /dev/null +++ b/packages/repo-intel/promotion_test.go @@ -0,0 +1,179 @@ +package repointel + +import ( + "strings" + "testing" + "time" +) + +func TestEvaluatePromotionAllowsCompleteRecallCandidate(t *testing.T) { + scenarios := []ScenarioBenchmark{ + { + ID: "proposal-flow", + Scores: []BenchmarkScore{ + {Backend: BackendCodebaseMemory, Recall: 1.0, Precision: 0.9, F1: 0.947, Latency: 250 * time.Millisecond}, + {Backend: BackendGitNexus, Recall: 0.8, Precision: 0.8, F1: 0.8, Latency: 120 * time.Millisecond}, + }, + }, + { + ID: "mir-callers", + Scores: []BenchmarkScore{ + {Backend: BackendCodebaseMemory, Recall: 1.0, Precision: 1.0, F1: 1.0, Latency: 300 * time.Millisecond}, + {Backend: BackendGitNexus, Recall: 1.0, Precision: 0.8, F1: 0.889, Latency: 100 * time.Millisecond}, + }, + }, + } + + decision := EvaluatePromotion( + BackendCodebaseMemory, + BackendGitNexus, + scenarios, + DefaultPromotionPolicy(), + ) + if !decision.Promote { + t.Fatalf("expected promotion, got reasons: %#v", decision.Reasons) + } +} + +func TestEvaluatePromotionAllowsLowerF1WhenRequiredRecallIsComplete(t *testing.T) { + scenarios := []ScenarioBenchmark{ + { + ID: "broad-but-complete", + Scores: []BenchmarkScore{ + {Backend: BackendCodebaseMemory, Recall: 1.0, Precision: 0.25, F1: 0.4, Latency: 300 * time.Millisecond}, + {Backend: BackendGitNexus, Recall: 1.0, Precision: 0.9, F1: 0.947, Latency: 100 * time.Millisecond}, + }, + }, + } + + decision := EvaluatePromotion( + BackendCodebaseMemory, + BackendGitNexus, + scenarios, + DefaultPromotionPolicy(), + ) + if !decision.Promote { + t.Fatalf("complete required recall should not be blocked by non-exhaustive-set F1: %#v", decision.Reasons) + } +} + +func TestEvaluatePromotionRejectsRecallRegressionEvenWhenCandidateIsFaster(t *testing.T) { + scenarios := []ScenarioBenchmark{ + { + ID: "proposal-flow", + Scores: []BenchmarkScore{ + {Backend: BackendCodebaseMemory, Recall: 0.8, Precision: 1.0, F1: 0.889, Latency: 20 * time.Millisecond}, + {Backend: BackendGitNexus, Recall: 1.0, Precision: 0.8, F1: 0.889, Latency: 200 * time.Millisecond}, + }, + }, + } + + decision := EvaluatePromotion( + BackendCodebaseMemory, + BackendGitNexus, + scenarios, + DefaultPromotionPolicy(), + ) + if decision.Promote { + t.Fatal("expected recall regression to block promotion") + } + assertContainsReason(t, decision.Reasons, "recall regression") +} + +func TestEvaluatePromotionRejectsFailedCandidate(t *testing.T) { + scenarios := []ScenarioBenchmark{ + { + ID: "runtime-boundary", + Scores: []BenchmarkScore{ + {Backend: BackendCodebaseMemory, Failed: true, Error: "stale index"}, + {Backend: BackendGitNexus, Recall: 1.0, Precision: 1.0, F1: 1.0}, + }, + }, + } + + decision := EvaluatePromotion( + BackendCodebaseMemory, + BackendGitNexus, + scenarios, + DefaultPromotionPolicy(), + ) + if decision.Promote { + t.Fatal("expected failed candidate to block promotion") + } + assertContainsReason(t, decision.Reasons, "candidate failed") +} + +func TestEvaluatePromotionRejectsFailedBaseline(t *testing.T) { + scenarios := []ScenarioBenchmark{ + { + ID: "runtime-boundary", + Scores: []BenchmarkScore{ + {Backend: BackendCodebaseMemory, Recall: 1.0, Precision: 1.0, F1: 1.0}, + {Backend: BackendGitNexus, Failed: true, Error: "index unavailable"}, + }, + }, + } + + decision := EvaluatePromotion( + BackendCodebaseMemory, + BackendGitNexus, + scenarios, + DefaultPromotionPolicy(), + ) + if decision.Promote { + t.Fatal("expected failed baseline to block comparative promotion") + } + assertContainsReason(t, decision.Reasons, "baseline failed") +} + +func TestEvaluatePromotionRejectsCandidateBelowAbsoluteRecallFloor(t *testing.T) { + policy := DefaultPromotionPolicy() + policy.MinRecall = 0.9 + scenarios := []ScenarioBenchmark{ + { + ID: "admission", + Scores: []BenchmarkScore{ + {Backend: BackendCodebaseMemory, Recall: 0.8, Precision: 1.0, F1: 0.889}, + {Backend: BackendGitNexus, Recall: 0.8, Precision: 1.0, F1: 0.889}, + }, + }, + } + + decision := EvaluatePromotion( + BackendCodebaseMemory, + BackendGitNexus, + scenarios, + policy, + ) + if decision.Promote { + t.Fatal("expected absolute recall floor to block equally incomplete backends") + } + assertContainsReason(t, decision.Reasons, "below minimum recall") +} + +func TestEvaluatePromotionRejectsMissingBaselineOrCandidate(t *testing.T) { + decision := EvaluatePromotion( + BackendCodebaseMemory, + BackendGitNexus, + []ScenarioBenchmark{{ + ID: "missing-baseline", + Scores: []BenchmarkScore{{Backend: BackendCodebaseMemory, Recall: 1, Precision: 1, F1: 1}}, + }}, + DefaultPromotionPolicy(), + ) + if decision.Promote { + t.Fatal("expected missing comparison backend to block promotion") + } + assertContainsReason(t, decision.Reasons, "missing baseline") +} + +func assertContainsReason(t *testing.T, reasons []string, want string) { + t.Helper() + want = strings.ToLower(want) + for _, reason := range reasons { + if strings.Contains(strings.ToLower(reason), want) { + return + } + } + t.Fatalf("reasons %#v do not contain %q", reasons, want) +} diff --git a/packages/repo-intel/router.go b/packages/repo-intel/router.go new file mode 100644 index 00000000..cb126765 --- /dev/null +++ b/packages/repo-intel/router.go @@ -0,0 +1,153 @@ +package repointel + +import "fmt" + +// Backend identifies a concrete code-intelligence or verification backend. +type Backend string + +const ( + BackendRipgrep Backend = "ripgrep" + BackendZoekt Backend = "zoekt" + BackendLSP Backend = "lsp" + BackendASTGrep Backend = "ast-grep" + BackendCodebaseMemory Backend = "codebase-memory" + BackendJoern Backend = "joern" + BackendGit Backend = "git" + BackendTests Backend = "tests" + BackendGitNexus Backend = "gitnexus" +) + +// Intent is the stable, vendor-neutral operation requested by an agent. +type Intent string + +const ( + IntentLocate Intent = "locate" + IntentUnderstand Intent = "understand" + IntentImpact Intent = "impact" + IntentRefactor Intent = "refactor" + IntentSecurity Intent = "security" + IntentHistory Intent = "history" + IntentVerify Intent = "verify" +) + +// Mode refines an intent enough to route it deterministically. +type Mode string + +const ( + ModeExact Mode = "exact" + ModeConceptual Mode = "conceptual" + ModeSymbol Mode = "symbol" + ModeArchitecture Mode = "architecture" + ModeStructural Mode = "structural" + ModeDataFlow Mode = "data-flow" +) + +// Scope identifies whether branch-local or fleet-wide state is required. +type Scope string + +const ( + ScopeWorktree Scope = "worktree" + ScopeFleet Scope = "fleet" +) + +// RouteRequest describes one code-intelligence operation. +type RouteRequest struct { + Intent Intent + Mode Mode + Scope Scope +} + +// RoutePlan describes the preferred backend, ordered fallbacks, and executable +// verification backends. Callers should not skip VerifyWith for mutating work. +type RoutePlan struct { + Primary Backend + Fallbacks []Backend + VerifyWith []Backend +} + +// Router maps stable operations to the cheapest precise backend. It is kept +// intentionally deterministic so agent behavior can be tested and audited. +type Router struct{} + +func NewRouter() *Router { return &Router{} } + +// Route returns the backend plan for a request. +func (r *Router) Route(req RouteRequest) (RoutePlan, error) { + if req.Scope != ScopeWorktree && req.Scope != ScopeFleet { + return RoutePlan{}, fmt.Errorf("unsupported scope %q", req.Scope) + } + + switch req.Intent { + case IntentLocate: + switch req.Mode { + case ModeExact: + if req.Scope == ScopeFleet { + return RoutePlan{Primary: BackendZoekt, Fallbacks: []Backend{BackendCodebaseMemory}}, nil + } + return RoutePlan{Primary: BackendRipgrep, Fallbacks: []Backend{BackendLSP}}, nil + case ModeConceptual: + return RoutePlan{Primary: BackendCodebaseMemory, Fallbacks: []Backend{BackendZoekt}}, nil + default: + return RoutePlan{}, unsupported(req) + } + + case IntentUnderstand: + switch req.Mode { + case ModeSymbol: + return RoutePlan{Primary: BackendLSP, Fallbacks: []Backend{BackendCodebaseMemory}}, nil + case ModeArchitecture: + return RoutePlan{Primary: BackendCodebaseMemory, Fallbacks: []Backend{BackendLSP}}, nil + default: + return RoutePlan{}, unsupported(req) + } + + case IntentImpact: + if req.Mode != ModeSymbol && req.Mode != ModeArchitecture { + return RoutePlan{}, unsupported(req) + } + return RoutePlan{ + Primary: BackendCodebaseMemory, + Fallbacks: []Backend{BackendLSP, BackendGit}, + VerifyWith: []Backend{BackendTests}, + }, nil + + case IntentRefactor: + if req.Mode != ModeStructural { + return RoutePlan{}, unsupported(req) + } + return RoutePlan{ + Primary: BackendASTGrep, + Fallbacks: []Backend{BackendLSP}, + VerifyWith: []Backend{BackendTests}, + }, nil + + case IntentSecurity: + if req.Mode != ModeDataFlow { + return RoutePlan{}, unsupported(req) + } + return RoutePlan{ + Primary: BackendJoern, + Fallbacks: []Backend{BackendCodebaseMemory}, + VerifyWith: []Backend{BackendTests}, + }, nil + + case IntentHistory: + if req.Mode != ModeExact && req.Mode != ModeSymbol { + return RoutePlan{}, unsupported(req) + } + return RoutePlan{Primary: BackendGit}, nil + + case IntentVerify: + if req.Mode != ModeExact && req.Mode != ModeSymbol && req.Mode != ModeArchitecture { + return RoutePlan{}, unsupported(req) + } + return RoutePlan{Primary: BackendTests}, nil + + default: + return RoutePlan{}, fmt.Errorf("unsupported intent %q", req.Intent) + } +} + +func unsupported(req RouteRequest) error { + return fmt.Errorf("unsupported code-intelligence route: intent=%q mode=%q scope=%q", req.Intent, req.Mode, req.Scope) +} diff --git a/packages/repo-intel/router_test.go b/packages/repo-intel/router_test.go new file mode 100644 index 00000000..ce0945d1 --- /dev/null +++ b/packages/repo-intel/router_test.go @@ -0,0 +1,68 @@ +package repointel + +import "testing" + +func TestRouterSelectsCheapestPreciseBackend(t *testing.T) { + router := NewRouter() + + tests := []struct { + name string + request RouteRequest + want Backend + }{ + {"exact worktree lookup uses ripgrep", RouteRequest{Intent: IntentLocate, Mode: ModeExact, Scope: ScopeWorktree}, BackendRipgrep}, + {"exact fleet lookup uses zoekt", RouteRequest{Intent: IntentLocate, Mode: ModeExact, Scope: ScopeFleet}, BackendZoekt}, + {"conceptual lookup uses codebase memory", RouteRequest{Intent: IntentLocate, Mode: ModeConceptual, Scope: ScopeFleet}, BackendCodebaseMemory}, + {"symbol understanding uses lsp", RouteRequest{Intent: IntentUnderstand, Mode: ModeSymbol, Scope: ScopeWorktree}, BackendLSP}, + {"architecture understanding uses codebase memory", RouteRequest{Intent: IntentUnderstand, Mode: ModeArchitecture, Scope: ScopeFleet}, BackendCodebaseMemory}, + {"impact uses codebase memory", RouteRequest{Intent: IntentImpact, Mode: ModeSymbol, Scope: ScopeFleet}, BackendCodebaseMemory}, + {"structural refactor uses ast grep", RouteRequest{Intent: IntentRefactor, Mode: ModeStructural, Scope: ScopeWorktree}, BackendASTGrep}, + {"security data flow uses joern", RouteRequest{Intent: IntentSecurity, Mode: ModeDataFlow, Scope: ScopeWorktree}, BackendJoern}, + {"history uses git", RouteRequest{Intent: IntentHistory, Mode: ModeExact, Scope: ScopeWorktree}, BackendGit}, + {"verification uses tests", RouteRequest{Intent: IntentVerify, Mode: ModeExact, Scope: ScopeWorktree}, BackendTests}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + plan, err := router.Route(tt.request) + if err != nil { + t.Fatalf("route: %v", err) + } + if plan.Primary != tt.want { + t.Fatalf("primary = %q, want %q; plan=%#v", plan.Primary, tt.want, plan) + } + }) + } +} + +func TestRouterAddsSemanticAndExecutableSafetyForImpact(t *testing.T) { + router := NewRouter() + plan, err := router.Route(RouteRequest{Intent: IntentImpact, Mode: ModeSymbol, Scope: ScopeFleet}) + if err != nil { + t.Fatalf("route: %v", err) + } + + if !containsBackend(plan.Fallbacks, BackendLSP) { + t.Fatalf("impact fallbacks = %#v, want LSP semantic check", plan.Fallbacks) + } + if !containsBackend(plan.VerifyWith, BackendTests) { + t.Fatalf("impact verification = %#v, want executable tests", plan.VerifyWith) + } +} + +func TestRouterRejectsUnsupportedCombinations(t *testing.T) { + router := NewRouter() + _, err := router.Route(RouteRequest{Intent: IntentHistory, Mode: ModeDataFlow, Scope: ScopeFleet}) + if err == nil { + t.Fatal("expected invalid routing combination to fail") + } +} + +func containsBackend(backends []Backend, want Backend) bool { + for _, backend := range backends { + if backend == want { + return true + } + } + return false +} diff --git a/scripts/codeintel_bakeoff.py b/scripts/codeintel_bakeoff.py new file mode 100644 index 00000000..098373d9 --- /dev/null +++ b/scripts/codeintel_bakeoff.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 +"""Run reproducible GitNexus vs codebase-memory retrieval benchmarks. + +The harness intentionally keeps indexing separate from query latency and scores only +repository file paths actually mentioned by each backend response. It uses MCP for +query execution so the benchmark exercises the same tool surface agents use. +""" + +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import queue +import shlex +import shutil +import subprocess +import sys +import threading +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any + +BACKEND_CODEBASE_MEMORY = "codebase-memory" +BACKEND_GITNEXUS = "gitnexus" +SUPPORTED_BACKENDS = (BACKEND_CODEBASE_MEMORY, BACKEND_GITNEXUS) + + +@dataclass +class Observation: + backend: str + results: list[str] + latency_ms: int + output_tokens: int + error: str = "" + raw_output: str = "" + index_ms: int = 0 + + def score_input(self) -> dict[str, Any]: + return { + "backend": self.backend, + "results": self.results, + "latency_ms": self.latency_ms, + "output_tokens": self.output_tokens, + "error": self.error, + } + + +class MCPClient: + def __init__(self, command: list[str], cwd: pathlib.Path, timeout_s: float = 60.0): + self.command = command + self.cwd = cwd + self.timeout_s = timeout_s + self._responses: queue.Queue[dict[str, Any]] = queue.Queue() + self._stderr: list[str] = [] + self._next_id = 1 + self._process = subprocess.Popen( + command, + cwd=str(cwd), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + bufsize=1, + ) + assert self._process.stdout is not None + assert self._process.stderr is not None + self._stdout_thread = threading.Thread(target=self._read_stdout, daemon=True) + self._stderr_thread = threading.Thread(target=self._read_stderr, daemon=True) + self._stdout_thread.start() + self._stderr_thread.start() + self._initialize() + + def _read_stdout(self) -> None: + assert self._process.stdout is not None + for line in self._process.stdout: + line = line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + self._responses.put(payload) + + def _read_stderr(self) -> None: + assert self._process.stderr is not None + for line in self._process.stderr: + self._stderr.append(line.rstrip()) + + def _send(self, payload: dict[str, Any]) -> None: + if self._process.poll() is not None: + raise RuntimeError(self._failure_message("MCP server exited before request")) + assert self._process.stdin is not None + self._process.stdin.write(json.dumps(payload, separators=(",", ":")) + "\n") + self._process.stdin.flush() + + def _request(self, method: str, params: dict[str, Any]) -> dict[str, Any]: + request_id = self._next_id + self._next_id += 1 + self._send( + { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": params, + } + ) + deadline = time.monotonic() + self.timeout_s + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(self._failure_message(f"timeout waiting for {method}")) + try: + payload = self._responses.get(timeout=min(remaining, 0.25)) + except queue.Empty: + if self._process.poll() is not None: + raise RuntimeError(self._failure_message(f"MCP server exited during {method}")) + continue + if payload.get("id") != request_id: + continue + if "error" in payload: + raise RuntimeError(f"{method}: {payload['error']}") + result = payload.get("result") + return result if isinstance(result, dict) else {"value": result} + + def _initialize(self) -> None: + self._request( + "initialize", + { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "dev-plane-codeintel-bakeoff", "version": "1"}, + }, + ) + self._send({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}) + + def call_tool(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + return self._request("tools/call", {"name": name, "arguments": arguments}) + + def close(self) -> None: + if self._process.poll() is None: + self._process.terminate() + try: + self._process.wait(timeout=3) + except subprocess.TimeoutExpired: + self._process.kill() + self._process.wait(timeout=3) + + def _failure_message(self, message: str) -> str: + stderr = "\n".join(self._stderr[-20:]).strip() + return f"{message}: {stderr}" if stderr else message + + def __enter__(self) -> "MCPClient": + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + +def extract_known_paths(text: str, known_paths: list[str]) -> list[str]: + return sorted({path for path in known_paths if path and path in text}) + + +def resolve_repo_path(root: pathlib.Path, repository: str) -> pathlib.Path: + basename = repository.rsplit("/", 1)[-1] + candidates = (root / basename, root / repository) + for candidate in candidates: + if candidate.is_dir(): + return candidate + raise FileNotFoundError( + f"repository {repository!r} not found under {root}; tried " + + ", ".join(str(candidate) for candidate in candidates) + ) + + +def validate_scenario(scenario: dict[str, Any]) -> None: + for field in ("id", "repository", "expected", "backends"): + if not scenario.get(field): + raise ValueError(f"scenario missing required field {field!r}: {scenario!r}") + backends = scenario["backends"] + if not isinstance(backends, dict): + raise ValueError(f"scenario backends must be an object: {scenario['id']}") + for backend in SUPPORTED_BACKENDS: + calls = backends.get(backend) + if not isinstance(calls, list) or not calls: + raise ValueError(f"scenario {scenario['id']} requires calls for {backend}") + for call in calls: + if not isinstance(call, dict) or not call.get("tool"): + raise ValueError(f"scenario {scenario['id']} has invalid {backend} call: {call!r}") + if "arguments" in call and not isinstance(call["arguments"], dict): + raise ValueError(f"scenario {scenario['id']} call arguments must be an object") + + +def mcp_result_text(payload: dict[str, Any]) -> str: + pieces: list[str] = [] + content = payload.get("content") + if isinstance(content, list): + for item in content: + if isinstance(item, dict) and isinstance(item.get("text"), str): + pieces.append(item["text"]) + structured = payload.get("structuredContent") + if structured is not None: + pieces.append(json.dumps(structured, sort_keys=True)) + if not pieces: + pieces.append(json.dumps(payload, sort_keys=True)) + return "\n".join(pieces) + + +def tracked_files(repo_path: pathlib.Path) -> list[str]: + completed = subprocess.run( + ["git", "-C", str(repo_path), "ls-files", "-z"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return sorted( + item.decode("utf-8", errors="replace") + for item in completed.stdout.split(b"\0") + if item + ) + + +def shell_command(env_name: str, default: str) -> list[str]: + value = os.environ.get(env_name, default) + command = shlex.split(value) + if not command: + raise ValueError(f"{env_name} resolved to an empty command") + return command + + +def command_available(command: list[str]) -> bool: + executable = command[0] + if os.path.sep in executable: + return pathlib.Path(executable).exists() + return shutil.which(executable) is not None + + +def run_index(backend: str, repo_path: pathlib.Path, timeout_s: float) -> tuple[int, str]: + if backend == BACKEND_CODEBASE_MEMORY: + command = shell_command("CODEINTEL_CODEBASE_MEMORY_CMD", "codebase-memory-mcp") + command += [ + "cli", + "index_repository", + json.dumps({"repo_path": str(repo_path.resolve())}, separators=(",", ":")), + ] + elif backend == BACKEND_GITNEXUS: + command = shell_command( + "CODEINTEL_GITNEXUS_ANALYZE_CMD", + "npx -y gitnexus@1.6.12 analyze --index-only", + ) + else: + raise ValueError(f"unsupported backend {backend}") + + if not command_available(command): + return 0, f"index command not found: {command[0]}" + + started = time.monotonic() + try: + completed = subprocess.run( + command, + cwd=str(repo_path), + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout_s, + ) + except subprocess.TimeoutExpired: + return int((time.monotonic() - started) * 1000), f"index timed out after {timeout_s:.0f}s" + elapsed_ms = int((time.monotonic() - started) * 1000) + if completed.returncode != 0: + detail = (completed.stderr or completed.stdout).strip() + return elapsed_ms, f"index failed ({completed.returncode}): {detail[-2000:]}" + return elapsed_ms, "" + + +def query_command(backend: str) -> list[str]: + if backend == BACKEND_CODEBASE_MEMORY: + return shell_command("CODEINTEL_CODEBASE_MEMORY_CMD", "codebase-memory-mcp") + if backend == BACKEND_GITNEXUS: + return shell_command( + "CODEINTEL_GITNEXUS_MCP_CMD", + "npx -y gitnexus@1.6.12 mcp", + ) + raise ValueError(f"unsupported backend {backend}") + + +def run_scenario_backend( + scenario: dict[str, Any], + backend: str, + repo_path: pathlib.Path, + known_paths: list[str], + index_ms: int, + index_error: str, + timeout_s: float, +) -> Observation: + if index_error: + return Observation( + backend=backend, + results=[], + latency_ms=0, + output_tokens=0, + error=index_error, + index_ms=index_ms, + ) + + command = query_command(backend) + if not command_available(command): + return Observation( + backend=backend, + results=[], + latency_ms=0, + output_tokens=0, + error=f"MCP command not found: {command[0]}", + index_ms=index_ms, + ) + + calls = scenario["backends"][backend] + raw_parts: list[str] = [] + query_started: float | None = None + try: + with MCPClient(command, repo_path, timeout_s=timeout_s) as client: + query_started = time.monotonic() + for call in calls: + payload = client.call_tool(call["tool"], call.get("arguments", {})) + raw_parts.append(mcp_result_text(payload)) + except Exception as exc: # benchmark failure should be recorded, not abort the suite + latency_ms = 0 + if query_started is not None: + latency_ms = int((time.monotonic() - query_started) * 1000) + return Observation( + backend=backend, + results=[], + latency_ms=latency_ms, + output_tokens=0, + error=str(exc), + index_ms=index_ms, + ) + + assert query_started is not None + latency_ms = int((time.monotonic() - query_started) * 1000) + raw = "\n".join(raw_parts) + paths = extract_known_paths(raw, known_paths) + return Observation( + backend=backend, + results=paths, + latency_ms=latency_ms, + output_tokens=max(1, (len(raw) + 3) // 4), + raw_output=raw, + index_ms=index_ms, + ) + + +def parse_repo_overrides(values: list[str]) -> dict[str, pathlib.Path]: + overrides: dict[str, pathlib.Path] = {} + for value in values: + if "=" not in value: + raise ValueError(f"--repo requires owner/name=/absolute/path, got {value!r}") + repository, raw_path = value.split("=", 1) + path = pathlib.Path(raw_path).expanduser().resolve() + if not path.is_dir(): + raise FileNotFoundError(f"override for {repository} is not a directory: {path}") + overrides[repository] = path + return overrides + + +def load_scenarios(path: pathlib.Path) -> list[dict[str, Any]]: + document = json.loads(path.read_text(encoding="utf-8")) + scenarios = document.get("scenarios") + if not isinstance(scenarios, list) or not scenarios: + raise ValueError(f"{path} does not contain a non-empty scenarios array") + for scenario in scenarios: + if not isinstance(scenario, dict): + raise ValueError(f"invalid scenario entry: {scenario!r}") + validate_scenario(scenario) + return scenarios + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--scenarios", + type=pathlib.Path, + default=pathlib.Path("benchmarks/code-intelligence/scenarios.json"), + ) + parser.add_argument( + "--repos-root", + type=pathlib.Path, + default=pathlib.Path(os.environ.get("CODEINTEL_REPOS_ROOT", "~/src")).expanduser(), + ) + parser.add_argument( + "--repo", + action="append", + default=[], + metavar="OWNER/NAME=/PATH", + help="override a repository checkout path; repeat as needed", + ) + parser.add_argument( + "--backend", + action="append", + choices=SUPPORTED_BACKENDS, + help="run only this backend; repeat to select both", + ) + parser.add_argument("--skip-index", action="store_true") + parser.add_argument("--timeout", type=float, default=300.0) + parser.add_argument( + "--output-dir", + type=pathlib.Path, + default=pathlib.Path("data/codeintel-bakeoff"), + ) + args = parser.parse_args(argv) + + scenarios = load_scenarios(args.scenarios) + backends = tuple(dict.fromkeys(args.backend or SUPPORTED_BACKENDS)) + overrides = parse_repo_overrides(args.repo) + output_dir = args.output_dir + output_dir.mkdir(parents=True, exist_ok=True) + + suite_output: dict[str, Any] = { + "version": 1, + "generated_at": datetime.now(timezone.utc).isoformat(), + "backends": list(backends), + "scenarios": [], + } + + had_failure = False + for scenario in scenarios: + repository = scenario["repository"] + try: + repo_path = overrides.get(repository) or resolve_repo_path(args.repos_root, repository) + known_paths = tracked_files(repo_path) + except Exception as exc: + print(f"[{scenario['id']}] repository setup failed: {exc}", file=sys.stderr) + had_failure = True + continue + + observations: list[Observation] = [] + for backend in backends: + index_ms, index_error = (0, "") + if not args.skip_index: + index_ms, index_error = run_index(backend, repo_path, args.timeout) + observation = run_scenario_backend( + scenario, + backend, + repo_path, + known_paths, + index_ms, + index_error, + args.timeout, + ) + observations.append(observation) + state = "FAIL" if observation.error else "OK" + print( + f"[{scenario['id']}] {backend}: {state} " + f"files={len(observation.results)} query={observation.latency_ms}ms " + f"index={observation.index_ms}ms", + file=sys.stderr, + ) + had_failure = had_failure or bool(observation.error) + + score_input = { + "scenario": scenario["id"], + "expected": scenario["expected"], + "runs": [observation.score_input() for observation in observations], + } + scenario_path = output_dir / f"{scenario['id']}.json" + scenario_path.write_text(json.dumps(score_input, indent=2) + "\n", encoding="utf-8") + + raw_dir = output_dir / "raw" / scenario["id"] + raw_dir.mkdir(parents=True, exist_ok=True) + for observation in observations: + if observation.raw_output: + (raw_dir / f"{observation.backend}.txt").write_text( + observation.raw_output, + encoding="utf-8", + ) + + suite_output["scenarios"].append( + { + "id": scenario["id"], + "repository": repository, + "repo_path": str(repo_path), + "expected": scenario["expected"], + "runs": [ + { + **observation.score_input(), + "index_ms": observation.index_ms, + } + for observation in observations + ], + "score_input": str(scenario_path), + } + ) + + summary_path = output_dir / "suite.json" + summary_path.write_text(json.dumps(suite_output, indent=2) + "\n", encoding="utf-8") + print(f"suite results: {summary_path}", file=sys.stderr) + return 1 if had_failure else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/codeintel_provenance.py b/scripts/codeintel_provenance.py new file mode 100644 index 00000000..2a79bc45 --- /dev/null +++ b/scripts/codeintel_provenance.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Attach immutable source, tool, and corpus provenance to a bake-off suite.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import shlex +import subprocess +from typing import Any + +DEFAULT_CORPUS = pathlib.Path("benchmarks/code-intelligence/scenarios.json") + + +def repo_revision(repo_path: pathlib.Path) -> str: + completed = subprocess.run( + ["git", "-C", str(repo_path), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + return completed.stdout.strip() + + +def file_sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def command_version(command: list[str]) -> str: + try: + completed = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + ) + except Exception as exc: + return f"error: {exc}" + + combined = "\n".join(part for part in (completed.stdout, completed.stderr) if part) + lines = [line.strip() for line in combined.splitlines() if line.strip()] + if completed.returncode != 0: + detail = lines[0] if lines else f"exit {completed.returncode}" + return f"error: {detail}" + return lines[0] if lines else "unknown" + + +def env_command(name: str, default: str) -> list[str]: + value = os.environ.get(name, default) + command = shlex.split(value) + if not command: + raise ValueError(f"{name} resolved to an empty command") + return command + + +def collect_tool_versions() -> dict[str, str]: + return { + "codebase-memory": command_version( + env_command( + "CODEINTEL_CODEBASE_MEMORY_VERSION_CMD", + "codebase-memory-mcp --version", + ) + ), + "gitnexus": command_version( + env_command( + "CODEINTEL_GITNEXUS_VERSION_CMD", + "npx -y gitnexus@1.6.12 --version", + ) + ), + "python": command_version(["python3", "--version"]), + "go": command_version(["go", "version"]), + "node": command_version(["node", "--version"]), + } + + +def safe_repo_revision(repo_path: pathlib.Path) -> str: + try: + return repo_revision(repo_path) + except Exception as exc: + return f"error: {exc}" + + +def corpus_scenarios_by_id(corpus_document: dict[str, Any]) -> dict[str, dict[str, Any]]: + scenarios = corpus_document.get("scenarios") + if not isinstance(scenarios, list): + raise ValueError("corpus.scenarios must be an array") + + by_id: dict[str, dict[str, Any]] = {} + for scenario in scenarios: + if not isinstance(scenario, dict): + raise ValueError(f"invalid corpus scenario entry: {scenario!r}") + scenario_id = scenario.get("id") + if not isinstance(scenario_id, str) or not scenario_id: + raise ValueError(f"corpus scenario id is required: {scenario!r}") + if scenario_id in by_id: + raise ValueError(f"duplicate corpus scenario id: {scenario_id}") + by_id[scenario_id] = scenario + return by_id + + +def attach_corpus_provenance( + suite: dict[str, Any], + corpus_document: dict[str, Any], + corpus_sha256: str, +) -> None: + corpus_by_id = corpus_scenarios_by_id(corpus_document) + scenarios = suite.get("scenarios") + if not isinstance(scenarios, list): + raise ValueError("suite.scenarios must be an array") + + for scenario in scenarios: + if not isinstance(scenario, dict): + raise ValueError(f"invalid suite scenario entry: {scenario!r}") + scenario_id = scenario.get("id") + if not isinstance(scenario_id, str) or not scenario_id: + raise ValueError(f"suite scenario id is required: {scenario!r}") + source = corpus_by_id.get(scenario_id) + if source is None: + raise ValueError(f"suite scenario {scenario_id!r} is missing from corpus") + if scenario.get("expected") != source.get("expected"): + raise ValueError(f"suite scenario {scenario_id!r} expected results differ from corpus") + + scenario["intent"] = source.get("intent") + scenario["prompt"] = source.get("prompt") + scenario["backend_calls"] = source.get("backends") + + suite["corpus"] = { + "version": corpus_document.get("version"), + "sha256": corpus_sha256, + } + + +def enrich_suite( + suite: dict[str, Any], + *, + corpus_document: dict[str, Any] | None = None, + corpus_sha256: str | None = None, +) -> dict[str, Any]: + scenarios = suite.get("scenarios") + if not isinstance(scenarios, list): + raise ValueError("suite.scenarios must be an array") + + repositories: dict[str, dict[str, str]] = {} + for scenario in scenarios: + if not isinstance(scenario, dict): + raise ValueError(f"invalid scenario entry: {scenario!r}") + repository = scenario.get("repository") + raw_path = scenario.get("repo_path") + if not isinstance(repository, str) or not repository: + continue + if not isinstance(raw_path, str) or not raw_path: + continue + + entry = repositories.get(repository) + if entry is None: + repo_path = pathlib.Path(raw_path) + entry = { + "path": raw_path, + "revision": safe_repo_revision(repo_path), + } + repositories[repository] = entry + scenario["revision"] = entry["revision"] + + suite["repositories"] = repositories + suite["tools"] = collect_tool_versions() + + if corpus_document is not None: + if not isinstance(corpus_sha256, str) or not corpus_sha256: + raise ValueError("corpus_sha256 is required when corpus_document is provided") + attach_corpus_provenance(suite, corpus_document, corpus_sha256) + + suite["provenance_version"] = 2 + return suite + + +def write_enriched_suite( + path: pathlib.Path, + *, + corpus_path: pathlib.Path = DEFAULT_CORPUS, +) -> None: + suite = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(suite, dict): + raise ValueError("suite document must be an object") + + corpus_document = json.loads(corpus_path.read_text(encoding="utf-8")) + if not isinstance(corpus_document, dict): + raise ValueError("corpus document must be an object") + + enriched = enrich_suite( + suite, + corpus_document=corpus_document, + corpus_sha256=file_sha256(corpus_path), + ) + enriched["corpus"]["path"] = str(corpus_path) + + tmp = path.with_name(path.name + ".tmp") + tmp.write_text(json.dumps(enriched, indent=2) + "\n", encoding="utf-8") + tmp.replace(path) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "suite", + nargs="?", + type=pathlib.Path, + default=pathlib.Path("data/codeintel-bakeoff/suite.json"), + ) + parser.add_argument( + "--corpus", + type=pathlib.Path, + default=DEFAULT_CORPUS, + help="benchmark corpus whose exact bytes and query definitions should be recorded", + ) + args = parser.parse_args() + if not args.suite.is_file(): + parser.error(f"suite file does not exist: {args.suite}") + if not args.corpus.is_file(): + parser.error(f"corpus file does not exist: {args.corpus}") + write_enriched_suite(args.suite, corpus_path=args.corpus) + print(args.suite) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_codeintel_bakeoff.py b/scripts/test_codeintel_bakeoff.py new file mode 100644 index 00000000..20dcf23f --- /dev/null +++ b/scripts/test_codeintel_bakeoff.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +import importlib.util +import pathlib +import sys +import tempfile +import unittest + +MODULE_PATH = pathlib.Path(__file__).with_name("codeintel_bakeoff.py") +spec = importlib.util.spec_from_file_location("codeintel_bakeoff", MODULE_PATH) +codeintel = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = codeintel +spec.loader.exec_module(codeintel) + + +class CodeIntelBakeoffTests(unittest.TestCase): + def test_extract_known_paths_returns_only_paths_present_in_backend_output(self): + known = [ + "apps/api/services/proposal-service.ts", + "apps/api/proposals/actions.ts", + "README.md", + ] + raw = ( + "Relevant: apps/api/services/proposal-service.ts\n" + "Caller: /tmp/adacavo/apps/api/proposals/actions.ts:42\n" + "Noise: https://example.com/README.md\n" + ) + + self.assertEqual( + codeintel.extract_known_paths(raw, known), + [ + "apps/api/services/proposal-service.ts", + "apps/api/proposals/actions.ts", + "README.md", + ], + ) + + def test_resolve_repo_path_uses_repository_basename(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + repo = root / "nulang-cloud" + repo.mkdir() + self.assertEqual( + codeintel.resolve_repo_path(root, "dporkka/nulang-cloud"), + repo, + ) + + def test_validate_scenario_requires_backend_calls(self): + scenario = { + "id": "s1", + "repository": "owner/repo", + "expected": ["a.go"], + "backends": { + "codebase-memory": [ + {"tool": "search_graph", "arguments": {"semantic_query": ["x"]}} + ], + "gitnexus": [ + {"tool": "query", "arguments": {"search_query": "x"}} + ], + }, + } + codeintel.validate_scenario(scenario) + + broken = dict(scenario) + broken["backends"] = {"codebase-memory": []} + with self.assertRaises(ValueError): + codeintel.validate_scenario(broken) + + def test_content_text_includes_structured_mcp_payload(self): + payload = { + "content": [{"type": "text", "text": "src/main.rs"}], + "structuredContent": {"files": ["src/repl.rs"]}, + } + text = codeintel.mcp_result_text(payload) + self.assertIn("src/main.rs", text) + self.assertIn("src/repl.rs", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_codeintel_corpus.py b/scripts/test_codeintel_corpus.py new file mode 100644 index 00000000..2f2a051f --- /dev/null +++ b/scripts/test_codeintel_corpus.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +import json +import pathlib +import unittest + +ROOT = pathlib.Path(__file__).resolve().parents[1] +CORPUS = ROOT / "benchmarks" / "code-intelligence" / "scenarios.json" + + +class CodeIntelCorpusTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.document = json.loads(CORPUS.read_text(encoding="utf-8")) + cls.scenarios = cls.document["scenarios"] + + def test_scenario_ids_and_expected_paths_are_unique(self): + ids = [scenario["id"] for scenario in self.scenarios] + self.assertEqual(len(ids), len(set(ids)), "scenario IDs must be unique") + + for scenario in self.scenarios: + expected = scenario["expected"] + self.assertTrue(expected, f"{scenario['id']}: expected paths must not be empty") + self.assertEqual( + len(expected), + len(set(expected)), + f"{scenario['id']}: expected paths must be unique", + ) + for path in expected: + self.assertIsInstance(path, str) + self.assertTrue(path.strip(), f"{scenario['id']}: empty expected path") + self.assertFalse(path.startswith("/"), f"{scenario['id']}: expected path must be repo-relative") + + def test_each_scenario_has_exactly_the_two_benchmark_backends(self): + for scenario in self.scenarios: + self.assertEqual( + set(scenario["backends"]), + {"codebase-memory", "gitnexus"}, + f"{scenario['id']}: backend set changed", + ) + for backend, calls in scenario["backends"].items(): + self.assertIsInstance(calls, list) + self.assertTrue(calls, f"{scenario['id']}: {backend} requires at least one call") + + def test_codebase_memory_calls_are_project_scoped_and_match_pinned_contract(self): + allowed_projects = { + "dporkka/adacavo": "adacavo", + "nulang-org/nulang": "nulang", + "dporkka/nulang-cloud": "nulang-cloud", + "dporkka/dev-plane": "dev-plane", + } + for scenario in self.scenarios: + expected_project = allowed_projects[scenario["repository"]] + for call in scenario["backends"]["codebase-memory"]: + tool = call["tool"] + args = call.get("arguments", {}) + self.assertEqual( + args.get("project"), + expected_project, + f"{scenario['id']}: codebase-memory call must target the indexed project explicitly", + ) + + if tool == "search_graph": + self.assertIsInstance(args.get("query"), str) + self.assertTrue(args["query"].strip()) + self.assertIsInstance(args.get("limit"), int) + self.assertGreater(args["limit"], 0) + self.assertNotIn("semantic_query", args) + elif tool == "trace_path": + self.assertIsInstance(args.get("function_name"), str) + self.assertTrue(args["function_name"].strip()) + self.assertIn(args.get("direction"), {"inbound", "outbound", "both"}) + else: + self.fail(f"{scenario['id']}: unsupported codebase-memory benchmark tool {tool!r}") + + def test_gitnexus_calls_match_pinned_contract(self): + for scenario in self.scenarios: + for call in scenario["backends"]["gitnexus"]: + tool = call["tool"] + args = call.get("arguments", {}) + if tool == "query": + self.assertIsInstance(args.get("search_query"), str) + self.assertTrue(args["search_query"].strip()) + self.assertIsInstance(args.get("maxTokens"), int) + self.assertGreater(args["maxTokens"], 0) + elif tool == "impact": + self.assertIsInstance(args.get("target"), str) + self.assertTrue(args["target"].strip()) + self.assertIn(args.get("direction"), {"upstream", "downstream"}) + self.assertIsInstance(args.get("summaryOnly"), bool) + self.assertIsInstance(args.get("maxTokens"), int) + self.assertGreater(args["maxTokens"], 0) + else: + self.fail(f"{scenario['id']}: unsupported GitNexus benchmark tool {tool!r}") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_codeintel_provenance.py b/scripts/test_codeintel_provenance.py new file mode 100644 index 00000000..1a291d90 --- /dev/null +++ b/scripts/test_codeintel_provenance.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.util +import json +import pathlib +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +MODULE_PATH = pathlib.Path(__file__).with_name("codeintel_provenance.py") +spec = importlib.util.spec_from_file_location("codeintel_provenance", MODULE_PATH) +provenance = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = provenance +spec.loader.exec_module(provenance) + + +class CodeIntelProvenanceTests(unittest.TestCase): + def test_repo_revision_returns_exact_head_sha(self): + with tempfile.TemporaryDirectory() as tmp: + repo = pathlib.Path(tmp) / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True) + subprocess.run(["git", "-C", str(repo), "config", "user.email", "test@example.com"], check=True) + subprocess.run(["git", "-C", str(repo), "config", "user.name", "Test"], check=True) + (repo / "a.txt").write_text("hello\n", encoding="utf-8") + subprocess.run(["git", "-C", str(repo), "add", "a.txt"], check=True) + subprocess.run(["git", "-C", str(repo), "commit", "-qm", "initial"], check=True) + expected = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + + self.assertEqual(provenance.repo_revision(repo), expected) + + def test_command_version_returns_first_nonempty_output_line(self): + completed = subprocess.CompletedProcess( + args=["tool", "--version"], + returncode=0, + stdout="\ncodebase-memory-mcp 0.11.0\nextra\n", + stderr="", + ) + with mock.patch.object(provenance.subprocess, "run", return_value=completed): + self.assertEqual( + provenance.command_version(["tool", "--version"]), + "codebase-memory-mcp 0.11.0", + ) + + def test_command_version_records_failure_without_raising(self): + completed = subprocess.CompletedProcess( + args=["tool", "--version"], + returncode=2, + stdout="", + stderr="boom\n", + ) + with mock.patch.object(provenance.subprocess, "run", return_value=completed): + self.assertEqual(provenance.command_version(["tool", "--version"]), "error: boom") + + def test_file_sha256_hashes_exact_bytes(self): + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) / "corpus.json" + payload = b'{"version":8}\n' + path.write_bytes(payload) + self.assertEqual(provenance.file_sha256(path), hashlib.sha256(payload).hexdigest()) + + def test_enrich_suite_records_versions_and_repository_revisions(self): + suite = { + "version": 1, + "scenarios": [ + { + "id": "s1", + "repository": "owner/repo", + "repo_path": "/repos/repo", + "runs": [], + }, + { + "id": "s2", + "repository": "owner/repo", + "repo_path": "/repos/repo", + "runs": [], + }, + ], + } + with mock.patch.object(provenance, "repo_revision", return_value="abc123"), mock.patch.object( + provenance, + "collect_tool_versions", + return_value={"codebase-memory": "0.11.0", "gitnexus": "1.6.12"}, + ): + enriched = provenance.enrich_suite(suite) + + self.assertEqual(enriched["repositories"], {"owner/repo": {"path": "/repos/repo", "revision": "abc123"}}) + self.assertEqual(enriched["tools"]["gitnexus"], "1.6.12") + self.assertEqual(enriched["scenarios"][0]["revision"], "abc123") + self.assertEqual(enriched["scenarios"][1]["revision"], "abc123") + + def test_enrich_suite_attaches_exact_corpus_definition(self): + suite = { + "version": 1, + "scenarios": [ + { + "id": "s1", + "repository": "owner/repo", + "repo_path": "/repos/repo", + "expected": ["src/a.go"], + "runs": [], + } + ], + } + corpus = { + "version": 8, + "scenarios": [ + { + "id": "s1", + "repository": "owner/repo", + "intent": "impact", + "prompt": "Find callers", + "expected": ["src/a.go"], + "backends": { + "codebase-memory": [{"tool": "trace_path", "arguments": {"project": "repo"}}], + "gitnexus": [{"tool": "impact", "arguments": {"target": "A"}}], + }, + } + ], + } + with mock.patch.object(provenance, "repo_revision", return_value="abc123"), mock.patch.object( + provenance, + "collect_tool_versions", + return_value={}, + ): + enriched = provenance.enrich_suite( + suite, + corpus_document=corpus, + corpus_sha256="deadbeef", + ) + + self.assertEqual(enriched["corpus"]["version"], 8) + self.assertEqual(enriched["corpus"]["sha256"], "deadbeef") + scenario = enriched["scenarios"][0] + self.assertEqual(scenario["intent"], "impact") + self.assertEqual(scenario["prompt"], "Find callers") + self.assertEqual(scenario["backend_calls"], corpus["scenarios"][0]["backends"]) + + def test_enrich_suite_rejects_scenario_missing_from_corpus(self): + suite = {"version": 1, "scenarios": [{"id": "missing", "runs": []}]} + corpus = {"version": 1, "scenarios": []} + with self.assertRaises(ValueError): + provenance.enrich_suite(suite, corpus_document=corpus, corpus_sha256="abc") + + def test_enrich_suite_rejects_expected_results_that_differ_from_corpus(self): + suite = { + "version": 1, + "scenarios": [{"id": "s1", "expected": ["wrong.go"], "runs": []}], + } + corpus = { + "version": 1, + "scenarios": [{"id": "s1", "expected": ["right.go"]}], + } + with self.assertRaises(ValueError): + provenance.enrich_suite(suite, corpus_document=corpus, corpus_sha256="abc") + + def test_write_enriched_suite_replaces_file_atomically(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + path = root / "suite.json" + corpus_path = root / "corpus.json" + path.write_text(json.dumps({"version": 1, "scenarios": []}), encoding="utf-8") + corpus_path.write_text(json.dumps({"version": 1, "scenarios": []}), encoding="utf-8") + mocked_enriched = { + "version": 2, + "scenarios": [], + "corpus": {"version": 1, "sha256": "abc"}, + } + with mock.patch.object(provenance, "enrich_suite", return_value=mocked_enriched): + provenance.write_enriched_suite(path, corpus_path=corpus_path) + written = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(written["version"], 2) + self.assertEqual(written["corpus"]["path"], str(corpus_path)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify-codeintel.sh b/scripts/verify-codeintel.sh new file mode 100644 index 00000000..657db5d4 --- /dev/null +++ b/scripts/verify-codeintel.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +printf '[codeintel] python harness tests\n' +python3 scripts/test_codeintel_bakeoff.py + +printf '[codeintel] corpus contract tests\n' +python3 scripts/test_codeintel_corpus.py + +printf '[codeintel] provenance tests\n' +python3 scripts/test_codeintel_provenance.py + +printf '[codeintel] repo-intel Go tests\n' +( + cd packages/repo-intel + go test ./... +) + +if [[ "${CODEINTEL_RUN_LIVE:-0}" == "1" ]]; then + printf '[codeintel] live backend bake-off\n' + set +e + python3 scripts/codeintel_bakeoff.py "$@" + bakeoff_status=$? + set -e + + provenance_status=1 + promotion_status=1 + if [[ -f data/codeintel-bakeoff/suite.json ]]; then + printf '[codeintel] capture provenance\n' + set +e + python3 scripts/codeintel_provenance.py data/codeintel-bakeoff/suite.json + provenance_status=$? + set -e + + printf '[codeintel] promotion verdict\n' + set +e + ( + cd packages/repo-intel + go run ./cmd/codeintel-promote ../../data/codeintel-bakeoff/suite.json + ) + promotion_status=$? + set -e + else + printf '[codeintel] provenance/promotion verdict unavailable: suite.json was not produced\n' >&2 + fi + + if [[ $bakeoff_status -ne 0 || $provenance_status -ne 0 || $promotion_status -ne 0 ]]; then + exit 1 + fi +else + printf '[codeintel] live bake-off skipped (set CODEINTEL_RUN_LIVE=1 to enable)\n' +fi diff --git a/tools/codeintel-benchmark/Dockerfile b/tools/codeintel-benchmark/Dockerfile new file mode 100644 index 00000000..02bf5310 --- /dev/null +++ b/tools/codeintel-benchmark/Dockerfile @@ -0,0 +1,45 @@ +FROM golang:1.26.8-bookworm AS go-toolchain + +FROM node:24-bookworm + +ARG CODEBASE_MEMORY_VERSION=0.11.0 +ARG GITNEXUS_VERSION=1.6.12 + +COPY --from=go-toolchain /usr/local/go /usr/local/go + +ENV PATH="/usr/local/go/bin:${PATH}" \ + CI=1 \ + HOME=/var/lib/codeintel \ + CBM_CACHE_DIR=/var/lib/codeintel/cbm \ + GITNEXUS_STORAGE_ROOT=/var/lib/codeintel/gitnexus \ + GITNEXUS_NO_UPDATE_NOTIFIER=1 \ + GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + git \ + python3 \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p \ + /var/lib/codeintel/cbm \ + /var/lib/codeintel/gitnexus \ + /workspace/dev-plane \ + /repos \ + && chmod 700 /var/lib/codeintel /var/lib/codeintel/cbm /var/lib/codeintel/gitnexus + +RUN npm install --global \ + "codebase-memory-mcp@${CODEBASE_MEMORY_VERSION}" \ + "gitnexus@${GITNEXUS_VERSION}" + +RUN codebase-memory-mcp --version \ + && gitnexus --version \ + && go version \ + && node --version \ + && python3 --version + +WORKDIR /workspace/dev-plane + +CMD ["bash", "scripts/verify-codeintel.sh"]