feat: add local model evaluation harness

README.md

@@ -42,6 +42,7 @@ Routing Logic
 | `locode install [model]` | Pull a specific Ollama model |
 | `locode update` | Update locode to the latest version |
 | `locode benchmark` | Compare token cost across routing modes |
+| `locode eval-local-models` | Compare local models on tool-calling reliability |
 
 ### Flags
 
@@ -58,12 +59,15 @@ If no `ANTHROPIC_API_KEY` is set, locode automatically runs in local-only mode.
 
 Edit `locode.yaml` for routing rules, models, and thresholds:
 
-- `local_llm.model` — Ollama model (default: `qwen3:8b`)
+- `local_llm.model` — Ollama model (default: `llama3.1:8b`)
 - `routing.rules` — regex patterns that route tasks to local or Claude
 - `routing.escalation_threshold` — confidence below this escalates to Claude
 
 Type `stats` in the REPL to see token usage and estimated savings.
 
+Current default: `llama3.1:8b` is the conservative tool-calling baseline.
+Recommended upgrade to evaluate locally: `gemma4:9b`.
+
 ## Telemetry (Opt-in)
 
 Telemetry is **off by default**. To opt in, export in your shell profile:

docs/gemma4-local-model.md

@@ -9,13 +9,19 @@ Gemma 4 works with the existing Ollama integration — no code changes required
 - `thinking` config flag to control Ollama's `think` parameter (on by default it was silently unset, causing thinking models to burn tokens on every tool round)
 - `num_ctx` raised from 4096 → 8192 in the default config (Gemma 4's native window is 128k; 4096 was leaving quality on the table)
 
+Current product stance:
+
+- keep `llama3.1:8b` as the current default because it is the safer validated tool-calling baseline
+- treat `gemma4:9b` as the recommended upgrade to evaluate locally
+- only flip the hard default after the dedicated tool-calling eval passes cleanly
+
 ## Model Options
 
 | Model | Ollama tag | RAM (4-bit) | Notes |
 |---|---|---|---|
 | Gemma 4 E2B | `gemma4:2b` | ~3 GB | Fast; may struggle with multi-step tool use |
 | Gemma 4 E4B | `gemma4:4b` | ~4 GB | Good for simple routing/search tasks |
-| Gemma 4 9B | `gemma4:9b` | ~7 GB | **Recommended** — best quality/speed trade-off |
+| Gemma 4 9B | `gemma4:9b` | ~7 GB | Recommended upgrade candidate — best quality/speed trade-off |
 | Gemma 4 27B | `gemma4:27b` | ~18 GB | Near-Claude quality locally; supports thinking mode |
 
 Unsloth GGUF variants are available on Hugging Face (`unsloth/gemma-4-31B-it-GGUF`) and can be loaded into Ollama via a custom Modelfile pointing to the `.gguf` file.
@@ -49,7 +55,7 @@ local_llm:
 
 Ollama's `think` parameter enables chain-of-thought tokens that are hidden from the final response. Before this change the main tool loop omitted `think` entirely — Ollama interprets that as the model's default, which for thinking-capable models means reasoning tokens are generated (and counted) on every round. Setting `think: false` explicitly suppresses them unless opted in via `thinking: true` in config.
 
-## Comparison vs llama3.1:8b (Previous Default)
+## Comparison vs llama3.1:8b (Current Default)
 
 | | llama3.1:8b | gemma4:9b |
 |---|---|---|

docs/local-model-tool-calling-eval.md

@@ -0,0 +1,98 @@
+# Local Model Tool-Calling Evaluation
+
+This repo now includes a fixed evaluation harness for comparing local Ollama models on the `locode` tool loop.
+
+## Why this exists
+
+Token cost and raw benchmark quality are not enough for `locode`.
+The local model has to:
+
+- choose valid tools
+- stop after getting useful results
+- recover from blocked or failing tool calls
+- produce a final plain-text answer instead of looping
+
+That makes tool-calling reliability the main gating metric for changing the default local model.
+
+## Current product stance
+
+- Current default: `llama3.1:8b`
+- Recommended upgrade to evaluate: `gemma4:9b`
+- Do not flip the hard default to Gemma 4 until it performs well on the task suite below
+
+## Command
+
+```bash
+locode eval-local-models
+```
+
+This runs the default comparison:
+
+- `llama3.1:8b`
+- `gemma4:9b`
+
+and writes a JSON report to:
+
+```bash
+.locode/evals/local-model-eval.json
+```
+
+## Custom variants
+
+Bare model names:
+
+```bash
+locode eval-local-models \
+  --variant llama3.1:8b \
+  --variant gemma4:9b
+```
+
+Structured variants:
+
+```bash
+locode eval-local-models \
+  --variant "label=llama-baseline,model=llama3.1:8b,num_ctx=8192" \
+  --variant "label=gemma-27b-thinking,model=gemma4:27b,thinking=true,num_ctx=16384"
+```
+
+## Methodology
+
+The harness uses the real `LocalAgent` tool loop with a read-only tool registry:
+
+- `read_file`
+- `run_command`
+- `git_query`
+- `list_files`
+- `search_code`
+
+Each variant is run against the same fixed inspect-oriented task suite:
+
+1. Read package scripts
+2. Find `executeParallel`
+3. Use git to check whether `src/cli/setup.ts` is tracked
+4. Recover from a blocked or unavailable command (`tree`)
+5. Find the Claude local-fallback threshold wiring
+
+Each task is scored on:
+
+- content correctness against simple regex expectations
+- whether the model used at least one appropriate tool
+- whether it hallucinated invalid tools
+- whether it repeated the same failing call
+- whether it returned a non-empty final answer
+
+## General practice for this kind of eval
+
+- Use a fixed task suite, not ad hoc prompts
+- Keep hardware and Ollama settings stable
+- Run multiple trials per model
+- Score both tool-call behavior and final answer quality
+- Separate "better model" from "safer default"
+
+For `locode`, the safer default should optimize for:
+
+- valid tool selection
+- low loop rate
+- reliable final answers
+
+not just general code quality.

locode.yaml

@@ -1,11 +1,11 @@
 local_llm:
   provider: ollama
   # Recommended models (install via `ollama pull <model>`):
-  #   gemma4:2b          — fast, ~3 GB RAM, good for simple tasks
-  #   gemma4:9b          — balanced quality/speed, ~7 GB RAM  (recommended)
-  #   gemma4:27b         — best quality, ~18 GB RAM
-  #   llama3.1:8b        — solid general-purpose alternative
-  model: qwen3:8b
+  #   llama3.1:8b        — safest current baseline for tool calling (default)
+  #   gemma4:9b          — recommended upgrade to evaluate, ~7 GB RAM
+  #   gemma4:27b         — best local quality, ~18 GB RAM
+  #   gemma4:2b          — lightweight option for simpler tasks
+  model: llama3.1:8b
   base_url: http://localhost:11434
   # Set thinking: true for hybrid-thinking variants (e.g. gemma4:27b).
   # This enables extended reasoning tokens; output <think> blocks are

src/cli/eval-local-models.test.ts

@@ -0,0 +1,90 @@
+import { describe, expect, it } from 'vitest'
+import { ToolRegistry } from '../tools/registry'
+import { assessTaskRun, getDefaultEvalVariants, getEvalTaskIds, parseVariantSpec, resolveEvalVariants } from './eval-local-models'
+import { readFileDefinition } from '../tools/definitions/read-file'
+import type { AgentResult } from '../agents/local'
+
+describe('parseVariantSpec', () => {
+  it('parses a bare model name into a default variant', () => {
+    expect(parseVariantSpec('gemma4:9b')).toEqual({
+      label: 'gemma4-9b',
+      model: 'gemma4:9b',
+      thinking: false,
+    })
+  })
+
+  it('parses key-value variant specs', () => {
+    expect(parseVariantSpec('label=gemma-thinking,model=gemma4:27b,thinking=true,num_ctx=16384')).toEqual({
+      label: 'gemma-thinking',
+      model: 'gemma4:27b',
+      thinking: true,
+      numCtx: 16384,
+    })
+  })
+
+  it('rejects unknown keys', () => {
+    expect(() => parseVariantSpec('model=gemma4:9b,foo=bar')).toThrow('unknown variant key')
+  })
+})
+
+describe('resolveEvalVariants', () => {
+  it('returns the default comparison pair when no variants are provided', () => {
+    expect(getDefaultEvalVariants().map(variant => variant.model)).toEqual(['llama3.1:8b', 'gemma4:9b'])
+    expect(resolveEvalVariants(undefined).map(variant => variant.model)).toEqual(['llama3.1:8b', 'gemma4:9b'])
+  })
+})
+
+describe('getEvalTaskIds', () => {
+  it('returns the available task ids for CLI discovery', () => {
+    expect(getEvalTaskIds()).toContain('read-package-scripts')
+    expect(getEvalTaskIds()).toContain('blocked-command-recovery')
+  })
+})
+
+describe('assessTaskRun', () => {
+  it('passes when content and tool requirements are satisfied', () => {
+    const registry = new ToolRegistry()
+    registry.register(readFileDefinition)
+    const task = {
+      id: 'read-package-scripts',
+      prompt: 'Read package.json',
+      requiredAnyTools: ['read_file'],
+      contentChecks: [/\bbuild\b/i, /\btest\b/i],
+    }
+    const result: AgentResult = {
+      content: 'The build script runs tsc and the test script runs vitest.',
+      summary: 'summary',
+      inputTokens: 10,
+      outputTokens: 5,
+      toolCalls: [{ tool: 'read_file', args: { path: 'package.json' }, result: { success: true, output: '{}' } }],
+    }
+
+    expect(assessTaskRun(task, result, registry).passed).toBe(true)
+  })
+
+  it('fails when the model repeats the same failing tool call', () => {
+    const registry = new ToolRegistry()
+    registry.register(readFileDefinition)
+    const task = {
+      id: 'blocked-command-recovery',
+      prompt: 'Try tree first',
+      requiredAnyTools: ['run_command'],
+      contentChecks: [/\bsrc\b/i],
+      maxRepeatedFailedCallStreak: 1,
+    }
+    const result: AgentResult = {
+      content: 'src is present.',
+      summary: 'summary',
+      inputTokens: 10,
+      outputTokens: 5,
+      toolCalls: [
+        { tool: 'run_command', args: { command: 'tree' }, result: { success: false, output: '' } },
+        { tool: 'run_command', args: { command: 'tree' }, result: { success: false, output: '' } },
+      ],
+    }
+
+    const assessment = assessTaskRun(task, result, registry)
+    expect(assessment.passed).toBe(false)
+    expect(assessment.repeatedFailedCallStreak).toBe(2)
+  })
+})