diff --git a/apply/anthropic_toolrole_test.go b/apply/anthropic_toolrole_test.go new file mode 100644 index 00000000..95aaaff7 --- /dev/null +++ b/apply/anthropic_toolrole_test.go @@ -0,0 +1,122 @@ +package apply_test + +import ( + "context" + "encoding/json" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/tidwall/gjson" + + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/store" +) + +// A count-changing component (summarize) TOGETHER WITH one that rewrites tool text (format) on +// ANTHROPIC wire bytes. This combination is what live traffic always runs, and it exposed a defect +// that neither component shows alone: +// +// 400 messages: Unexpected role "tool" +// +// A synthetic role=tool message is an internal representation only; Anthropic has no such role. The +// rebuild emits a message verbatim only when it byte-matches its pre-pipeline form, so once format +// rewrote a tool message's text it fell through to a fresh marshal and leaked role="tool" onto the +// wire. Measured live at 12 of 39 runs, and it only became visible after the tool-call-id fix stopped +// these messages from being deleted outright -- one defect had been masking the other. +func TestAnthropicCountChangeNeverLeaksToolRole(t *testing.T) { + rows := make([]map[string]any, 0, 40) + for i := 0; i < 40; i++ { + rows = append(rows, map[string]any{ + "id": 100000 + i, "path": "src/auth.py", "sym": "TOKEN_GRACE_41ab", + "status": "ok", "note": "row that format can restructure", + }) + } + bigB, _ := json.MarshalIndent(rows, "", " ") + big := string(bigB) + msgs := []map[string]any{{"role": "user", "content": "start"}} + for i := 0; i < 8; i++ { + a, b := "pa_"+string(rune('a'+i)), "pb_"+string(rune('a'+i)) + msgs = append(msgs, + map[string]any{"role": "assistant", "content": []map[string]any{ + {"type": "text", "text": "calling two"}, + {"type": "tool_use", "id": a, "name": "Read", "input": map[string]any{}}, + {"type": "tool_use", "id": b, "name": "Read", "input": map[string]any{}}, + }}, + map[string]any{"role": "user", "content": []map[string]any{ + {"type": "tool_result", "tool_use_id": a, "content": big}, + {"type": "tool_result", "tool_use_id": b, "content": big}, + }}) + } + msgs = append(msgs, map[string]any{"role": "user", "content": "final question"}) + body, _ := json.Marshal(map[string]any{"model": "claude-x", "messages": msgs}) + + rewroteAny := false + for _, keep := range []int{1, 2, 3, 4} { + cfg := pipe(t, "pipeline: [format, summarize]\ncomponents:\n summarize: {keep_last: "+ + string(rune('0'+keep))+", start_from_message: 0, min_tokens: 1}\n") + p, _ := cfg.Build(nil) + out, changed := apply.BodyWithModel(context.Background(), p, + store.NewMemory(store.Options{}), bschemas.Anthropic, body, "", false, + components.ModelSpec{Incoming: stubModel{resp: "essential facts"}}) + if !changed { + continue + } + // PRECONDITION. If no tool_result text was rewritten, the failing path is never entered and + // a pass proves nothing -- the first version of this test passed with the fix removed for + // exactly that reason. Fail loudly rather than pass vacuously. + rewrote := false + gjson.GetBytes(out, "messages").ForEach(func(_, m gjson.Result) bool { + m.Get("content").ForEach(func(_, blk gjson.Result) bool { + if blk.Get("type").String() == "tool_result" && + blk.Get("content").String() != big && blk.Get("content").Exists() { + rewrote = true + } + return true + }) + return true + }) + if rewrote { + rewroteAny = true + } + arr := gjson.GetBytes(out, "messages").Array() + for i, m := range arr { + if r := m.Get("role").String(); r != "user" && r != "assistant" && r != "system" { + t.Fatalf("keep_last=%d: wire message %d has role %q -- Anthropic rejects this", + keep, i, r) + } + var uses []string + m.Get("content").ForEach(func(_, blk gjson.Result) bool { + if blk.Get("type").String() == "tool_use" { + uses = append(uses, blk.Get("id").String()) + } + return true + }) + if len(uses) == 0 { + continue + } + answered := map[string]bool{} + if i+1 < len(arr) { + arr[i+1].Get("content").ForEach(func(_, blk gjson.Result) bool { + if blk.Get("type").String() == "tool_result" { + answered[blk.Get("tool_use_id").String()] = true + } + return true + }) + } + for _, u := range uses { + if !answered[u] { + t.Fatalf("keep_last=%d: wire message %d declares %q unanswered", keep, i, u) + } + } + } + } + // Asserted ONCE, across all keep_last values: at least one case must have carried a rewritten + // tool_result through a count change, or the role-leak path was never entered and a pass proves + // nothing. The first version of this test passed with the fix removed for exactly that reason, + // which is the same vacuous-check trap recorded twice in docs/experiments/loca/. + if !rewroteAny { + t.Fatal("no case produced a rewritten tool_result surviving a count change, so this test " + + "cannot detect the defect it exists for; fix the fixture") + } +} diff --git a/apply/apply.go b/apply/apply.go index 515c4823..74951b6c 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -315,6 +315,7 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o // provider's cap of four counts them all (issue #32, defect 2). ExistingBreakpoints: wireBreakpoints(body), Mode: mode, + PrefixAsk: o.PrefixAsk, } tr.Session, tr.CacheAware, tr.MaxCachedIdx, tr.Messages = sessionID, cacheAware, maxCachedIdx, len(norm) // The eligible (attempted) denominator: what age/supersession offloaders were @@ -343,7 +344,37 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o // retained message's ORIGINAL raw bytes (byte-lossless, incl. Anthropic // tool_result) and marshaling only genuinely new messages (the summary). if len(chat.Input) != len(norm) { - nb, ok := rebuildCountChanged(body, msgsRaw.Array(), normPre, slots, chat.Input) + // Write Anthropic tool_result text rewrites into the BODY before rebuilding. + // + // A synthetic role=tool message cannot be marshaled into an Anthropic request — the + // provider answers `messages: Unexpected role "tool"`. The rebuild below emits a message + // verbatim only when it byte-matches its pre-pipeline form, so a tool message whose text a + // component rewrote (format rewrites hundreds per run) would fall through to a fresh + // marshal and produce exactly that rejection. Measured live: 12 of 39 runs, immediately + // after the tool-call-id fix stopped these messages from being silently deleted — one + // defect had been masking the other. + // + // The write-back is the same mechanism the equal-count path uses (sjson at the block's + // exact path), applied here first so the rebuild only ever has to decide which messages + // to keep, never how to serialize one. + pre := body + for i := range norm { + if i >= len(slots) || slots[i].kind != anthropicToolText { + continue + } + id := norm[i].ChatToolMessage + if id == nil || id.ToolCallID == nil { + continue + } + newText, found := toolTextByID(chat.Input, *id.ToolCallID) + if !found || newText == slots[i].preText { + continue + } + if nb, err := sjson.SetBytes(pre, slots[i].path, newText); err == nil { + pre = nb + } + } + nb, ok := rebuildCountChanged(pre, gjson.GetBytes(pre, "messages").Array(), normPre, slots, norm, chat.Input) if !ok && systemSplit { res.Body, res.Changed = body, true // keep the split even when the rebuild declined return res @@ -670,6 +701,56 @@ func normalize(provider bschemas.ModelProvider, arr []gjson.Result) (norm []bsch if err := json.Unmarshal([]byte(m.Raw), &cm); err != nil { continue // unparseable message — leave it in the body untouched } + // Recover ANTHROPIC tool calls, which bifrost's schema cannot represent. + // + // An Anthropic assistant turn carries its calls as `tool_use` CONTENT BLOCKS, and + // bschemas.ChatContentBlock has no such type — its Type enum is text/image_url/ + // input_audio/file/refusal — so the ids are simply absent from the unmarshaled message. + // Anything that reasons about tool pairing therefore saw an assistant message with ZERO + // calls on every Anthropic request, which produced two defects from one cause: + // + // - dropOrphanedToolResults builds its answerable set from ToolCalls alone, so on + // Anthropic traffic the set was always empty, EVERY tool_result looked orphaned, and + // the "repair" deleted all of them. The provider then rejected the request for the + // unanswered calls that remained. Measured live: 28 of 75 runs + // (docs/experiments/loca/iter011/results.md). + // - schema.ValidateShape was blind to the same ids, so it could not see the breakage — + // which is why a test asserting all 11 presets emit shape-valid requests stayed green + // while live Anthropic traffic failed. + // + // Populating ToolCalls here fixes the whole class at the point where the dialect is still + // known, rather than teaching each consumer about Anthropic. The ids are what pairing + // needs; arguments are carried for completeness. Note this deliberately makes the + // `lossless` check below FALSE for such messages, which is honest — bifrost genuinely + // cannot round-trip them — and it only makes the write-back guard more conservative. + if provider == bschemas.Anthropic && cm.Role == bschemas.ChatMessageRoleAssistant && + m.Get("content").IsArray() { + var calls []bschemas.ChatAssistantMessageToolCall + for _, blk := range m.Get("content").Array() { + if blk.Get("type").String() != "tool_use" { + continue + } + id, name := blk.Get("id").String(), blk.Get("name").String() + if id == "" { + continue + } + idc, namec, args := id, name, blk.Get("input").Raw + calls = append(calls, bschemas.ChatAssistantMessageToolCall{ + ID: &idc, + Function: bschemas.ChatAssistantMessageToolCallFunction{ + Name: &namec, Arguments: args, + }, + }) + } + if len(calls) > 0 { + if cm.ChatAssistantMessage == nil { + cm.ChatAssistantMessage = &bschemas.ChatAssistantMessage{} + } + if len(cm.ChatAssistantMessage.ToolCalls) == 0 { + cm.ChatAssistantMessage.ToolCalls = calls + } + } + } preMarshal, _ := json.Marshal(cm) norm = append(norm, cm) slots = append(slots, slot{ @@ -713,7 +794,18 @@ func jsonEqual(a, b []byte) bool { // normalized message (a survivor) is emitted as its ORIGINAL body raw bytes // (byte-lossless); genuinely new messages (the summary) are marshaled fresh. // Fail-open (returns body,false) if any survivor can't be mapped to the body. -func rebuildCountChanged(body []byte, orig []gjson.Result, normPre [][]byte, slots []slot, out []bschemas.ChatMessage) ([]byte, bool) { +// toolTextByID finds the post-pipeline text of the synthetic tool message answering id. +func toolTextByID(msgs []bschemas.ChatMessage, id string) (string, bool) { + for i := range msgs { + tm := msgs[i].ChatToolMessage + if tm != nil && tm.ToolCallID != nil && *tm.ToolCallID == id { + return schema.MessageText(msgs[i]), true + } + } + return "", false +} + +func rebuildCountChanged(body []byte, orig []gjson.Result, normPre [][]byte, slots []slot, norm, out []bschemas.ChatMessage) ([]byte, bool) { // The rebuild emits ONLY slot-mapped messages, so a body message normalize skipped // (unparseable — it has no slot) would be silently DELETED from the forwarded // request. Deleting a message is an ALTERED request, not a fail-open one, so decline @@ -740,10 +832,23 @@ func rebuildCountChanged(body []byte, orig []gjson.Result, normPre [][]byte, slo return body, false } matched := -1 - for k := range normPre { + // A synthetic Anthropic tool message is matched by tool_call_id, NOT by bytes: its text may + // have been rewritten by a component, and that rewrite has already been written into the + // body above. Byte-matching it would fail and send it down the fresh-marshal path, which + // emits `role: "tool"` and is rejected outright by Anthropic. + if tm := out[i].ChatToolMessage; tm != nil && tm.ToolCallID != nil { + for k := range norm { + ntm := norm[k].ChatToolMessage + if !used[k] && ntm != nil && ntm.ToolCallID != nil && + *ntm.ToolCallID == *tm.ToolCallID { + matched = k + break + } + } + } + for k := 0; matched < 0 && k < len(normPre); k++ { if !used[k] && bytes.Equal(mb, normPre[k]) { matched = k - break } } if matched < 0 { diff --git a/apply/ctxmgmt_preserved_test.go b/apply/ctxmgmt_preserved_test.go new file mode 100644 index 00000000..91b0c105 --- /dev/null +++ b/apply/ctxmgmt_preserved_test.go @@ -0,0 +1,66 @@ +package apply_test + +import ( + "context" + "encoding/json" + "strings" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/tidwall/gjson" + + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/store" +) + +// LOCA's context clearing is Anthropic's SERVER-SIDE context_management feature, passed as a request +// parameter (clear_tool_uses_20250919 with trigger input_tokens). If a count-changing component +// dropped that parameter while rewriting the body, the provider's clearing would be silently +// disabled -- and since the proxy also shrinks the request below the trigger, nothing would keep an +// oversized transcript legal. Five runs in the 128k arms died on "prompt is too long" at up to 6M +// tokens, so whether this survives is a product question, not a formality. +func TestContextManagementParamSurvivesCountChange(t *testing.T) { + big := strings.Repeat("verbose tool output line\n", 200) + msgs := []map[string]any{{"role": "user", "content": "go"}} + for i := 0; i < 6; i++ { + id := "t" + string(rune('a'+i)) + msgs = append(msgs, + map[string]any{"role": "assistant", "content": []map[string]any{ + {"type": "text", "text": "calling"}, + {"type": "tool_use", "id": id, "name": "Read", "input": map[string]any{}}}}, + map[string]any{"role": "user", "content": []map[string]any{ + {"type": "tool_result", "tool_use_id": id, "content": big}}}) + } + msgs = append(msgs, map[string]any{"role": "user", "content": "final"}) + body, _ := json.Marshal(map[string]any{ + "model": "claude-x", + "messages": msgs, + "context_management": map[string]any{"edits": []map[string]any{{ + "type": "clear_tool_uses_20250919", + "trigger": map[string]any{"type": "input_tokens", "value": 128000}, + "keep": map[string]any{"type": "tool_uses", "value": 3}, + "clear_at_least": map[string]any{"type": "input_tokens", "value": 20000}, + }}}, + }) + cfg := pipe(t, "pipeline: [format, summarize]\ncomponents:\n summarize: {keep_last: 2, start_from_message: 0, min_tokens: 1}\n") + p, _ := cfg.Build(nil) + out, changed := apply.BodyWithModel(context.Background(), p, store.NewMemory(store.Options{}), + bschemas.Anthropic, body, "", false, + components.ModelSpec{Incoming: stubModel{resp: "essential facts"}}) + if !changed { + t.Skip("summarize did not act") + } + got := gjson.GetBytes(out, "context_management") + if !got.Exists() { + t.Fatal("context_management was DROPPED by the rewrite: the provider's own clearing would be " + + "silently disabled while the proxy also keeps requests under its trigger") + } + want := gjson.GetBytes(body, "context_management") + if got.Raw != want.Raw { + t.Errorf("context_management was altered:\n got %s\nwant %s", got.Raw, want.Raw) + } + if v := got.Get("edits.0.trigger.value").Int(); v != 128000 { + t.Errorf("trigger value changed to %d", v) + } +} diff --git a/apply/normalize_toolcalls_test.go b/apply/normalize_toolcalls_test.go new file mode 100644 index 00000000..ae0e3c6c --- /dev/null +++ b/apply/normalize_toolcalls_test.go @@ -0,0 +1,57 @@ +package apply + +import ( + "encoding/json" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/tidwall/gjson" +) + +// Does normalize populate ChatAssistantMessage.ToolCalls for an ANTHROPIC assistant message whose +// tool calls arrive as CONTENT BLOCKS? dropOrphanedToolResults builds its "answerable" +// set exclusively from that field, so if it is empty on Anthropic traffic then every tool_result +// looks orphaned and the repair DELETES them all -- which is exactly what the instrumented rebuild +// showed (out contained no tool messages at all). +// +// The offload-level tests passed because they hand-build ChatMessages with ToolCalls populated, i.e. +// the OpenAI-shaped representation. That is why unit tests were green while live Anthropic traffic +// failed 28/75. +func TestNormalizePopulatesToolCallsForAnthropicBlocks(t *testing.T) { + body, _ := json.Marshal(map[string]any{ + "model": "claude-x", + "messages": []map[string]any{ + {"role": "user", "content": "go"}, + {"role": "assistant", "content": []map[string]any{ + {"type": "text", "text": "calling"}, + {"type": "tool_use", "id": "t1", "name": "Read", "input": map[string]any{}}, + }}, + {"role": "user", "content": []map[string]any{ + {"type": "tool_result", "tool_use_id": "t1", "content": "the output"}, + }}, + }, + }) + arr := gjson.GetBytes(body, "messages").Array() + norm, _ := normalize(bschemas.Anthropic, arr) + for i, m := range norm { + n := 0 + if m.ChatAssistantMessage != nil { + n = len(m.ChatAssistantMessage.ToolCalls) + } + id := "" + if m.ChatToolMessage != nil && m.ChatToolMessage.ToolCallID != nil { + id = *m.ChatToolMessage.ToolCallID + } + t.Logf("norm[%d] role=%-9s ToolCalls=%d answers=%q", i, m.Role, n, id) + } + var calls int + for _, m := range norm { + if m.ChatAssistantMessage != nil { + calls += len(m.ChatAssistantMessage.ToolCalls) + } + } + if calls == 0 { + t.Errorf("no assistant ToolCalls recovered from Anthropic tool_use blocks: "+ + "dropOrphanedToolResults will treat every tool_result as an orphan and delete it") + } +} diff --git a/apply/opts.go b/apply/opts.go index cdb14462..a04c14f8 100644 --- a/apply/opts.go +++ b/apply/opts.go @@ -37,6 +37,10 @@ type Opts struct { // Mode is the operating mode. Empty means components.ModeSync, so a caller that does // not know about modes gets exactly today's behavior. Mode components.Mode + // PrefixAsk, when set, lets a component put a question to the request's own model with the + // previous turn's SENT body as the cached prefix. See components.PrefixAsker. nil => components + // fall back to Model.Complete, which is today's behaviour for every caller that does not set it. + PrefixAsk components.PrefixAsker // Tracker, when set, owns the per-session cached-prefix boundary. Supplying it also // removes the concurrent-turn race in the legacy read-then-deferred-write of prevLen. // nil => the legacy store-backed path, unchanged for library callers and /compact. diff --git a/apply/parallel_wire_test.go b/apply/parallel_wire_test.go new file mode 100644 index 00000000..5abb4b84 --- /dev/null +++ b/apply/parallel_wire_test.go @@ -0,0 +1,108 @@ +package apply_test + +import ( + "context" + "encoding/json" + "strings" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/tidwall/gjson" + + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/store" +) + +// THE SHAPE LIVE TRAFFIC CARRIES. In Anthropic's wire format a PARALLEL tool call is one assistant +// message with several tool_use blocks, answered by ONE user message holding several tool_result +// blocks. apply normalizes that user message into several synthetic role=tool messages, so N +// normalized messages share ONE body index -- the case the guard in apply exists for. +// +// Live LOCA runs failed 28 of 75 with the provider reporting an unanswered tool_use, and a capture +// hop between the proxy and the gateway showed a trailing assistant(tool_use, tool_use) with nothing +// after it. summarize is clean on the bifrost message list at every keep_last from 1 to 5 (see +// components/offload/summarize_parallel_test.go), so the wire rebuild is the remaining suspect. +// +// CONFIRMED: this test FAILS at keep_last=4 while the message-list test passes with the IDENTICAL +// summarize config, which localises the defect to rebuildCountChanged rather than to summarize. The +// emitted wire is [user, summary, assistant(tool_use pa_a, tool_use pb_a), user("final question")] -- +// the body message holding BOTH tool_results is dropped entirely, so the assistant's parallel call +// goes unanswered and the provider rejects the request. +// +// Not yet fixed, deliberately. The suspect area is the slot/body-index mapping that lets several +// normalized messages share one body index (the `emitted` guard), and a careless change there risks +// the byte-losslessness guarantee this package exists to provide. This component family has produced +// four shape defects already; a blind fifth fix is not the way to close it. The reproduction is the +// deliverable. +func TestSummarizeKeepsParallelToolResultsOnTheWire(t *testing.T) { + big := strings.Repeat("verbose parallel tool output\n", 60) + msgs := []map[string]any{ + {"role": "user", "content": "start the task"}, + } + for i := 0; i < 8; i++ { + a, b := "pa_"+string(rune('a'+i)), "pb_"+string(rune('a'+i)) + msgs = append(msgs, + map[string]any{"role": "assistant", "content": []map[string]any{ + {"type": "text", "text": "calling two"}, + {"type": "tool_use", "id": a, "name": "Read", "input": map[string]any{}}, + {"type": "tool_use", "id": b, "name": "Read", "input": map[string]any{}}, + }}, + // BOTH results in ONE user message -- Anthropic's requirement for a parallel call. + map[string]any{"role": "user", "content": []map[string]any{ + {"type": "tool_result", "tool_use_id": a, "content": big}, + {"type": "tool_result", "tool_use_id": b, "content": big}, + }}, + ) + } + msgs = append(msgs, map[string]any{"role": "user", "content": "final question"}) + body, _ := json.Marshal(map[string]any{"model": "claude-x", "messages": msgs}) + + for _, keep := range []int{1, 2, 3, 4} { + t.Run("keep_last", func(t *testing.T) { + cfg := pipe(t, "pipeline: [summarize]\ncomponents:\n summarize: {keep_last: "+ + string(rune('0'+keep))+", start_from_message: 0, min_tokens: 1}\n") + p, _ := cfg.Build(nil) + out, changed := apply.BodyWithModel(context.Background(), p, + store.NewMemory(store.Options{}), bschemas.Anthropic, body, "", false, + components.ModelSpec{Incoming: stubModel{resp: "essential facts"}}) + if !changed { + t.Skip("summarize did not act") + } + // Walk the EMITTED wire messages and check every tool_use is answered in the next one. + arr := gjson.GetBytes(out, "messages").Array() + for i, m := range arr { + var uses []string + m.Get("content").ForEach(func(_, blk gjson.Result) bool { + if blk.Get("type").String() == "tool_use" { + uses = append(uses, blk.Get("id").String()) + } + return true + }) + if len(uses) == 0 { + continue + } + answered := map[string]bool{} + if i+1 < len(arr) { + arr[i+1].Get("content").ForEach(func(_, blk gjson.Result) bool { + if blk.Get("type").String() == "tool_result" { + answered[blk.Get("tool_use_id").String()] = true + } + return true + }) + } + for _, u := range uses { + if !answered[u] { + t.Errorf("keep_last=%d: wire message %d declares tool_use %q with no "+ + "tool_result immediately after -- the provider rejects this", keep, i, u) + for k, mm := range arr { + t.Errorf(" [%d] role=%s content_head=%.90s", k, + mm.Get("role").String(), mm.Get("content").Raw) + } + return + } + } + } + }) + } +} diff --git a/components/all/more_test.go b/components/all/more_test.go index b9e3c77f..24b18907 100644 --- a/components/all/more_test.go +++ b/components/all/more_test.go @@ -227,7 +227,7 @@ func TestKeptVerbatimNotRecompacted(t *testing.T) { } st := store.NewMemory(store.Options{}) big := strings.Repeat("recoverable content the agent just expanded and needs verbatim\n", 30) - offload.MarkKeptVerbatim(st, big) // simulate the proxy's expand loop + offload.MarkKeptVerbatim(st, "s", big) // simulate the proxy's expand loop (session "s") req := &schemas.BifrostChatRequest{Input: []schemas.ChatMessage{toolMsg(big), toolMsg("newest tiny output")}} c := &components.Ctx{Ctx: context.Background(), Session: "s", Store: st} diff --git a/components/all/schema_shape_test.go b/components/all/schema_shape_test.go new file mode 100644 index 00000000..4c6b60e1 --- /dev/null +++ b/components/all/schema_shape_test.go @@ -0,0 +1,105 @@ +package all_test + +import ( + "context" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/context-guru/components" + _ "github.com/rossoctl/context-guru/components/all" + "github.com/rossoctl/context-guru/config" + "github.com/rossoctl/context-guru/schema" + "github.com/rossoctl/context-guru/store" +) + +// EVERY SHIPPED PRESET MUST EMIT A SENDABLE REQUEST. +// +// This is the guard that was missing. Three message-shape violations shipped in `summarize` +// and were found only by sending live traffic to a real API, one at a time, each masked by +// the previous one: +// +// 400 messages.1: role 'system' must precede an 'assistant' message or end the array +// 400 messages.N.content.M: unexpected `tool_use_id` found in `tool_result` blocks +// 400 messages.N: `tool_use` ids were found without `tool_result` blocks immediately after +// +// They were invisible to this repo's two verification methods. No test asserted shape, and +// every offline measurement replayed through `/compact`, which runs the pipeline and returns +// the rewritten body WITHOUT forwarding upstream — so no provider ever validated it. Replay +// measures what a component removed; it cannot tell you the result is sendable. +// +// All three are statically checkable (schema.ValidateShape), so this test closes the gap for +// free: a table over every registered preset, run on a tool-heavy transcript, asserting the +// output breaks no shape invariant. +func TestEveryPresetEmitsAShapeValidRequest(t *testing.T) { + // A transcript with the features that break naive restructuring: a system head, several + // complete tool exchanges, and enough mass that size-gated components actually fire. + fixture := func() *bschemas.BifrostChatRequest { + big := "" + for i := 0; i < 60; i++ { + big += "line of tool output referencing src/auth.py and TOKEN_GRACE_SECONDS_41ab\n" + } + msgs := []bschemas.ChatMessage{ + {Role: bschemas.ChatMessageRoleSystem, + Content: &bschemas.ChatMessageContent{ContentStr: strp("you are a coding agent")}}, + {Role: bschemas.ChatMessageRoleUser, + Content: &bschemas.ChatMessageContent{ContentStr: strp("Fix test_auth_expiry in src/auth.py")}}, + } + for i := 0; i < 6; i++ { + id := "call_" + string(rune('a'+i)) + name := "Read" + args := `{"path":"src/auth.py"}` + msgs = append(msgs, + bschemas.ChatMessage{Role: bschemas.ChatMessageRoleAssistant, + Content: &bschemas.ChatMessageContent{ContentStr: strp("Reading the module.")}, + ChatAssistantMessage: &bschemas.ChatAssistantMessage{ + ToolCalls: []bschemas.ChatAssistantMessageToolCall{ + {ID: &id, Function: bschemas.ChatAssistantMessageToolCallFunction{ + Name: &name, Arguments: args}}}}}, + bschemas.ChatMessage{Role: bschemas.ChatMessageRoleTool, + Content: &bschemas.ChatMessageContent{ContentStr: strp(big)}, + ChatToolMessage: &bschemas.ChatToolMessage{ToolCallID: &id}}, + ) + } + msgs = append(msgs, bschemas.ChatMessage{Role: bschemas.ChatMessageRoleUser, + Content: &bschemas.ChatMessageContent{ContentStr: strp("keep going")}}) + return &bschemas.BifrostChatRequest{Input: msgs} + } + + // Sanity: the fixture itself must be valid, or a failure below proves nothing. + if v := schema.ValidateShape(fixture().Input); len(v) != 0 { + t.Fatalf("fixture is not shape-valid, so this test cannot attribute violations: %v", v) + } + + for _, name := range []string{ + "off", "safe", "balanced", "aggressive", "coding", "mcp", + "agent", "general", "summarize", "codesmart", "codesafe", + } { + t.Run(name, func(t *testing.T) { + names, ok := config.PresetPipeline(name) + if !ok { + t.Fatalf("preset %q is not registered", name) + } + cfg := &config.Config{Pipeline: names} + pipe, err := cfg.Build(nil) + if err != nil { + t.Skipf("preset %q needs config this test does not supply: %v", name, err) + } + req := fixture() + c := &components.Ctx{ + Ctx: context.Background(), Session: "shape-" + name, + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + // A stub model so the LLM-driven components actually restructure rather + // than degrading to a no-op — a skipped component proves nothing here. + Model: components.ModelSpec{Incoming: stubModel{resp: "SUMMARY: read the auth module."}, + Static: stubModel{resp: "SUMMARY: read the auth module."}}, + } + pipe.Run(req, c) + if v := schema.ValidateShape(req.Input); len(v) != 0 { + t.Errorf("preset %q emitted a request a provider would reject:", name) + for _, x := range v { + t.Errorf(" %s", x) + } + } + }) + } +} diff --git a/components/component.go b/components/component.go index f0afd23f..c868ecaa 100644 --- a/components/component.go +++ b/components/component.go @@ -74,6 +74,41 @@ type Model interface { Complete(ctx context.Context, prompt string) (string, error) } +// PrefixUsage reports what one PrefixAsk cost, straight from the provider's usage block. It exists +// because the whole point of a prefix ask is the cache read, and a cache read that silently is not +// happening looks identical to one that is -- except on the bill. Surfaced in /stats. +type PrefixUsage struct { + CacheRead int + CacheWrite int + Fresh int + Output int +} + +// PrefixAsker completes `ask` as a trailing user message appended to the EXACT body this session +// sent upstream on the previous turn. +// +// Why the previous turn's SENT body and not the incoming one: the provider's prompt cache was +// populated from what context-guru emitted, which is the compacted form. The incoming body is +// uncompacted, so it diverges from the cached bytes at the first thing any component removed, and +// everything after that point is a miss. Appending to the bytes that were actually sent is the only +// construction that reliably reads the cache. +// +// The consequence is that the ask sees the transcript as of the previous turn. That is acceptable for +// the judgement it serves -- the missing part is the newest tool output, which is tail content that +// has had no turns in which to be superseded and would be kept anyway -- and it has the side benefit +// of keeping a large model call off the agent's critical path. +// +// nil when the host cannot support it (no stashed body for this session yet, first turn, or the +// feature is off). Callers MUST fall back to Model.Complete rather than skipping their work. +// The session is passed at CALL time, not bound at construction. It has to be: the host builds the +// asker before the pipeline runs, and the RESOLVED session id (a content hash when the caller sends +// no session header, then tenant-scoped) is only known inside the pipeline. Binding the host's +// pre-pipeline value stashed under one key and looked up under another -- observed live as +// prefix_ask_used = 0 with no failures recorded, i.e. a feature that silently never ran. +type PrefixAsker interface { + Ask(ctx context.Context, session, ask string) (reply string, usage PrefixUsage, err error) +} + // ModelSpec carries the LLM clients a NeedsModel component may use, resolved per // request by the host adapter. Incoming is the proxied request's own model + // credentials (nil when unavailable, e.g. the AuthBridge host); Static is a @@ -156,6 +191,10 @@ type Ctx struct { // -1 = unknown/first turn/cache off ⇒ no tail restriction. Only meaningful when // CacheAware is true. MaxCachedIdx int + // PrefixAsk, when non-nil, lets a component put a question to the request's own model with the + // previous turn's SENT body as the prefix, so the provider reads its prompt cache instead of + // being re-sent the transcript. See PrefixAsker for why that body and not the incoming one. + PrefixAsk PrefixAsker // FilterStats receives cmdfilter's per-filter ledger (which command families pay // off, and which output shapes matched nothing). nil = not recording. // @@ -286,6 +325,23 @@ func (r *Report) Gate(name string) { r.Gates[name]++ } +// GateN adds n to the named counter in one call, for the counts that are not +// one-candidate-declined-per-increment. +// +// It exists because a per-candidate Gate() cannot express "this call was OFFERED twelve +// candidates". Deriving batch size from verdict counts instead was measured wrong: verdicts count +// what the model chose to ANSWER, so a starved batch and a model that ignores two thirds of a full +// batch produce the same number. Distinguishing them needs the offered count recorded directly. +func (r *Report) GateN(name string, n int) { + if r == nil || n == 0 { + return + } + if r.Gates == nil { + r.Gates = map[string]int{} + } + r.Gates[name] += n +} + // Saved returns non-negative tokens saved by this component. func (r Report) Saved() int { if r.TokensAfter > r.TokensBefore { diff --git a/components/offload/coref.go b/components/offload/coref.go new file mode 100644 index 00000000..3cc5fd69 --- /dev/null +++ b/components/offload/coref.go @@ -0,0 +1,441 @@ +package offload + +import ( + "math" + "strconv" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/expand" + "github.com/rossoctl/context-guru/internal/coref" + "github.com/rossoctl/context-guru/schema" + "gopkg.in/yaml.v3" +) + +func init() { components.Register("coref", newCoref) } + +// Coref is co-reference-aware compaction: at a threshold crossing it cuts the tool +// outputs that later turns never carried anything forward from, in ONE batched pass. +// Design and derivation: docs/proposals/coref-compaction.md. +// +// It differs from every other offloader here in one deliberate way, and the whole shape +// of the component follows from it: it MUTATES THE CACHED PREFIX on purpose. Age-based +// offloaders refuse to (Ctx.TailOnly) because breaking the prefix hash at index i +// cache-WRITES the suffix, at 11.5x a cache-read. So: +// +// - The cut is BATCHED. One rewrite has to serve every cut in the pass, because a +// single early cut can never repay its own rewrite on tokens: at 5k cut from 20% +// depth of a 150k transcript the break-even is T > 276 turns. Batched, 60k of the +// same transcript needs T > 23, which a long session actually reaches. Hence +// min_batch_frac, and hence "rare and certain" rather than "per output per turn". +// - The spend is BUDGETED and reported, not incidental (rewrite_budget). A component +// that spends cache-writes on purpose has to be answerable for how many. +// - The decision is LATCHED and replayed, never re-derived. See the freeze note in +// Offload; this is the constraint that rules out repairLostFreeze. +// +// The default cut set is `unreferenced` only — outputs no later turn exactly reuses +// anything from. That is the honest ceiling of a zero-LLM implementation and needs no +// calibrated threshold. `closed` (the case-A large cut: referenced once or twice, long +// ago) is off by default because its two thresholds are the OUTPUT of the measurement +// pass in deploy/harbor/coref.py, which has not yet run on real traffic; shipping a +// guessed closed_dist would be shipping the one number the proposal says must be +// measured. Turn it on with cut_closed once there are numbers. +type Coref struct { + trigger components.Trigger + minTokens int + closedDist int + openReps int + minLaterTurns int + cutUnreferenced bool + cutClosed bool + rewriteBudget int + minBatchFrac float64 + breakEven bool + keepHeadChars int + mode markerMode +} + +type corefConfig struct { + Trigger components.Trigger `yaml:"trigger"` + // MinTokens is the per-output floor; matches coref.py's min_output default so the + // component and the measurement consider the same population. + MinTokens int `yaml:"min_tokens"` + // ClosedDist / OpenReps are the open-vs-closed thresholds. Defaults mirror + // coref.py's, and are placeholders until it runs on captured traffic. + ClosedDist int `yaml:"closed_dist"` + OpenReps int `yaml:"open_reps"` + // MinLaterTurns is the opportunity floor: an output with fewer model turns after it + // than this is never cut, because it has not yet had a chance to be referenced. + MinLaterTurns *int `yaml:"min_later_turns"` + // CutUnreferenced / CutClosed select the cut set (see Coref). + CutUnreferenced *bool `yaml:"cut_unreferenced"` + CutClosed *bool `yaml:"cut_closed"` + // RewriteBudget caps prefix-rewrite passes per session. 0 disables the component's + // only cache-spending path entirely (replay of already-latched decisions continues). + RewriteBudget *int `yaml:"rewrite_budget"` + // MinBatchFrac is the batching constraint: the pass must cut at least this fraction + // of the request before it is worth a rewrite. + // + // The default was 0.15, derived from the illustrative arithmetic in the proposal's §4 + // and never checked against how much mass is actually available. Measured, Tier-1 + // matching finds a mean 4.4% of the request (`unreferenced`) or 9.6% (`+closed`) on real + // long sessions — so 0.15 admitted ONE of nineteen sessions past the agent's compaction + // threshold, and zero at the shipped cut set. A gate no traffic can clear is not a + // conservative default, it is an off switch that looks like a threshold. 0.05 admits + // 16/19; the honest position is that the right value is an experimental result and this + // is a starting point, not a claim. + MinBatchFrac *float64 `yaml:"min_batch_frac"` + // BreakEven applies the S*T > 11.5*W inequality with an estimated T. Ignored when + // the context window is unknown, like every other fraction-based threshold here. + BreakEven *bool `yaml:"break_even"` + // KeepHeadChars leaves a one-line peek inside the marker so the model knows what was + // cut without a blind expand round-trip; 0 disables. + KeepHeadChars *int `yaml:"keep_head_chars"` + MarkerMode string `yaml:"marker_mode"` // full (default) | summary | off +} + +func newCoref(raw []byte) (components.Component, error) { + cfg := corefConfig{MinTokens: 300, ClosedDist: corefClosedDistDefault, OpenReps: corefOpenRepsDefault} + if len(raw) > 0 { + if err := yaml.Unmarshal(raw, &cfg); err != nil { + return nil, err + } + } + cf := &Coref{ + trigger: cfg.Trigger, + minTokens: cfg.MinTokens, + closedDist: cfg.ClosedDist, + openReps: cfg.OpenReps, + minLaterTurns: corefMinLaterDefault, + cutUnreferenced: true, + cutClosed: false, + rewriteBudget: 3, + minBatchFrac: 0.05, + breakEven: true, + keepHeadChars: 96, + mode: parseMarkerMode(cfg.MarkerMode), + } + if cfg.MinLaterTurns != nil { + cf.minLaterTurns = *cfg.MinLaterTurns + } + if cfg.CutUnreferenced != nil { + cf.cutUnreferenced = *cfg.CutUnreferenced + } + if cfg.CutClosed != nil { + cf.cutClosed = *cfg.CutClosed + } + if cfg.RewriteBudget != nil { + cf.rewriteBudget = *cfg.RewriteBudget + } + if cfg.MinBatchFrac != nil { + cf.minBatchFrac = *cfg.MinBatchFrac + } + if cfg.BreakEven != nil { + cf.breakEven = *cfg.BreakEven + } + if cfg.KeepHeadChars != nil { + cf.keepHeadChars = *cfg.KeepHeadChars + } + return cf, nil +} + +func (Coref) Name() string { return "coref" } +func (Coref) Enabled(*components.Ctx) bool { return true } + +// plannedCut is one accepted candidate, held until the whole batch clears its gates — +// nothing is stashed or rewritten before then, so a batch that fails a gate leaves the +// request byte-identical. +type plannedCut struct { + idx int + original string + newText string + key string + eff markerMode + saved int +} + +func (cf *Coref) Offload(req *bschemas.BifrostChatRequest, rep *components.Report, c *components.Ctx) ([]string, error) { + // The trigger is pure request shape, so ask before paying for the index. Replay + // (below) is NOT gated on it: a latched cut must be re-applied on every turn whether + // or not this turn would have decided to cut anything, or the output flips + // cut→full→cut and churns the very cache this component is budgeting. + fires := cf.trigger.Fires(req, c.CtxWindow) + + // Index the PRISTINE request, before any replay rewrites a message. Two reasons, and + // the second is the load-bearing one: the agent re-sends originals every turn, so the + // pristine transcript is the stable input the offline measurement also reads — and if + // the index were built after replay, an earlier cut would remove identifiers from the + // exclusion sets and silently reclassify unrelated outputs. That is history dependence + // on our OWN past output, which is how a "keep" turns into a "cut" turns into a + // different set of bytes at the same index. + classes := map[int]coref.Class{} + if fires { + for _, r := range coref.Index(flattenForCoref(req), cf.trigger.OutputFloor(c.CtxWindow, cf.minTokens), schema.TextTokens) { + classes[r.Idx] = coref.Classify(r, cf.closedDist, cf.openReps, cf.minLaterTurns) + } + } + + pristineTokens := schema.MessagesTokens(req) + var keys []string + changed := 0 + + // Replay latched decisions, on every tool output, every turn, at any depth. + replayed := map[int]bool{} + for _, i := range toolIndices(req) { + m := &req.Input[i] + if !schema.Rewritable(*m) || schema.MessageText(*m) == "" { + continue + } + if fk, _, ok := reapplyFrozen(c, cf.Name(), m); ok { + replayed[i] = true + changed++ + keys = append(keys, fk...) + } + } + + if !fires { + rep.Gate("trigger") + if changed == 0 { + rep.Skipped = true + } + return keys, nil + } + + // A new pass spends a cache-write. Ask before planning one. + spent := corefRewrites(c) + if cf.rewriteBudget <= 0 || spent >= cf.rewriteBudget { + rep.Gate("rewrite_budget") + if changed == 0 { + rep.Skipped = true + } + return keys, nil + } + + plan, err := cf.planCuts(req, rep, c, classes, replayed) + if err != nil { + return keys, err + } + if len(plan) == 0 { + if changed == 0 { + rep.Skipped = true + } + return keys, nil + } + + saved := 0 + for _, p := range plan { + saved += p.saved + } + // Batching: one rewrite must serve the whole pass, so the pass has to be big enough + // to be worth one. A batch below the floor is not a small win, it is a loss. + if cf.minBatchFrac > 0 && pristineTokens > 0 && + float64(saved) < cf.minBatchFrac*float64(pristineTokens) { + rep.Gate("batch_too_small") + if changed == 0 { + rep.Skipped = true + } + return keys, nil + } + if cf.breakEven { + if _, _, ok := cf.breakEvenTurns(req, plan, c); !ok { + rep.Gate("break_even") + if changed == 0 { + rep.Skipped = true + } + return keys, nil + } + } + + // Commit the whole batch, then charge the session exactly one rewrite for it. + for _, p := range plan { + commitMark(c, rep, p.eff, p.key, p.original) + schema.SetMessageText(&req.Input[p.idx], p.newText) + // Latch. From here the bytes for this content are fixed for the session: the next + // turn replays them rather than re-deciding, because the decision is a function of + // the transcript that existed when it was taken, not of the content alone. + // + // This is also why coref must NOT consult repairLostFreeze, which mask and + // failed_run legitimately do. That repair re-derives a lost decision at depth on the + // grounds that the replacement is a pure function of (content, config) and so + // reproduces the bytes the provider already cached. A co-reference decision is + // history-dependent by construction — re-deriving it against a longer transcript can + // yield a DIFFERENT class and different bytes, which is precisely the prefix flip the + // repair exists to avoid. A lost coref freeze therefore just declines. + freeze(c, cf.Name(), p.original, p.newText) + changed++ + if p.key != "" { + keys = append(keys, p.key) + } + } + setCorefRewrites(c, spent+1) + + if changed == 0 { + rep.Skipped = true + } + return keys, nil +} + +// planCuts builds the batch without side effects: every candidate is size-checked with +// its marker included (tryMark stashes nothing), so a batch that later fails a gate +// leaves the request untouched. +func (cf *Coref) planCuts(req *bschemas.BifrostChatRequest, rep *components.Report, c *components.Ctx, + classes map[int]coref.Class, replayed map[int]bool) ([]plannedCut, error) { + + floor := cf.trigger.OutputFloor(c.CtxWindow, cf.minTokens) + var plan []plannedCut + for _, i := range toolIndices(req) { + if replayed[i] { + continue // already latched; counted above, never re-decided + } + m := req.Input[i] + if !schema.Rewritable(m) { + rep.Gate("non_text_blocks") + continue + } + content := schema.MessageText(m) + if content == "" || schema.TextTokens(content) < floor { + rep.Gate("below_min_tokens") + continue + } + if skipReduce(c, content) { + rep.Gate("marker_or_kept_verbatim") + continue + } + class, ok := classes[i] + if !ok { + rep.Gate("not_indexed") // below the index floor, or not a recorded output + continue + } + if !(class == coref.Unreferenced && cf.cutUnreferenced) && + !(class == coref.Closed && cf.cutClosed) { + rep.Gate("class_" + string(class)) + continue + } + // The marker states WHAT was removed and never why it was safe to remove. The + // earlier wording ("no later turn referred back to it") asserted the one claim that + // is false whenever the reference was transformed or semantic — tiers 2 and 3, which + // this index cannot see — so it read as reassurance and discouraged the expand call + // that would have repaired the mistake. Since only the model can initiate recovery, a + // marker that talks it out of recovering is worse than an opaque one. + note := "[tool output compacted" + if stub := corefStub(content); stub != "" { + // Structured content: describe the shape, which is addressable. "200 records, + // fields: address, id, name" tells a model hunting for an address that this is + // where addresses live; a head peek of one arbitrary row does not. + note += "; " + stub + } else if peek := headPeek(content, cf.keepHeadChars); peek != "" { + note += "; starts: " + peek + } + note += "] " + newText, key, eff, ok := tryMark(c, cf.mode, content, " [full output: call "+expand.ToolName+"]", + func(tok string) string { return note + tok }) + if !ok { + rep.Gate("marker_no_win") + continue + } + plan = append(plan, plannedCut{ + idx: i, original: content, newText: newText, key: key, eff: eff, + saved: schema.TextTokens(content) - schema.TextTokens(newText), + }) + } + return plan, nil +} + +// breakEvenTurns applies the inequality from the proposal's §4: +// +// cost = W x (2.50 - 0.20) = 11.5 x W (in cache-read-equivalents) +// benefit = S x T x 0.20 = S x T +// worth it when S x T > 11.5 x W +// +// S is what the batch cuts; W is the suffix this cut forces the provider to re-write — +// counted from the shallowest cut index to the CACHED boundary, because content past it +// was never cached and would be written on this turn regardless; T is how many more +// turns the session has to collect the saving on, which is the quantity nobody has, so +// it is estimated from how fast the transcript has been growing. +// +// The consequence is counter-intuitive and worth stating: firing at 90% of the window +// means T is nearly zero, i.e. paying a rewrite for a saving that will be collected +// once. The profitable moment to compact is EARLIER than the moment of maximum pressure. +// +// Returns (needed T, estimated T, whether it clears). Always clears when the context +// window is unknown — same convention as every fraction-based threshold here: an +// unresolvable threshold imposes no constraint rather than silently disabling the pass. +func (cf *Coref) breakEvenTurns(req *bschemas.BifrostChatRequest, plan []plannedCut, c *components.Ctx) (need, have int, ok bool) { + saved := 0 + shallowest := len(req.Input) + for _, p := range plan { + saved += p.saved + if p.idx < shallowest { + shallowest = p.idx + } + } + // The arithmetic itself lives in prefix_econ.go, shared with extract_llm's + // allow_cached_prefix path so the two components cannot price the same cache-write + // differently. + return prefixRewritePays(req, saved, shallowest, c) +} + +// flattenForCoref projects a request onto the neutral message list internal/coref +// indexes, 1:1 with req.Input indices so a Record points back at its message. +// +// The split is what defines a reference: a tool message is MASS (Results), everything +// else is a reference-bearing SURFACE (Texts) — prose plus the tool-call name and +// arguments, which is where a model names the path/symbol/id it took from an earlier +// output. A later tool result echoing a token is the environment repeating itself, not +// the model using the value, so it never counts as a reference. +func flattenForCoref(req *bschemas.BifrostChatRequest) []coref.Message { + out := make([]coref.Message, len(req.Input)) + for i := range req.Input { + m := req.Input[i] + if m.Role == bschemas.ChatMessageRoleTool { + id := "" + if m.ChatToolMessage != nil && m.ChatToolMessage.ToolCallID != nil { + id = *m.ChatToolMessage.ToolCallID + } + out[i] = coref.Message{Results: []coref.Result{{ID: id, Text: schema.MessageText(m)}}} + continue + } + texts := []string{} + if t := schema.MessageText(m); t != "" { + texts = append(texts, t) + } + if m.ChatAssistantMessage != nil { + for _, tc := range m.ChatAssistantMessage.ToolCalls { + name := "" + if tc.Function.Name != nil { + name = *tc.Function.Name + } + texts = append(texts, name+" "+tc.Function.Arguments) + } + } + out[i] = coref.Message{Texts: texts} + } + return out +} + +// --- per-session rewrite budget --------------------------------------------- +// +// The one number this component is answerable for. Every other offloader's cache +// discipline is "never touch the prefix"; coref's is "touch it at most N times", so N +// has to be counted somewhere durable rather than inferred from the savings. + +func corefRewritesKey(session string) string { return "cg:coref:rw:" + session } + +func corefRewrites(c *components.Ctx) int { + b, ok := c.Store.Get(corefRewritesKey(c.Session)) + if !ok { + return 0 + } + n, err := strconv.Atoi(string(b)) + if err != nil || n < 0 { + // Unreadable counter reads as EXHAUSTED, not as zero. Fail-open here means fail + // open on the request (which is unaffected — no cut is taken), not fail open on an + // unbounded cache spend. + return math.MaxInt32 + } + return n +} + +func setCorefRewrites(c *components.Ctx, n int) { + c.Store.Put(corefRewritesKey(c.Session), []byte(strconv.Itoa(n))) +} diff --git a/components/offload/coref_test.go b/components/offload/coref_test.go new file mode 100644 index 00000000..f889ef3e --- /dev/null +++ b/components/offload/coref_test.go @@ -0,0 +1,649 @@ +package offload + +import ( + "fmt" + "strings" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/expand" + "github.com/rossoctl/context-guru/schema" + "github.com/rossoctl/context-guru/store" +) + +// --- fixture helpers --------------------------------------------------------- + +func corefUser(text string) bschemas.ChatMessage { + t := text + return bschemas.ChatMessage{Role: bschemas.ChatMessageRoleUser, + Content: &bschemas.ChatMessageContent{ContentStr: &t}} +} + +// corefAsst builds a model turn: prose plus a tool call, which together are the +// reference-bearing surface the index reads. +func corefAsst(text, name, args string) bschemas.ChatMessage { + t, n := text, name + return bschemas.ChatMessage{ + Role: bschemas.ChatMessageRoleAssistant, + Content: &bschemas.ChatMessageContent{ContentStr: &t}, + ChatAssistantMessage: &bschemas.ChatAssistantMessage{ + ToolCalls: []bschemas.ChatAssistantMessageToolCall{ + {Function: bschemas.ChatAssistantMessageToolCallFunction{Name: &n, Arguments: args}}, + }, + }, + } +} + +func corefTool(id, text string) bschemas.ChatMessage { + t, i := text, id + return bschemas.ChatMessage{Role: bschemas.ChatMessageRoleTool, + Content: &bschemas.ChatMessageContent{ContentStr: &t}, + ChatToolMessage: &bschemas.ChatToolMessage{ToolCallID: &i}} +} + +// corefBody is a distinct multi-line output; distinct because shared filler would be +// discarded as session boilerplate by the index. +func corefBody(tag string) string { + var b strings.Builder + for i := 0; i < 60; i++ { + fmt.Fprintf(&b, "%4d\t%s_line_%d = compute_%s_%d(arg_%d)\n", i, tag, i, tag, i, i) + } + return b.String() +} + +// Sentinels that say whether an output is still verbatim. They sit at the END of their +// output on purpose: the marker carries a head peek of what it replaced, so a sentinel at +// the head survives the cut and "is it still there?" stops meaning "was it kept?". +const ( + corefNovelUsed = "TOKEN_GRACE_SECONDS_41ab" // introduced by the read, then carried forward + corefNovelUnused = "TREE_SCAN_MARKER_9d7c" // introduced by the listing, never used again + corefNovelFresh = "FRESH_SCAN_MARKER_5e1f" // introduced by a later listing, never used +) + +// corefReq is a transcript with exactly two large tool outputs: index 2 is REFERENCED by +// the following model turn (must survive) and index 5 is referenced by nothing (the cut). +func corefReq() *bschemas.BifrostChatRequest { + return &bschemas.BifrostChatRequest{Input: []bschemas.ChatMessage{ + corefUser("Fix the failing test test_auth_expiry in src/auth.py"), + corefAsst("Reading the auth module.", "Read", `{"path":"src/auth.py"}`), + corefTool("t1", "src/auth.py\n"+corefBody("auth")+"\n"+corefNovelUsed+" = 0\n"), + corefAsst("The bug is "+corefNovelUsed+"; it must be 300.", "Edit", + `{"path":"src/auth.py","old":"`+corefNovelUsed+` = 0"}`), + corefAsst("Surveying the tree.", "Bash", `{"cmd":"ls -R"}`), + corefTool("t2", corefBody("tree")+"\n"+corefNovelUnused+"\n"), + }} +} + +// corefWithSecondListing appends a fresh model turn plus a second unreferenced output — +// a new candidate for a pass that must be turned away. +func corefWithSecondListing() *bschemas.BifrostChatRequest { + req := corefReq() + req.Input = append(req.Input, + corefAsst("Listing again.", "Bash", `{"cmd":"ls /srv"}`), + corefTool("t3", corefBody("srv")+"\n"+corefNovelFresh+"\n"), + ) + return req +} + +const ( + corefRefIdx = 2 // referenced output — must be left verbatim + corefCutIdx = 5 // unreferenced output — the cut + corefFreshIdx = 7 // the second listing's output (see corefWithSecondListing) +) + +func corefFor(t *testing.T, extraYAML string) *Coref { + t.Helper() + // The batch floor is the gate under test in exactly one case, so default it out of the + // way here rather than repeating it — and never define it twice (yaml rejects that). + // Defaults that would otherwise dominate every case: the batch floor, and the + // opportunity floor (the fixture is a short transcript whose cut candidate sits at the + // tail). Each is the gate under test in exactly one place, so a default is only added + // when the case does not set it — yaml rejects a duplicated key. + base := "min_tokens: 20\n" + for k, v := range map[string]string{"min_batch_frac": "0", "min_later_turns": "0"} { + if !strings.Contains(extraYAML, k) { + base += k + ": " + v + "\n" + } + } + comp, err := newCoref([]byte(base + extraYAML)) + if err != nil { + t.Fatal(err) + } + return comp.(*Coref) +} + +func corefCtx(st store.Store) *components.Ctx { + return &components.Ctx{Session: "s", Store: st} +} + +// --- behaviour --------------------------------------------------------------- + +// The deterministic ceiling: cut what nothing referred back to, keep what was referred +// to. Nothing else in the request may move. +func TestCorefCutsOnlyUnreferencedOutputs(t *testing.T) { + cf := corefFor(t, "") + req, orig := corefReq(), corefReq() + c := corefCtx(store.NewMemory(store.Options{})) + var rep components.Report + + keys, err := cf.Offload(req, &rep, c) + if err != nil { + t.Fatal(err) + } + if rep.Skipped { + t.Fatal("component skipped; expected the unreferenced output to be cut") + } + if got := schema.MessageText(req.Input[corefRefIdx]); got != schema.MessageText(orig.Input[corefRefIdx]) { + t.Errorf("the REFERENCED output was modified; it is load-bearing:\n%q", got) + } + cut := schema.MessageText(req.Input[corefCutIdx]) + if cut == schema.MessageText(orig.Input[corefCutIdx]) { + t.Fatal("the unreferenced output was not cut") + } + if !strings.Contains(cut, "tool output compacted") { + t.Errorf("marker does not say what happened: %q", cut) + } + assertMarkerMakesNoSafetyClaim(t, cut) + for i := range req.Input { + if i == corefCutIdx { + continue + } + if schema.MessageText(req.Input[i]) != schema.MessageText(orig.Input[i]) { + t.Errorf("message %d changed; only the cut output may move", i) + } + } + // Reversible: the original must be retrievable under the returned key. + if len(keys) != 1 { + t.Fatalf("cache keys = %v, want exactly one stashed original", keys) + } + got, ok := c.Store.Get(keys[0]) + if !ok || string(got) != schema.MessageText(orig.Input[corefCutIdx]) { + t.Error("the stashed original does not round-trip; the cut is not reversible") + } + if ms := expand.ParseMarkers(cut); len(ms) != 1 || ms[0] != keys[0] { + t.Errorf("marker in the cut text = %v, want the stash key %q", ms, keys[0]) + } +} + +// Latching, and the monotonicity that pays for it. Once a cut is taken, later turns +// replay the SAME BYTES even when fresh evidence would now classify the output as open. +// Re-deciding is what rewrites the prefix a second time, so new evidence may never +// resurrect a span — keep→cut only, in one direction. +func TestCorefLatchesAndNeverResurrects(t *testing.T) { + cf := corefFor(t, "") + st := store.NewMemory(store.Options{}) + + req := corefReq() + var rep components.Report + if _, err := cf.Offload(req, &rep, corefCtx(st)); err != nil { + t.Fatal(err) + } + latched := schema.MessageText(req.Input[corefCutIdx]) + if latched == schema.MessageText(corefReq().Input[corefCutIdx]) { + t.Fatal("nothing was cut on the first turn") + } + + // A later turn now references the previously-unreferenced output, repeatedly and + // recently — i.e. it would classify as OPEN if the decision were re-derived. + for turn := 1; turn <= 3; turn++ { + next := corefReq() // the agent re-sends the originals every turn + for k := 0; k < turn; k++ { + next.Input = append(next.Input, + corefAsst(corefNovelUnused+" again, attempt "+fmt.Sprint(k), "Bash", `{"cmd":"ls -R"}`)) + } + var r components.Report + if _, err := cf.Offload(next, &r, corefCtx(st)); err != nil { + t.Fatal(err) + } + if got := schema.MessageText(next.Input[corefCutIdx]); got != latched { + t.Fatalf("turn %d re-derived the decision:\n got %q\nwant %q", turn, got, latched) + } + } +} + +// The budget is the component's answer for the cache-writes it spends on purpose. Once +// spent, further passes decline — while already-latched decisions keep being replayed, +// because NOT replaying them is itself the cache-destructive move. +func TestCorefRewriteBudgetIsSpentOnceAndEnforced(t *testing.T) { + cf := corefFor(t, "rewrite_budget: 1\n") + st := store.NewMemory(store.Options{}) + + req := corefReq() + var rep components.Report + if _, err := cf.Offload(req, &rep, corefCtx(st)); err != nil { + t.Fatal(err) + } + latched := schema.MessageText(req.Input[corefCutIdx]) + if latched == schema.MessageText(corefReq().Input[corefCutIdx]) { + t.Fatal("nothing was cut on the first turn") + } + if n := corefRewrites(corefCtx(st)); n != 1 { + t.Errorf("rewrites charged = %d, want exactly 1 for the whole batch", n) + } + + // A second pass with a brand-new unreferenced output must decline: the budget is gone. + next := corefWithSecondListing() + var r2 components.Report + if _, err := cf.Offload(next, &r2, corefCtx(st)); err != nil { + t.Fatal(err) + } + if r2.Gates["rewrite_budget"] == 0 { + t.Error("expected the rewrite_budget gate to turn the second pass away") + } + if got := schema.MessageText(next.Input[corefFreshIdx]); !strings.Contains(got, corefNovelFresh) { + t.Error("the new output was cut despite an exhausted budget") + } + if got := schema.MessageText(next.Input[corefCutIdx]); got != latched { + t.Error("an exhausted budget stopped the replay of an already-latched decision, " + + "which is the flip the budget exists to prevent") + } + if n := corefRewrites(corefCtx(st)); n != 1 { + t.Errorf("rewrites charged = %d after a declined pass, want 1", n) + } +} + +// Batching is the reason this component exists in this shape: one rewrite has to serve +// the whole pass. A batch below the floor leaves the request byte-identical rather than +// taking a small, losing cut. +func TestCorefDeclinesABatchTooSmallToPayForItsRewrite(t *testing.T) { + cf := corefFor(t, "min_batch_frac: 0.9\n") + req, orig := corefReq(), corefReq() + var rep components.Report + if _, err := cf.Offload(req, &rep, corefCtx(store.NewMemory(store.Options{}))); err != nil { + t.Fatal(err) + } + if rep.Gates["batch_too_small"] == 0 { + t.Error("expected the batch_too_small gate") + } + if !rep.Skipped { + t.Error("a declined pass must report Skipped") + } + for i := range req.Input { + if schema.MessageText(req.Input[i]) != schema.MessageText(orig.Input[i]) { + t.Fatalf("message %d was modified by a pass that declined; planning must be side-effect free", i) + } + } +} + +// Firing at maximum pressure is firing when there is almost nothing left to collect the +// saving on. The break-even inequality has to say no there, otherwise the component pays +// a cache-write for a single turn of savings. +func TestCorefBreakEvenDeclinesAtTheWindowEdge(t *testing.T) { + cf := corefFor(t, "") + req := corefReq() + // A window barely above the current request: the estimated turns remaining collapses + // to ~0, so no cut can repay the rewrite. + c := corefCtx(store.NewMemory(store.Options{})) + c.CtxWindow = schema.MessagesTokens(req) + 1 + var rep components.Report + if _, err := cf.Offload(req, &rep, c); err != nil { + t.Fatal(err) + } + if rep.Gates["break_even"] == 0 { + t.Fatalf("expected the break_even gate at the window edge; gates=%v", rep.Gates) + } + if got := schema.MessageText(req.Input[corefCutIdx]); !strings.Contains(got, corefNovelUnused) { + t.Error("a cut was taken that cannot repay its own cache-write") + } + + // With room to run, the same request and the same cut must clear. + roomy := corefReq() + c2 := corefCtx(store.NewMemory(store.Options{})) + c2.CtxWindow = schema.MessagesTokens(roomy) * 50 + var rep2 components.Report + if _, err := cf.Offload(roomy, &rep2, c2); err != nil { + t.Fatal(err) + } + if rep2.Gates["break_even"] != 0 { + t.Errorf("break_even declined with 50x the window headroom; gates=%v", rep2.Gates) + } +} + +// coref is the one offloader that mutates the already-cached prefix on purpose — that is +// its entire function, and it is why the spend is budgeted instead of forbidden. A tail +// restriction here would make the component a no-op on exactly the transcripts it exists +// for, since by the time a session crosses the threshold the mass is all in the prefix. +func TestCorefDeliberatelyCutsInsideTheCachedPrefix(t *testing.T) { + cf := corefFor(t, "") + req := corefReq() + c := corefCtx(store.NewMemory(store.Options{})) + c.CacheAware, c.MaxCachedIdx = true, len(req.Input)-1 // everything already cached + var rep components.Report + if _, err := cf.Offload(req, &rep, c); err != nil { + t.Fatal(err) + } + if got := schema.MessageText(req.Input[corefCutIdx]); strings.Contains(got, corefNovelUnused) { + t.Fatal("coref respected the cache tail; it is supposed to spend the rewrite, " + + "under budget, or it can never act on a long session") + } +} + +// The trigger gates only NEW decisions. Replay is unconditional, because a latched cut +// that stops being replayed flips cut→full inside the cached prefix. +func TestCorefTriggerGatesNewCutsButNotReplay(t *testing.T) { + st := store.NewMemory(store.Options{}) + open := corefFor(t, "") + req := corefReq() + var rep components.Report + if _, err := open.Offload(req, &rep, corefCtx(st)); err != nil { + t.Fatal(err) + } + latched := schema.MessageText(req.Input[corefCutIdx]) + if latched == schema.MessageText(corefReq().Input[corefCutIdx]) { + t.Fatal("nothing was cut on the first turn") + } + + // Same store, but a trigger that cannot fire on this request shape. + shut := corefFor(t, "trigger:\n min_messages: 9999\n") + next := corefWithSecondListing() + var r2 components.Report + if _, err := shut.Offload(next, &r2, corefCtx(st)); err != nil { + t.Fatal(err) + } + if r2.Gates["trigger"] == 0 { + t.Error("expected the trigger gate") + } + if got := schema.MessageText(next.Input[corefFreshIdx]); !strings.Contains(got, corefNovelFresh) { + t.Error("a new cut was taken while the trigger was shut") + } + if got := schema.MessageText(next.Input[corefCutIdx]); got != latched { + t.Error("a shut trigger suppressed the replay of a latched decision") + } + if r2.Skipped { + t.Error("a turn that replayed a latched decision did act; Skipped is wrong") + } +} + +// The kept-verbatim exemption must NOT leak across sessions. One agent expanding a config +// dump, a manifest or a schema used to exempt that byte-identical content in every session +// thereafter — permanently, silently, and preferentially on the content most worth cutting, +// because recurring-across-sessions is exactly what makes content valuable to compact. +// +// This is the negative half of TestCorefLeavesExpandedContentAlone, and it is the half that +// fails on the old key layout: the guard is real within its session and absent outside it. +func TestKeptVerbatimDoesNotLeakAcrossSessions(t *testing.T) { + cf := corefFor(t, "") + st := store.NewMemory(store.Options{}) + req := corefReq() + MarkKeptVerbatim(st, "session-a", schema.MessageText(req.Input[corefCutIdx])) + + // A DIFFERENT session sends the same bytes. It has never expanded anything, so it cannot + // be in an expand loop and must not inherit session-a's exemption. + var rep components.Report + if _, err := cf.Offload(req, &rep, &components.Ctx{Session: "session-b", Store: st}); err != nil { + t.Fatal(err) + } + if got := schema.MessageText(req.Input[corefCutIdx]); strings.Contains(got, corefNovelUnused) { + t.Fatal("session-b inherited session-a's kept-verbatim exemption; the guard is leaking") + } + + // And the guard still holds for the session that earned it. + req2 := corefReq() + var rep2 components.Report + if _, err := cf.Offload(req2, &rep2, &components.Ctx{Session: "session-a", Store: st}); err != nil { + t.Fatal(err) + } + if got := schema.MessageText(req2.Input[corefCutIdx]); !strings.Contains(got, corefNovelUnused) { + t.Fatal("session-a lost its own exemption; scoping broke the guard it was meant to keep") + } +} + +// An empty session must not fall back to a global mark — that is the leak, reinstated. +func TestMarkKeptVerbatimIgnoresAnEmptySession(t *testing.T) { + st := store.NewMemory(store.Options{}) + MarkKeptVerbatim(st, "", "content expanded by nobody in particular") + if _, ok := st.Get(keptKey("", contentKey("content expanded by nobody in particular"))); ok { + t.Fatal("an empty session wrote a mark; it must be a no-op") + } +} + +// An output the agent expanded must never be re-cut: doing so just makes it expand again, +// once per turn, paying a round-trip and a cache-write each time. +func TestCorefLeavesExpandedContentAlone(t *testing.T) { + cf := corefFor(t, "") + st := store.NewMemory(store.Options{}) + req := corefReq() + MarkKeptVerbatim(st, "s", schema.MessageText(req.Input[corefCutIdx])) + + var rep components.Report + if _, err := cf.Offload(req, &rep, corefCtx(st)); err != nil { + t.Fatal(err) + } + if got := schema.MessageText(req.Input[corefCutIdx]); !strings.Contains(got, corefNovelUnused) { + t.Fatal("re-cut content the agent had expanded; that is the expand bounce loop") + } + if rep.Gates["marker_or_kept_verbatim"] == 0 { + t.Error("expected the kept-verbatim gate to record the declined candidate") + } +} + +// cut_closed is off by default: its thresholds are the OUTPUT of the measurement pass, +// so until that has run the component must not take the large case-A cut. Enabling it +// must then actually take it. +func TestCorefClosedCutIsOptIn(t *testing.T) { + req := corefReq() + // Push the reference far enough into the past that the referenced output is `closed` + // rather than `open` (recency is measured from the head). + for k := 0; k < 20; k++ { + req.Input = append(req.Input, corefAsst("thinking "+fmt.Sprint(k), "Bash", `{"cmd":"true"}`)) + } + + off := corefFor(t, "") + var rep components.Report + if _, err := off.Offload(req, &rep, corefCtx(store.NewMemory(store.Options{}))); err != nil { + t.Fatal(err) + } + if got := schema.MessageText(req.Input[corefRefIdx]); !strings.Contains(got, corefNovelUsed) { + t.Fatal("the closed output was cut with cut_closed off (the default)") + } + if rep.Gates["class_closed"] == 0 { + t.Fatalf("expected a declined closed candidate; gates=%v", rep.Gates) + } + + on := corefFor(t, "cut_closed: true\n") + req2 := corefReq() + for k := 0; k < 20; k++ { + req2.Input = append(req2.Input, corefAsst("thinking "+fmt.Sprint(k), "Bash", `{"cmd":"true"}`)) + } + var rep2 components.Report + if _, err := on.Offload(req2, &rep2, corefCtx(store.NewMemory(store.Options{}))); err != nil { + t.Fatal(err) + } + got := schema.MessageText(req2.Input[corefRefIdx]) + if strings.Contains(got, corefNovelUsed) { + t.Fatalf("cut_closed: true did not take the closed cut; gates=%v", rep2.Gates) + } + assertMarkerMakesNoSafetyClaim(t, got) +} + +// An unreadable budget counter must read as EXHAUSTED. Failing open on the request (no +// cut taken) is correct; failing open on an unbounded cache spend is not. +func TestCorefUnreadableBudgetCounterDeclines(t *testing.T) { + st := store.NewMemory(store.Options{}) + st.Put(corefRewritesKey("s"), []byte("not-a-number")) + cf := corefFor(t, "") + req := corefReq() + var rep components.Report + if _, err := cf.Offload(req, &rep, corefCtx(st)); err != nil { + t.Fatal(err) + } + if rep.Gates["rewrite_budget"] == 0 { + t.Error("a corrupt counter must read as exhausted, not as zero spent") + } + if got := schema.MessageText(req.Input[corefCutIdx]); !strings.Contains(got, corefNovelUnused) { + t.Error("a cut was taken against an unreadable budget") + } +} + +func TestCorefEstimateTurnsRemaining(t *testing.T) { + for _, tc := range []struct { + name string + reqTokens, turns, window int + want int + }{ + {"unknown window imposes nothing", 1000, 10, 0, 0}, + {"already at the window", 1000, 10, 1000, 0}, + {"half full, 100/turn", 1000, 10, 2000, 10}, + {"early in a long session", 1000, 10, 11000, 100}, + {"no turns yet", 1000, 0, 5000, 0}, + } { + if got := estimateTurnsRemaining(tc.reqTokens, tc.turns, tc.window); got != tc.want { + t.Errorf("%s: got %d, want %d", tc.name, got, tc.want) + } + } +} + +func TestCorefEmptyRequestIsANoOp(t *testing.T) { + cf := corefFor(t, "") + req := &bschemas.BifrostChatRequest{} + var rep components.Report + keys, err := cf.Offload(req, &rep, corefCtx(store.NewMemory(store.Options{}))) + if err != nil || len(keys) != 0 || !rep.Skipped { + t.Errorf("empty request: keys=%v skipped=%v err=%v", keys, rep.Skipped, err) + } +} + +// The opportunity floor, at the component level: an output too new to have been referenced +// must survive the pass. Without it a batched cut would preferentially remove the most +// RECENT context, since recency and "no references yet" are the same thing at the tail. +func TestCorefOpportunityFloorProtectsTheTail(t *testing.T) { + cf := corefFor(t, "min_later_turns: 8\n") + req := corefReq() + var rep components.Report + if _, err := cf.Offload(req, &rep, corefCtx(store.NewMemory(store.Options{}))); err != nil { + t.Fatal(err) + } + if got := schema.MessageText(req.Input[corefCutIdx]); !strings.Contains(got, corefNovelUnused) { + t.Fatal("cut a tail output that had had no chance to be referenced") + } + if rep.Gates["class_open"] == 0 { + t.Errorf("expected the tail output to be declined as open; gates=%v", rep.Gates) + } +} + +// An output whose values the index cannot see (records of plain names/ids) must never be +// cut by the DEFAULT config. Raised in review on PR #80. +func TestCorefNeverCutsOpaqueOutputs(t *testing.T) { + people := strings.Repeat( + `[{"name":"david","id":123,"address":"foobarbaz"},{"name":"osher","id":235,"address":"banana"}]`, 60) + req := &bschemas.BifrostChatRequest{Input: []bschemas.ChatMessage{ + corefUser("look up the people directory"), + corefAsst("Querying.", "query_people", `{}`), + corefTool("p1", people), + corefAsst("I need to remember david 123 address.", "Bash", `{"cmd":"true"}`), + }} + for k := 0; k < 20; k++ { + req.Input = append(req.Input, corefAsst(fmt.Sprintf("step %d", k), "Bash", `{"cmd":"true"}`)) + } + cf := corefFor(t, "") + var rep components.Report + if _, err := cf.Offload(req, &rep, corefCtx(store.NewMemory(store.Options{}))); err != nil { + t.Fatal(err) + } + if got := schema.MessageText(req.Input[2]); !strings.Contains(got, "foobarbaz") { + t.Fatal("cut an output the index has no evidence about; the agent had just said it " + + "needs david's address, and the payload was never copied into a model turn") + } + if rep.Gates["class_opaque"] == 0 { + t.Errorf("expected the candidate to be declined as opaque; gates=%v", rep.Gates) + } +} + +// A marker may say WHAT was removed. It may never claim the removal was safe. +// +// The wording it replaced ("no later turn referred back to it") asserted exactly the claim +// that is false whenever the reference was transformed or semantic — tiers 2 and 3, which +// this index cannot see. Only the model can initiate recovery, so a marker that reads as +// reassurance suppresses the expand call that would have repaired the mistake. That failure +// is silent: no counter this component keeps can distinguish "never needed" from "needed and +// never asked for". +func assertMarkerMakesNoSafetyClaim(t *testing.T, marker string) { + t.Helper() + for _, claim := range []string{ + "no later turn referred back", + "survives in a later turn", + "nothing referred", + "safe to", + "not needed", + "no longer needed", + } { + if strings.Contains(strings.ToLower(marker), claim) { + t.Errorf("marker asserts its own safety (%q), which discourages recovery: %q", claim, marker) + } + } +} + +// For structured output the residue must be ADDRESSABLE — the shape, not one arbitrary row. +// An agent looking for someone's address has to be able to tell from the marker alone that +// this is the output where addresses live. +func TestCorefMarkerDescribesStructuredShape(t *testing.T) { + people := strings.Repeat( + `{"name":"david","id":123456,"address":"foobarbaz","city":"haifa"},`, 200) + body := "[" + strings.TrimSuffix(people, ",") + "]" + + req := &bschemas.BifrostChatRequest{Input: []bschemas.ChatMessage{ + corefUser("load the directory"), + corefAsst("Loading.", "query_people", `{"q":"all"}`), + corefTool("p1", body), + corefAsst("Loaded; moving on to unrelated work.", "Bash", `{"cmd":"true"}`), + }} + for k := 0; k < 20; k++ { + req.Input = append(req.Input, corefAsst(fmt.Sprintf("step %d", k), "Bash", `{"cmd":"true"}`)) + } + cf := corefFor(t, "") + var rep components.Report + if _, err := cf.Offload(req, &rep, corefCtx(store.NewMemory(store.Options{}))); err != nil { + t.Fatal(err) + } + got := schema.MessageText(req.Input[2]) + if strings.Contains(got, "foobarbaz") { + t.Fatalf("not cut, so there is no marker to check; gates=%v", rep.Gates) + } + for _, want := range []string{"200 records", "address", "name"} { + if !strings.Contains(got, want) { + t.Errorf("marker omits %q, so the model cannot tell what is in here: %q", want, got) + } + } + assertMarkerMakesNoSafetyClaim(t, got) +} + +func TestCorefStub(t *testing.T) { + for _, tc := range []struct{ name, in, want string }{ + {"array of records", `[{"b":1,"a":2},{"a":3,"b":4}]`, "2 records, fields: a, b"}, + {"array of scalars", `[1,2,3]`, "3 records"}, + {"wrapped collection", `{"total":2,"rows":[{"x":1},{"x":2}]}`, "object, fields: rows, total (rows: 2 items)"}, + {"plain text has no shape", "Traceback (most recent call last):", ""}, + {"empty", "", ""}, + {"malformed json", `[{"a":`, ""}, + } { + if got := corefStub(tc.in); got != tc.want { + t.Errorf("%s: corefStub(%q) = %q, want %q", tc.name, tc.in, got, tc.want) + } + } + // Field order must be stable: the marker text is replayed byte-for-byte every later + // turn, so a map-iteration-ordered descriptor would flip the prefix and cache-write it. + first := corefStub(`[{"z":1,"a":2,"m":3}]`) + for i := 0; i < 50; i++ { + if got := corefStub(`[{"z":1,"a":2,"m":3}]`); got != first { + t.Fatalf("descriptor is not deterministic: %q vs %q", got, first) + } + } + // A wide record must not turn the marker into a schema dump. + var wide strings.Builder + wide.WriteString("[{") + for i := 0; i < 40; i++ { + if i > 0 { + wide.WriteString(",") + } + fmt.Fprintf(&wide, `"field%02d":%d`, i, i) + } + wide.WriteString("}]") + got := corefStub(wide.String()) + if !strings.Contains(got, "…+28") { + t.Errorf("wide record not truncated: %q", got) + } + if len([]rune(got)) > stubCap { + t.Errorf("descriptor exceeds stubCap: %d runes", len([]rune(got))) + } +} diff --git a/components/offload/corefstub.go b/components/offload/corefstub.go new file mode 100644 index 00000000..a5536373 --- /dev/null +++ b/components/offload/corefstub.go @@ -0,0 +1,144 @@ +package offload + +import ( + "encoding/json" + "sort" + "strconv" + "strings" +) + +// The residue a cut leaves behind, and why it is more than a cosmetic choice. +// +// Reversibility is a CAPABILITY, not a guarantee. The stash guarantees the bytes can be +// recovered; only the model can decide to recover them, by calling the expand tool. So a +// wrong cut has three outcomes, not one: +// +// 1. the model notices and expands the right marker — one round-trip plus a cache-write; +// 2. it notices something is missing but cannot tell WHICH marker holds it — several +// expands, or it gives up; +// 3. it never notices, and answers from less information than it had. +// +// Only (1) is the cost the design originally claimed. (3) is silent, and no counter this +// component keeps can see it — expand-rate measures noticed errors only, which is why +// reward is the sole instrument that detects it. +// +// What the residue can actually influence is the gap between (1) and (2): whether the +// model can tell, without expanding, that THIS marker is where the thing it wants lives. +// A head peek — the first ~96 characters — does that well for a file read or a traceback, +// where the head identifies the whole. It does it badly for a record set, where the head +// is one arbitrary row: an agent hunting for someone's address cannot tell from +// `[{"name":"david","id":123,...` whether addresses are in here at all, let alone whose. +// +// So for structured content the residue describes the SHAPE instead: how many records, and +// what fields they carry. That is addressable — "records with keys name/id/address, 200 of +// them" tells the model where to look — where a peek is merely evocative. + +// stubCap bounds the descriptor so the marker can never dominate the message it replaces +// (tryMark's never-worse check would drop the rewrite anyway, but a cut that fails to +// shrink is a wasted candidate rather than a bug). +const stubCap = 200 + +// maxStubKeys bounds how many field names the descriptor lists. Enough to identify what +// the records hold; not a schema dump. +const maxStubKeys = 12 + +// corefStub describes what was cut, in the terms most likely to let the model decide +// whether it needs it back. Returns "" when it can say nothing useful, in which case the +// caller falls back to a head peek. +// +// Deliberately structural and never evaluative: it says what the content IS, never what it +// was worth. An earlier version of this component wrote "no later turn referred back to +// it" into the marker, which is precisely the claim that is FALSE whenever the reference +// was transformed or semantic (tiers 2 and 3) — so it read as reassurance and discouraged +// the expand call that would have repaired the mistake. A marker must not talk the model +// out of recovering content. +func corefStub(content string) string { + t := strings.TrimSpace(content) + if len(t) == 0 { + return "" + } + switch t[0] { + case '[': + return stubArray(t) + case '{': + return stubObject(t) + } + return "" +} + +// stubArray describes a JSON array: its length, and the union of keys across the records +// it holds (sampled — a 10k-element array does not need a full scan to be described). +func stubArray(t string) string { + var items []json.RawMessage + if json.Unmarshal([]byte(t), &items) != nil { + return "" + } + if len(items) == 0 { + return "" + } + keys := map[string]struct{}{} + sampled := 0 + for _, it := range items { + if sampled >= 32 { + break + } + var obj map[string]json.RawMessage + if json.Unmarshal(it, &obj) != nil { + continue // scalar or nested array: no field names to report + } + sampled++ + for k := range obj { + keys[k] = struct{}{} + } + } + out := strconv.Itoa(len(items)) + " records" + if ks := sortedKeys(keys); len(ks) > 0 { + out += ", fields: " + joinKeys(ks) + } + return clipRunes(out, stubCap) +} + +// stubObject describes a JSON object by its top-level keys, and — the common shape for a +// tool that wraps its payload — the length of the one array it contains. +func stubObject(t string) string { + var obj map[string]json.RawMessage + if json.Unmarshal([]byte(t), &obj) != nil { + return "" + } + if len(obj) == 0 { + return "" + } + keys := map[string]struct{}{} + for k := range obj { + keys[k] = struct{}{} + } + out := "object, fields: " + joinKeys(sortedKeys(keys)) + // A single wrapped collection is worth counting: "rows: 400" is the fact that decides + // whether this is the output holding what the model is looking for. + for _, k := range sortedKeys(keys) { + var arr []json.RawMessage + if json.Unmarshal(obj[k], &arr) == nil && len(arr) > 0 { + out += " (" + k + ": " + strconv.Itoa(len(arr)) + " items)" + break + } + } + return clipRunes(out, stubCap) +} + +func sortedKeys(m map[string]struct{}) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) // stable order: the marker text must be byte-identical on replay + return out +} + +// joinKeys lists field names, truncating past maxStubKeys so a wide record does not turn +// the marker into a schema dump. +func joinKeys(ks []string) string { + if len(ks) > maxStubKeys { + return strings.Join(ks[:maxStubKeys], ", ") + ", …+" + strconv.Itoa(len(ks)-maxStubKeys) + } + return strings.Join(ks, ", ") +} diff --git a/components/offload/extract_econ.go b/components/offload/extract_econ.go index 3c57c9d2..92975630 100644 --- a/components/offload/extract_econ.go +++ b/components/offload/extract_econ.go @@ -63,8 +63,13 @@ import ( // tokenValue is the dollars-per-token a SAVED token is worth, and the reason the gate // exists. Both rates are per single token (not per million). type tokenValue struct { + // perToken is what a saved token is worth on each REPLAY turn — every turn after the + // one the compaction is first applied on. perToken float64 - cached bool // true when priced at the cache-read rate + // firstToken is what it is worth on the turn the compaction IS applied, which is a + // different rate on a caching backend and was previously conflated with perToken. + firstToken float64 + cached bool // true when replays are priced at the cache-read rate } // Default agent-model rates (claude-sonnet-5 class, $3/$15 per MTok, cache read 0.1x). @@ -73,17 +78,57 @@ type tokenValue struct { const ( agentFreshPerMTok = 3.00 agentCacheReadPerMTok = 0.30 // 0.1x fresh, the standard Anthropic cache-read multiplier + // 1.25x fresh, the standard Anthropic cache-WRITE multiplier. Needed because the turn a + // compaction is applied on is not a cache-read turn — see savedTokenValue. + agentCacheWritePerMTok = 3.75 ) -// savedTokenValue prices one saved token for THIS request. When the request goes to a -// prompt-caching backend, content the agent re-sends every turn is already in the cached -// prefix, so removing it saves the cache-read rate — the 10x haircut that sinks the -// component's economics. +// savedTokenValue prices one saved token for THIS request, distinguishing the turn the +// compaction is APPLIED on from the turns it is REPLAYED on. Those are different rates on +// a caching backend, and conflating them undervalued the component by ~3.3x. +// +// The old version priced every saved token at the cache-read rate whenever the request was +// cache-aware, reasoning that "content the agent re-sends every turn is already in the +// cached prefix". That is true of a REPLAY turn and false of the turn the cut is made — +// and for extract_llm it is false in the only place the component can act at all. When +// cache-aware it is confined to the TAIL (see the TailOnly check in extract_llm.go), and +// tail content is by definition new this turn: it has never been cached, so on this turn +// it is billed as a cache-WRITE (1.25x fresh) or as plain fresh input if it falls after the +// last cache_control breakpoint — either way 10-12.5x the cache-read rate it was assigned. +// +// Confirmed on live traffic rather than argued: a real SWE-bench trial reported 52,561 +// cache_creation tokens against 746,047 cache_read across 18 turns. New tail content is +// cache-CREATED every turn; it is not read. +// +// The correction matters because the ~30,500 tokens/output break-even quoted in +// evaluateGate — the stated justification for disabling this component on caching backends +// by default — was computed with the old pricing. Directly recomputed against a flat +// $0.00766 haiku call cost and the 0.12 default ratio: +// +// recurring (reuses=6): 30,397 -> 11,550 tokens/output (2.63x) +// first sight (reuses=4): 42,556 -> 12,900 tokens/output (3.30x) +// +// That does not rescue small-output workloads — SWE-bench's largest measured tool output is +// 2,760 tokens, still an order of magnitude short — so the shipping VERDICT for those is +// unchanged even though the number was wrong. It matters on large-output workloads: on +// LOCA-bench captures the eligible set goes from 7 to 31 of 1,639 outputs. +// +// One consequence worth knowing before tuning: because the applied turn now dominates the +// sum, RECURRENCE is a much weaker lever than the old pricing implied — it multiplies the +// expected saving by 1.12x, not 1.40x. +// +// The non-caching path is unchanged by construction: firstToken == perToken == fresh, so +// the total stays (1 + reuses) x fresh exactly as before. func savedTokenValue(c *components.Ctx) tokenValue { if c != nil && c.CacheAware { - return tokenValue{perToken: agentCacheReadPerMTok / 1_000_000, cached: true} + return tokenValue{ + perToken: agentCacheReadPerMTok / 1_000_000, + firstToken: agentCacheWritePerMTok / 1_000_000, + cached: true, + } } - return tokenValue{perToken: agentFreshPerMTok / 1_000_000, cached: false} + fresh := agentFreshPerMTok / 1_000_000 + return tokenValue{perToken: fresh, firstToken: fresh, cached: false} } // priorCallCost is a last-resort per-call cost estimate (~the Terminal-Bench average). @@ -188,7 +233,9 @@ func expectedReuses(seenBefore bool, turnsSoFar int) float64 { // evaluateGate decides whether one candidate output is worth an extraction call. // -// expected saving = tokens we expect to remove x (1 + expected future reuses) x per-token value +// expected saving = tokens we expect to remove x (first-turn value + expected future +// reuses x replay value). The two rates differ on a caching backend: the turn the cut is +// made is a cache-write (or fresh input), only later turns are cache-reads. // expected cost = observed mean cost of one extraction call // // Allow only when saving strictly exceeds cost. Every suppression carries a reason. @@ -198,7 +245,9 @@ func evaluateGate(sizeTokens int, ratio float64, val tokenValue, cost float64, expectedRemoved := float64(sizeTokens) * ratio reuses := expectedReuses(seenBefore, turnsSoFar) // The compaction is applied on this turn AND replayed on each expected future turn. - saving := expectedRemoved * (1 + reuses) * val.perToken + // Applied once at firstToken, then replayed at perToken. On a non-caching backend the + // two rates are equal and this is identical to the old (1 + reuses) x perToken. + saving := expectedRemoved * (val.firstToken + reuses*val.perToken) d := gateDecision{expSaving: saving, expCost: cost} // Hard decline on a caching backend unless explicitly forced. This is the SHIPPING diff --git a/components/offload/extract_econ_test.go b/components/offload/extract_econ_test.go index 7f5bfdf9..3962e4e0 100644 --- a/components/offload/extract_econ_test.go +++ b/components/offload/extract_econ_test.go @@ -43,9 +43,17 @@ func TestGatePermitsOnNonCachingBackend(t *testing.T) { t.Fatalf("non-caching backend must permit a %d-token output: saving=$%.5f cost=$%.5f", size, fresh.expSaving, fresh.expCost) } - // The 10x rate difference must show up in the valuation, not just the verdict. - if ratio := fresh.expSaving / cached.expSaving; math.Abs(ratio-10) > 0.01 { - t.Fatalf("fresh tokens must be worth 10x cached ones, got %.3fx", ratio) + // The rate difference must show up in the valuation, not just the verdict — but it is + // ~3x, not 10x, and the difference is the point of the first-turn split. + // + // 10x is the ratio between a cache-READ and fresh input, so it is the right factor for a + // REPLAY turn. It is wrong for the turn the cut is applied on: that content is new, so it + // is billed as a cache-write ($3.75/MTok) which is actually DEARER than fresh input + // ($3.00). Blended over reuses=4 the totals are 15.00 vs 4.95 per MTok => 3.03x. An + // assertion of 10x here was pinning the old conflation of the two rates. + if ratio := fresh.expSaving / cached.expSaving; math.Abs(ratio-3.0303) > 0.01 { + t.Fatalf("fresh/cached total saving must be ~3.03x (applied turn is a cache-write, "+ + "only replays take the 10x haircut), got %.4fx", ratio) } } @@ -54,10 +62,18 @@ func TestGatePermitsOnNonCachingBackend(t *testing.T) { // 82/103 across sessions, so this is the common case, not an edge case. func TestGatePermitsHighReuseContent(t *testing.T) { val := savedTokenValue(&components.Ctx{CacheAware: true}) - // 34000 tokens: above the ~30.5k cached-RECURRING break-even, below the ~42.6k - // cached-once one. This size is the gate's whole thesis in one fixture — recurrence is - // what tips an otherwise-losing call into profit, so the SAME size goes both ways. - size := 34000 + // 12000 tokens: above the cached-RECURRING break-even (~11,550) and below the + // cached-once one (~12,900). This size is the gate's whole thesis in one fixture — + // recurrence is what tips an otherwise-losing call into profit, so the SAME size goes + // both ways. + // + // It used to be 34000, against break-evens of ~30.5k and ~42.6k. Correcting the applied + // turn from the cache-read rate to the cache-write rate moved both down ~2.6-3.3x AND + // narrowed the gap between them: recurrence now multiplies the saving by 1.12x rather + // than 1.40x, because the applied turn dominates the sum. So the window this fixture has + // to sit in is ~11,550-12,900 — genuinely tight, and the reason the bound is asserted + // here rather than left implicit. + size := 12000 once := evaluateGate(size, defaultCompressionRatio, val, callCost(cheapmodel.HaikuPricing(), size), false, 5, false, true) recur := evaluateGate(size, defaultCompressionRatio, val, callCost(cheapmodel.HaikuPricing(), size), true, 5, false, true) @@ -91,17 +107,20 @@ func TestBreakEvenSizesMatchTheDocumentedVerdict(t *testing.T) { } cachedRecur := breakEven(true, true) freshRecur := breakEven(false, true) - if cachedRecur < 26_000 || cachedRecur > 35_000 { - t.Errorf("cached+recurring break-even = %d tokens, expected ~30,500 "+ + if cachedRecur < 10_000 || cachedRecur > 13_500 { + t.Errorf("cached+recurring break-even = %d tokens, expected ~11,600 "+ "(docs/components/extract_llm.md states this figure)", cachedRecur) } if freshRecur < 1_400 || freshRecur > 2_400 { t.Errorf("fresh+recurring break-even = %d tokens, expected ~1,800", freshRecur) } - // The gap is WIDER than the bare 10x rate haircut, because cost stops growing once the - // prompt hits the shown-content cap while value keeps scaling with output size. - if ratio := float64(cachedRecur) / float64(freshRecur); ratio < 12 || ratio > 30 { - t.Errorf("cached/fresh break-even ratio = %.1fx, expected ~20x", ratio) + // The gap used to be ~20x — WIDER than the bare 10x rate haircut, because cost stops + // growing once the prompt hits the shown-content cap while value keeps scaling. Pricing + // the APPLIED turn as a cache-write rather than a cache-read cut it to ~6.4x: the applied + // turn is billed at nearly the fresh rate in both regimes, so only the replay turns still + // take the haircut. + if ratio := float64(cachedRecur) / float64(freshRecur); ratio < 5 || ratio > 8 { + t.Errorf("cached/fresh break-even ratio = %.1fx, expected ~6.4x", ratio) } } @@ -415,3 +434,75 @@ func TestTooSlowToExplore(t *testing.T) { t.Error("PR #37's measured latency must stop exploration") } } + +// --- First-turn vs replay pricing (the tail is not a cache-read) --------------- + +// The turn a compaction is APPLIED on is not a cache-read turn. When cache-aware, +// extract_llm can only act on the TAIL, and tail content has never been cached — on that +// turn it is billed as a cache-write (or fresh input), 10-12.5x the cache-read rate. The +// old pricing charged every turn at cache-read and so undervalued the component ~3.3x. +// +// This pins the arithmetic rather than the conclusion: the ratio between the two formulas +// is what the shipping decision (disabled on caching backends, justified by a ~30,500 +// tok/output break-even) rests on. +func TestCachedSavingPricesTheAppliedTurnAsAWrite(t *testing.T) { + val := savedTokenValue(&components.Ctx{CacheAware: true}) + if val.firstToken <= val.perToken { + t.Fatalf("applied turn must be worth MORE than a replay turn on a caching backend: "+ + "first=%v replay=%v", val.firstToken, val.perToken) + } + if got, want := val.firstToken*1e6, agentCacheWritePerMTok; got != want { + t.Errorf("applied turn priced at $%.2f/MTok, want the cache-write rate $%.2f", got, want) + } + if got, want := val.perToken*1e6, agentCacheReadPerMTok; got != want { + t.Errorf("replay turn priced at $%.2f/MTok, want the cache-read rate $%.2f", got, want) + } + + // The correction's size, at the default first-sight reuse prior (4). + const reuses = 4.0 + old := (1 + reuses) * agentCacheReadPerMTok + corrected := agentCacheWritePerMTok + reuses*agentCacheReadPerMTok + if ratio := corrected / old; ratio < 3.0 || ratio > 3.6 { + t.Errorf("correction is %.2fx; expected ~3.3x — if the rates changed, re-derive the "+ + "break-even quoted in evaluateGate and savedTokenValue before editing this bound", ratio) + } +} + +// The non-caching path must be BYTE-identical to the old single-rate formula. The fix is +// meant to change the cached case only, and this is the guard that keeps a future edit from +// silently repricing every non-caching workload the published numbers came from. +func TestNonCachingPricingIsUnchangedByTheFirstTurnSplit(t *testing.T) { + val := savedTokenValue(&components.Ctx{CacheAware: false}) + if val.firstToken != val.perToken { + t.Fatalf("non-caching: both rates must be fresh input; first=%v replay=%v", + val.firstToken, val.perToken) + } + for _, reuses := range []float64{3, 4, 6} { + oldWay := (1 + reuses) * val.perToken + newWay := val.firstToken + reuses*val.perToken + if oldWay != newWay { + t.Errorf("reuses=%v: non-caching saving changed (%v -> %v)", reuses, oldWay, newWay) + } + } +} + +// A cached output that the OLD pricing suppressed and the CORRECTED pricing permits: the +// whole point of the fix. Sized between the two break-evens (~9.2k corrected, ~30.5k old). +func TestCorrectedPricingPermitsMidSizedCachedOutput(t *testing.T) { + const size = 20000 // above the corrected break-even, below the old one + cost := callCost(cheapmodel.HaikuPricing(), size) + val := savedTokenValue(&components.Ctx{CacheAware: true}) + + removed := float64(size) * defaultCompressionRatio + oldSaving := removed * (1 + 4) * val.perToken // the pre-fix formula + newSaving := removed * (val.firstToken + 4*val.perToken) // what the gate computes now + + if !(oldSaving < cost) { + t.Skipf("fixture no longer straddles the break-even (old saving $%.5f vs cost $%.5f); "+ + "resize it rather than deleting this test", oldSaving, cost) + } + if !(newSaving > cost) { + t.Fatalf("corrected pricing still cannot pay for a %d-token cached output: "+ + "saving=$%.5f cost=$%.5f", size, newSaving, cost) + } +} diff --git a/components/offload/extract_llm.go b/components/offload/extract_llm.go index cc62692e..6f86acb1 100644 --- a/components/offload/extract_llm.go +++ b/components/offload/extract_llm.go @@ -3,6 +3,7 @@ package offload import ( "context" "errors" + "fmt" "log/slog" "os" "regexp" @@ -15,6 +16,7 @@ import ( "github.com/rossoctl/context-guru/components" "github.com/rossoctl/context-guru/expand" "github.com/rossoctl/context-guru/internal/cheapmodel" + "github.com/rossoctl/context-guru/internal/coref" "github.com/rossoctl/context-guru/internal/extract" "github.com/rossoctl/context-guru/internal/modelinfo" "github.com/rossoctl/context-guru/metrics" @@ -126,6 +128,11 @@ type ExtractLLM struct { mu sync.Mutex llmSeen map[string]int // session -> count of qualifying (LLM-eligible) requests + // selectionMode picks HOW the model decides. "" (default) = the per-output trim loop. + // "merged" = ONE bulk adjudication carrying the co-reference criterion; see + // extract_llm_merged.go, and note the prior is negative -- the per-output form of the + // merged idea is refuted and the deterministic index beat every model arm measured. + selectionMode string // minTokensSet records whether the operator pinned min_tokens / trigger explicitly. // When they did, their threshold governs (backward compatibility). When they did not, // the derived pressure-based trigger is the default — no per-workload tuning (#28 E). @@ -136,6 +143,13 @@ type ExtractLLM struct { // allowCached permits extraction on prompt-caching backends. Default FALSE — see // extractLLMConfig.AllowOnCachingBackend for why the default ships disabled there. allowCached bool + // allowPrefix lets this component reach the provider's already-cached prefix, paying a + // cache-write for it. See extractLLMConfig.AllowCachedPrefix. + allowPrefix bool + // prefixMinLater is the opportunity floor applied to prefix candidates only. + prefixMinLater int + // prefixClasses are the co-reference verdicts eligible for prefix work. + prefixClasses map[coref.Class]bool // pricing prices the extraction model's tokens for the gate's cost side (#28 D). pricing cheapmodel.Pricing // ratios learns this workload's real compression ratio instead of assuming one. @@ -259,6 +273,63 @@ type extractLLMConfig struct { // defensible default. Set true if your outputs are genuinely huge; the gate's // economics then decide each call as normal. AllowOnCachingBackend *bool `yaml:"allow_on_caching_backend"` + // AllowCachedPrefix lets extraction reach content the provider has ALREADY CACHED, + // rather than only the uncached tail. Unset = FALSE, because doing so deliberately + // breaks the prefix hash and forces a cache-write of the suffix — the one cost every + // other offloader refuses to pay (see prefix_econ.go). + // + // This is the "full body" reach. Lifting the tail restriction has a cache cost, which is + // why enabling this switches on two extra gates that do not apply to tail work — but the + // cost is not the only thing it changes. + // + // CORRECTED (iteration 019): an earlier version of this comment claimed the tail + // restriction "is not a safety property of the model call, it is a cache-cost property". + // That is wrong. It is also an INFORMATION property. Need is relevance minus whatever has + // already been captured elsewhere, and that second term lives in the turns AFTER the + // output — which the model is not shown. On tail content there are few later turns, so + // almost nothing can have superseded it and relevance ≈ need; the local prompt is sound + // precisely because of the restriction. Lifting it moves the model onto old outputs, where + // the gap between relevance and need is widest, while still asking the local question. + // The index pre-filter below is what compensates — so a selection mode that bypasses that + // pre-filter (see selectionMode == "merged") has neither the index nor the window. + // Measured in docs/experiments/loca/iter019/results.md §4. + // + // The two extra gates: + // + // 1. the co-reference index as an eligibility pre-filter, so the component never pays + // a model call to look at prefix content a cheap deterministic pass can already + // show is still referenced; and + // 2. the S*T > 11.5*W break-even from prefix_econ.go, so a prefix rewrite has to be + // repaid by the turns remaining in the session. + // + // The division of labour is the point: the index looks BACKWARD (what has already been + // referenced and is therefore spent) and the model looks FORWARD (how much of what + // remains will still be needed). Neither sees what the other sees. + AllowCachedPrefix *bool `yaml:"allow_cached_prefix"` + // PrefixMinLaterTurns is the opportunity floor for prefix candidates, mirroring + // coref's: an output with fewer model turns after it has not yet HAD a chance to be + // referenced, so absence of references says nothing about it. + PrefixMinLaterTurns *int `yaml:"prefix_min_later_turns"` + // PrefixClasses selects which co-reference verdicts become prefix candidates. Default + // ["unreferenced", "closed"], and the choice is a real trade-off rather than a tuning + // detail: + // + // unreferenced — the index found no later EXACT reuse. That is not "unused": the index + // matches identifiers, so a value the model summed, converted or reworded leaves no + // substring behind (tiers 2 and 3). Handing this class to the model is therefore not + // asking it to trim, it is asking it to VETO — to notice an implicit reference the + // index structurally cannot see. It yields little when the index was right, and it is + // the only mechanism available for catching when it was wrong. + // + // closed — referenced once or twice, long ago. Something WAS taken out of it, and an + // exact matcher cannot tell "took the value, rest is chaff" from "took an ANCHOR and + // still needs the payload it points at". That ambiguity is why coref's cut_closed + // ships off, and it is exactly a judgement call: a model can read the output, see the + // reference was a name or an id, and keep the payload a blind cut would lose. + // + // So closed is where trimming earns its call, and unreferenced is where a veto might. + // Which of those is worth paying for is an experimental question, hence a list. + PrefixClasses []string `yaml:"prefix_classes"` // EconomicGate opts out of the expected-value gate (#28 D). Unset = ON (the default): // only call the LLM when the expected saving exceeds the expected call cost, priced // from real model rates and the cache-awareness of the traffic. Set false to restore @@ -268,6 +339,18 @@ type extractLLMConfig struct { // static table cannot name (a self-hosted id like `qwen3-coder-30b`, or a gateway alias // that hides the real model). Unset = resolved per model, see (*ExtractLLM).inputLimit. ModelMaxInput int `yaml:"model_max_input_tokens"` + // SelectionMode chooses the decision shape. Unset (default) = per-output trimming, one call + // per output. "merged" = a single BULK adjudication over all candidates together, carrying the + // co-reference evidence in the prompt, so one call makes both the backward-looking and + // forward-looking judgement. + // + // The default is not a preference, it is a measurement. Per-output merged judgement is REFUTED + // (6% live-kept, inside the drop-everything null model's error bar), bulk is the only model + // shape that worked (58%), and the free deterministic index still beat both (95% at 11% + // false-drop). See docs/results/coref-selection-experiment.md and + // docs/experiments/loca/iter009/results.md. `merged` exists to test REWARD, which no + // decision-quality experiment can speak to. + SelectionMode string `yaml:"selection_mode"` // SkipFileReads controls whether line-numbered source-file dumps are left verbatim. // Tri-state: unset = AUTO (skip when the request is prompt-cached, reduce otherwise); // true = always skip; false = always reduce. Rationale (measured, SWE-bench 50): @@ -320,6 +403,48 @@ func newExtractLLM(raw []byte) (components.Component, error) { if cfg.AllowOnCachingBackend != nil { allowCached = *cfg.AllowOnCachingBackend } + // Prefix reach is off unless asked for, and asking for it implies the caching backend + // is in play at all — there is no cached prefix to reach otherwise. + allowPrefix := false + if cfg.AllowCachedPrefix != nil { + allowPrefix = *cfg.AllowCachedPrefix + } + if allowPrefix { + allowCached = true + } + prefixMinLater := corefMinLaterDefault // the same floor coref applies + prefixClasses := map[coref.Class]bool{coref.Unreferenced: true, coref.Closed: true} + if len(cfg.PrefixClasses) > 0 { + prefixClasses = map[coref.Class]bool{} + for _, n := range cfg.PrefixClasses { + switch cls := coref.Class(strings.ToLower(strings.TrimSpace(n))); cls { + case coref.Unreferenced, coref.Closed: + prefixClasses[cls] = true + case coref.Open, coref.Opaque: + // Refused rather than honoured. `open` is content a later turn demonstrably + // still uses and `opaque` is content the index cannot see into at all — for + // neither is there evidence of being spent, so admitting them would turn the + // pre-filter into "consider everything" and lose the one property that makes + // prefix reach affordable. + return nil, fmt.Errorf("extract_llm: prefix_classes may not include %q: it is "+ + "not evidence of spent content", cls) + default: + return nil, fmt.Errorf("extract_llm: unknown prefix_classes entry %q", n) + } + } + } + if cfg.PrefixMinLaterTurns != nil { + prefixMinLater = *cfg.PrefixMinLaterTurns + } + switch strings.ToLower(strings.TrimSpace(cfg.SelectionMode)) { + case "", "merged": + default: + // Reject at construction rather than silently falling back: a mistyped selection_mode + // would otherwise run the DEFAULT shape while the operator believed they were measuring + // the merged one, and the arms are indistinguishable from the outside. 15 of this repo's + // 23 components parse their own YAML non-strictly, so a typo here is otherwise invisible. + return nil, fmt.Errorf("extract_llm: selection_mode %q is not one of \"\" (per-output) or \"merged\"", cfg.SelectionMode) + } return &ExtractLLM{ minTokens: cfg.MinTokens, strategy: cfg.Strategy, modelSource: cfg.Model.Source, modelClient: cfg.Model.Client(), @@ -327,9 +452,11 @@ func newExtractLLM(raw []byte) (components.Component, error) { llmEveryN: cfg.LLMEveryN, llmMaxPerReq: cfg.LLMMaxPerReq, skipFileReads: cfg.SkipFileReads, llmSeen: map[string]int{}, minTokensSet: explicit, gate: gate, allowCached: allowCached, - pricing: cheapmodel.PricingFromEnv(), - prevTokens: map[string]int{}, modelName: cfg.Model.Model, - modelMaxInput: cfg.ModelMaxInput, + allowPrefix: allowPrefix, prefixMinLater: prefixMinLater, + prefixClasses: prefixClasses, + pricing: cheapmodel.PricingFromEnv(), + prevTokens: map[string]int{}, modelName: cfg.Model.Model, + modelMaxInput: cfg.ModelMaxInput, selectionMode: strings.ToLower(strings.TrimSpace(cfg.SelectionMode)), }, nil } @@ -474,6 +601,32 @@ func (e *ExtractLLM) Offload(req *bschemas.BifrostChatRequest, rep *components.R content string id string } + // PREFIX ELIGIBILITY — the backward-looking half of the full-body pass. + // + // Reaching the cached prefix costs a cache-write, so the component must not pay a model + // call to discover that prefix content is still in use. The co-reference index answers + // that deterministically and for free: an output is eligible only if it introduced + // identifiers AND no later model turn carried any of them forward. Anything still + // referenced (open), or that the index cannot see into at all (opaque), is left alone + // without a call. + // + // Indexed from the PRISTINE request for the same reason coref does it: an earlier + // replay would have removed identifiers from the exclusion sets and silently + // reclassified unrelated outputs. + prefixSpent := map[int]bool{} + if e.allowPrefix && c.CacheAware { + for _, r := range coref.Index(flattenForCoref(req), floor, schema.TextTokens) { + cls := coref.Classify(r, corefClosedDistDefault, corefOpenRepsDefault, e.prefixMinLater) + if e.prefixClasses[cls] { + prefixSpent[r.Idx] = true + } + } + } + // Prefix candidates are tracked separately: their rewrite has to clear break-even as a + // BATCH (one write serves all of them), which cannot be decided per candidate. + prefixIdx := []int{} + prefixExpSaved := 0.0 + var cands []cand skipFR := false if e.skipFileReads != nil { @@ -531,13 +684,31 @@ func (e *ExtractLLM) Offload(req *bschemas.BifrostChatRequest, rep *components.R dbgMaxSz = sz } if c.CacheAware && !c.TailOnly(i) { - dbgTail++ - if sz >= floor { - dbgBigTailBlocked++ // a large output we skipped ONLY because it's not in the tail - rep.Gate("cached_prefix_above_floor") + if !e.allowPrefix { + dbgTail++ + if sz >= floor { + dbgBigTailBlocked++ // large, skipped ONLY because it's not in the tail + rep.Gate("cached_prefix_above_floor") + } + rep.Gate("cached_prefix") + continue + } + // Prefix reach is on. The co-reference index is the cheapest gate there is, so + // it goes first: no model call, no economics, nothing, for content a free + // deterministic pass can already show is still in use. + // + // EXCEPT in merged mode, where this gate is self-defeating twice over. Measured: it + // removed 149,681 candidates in one arm, leaving ~1 per request, so the "bulk" + // adjudication returned 1.02 verdicts per call and was really the PER-OUTPUT design -- + // the one refuted at 6% live-kept. And filtering by the index means the model only ever + // sees what the index has ALREADY judged spent, which destroys the one thing merged mode + // exists to provide: a veto on the index's Tier-1 blind spot. The index's verdict still + // reaches the model, but as EVIDENCE inside the prompt (see renderEvidence) rather than + // as a gate that pre-decides the answer. + if !prefixSpent[i] && e.selectionMode != "merged" { + rep.Gate("prefix_still_referenced") + continue } - rep.Gate("cached_prefix") - continue } // CROSS-SESSION reuse (#28 C), deliberately placed AFTER the tail gate — the // invariant TestGlobalCacheHitIsNotSplicedAtDepth exists to protect. An extraction is @@ -643,6 +814,11 @@ func (e *ExtractLLM) Offload(req *bschemas.BifrostChatRequest, rep *components.R } metrics.RecordExtractionReason(d.reason) } + if e.allowPrefix && c.CacheAware && !c.TailOnly(i) { + // Eligibility was decided at the tail gate; record it for the batch break-even. + prefixIdx = append(prefixIdx, i) + prefixExpSaved += float64(sz) * ratio + } cands = append(cands, cand{i, content, id}) } if debugExtractLLM && len(tools) > 0 { @@ -665,78 +841,140 @@ func (e *ExtractLLM) Offload(req *bschemas.BifrostChatRequest, rep *components.R // running them concurrently (bounded) keeps a turn's cost to ~one call's wall time — // so parallel beats a single-call batch on tokens AND latency. Each output fails open // independently (a miss leaves that one verbatim). + // PREFIX BREAK-EVEN — one cache-write serves the whole prefix batch, so it is priced + // once, on the batch, exactly as coref prices its pass. Expected (not realised) saving, + // because the model has not run yet; that is the same estimate the economic gate above + // already reasons with. + if len(prefixIdx) > 0 { + shallowest := prefixIdx[0] + for _, i := range prefixIdx { + if i < shallowest { + shallowest = i + } + } + need, have, ok := prefixRewritePays(req, int(prefixExpSaved), shallowest, c) + if !ok { + // Drop the prefix candidates and keep any tail ones: the tail costs no write, so + // a failed prefix batch must not suppress work that was already free. + keep := cands[:0] + inPrefix := map[int]bool{} + for _, i := range prefixIdx { + inPrefix[i] = true + } + for _, cd := range cands { + if !inPrefix[cd.i] { + keep = append(keep, cd) + } + } + cands = keep + rep.Gate("prefix_rewrite_not_repaid") + if debugExtractLLM { + slog.Info("cg.debug.extract_llm.prefix", "decision", "decline", + "needTurns", need, "haveTurns", have, "expSaved", int(prefixExpSaved), + "shallowest", shallowest, "candidates", len(prefixIdx)) + } + } else if debugExtractLLM { + slog.Info("cg.debug.extract_llm.prefix", "decision", "allow", + "needTurns", need, "haveTurns", have, "expSaved", int(prefixExpSaved), + "candidates", len(prefixIdx)) + } + } + if len(cands) > 0 { type outT struct{ projected, summary string } out := make([]outT, len(cands)) - sem := make(chan struct{}, llmConcurrency) - var wg sync.WaitGroup - for k := range cands { - wg.Add(1) - go func(k int) { - defer wg.Done() - sem <- struct{}{} - defer func() { <-sem }() - ctx, cancel := context.WithTimeout(c.Ctx, llmCallTimeout) - defer cancel() - before := schema.TextTokens(cands[k].content) - start := time.Now() - res, sum, _ := extract.RunExtractionSummary(ctx, cands[k].content, goal, keepIDs, before, extCfg, model) - metrics.RecordExtractionCall(float64(time.Since(start).Milliseconds())) - // CLASSIFY THE SILENT FAILURE — and classify it INDEPENDENTLY of whether - // a result came back. RunExtractionSummary returns ("", "", "none") for every - // failure mode, so timeout / sandbox rejection / "nothing shrank" are - // indistinguishable in its return value. Our own ctx is the one reliable - // signal: if its deadline expired, THIS call was abandoned. - // - // Do NOT fold this into an `else` of the success check. In `code` mode the - // deterministic strategy runs as a fallback (extract.go:367-368), so a call - // whose LLM leg timed out can still return a smaller `res` — and an `else` - // would then record nothing. That is exactly the shape of the bug these - // counters exist to expose: the arm keeps compacting a little, so no - // dashboard looks broken while the expensive path has silently stopped. - // - // Fail-open behaviour is unchanged either way — this only records. - timedOut := errors.Is(ctx.Err(), context.DeadlineExceeded) - if ctx.Err() != nil { - if timedOut { - atomic.AddInt64(&llmTimeouts, 1) - } else { - atomic.AddInt64(&llmErrors, 1) - } - } - if res != "" && res != cands[k].content { - out[k] = outT{res, sum} - // Feed the observed ratio so the gate prices future calls on what this - // workload actually achieves, not on an assumption. - e.ratios.observe(before-schema.TextTokens(res), before) - metrics.RecordExtractionSaving(before - schema.TextTokens(res)) - } else if !timedOut { - e.ratios.observe(0, before) // a miss is real evidence: ratio 0 + + // MERGED MODE: one bulk adjudication instead of one call per output. It fills the same + // `out` slots, so phase 3 below -- freeze, marker, store, never-worse, every counter -- is + // shared verbatim. Only the decision differs, which is what makes the two arms comparable. + if e.selectionMode == "merged" { + in := make([]mergedInput, 0, len(cands)) + for _, cd := range cands { + in = append(in, mergedInput{Idx: cd.i, Content: cd.content, ID: cd.id}) + } + // ONE call for the whole batch -- that is the entire point of the merged shape, and + // hoisting it out of the loop is load-bearing rather than stylistic: per-candidate + // calls would be the per-output design that measured 6% live-kept. + dec := e.adjudicateMerged(req, rep, c, in, goal, model) + for k := range cands { + if d, ok := dec[cands[k].i]; ok { + out[k] = outT{d.Projected, d.Summary} } - // TIMED OUT WITH NOTHING BACK => DELIBERATELY NOT OBSERVED. A ratio-0 - // observation means "the model looked at this output and could not shrink - // it", which is real evidence about the workload. A deadline means the call - // never finished — evidence about SERVER LATENCY, not compressibility — and - // feeding it to the tracker makes the gate shut itself permanently on - // exactly the deployment where the budget is already too small: - // - // minRatioSampleTokens is 1500, so ONE timed-out medium output both ends - // this session's exploration (r.total >= the sample floor => exploring() - // returns false) and starts dragging ratio() down from the 0.12 prior. A - // few more and expectedRemoved falls below call cost for everything, so - // evaluateGate suppresses every call — and the tracker lives on the - // Pipeline for the proxy's LIFETIME, so nothing revises it afterwards. - // - // That is the self-justifying prior extract_econ.go's exploration budget - // exists to prevent, re-entered through the timeout path. MEASURED: 13 - // timeouts in one 50-task arm at the 90s budget on a KV-pressured TP=1 - // server, i.e. this is a live regime, not a hypothetical. Skipping the - // observation leaves the gate's estimate untouched; the timeouts are still - // counted (above) and still brake exploration via slowCallMs, which is the - // latency-aware layer that SHOULD react to a slow server. - }(k) - } - wg.Wait() + } + goto splice + } + { + sem := make(chan struct{}, llmConcurrency) + var wg sync.WaitGroup + for k := range cands { + wg.Add(1) + go func(k int) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + ctx, cancel := context.WithTimeout(c.Ctx, llmCallTimeout) + defer cancel() + before := schema.TextTokens(cands[k].content) + start := time.Now() + res, sum, _ := extract.RunExtractionSummary(ctx, cands[k].content, goal, keepIDs, before, extCfg, model) + metrics.RecordExtractionCall(float64(time.Since(start).Milliseconds())) + // CLASSIFY THE SILENT FAILURE — and classify it INDEPENDENTLY of whether + // a result came back. RunExtractionSummary returns ("", "", "none") for every + // failure mode, so timeout / sandbox rejection / "nothing shrank" are + // indistinguishable in its return value. Our own ctx is the one reliable + // signal: if its deadline expired, THIS call was abandoned. + // + // Do NOT fold this into an `else` of the success check. In `code` mode the + // deterministic strategy runs as a fallback (extract.go:367-368), so a call + // whose LLM leg timed out can still return a smaller `res` — and an `else` + // would then record nothing. That is exactly the shape of the bug these + // counters exist to expose: the arm keeps compacting a little, so no + // dashboard looks broken while the expensive path has silently stopped. + // + // Fail-open behaviour is unchanged either way — this only records. + timedOut := errors.Is(ctx.Err(), context.DeadlineExceeded) + if ctx.Err() != nil { + if timedOut { + atomic.AddInt64(&llmTimeouts, 1) + } else { + atomic.AddInt64(&llmErrors, 1) + } + } + if res != "" && res != cands[k].content { + out[k] = outT{res, sum} + // Feed the observed ratio so the gate prices future calls on what this + // workload actually achieves, not on an assumption. + e.ratios.observe(before-schema.TextTokens(res), before) + metrics.RecordExtractionSaving(before - schema.TextTokens(res)) + } else if !timedOut { + e.ratios.observe(0, before) // a miss is real evidence: ratio 0 + } + // TIMED OUT WITH NOTHING BACK => DELIBERATELY NOT OBSERVED. A ratio-0 + // observation means "the model looked at this output and could not shrink + // it", which is real evidence about the workload. A deadline means the call + // never finished — evidence about SERVER LATENCY, not compressibility — and + // feeding it to the tracker makes the gate shut itself permanently on + // exactly the deployment where the budget is already too small: + // + // minRatioSampleTokens is 1500, so ONE timed-out medium output both ends + // this session's exploration (r.total >= the sample floor => exploring() + // returns false) and starts dragging ratio() down from the 0.12 prior. A + // few more and expectedRemoved falls below call cost for everything, so + // evaluateGate suppresses every call — and the tracker lives on the + // Pipeline for the proxy's LIFETIME, so nothing revises it afterwards. + // + // That is the self-justifying prior extract_econ.go's exploration budget + // exists to prevent, re-entered through the timeout path. MEASURED: 13 + // timeouts in one 50-task arm at the 90s budget on a KV-pressured TP=1 + // server, i.e. this is a live regime, not a hypothetical. Skipping the + // observation leaves the gate's estimate untouched; the timeouts are still + // counted (above) and still brake exploration via slowCallMs, which is the + // latency-aware layer that SHOULD react to a slow server. + }(k) + } + wg.Wait() + } + splice: for k := range cands { // Phase 3 (serial): freeze + splice. if out[k].projected == "" { continue diff --git a/components/offload/extract_llm_merged.go b/components/offload/extract_llm_merged.go new file mode 100644 index 00000000..a4bdd1d8 --- /dev/null +++ b/components/offload/extract_llm_merged.go @@ -0,0 +1,354 @@ +package offload + +import ( + "context" + "fmt" + "log/slog" + "strings" + "sync/atomic" + "time" + + bschemas "github.com/maximhq/bifrost/core/schemas" + + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/internal/coref" + "github.com/rossoctl/context-guru/internal/extract" + "github.com/rossoctl/context-guru/metrics" + "github.com/rossoctl/context-guru/schema" +) + +// THE MERGED DESIGN: one model call that carries the co-reference criterion, replacing the +// per-output trim loop. +// +// The idea, in the form it was originally put: once you are already paying for an LLM call, that +// call can also decide what has been referenced and is spent — so the backward-looking index and the +// forward-looking model stop being two passes. The value would be in the cases an exact matcher +// structurally cannot judge: Tier-2/3 reuse (a value transformed before being restated leaves no +// substring) and the anchor-vs-payload ambiguity (was the reference a pointer, or the payload +// itself?). +// +// WHAT THE EVIDENCE ALREADY SAYS, so this is not built naively: +// +// * The PER-OUTPUT form of this is REFUTED — 6% live-kept, inside the null model's error bar +// (docs/results/coref-selection-experiment.md finding 1). Shown one output at a time, a model +// just drops it. So this implementation is BULK: all candidates in one call, judged +// comparatively, which measured 58%. +// * The deterministic index still beat every model arm (95% live-kept at 11% false-drop, free), +// and neither floor-symmetry nor a widened Tier-2 ground truth closed the gap +// (docs/experiments/loca/iter009/results.md). So the prior here is NEGATIVE. +// * Those experiments measured decision quality on captured traffic and explicitly could not speak +// to reward. Reward is the one axis left, and the only reason to build this. +// +// Opt-in via `selection_mode: merged`. Default stays the per-output trim loop. +// +// The integration point is deliberately narrow: this fills the SAME projected/summary slots the +// parallel per-output loop fills, so freezing, marker creation, the store, the never-worse check and +// every counter downstream are untouched and shared. Only the decision changes, not the mechanics. + +// unusableSamples bounds how many unparseable replies get logged in full. Process-wide, because the +// question it answers ("what is the model actually emitting?") is answered by the first few and the +// rest would only be log volume. +var unusableSamples atomic.Int64 + +const maxUnusableSamples = 5 + +// mergedSampleChars bounds each output shown in the prompt. The whole point is comparative +// judgement across many outputs, so the per-output budget must stay small enough that ~15 of them +// plus the contract still fit the extraction model's window. +const mergedSampleChars = 4000 + +// mergedMaxItems caps one adjudication. +// +// 12, not 15. Batch size is a YIELD/SAFETY trade-off, measured in +// docs/experiments/loca/iter019/results.md §"batch size": at batch 3-6 the model dropped a +// genuinely-spent output only half the time -- small batches do not make it wrong, they make it +// unwilling to act, which is what a 94% keep rate at a live batch of 2.63 looks like. At batch 10 it +// cleared 100% of the genuinely-spent candidates. But at batch 16 the transport burden started to +// tell: 4 of 37 required quotes came back non-verbatim, against 0 of 16 at batch 10. So the ceiling +// sits between 10 and 16 and this picks the conservative end of it. +const mergedMaxItems = 12 + +// renderEvidence formats one output's co-reference record for the prompt. Counts only — no +// identifier lists — because the measured win came from comparative ranking, not from more detail, +// and every extra token here is paid on every candidate. +func renderEvidence(r *coref.Record, laterTurns int) string { + if r == nil { + // No record: the output was below the index's size floor, so the index has no opinion. Say + // so plainly rather than emitting zeros, which would read as "nothing referenced it". + return fmt.Sprintf("no index record (below size floor); later_turns=%d", laterTurns) + } + age := "never" + if r.RefAge >= 0 { + age = fmt.Sprintf("%d messages ago", r.RefAge) + } + return fmt.Sprintf("novel=%d refs=%d ref_age=%s used_frac=%.2f later_turns=%d verdict_of_index=%s", + r.Novel, r.Refs, age, r.UsedFrac, r.LaterTurns, + coref.Classify(*r, corefClosedDistDefault, corefOpenRepsDefault, corefMinLaterDefault)) +} + +// mergedInput is one candidate offered for adjudication. A package-level type because +// extract_llm.go's `cand` is function-local, and coupling to it would drag this whole decision path +// back inside Offload. +type mergedInput struct { + Idx int + Content string + ID string +} + +// mergedDecision is what phase 3 needs: the projection to splice and its summary segment. Keyed by +// message index on return so the caller maps it back without either side knowing the other's types. +type mergedDecision struct { + Projected string + Summary string +} + +// adjudicateMerged runs ONE model call over ALL candidates and returns the decisions keyed by +// message index. Any failure returns nil, which the caller treats as "change nothing" — the +// fail-open direction the whole pipeline uses. +func (e *ExtractLLM) adjudicateMerged( + req *bschemas.BifrostChatRequest, rep *components.Report, c *components.Ctx, + cands []mergedInput, goal string, model components.Model, +) map[int]mergedDecision { + if model == nil || len(cands) == 0 { + return nil + } + // OFFERED, counted separately from ANSWERED. verdicts/calls was used as the batch-size metric + // and it is not one: it counts what the model chose to answer, not what it was shown. Live, that + // read 2.80 while merged_batch_truncated fired 43 times in 162 calls -- arithmetically impossible + // for offered batches, and the resolution is that the model silently omits labels. Without this + // counter, "the batch is starved" and "the model answered for a third of the batch" are the same + // number, and the first reading cost three iterations. + rep.GateN("merged_offered", len(cands)) + if len(cands) > mergedMaxItems { + // NO SILENT CAPS. A truncated batch is a bounded-coverage decision and must be visible in + // the counters, or "we judged everything" and "we judged the first twelve" read identically. + rep.Gate("merged_batch_truncated") + cands = cands[:mergedMaxItems] + } + + // The index's own measurements, keyed by message index, so the model sees what the + // deterministic pass concluded and can veto it rather than duplicate it. + // The transcript as one string, so a claimed obligation quote can be verified against what the + // agent was actually told rather than trusted. + var fb strings.Builder + for _, m := range flattenForCoref(req) { + for _, t := range m.Texts { + fb.WriteString(t) + fb.WriteByte('\n') + } + for _, r := range m.Results { + fb.WriteString(r.Text) + fb.WriteByte('\n') + } + } + flat := fb.String() + + recs := map[int]*coref.Record{} + for _, r := range coref.Index(flattenForCoref(req), e.minTokens, schema.TextTokens) { + rr := r + recs[r.Idx] = &rr + } + laterTurns := func(i int) int { + n := 0 + for j := i + 1; j < len(req.Input); j++ { + if req.Input[j].Role == bschemas.ChatMessageRoleAssistant { + n++ + } + } + return n + } + + items := make([]extract.BulkItem, 0, len(cands)) + for _, cd := range cands { + items = append(items, extract.BulkItem{ + Index: cd.Idx, + ID: cd.ID, + SizeTokens: schema.TextTokens(cd.Content), + Evidence: renderEvidence(recs[cd.Idx], laterTurns(cd.Idx)), + Sample: truncateForPrompt(cd.Content, mergedSampleChars), + }) + } + + ctx, cancel := context.WithTimeout(c.Ctx, llmCallTimeout) + defer cancel() + start := time.Now() + + // PREFER A PREFIX ASK. With the transcript above the question, the model reads the outputs in + // full from the provider's cache instead of judging truncated samples we paid fresh to ship -- + // and it can finally see the later turns, which is where "already captured elsewhere" lives. A + // bare completion cannot show it either of those things. + // + // Falls back to the plain completion on the first turn of a session (nothing forwarded yet), when + // the feature is off, and on any error. Falling back rather than skipping matters: treating "no + // prefix" as "no verdicts" would silently turn the component off for the first turn of every + // session and look like a model that declined to act. + var reply string + var err error + if c.PrefixAsk != nil { + var u components.PrefixUsage + reply, u, err = c.PrefixAsk.Ask(ctx, c.Session, extract.BuildPrefixAsk(goal, items)) + if err != nil { + rep.Gate("prefix_ask_failed") + reply, err = model.Complete(ctx, extract.BuildBulkPrompt(goal, items)) + } else { + rep.Gate("prefix_ask_used") + // THE justification for the whole mechanism, so it is counted rather than assumed. A + // prefix ask that reads nothing from cache is paying fresh for the transcript -- roughly + // ten times the intended cost -- and is indistinguishable from a working one except on + // the bill. + if u.CacheRead == 0 { + rep.Gate("prefix_ask_cache_read_ZERO") + } + } + } else { + reply, err = model.Complete(ctx, extract.BuildBulkPrompt(goal, items)) + } + metrics.RecordExtractionCall(float64(time.Since(start).Milliseconds())) + if err != nil { + rep.Gate("merged_call_failed") + return nil + } + verdicts, parsed := extract.ParseBulkVerdicts(reply) + if !parsed { + // TRUNCATION IS NOT JUNK, and folding them together hid a 70%-of-calls failure behind a name + // that reads as "the prompt is wrong". A reply that opened the array but never closed it ran + // out of output budget; one that never opened it is a genuine format failure. The remedies + // are opposite -- raise max_tokens versus fix the prompt -- so they get separate counters. + if strings.Contains(reply, "[") && !strings.Contains(reply, "]") { + rep.Gate("merged_reply_truncated") + } else { + rep.Gate("merged_unparseable") + } + // LOG A BOUNDED SAMPLE OF THE ACTUAL TEXT. + // + // Six rounds of this component's failures were diagnosed by inferring a cause from gate + // counters, and each inference was at least partly wrong: the pre-filter, the per-request call + // cap, the economic gate, an always-firing trigger, a batch metric that counted answers rather + // than offers, and a reply cut off by the output budget. A counter can say THAT a reply was + // unusable; only the text says WHY. Reading one is cheaper than a seventh inference. + // + // Bounded in count and length, because a systematic failure would otherwise flood the log with + // transcript content lifted from the reply. + if unusableSamples.Add(1) <= maxUnusableSamples { + head := reply + if len(head) > 500 { + head = head[:500] + } + slog.Warn("cg.merged.unusable_reply", "reply_len", len(reply), + "offered", len(cands), "head", head) + } + return nil + } + if len(verdicts) == 0 { + // A well-formed empty answer: the model read the batch and kept all of it. The contract + // invites that explicitly, so it must not be filed as a failure. + rep.Gate("merged_kept_whole_batch") + return nil + } + + byIdx := map[int]int{} // message index -> position in cands + for k := range cands { + byIdx[cands[k].Idx] = k + } + dec := map[int]mergedDecision{} + for _, v := range verdicts { + k, ok := byIdx[v.Index] + if !ok { + continue // a verdict for something we did not offer + } + content := cands[k].Content + before := schema.TextTokens(content) + nb := strings.ToLower(strings.TrimSpace(v.NeededBy)) + + // THE FORCED EVIDENCE, CHECKED. The model must say which obligation still needs the output + // and quote the transcript text creating it. Verifying the quote is cheap and turns + // "did it make that up?" from a worry into a counter -- the same discipline the old trim + // containment check applied, which caught 8 inventions in 9 attempts. + if q := strings.TrimSpace(v.Quote); q != "" { + if !strings.Contains(flat, q) { + // A fabricated obligation argues for KEEPING, so this is not a dangerous failure -- + // but it is unreliability, and it grew with batch size when measured, so it is the + // signal that says the batch is too large. + rep.Gate("merged_quote_not_verbatim") + } + } else if nb != "" && nb != "none" { + rep.Gate("merged_obligation_unquoted") + } + + var projected, summary string + switch v.Verdict { + case "drop": + // COHERENCE, and it points the dangerous way. The criterion states that a drop requires + // needed_by "none"; a verdict that names an outstanding obligation and then drops the + // output anyway contradicts itself in the direction of silent loss. Refuse it. + if nb != "" && nb != "none" { + rep.Gate("merged_drop_contradicts_obligation") + continue + } + if nb == "" { + // The field was not answered at all. Tolerated rather than refused -- requiring it + // would collapse yield to zero against a model that omits it -- but counted, because + // an unanswered criterion means the forcing function did not run for this verdict + // and the drop carries none of the protection it was measured to provide. + rep.Gate("merged_drop_unjustified") + } + // The residue is the SHAPE descriptor, not a head peek: for a record set the first + // rows say nothing about whether the field you want is in there. See corefstub.go. + projected, summary = mergedResidue(content), "adjudicated spent" + rep.Gate("merged_drop") + default: + rep.Gate("merged_keep") + continue + } + if projected == "" { + continue + } + after := schema.TextTokens(projected) + if after >= before { // never-worse; phase 3 would discard it anyway + continue + } + // Feed the observed ratio so the economic gate prices FUTURE calls on what this workload + // actually achieves. The per-output loop does the same; without it the merged arm would be + // gated on the other shape's history. + e.ratios.observe(before-after, before) + metrics.RecordExtractionSaving(before - after) + dec[v.Index] = mergedDecision{projected, summary} + } + return dec +} + +// mergedResidue is what a dropped output leaves behind. +// +// corefStub gives the SHAPE for structured content ("200 records, fields: name/id/address"), which +// corefstub.go argues is the right residue for a record set: its first rows say nothing about +// whether the field you want is in there. But it returns "" for anything that is not a JSON array +// or object — and that is most real tool output: logs, file reads, tracebacks. Measured: "" for +// newline-delimited JSON and for plain text. +// +// Without a fallback the merged arm would silently fail to act on those: the drop verdict is +// recorded in the gate counters while the projection comes back empty and phase 3 skips it, so the +// arm would look like it was deciding and removing nothing. That is exactly the "acted: 0 is not +// diagnosable" failure the Report.Gates comment warns about, one layer further in. +// +// For unstructured content a HEAD PEEK is the right residue, by corefstub.go's own reasoning +// applied in the other direction: the head identifies the whole for a file read or a traceback, and +// is only misleading for record sets — which corefStub already covers. +func mergedResidue(content string) string { + if s := corefStub(content); s != "" { + return s + } + head := strings.TrimSpace(content) + if len(head) > 96 { + head = head[:96] + } + return fmt.Sprintf("%s… [%d chars omitted]", head, len(content)-len(head)) +} + +// truncateForPrompt bounds one output shown in the bulk prompt, marking the cut so the model knows +// it is judging an excerpt and must not "complete" the missing part in a trim. +func truncateForPrompt(s string, max int) string { + if len(s) <= max { + return s + } + return s[:max] + "\n…[excerpt truncated for adjudication; a trim must copy only text shown above]" +} diff --git a/components/offload/extract_llm_merged_test.go b/components/offload/extract_llm_merged_test.go new file mode 100644 index 00000000..cde91849 --- /dev/null +++ b/components/offload/extract_llm_merged_test.go @@ -0,0 +1,279 @@ +package offload + +import ( + "context" + "encoding/json" + "strings" + "sync/atomic" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/internal/extract" + "github.com/rossoctl/context-guru/schema" + "github.com/rossoctl/context-guru/store" +) + +// countingModel records how many times it was called and returns a canned adjudication. +type countingModel struct { + calls int64 + reply string + lastAsk string +} + +func (m *countingModel) Complete(ctx context.Context, prompt string) (string, error) { + atomic.AddInt64(&m.calls, 1) + m.lastAsk = prompt + return m.reply, nil +} + +func mkToolMsg(id, text string) bschemas.ChatMessage { + i := id + return bschemas.ChatMessage{Role: bschemas.ChatMessageRoleTool, + Content: &bschemas.ChatMessageContent{ContentStr: &text}, + ChatToolMessage: &bschemas.ChatToolMessage{ToolCallID: &i}} +} + +func mergedReq(n int, body string) *bschemas.BifrostChatRequest { + u := "find the failing test in src/auth.py" + msgs := []bschemas.ChatMessage{{Role: bschemas.ChatMessageRoleUser, + Content: &bschemas.ChatMessageContent{ContentStr: &u}}} + for i := 0; i < n; i++ { + id := "call_" + string(rune('a'+i)) + nm := "Read" + msgs = append(msgs, + bschemas.ChatMessage{Role: bschemas.ChatMessageRoleAssistant, + Content: &bschemas.ChatMessageContent{ContentStr: &nm}, + ChatAssistantMessage: &bschemas.ChatAssistantMessage{ + ToolCalls: []bschemas.ChatAssistantMessageToolCall{{ID: &id, + Function: bschemas.ChatAssistantMessageToolCallFunction{Name: &nm}}}}}, + mkToolMsg(id, body)) + } + tail := "keep going" + msgs = append(msgs, bschemas.ChatMessage{Role: bschemas.ChatMessageRoleUser, + Content: &bschemas.ChatMessageContent{ContentStr: &tail}}) + return &bschemas.BifrostChatRequest{Input: msgs} +} + +// The merged shape's defining property: ONE model call for the whole batch. Per-candidate calls +// would be the per-output design that measured 6% live-kept, so this is a correctness property and +// not a performance nicety. +func TestMergedMakesExactlyOneModelCall(t *testing.T) { + body := "[" + strings.TrimSuffix(strings.Repeat("{\"row\":\"value src/auth.py TOKEN_GRACE_41ab\"},", 400), ",") + "]" + req := mergedReq(6, body) + // Adjudicate: drop the first two tool outputs (message indices 2 and 4), keep the rest. + vs := []extract.BulkVerdict{{Index: 2, Verdict: "drop"}, {Index: 4, Verdict: "drop"}} + raw, _ := json.Marshal(vs) + m := &countingModel{reply: string(raw)} + + e, err := newExtractLLM([]byte("{\"selection_mode\":\"merged\",\"min_tokens\":300,\"allow_on_caching_backend\":true,\"economic_gate\":false}")) + if err != nil { + t.Fatalf("newExtractLLM: %v", err) + } + off, ok := e.(components.Offload) + if !ok { + t.Fatal("not an Offload") + } + c := &components.Ctx{Ctx: context.Background(), Session: "merged-1", + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + Model: components.ModelSpec{Incoming: m, Static: m}} + before := schema.MessagesTokens(req) + rep := &components.Report{} + if _, err := off.Offload(req, rep, c); err != nil { + t.Fatalf("Offload: %v", err) + } + t.Logf("gates=%v skipped=%v", rep.Gates, rep.Skipped) + if got := atomic.LoadInt64(&m.calls); got != 1 { + t.Errorf("merged mode made %d model calls, want exactly 1 -- per-candidate calls are the "+ + "refuted per-output design", got) + } + after := schema.MessagesTokens(req) + if after >= before { + t.Errorf("merged mode removed nothing: %d -> %d tokens", before, after) + } + // The prompt must carry BOTH the co-reference evidence and the cost-honest framing; those are + // the two measured ingredients, and a prompt missing either is a different experiment. + if !strings.Contains(m.lastAsk, "novel=") || !strings.Contains(m.lastAsk, "refs=") { + t.Error("prompt carries no co-reference evidence, so it is not the merged design") + } + if !strings.Contains(m.lastAsk, "NOT notice the gap") { + t.Error("prompt lacks the cost-honest framing, worth ~26 points of live-kept when measured") + } + if strings.Contains(strings.ToLower(m.lastAsk), "recoverable") { + t.Error("prompt reassures the model that removals are recoverable -- the exact clause that " + + "measured 91% removal at 6% live-kept") + } +} + +// TRIM IS GONE, and a model that answers with it anyway must degrade to the SAFE direction. Dropping +// the verdict outright would leave the output unjudged, which reads identically to "the model said +// nothing about it" in the counters -- the same indistinguishability the Report.Gates comment warns +// about. Trim was removed because it was chosen zero times in 21 probe opportunities and, in +// production, accepted once against eight rejected as invented. +func TestMergedTrimDegradesToKeep(t *testing.T) { + body := strings.Repeat("{\"row\":\"real value here\"}\n", 400) + req := mergedReq(3, body) + raw, _ := json.Marshal([]map[string]any{{"i": 2, "verdict": "trim", "kept": "text the model invented"}}) + m := &countingModel{reply: string(raw)} + e, _ := newExtractLLM([]byte("{\"selection_mode\":\"merged\",\"min_tokens\":300,\"allow_on_caching_backend\":true,\"economic_gate\":false}")) + off := e.(components.Offload) + c := &components.Ctx{Ctx: context.Background(), Session: "merged-2", + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + Model: components.ModelSpec{Incoming: m, Static: m}} + before := schema.MessagesTokens(req) + rep := &components.Report{} + off.Offload(req, rep, c) + if schema.MessagesTokens(req) != before { + t.Error("a trim verdict changed the request; trim is no longer a supported action and must " + + "degrade to keep rather than splice model-authored text") + } + if rep.Gates["merged_keep"] == 0 { + t.Errorf("a trim verdict was not counted as a keep; gates=%v", rep.Gates) + } +} + +// A drop that CONTRADICTS its own obligation answer must be refused. This is the one verification +// that points the dangerous way: the model has said an outstanding obligation still needs the output +// and then asked to remove it anyway. +func TestMergedRefusesDropThatNamesAnObligation(t *testing.T) { + body := strings.Repeat("{\"row\":\"real value here\"}\n", 400) + req := mergedReq(3, body) + raw, _ := json.Marshal([]map[string]any{{"i": 2, "verdict": "drop", + "needed_by": "b", "quote": "find the failing test in src/auth.py"}}) + m := &countingModel{reply: string(raw)} + e, _ := newExtractLLM([]byte("{\"selection_mode\":\"merged\",\"min_tokens\":300,\"allow_on_caching_backend\":true,\"economic_gate\":false}")) + off := e.(components.Offload) + c := &components.Ctx{Ctx: context.Background(), Session: "merged-4", + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + Model: components.ModelSpec{Incoming: m, Static: m}} + before := schema.MessagesTokens(req) + rep := &components.Report{} + off.Offload(req, rep, c) + if schema.MessagesTokens(req) != before { + t.Errorf("an output was dropped although the model said obligation (b) still needs it; "+ + "gates=%v", rep.Gates) + } + if rep.Gates["merged_drop_contradicts_obligation"] == 0 { + t.Errorf("the contradiction was not counted; gates=%v", rep.Gates) + } +} + +// A FABRICATED obligation quote must be counted. It argues for keeping, so it is not dangerous, but +// quote fidelity degraded with batch size when measured (4 of 37 non-verbatim at batch 16 against 0 +// of 16 at batch 10), which makes this counter the signal that the batch is too large. +func TestMergedCountsFabricatedObligationQuote(t *testing.T) { + body := strings.Repeat("{\"row\":\"real value here\"}\n", 400) + req := mergedReq(3, body) + raw, _ := json.Marshal([]map[string]any{{"i": 2, "verdict": "keep", + "needed_by": "b", "quote": "a sentence that appears nowhere in the transcript at all"}}) + m := &countingModel{reply: string(raw)} + e, _ := newExtractLLM([]byte("{\"selection_mode\":\"merged\",\"min_tokens\":300,\"allow_on_caching_backend\":true,\"economic_gate\":false}")) + off := e.(components.Offload) + c := &components.Ctx{Ctx: context.Background(), Session: "merged-5", + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + Model: components.ModelSpec{Incoming: m, Static: m}} + rep := &components.Report{} + off.Offload(req, rep, c) + if rep.Gates["merged_quote_not_verbatim"] == 0 { + t.Errorf("an invented obligation quote was accepted without being counted; gates=%v", rep.Gates) + } +} + +// The prompt must carry the CRITERION and demand the obligation evidence. Arms with an identical +// criterion but no required evidence field measured 4/4 false drops against 2/4 -- so a prompt that +// states the rule without forcing the answer is a different, measurably worse experiment. +func TestMergedPromptForcesObligationEvidence(t *testing.T) { + body := strings.Repeat("{\"row\":\"real value here\"}\n", 400) + req := mergedReq(3, body) + m := &countingModel{reply: "[]"} + e, _ := newExtractLLM([]byte("{\"selection_mode\":\"merged\",\"min_tokens\":300,\"allow_on_caching_backend\":true,\"economic_gate\":false}")) + off := e.(components.Offload) + c := &components.Ctx{Ctx: context.Background(), Session: "merged-6", + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + Model: components.ModelSpec{Incoming: m, Static: m}} + off.Offload(req, &components.Report{}, c) + for _, want := range []string{"NOT YET COMPLETE", "needed_by", "quote", "VERBATIM"} { + if !strings.Contains(m.lastAsk, want) { + t.Errorf("prompt is missing %q, so the forcing function measured to halve false drops "+ + "is not present", want) + } + } + if strings.Contains(m.lastAsk, "\"trim\"") { + t.Error("prompt still offers trim, which was removed") + } +} + +// A mistyped selection_mode must fail at construction, not silently run the default shape. +func TestMergedRejectsUnknownSelectionMode(t *testing.T) { + if _, err := newExtractLLM([]byte("{\"selection_mode\":\"bluk\"}")); err == nil { + t.Error("a typo in selection_mode was accepted; the arm would silently measure the default") + } +} + +// PLAIN TEXT must be droppable too. corefStub returns "" for anything that is not a JSON array or +// object, which is most real tool output -- logs, file reads, tracebacks. Before mergedResidue's +// fallback the drop verdict was recorded in the gate counters while the projection came back empty +// and phase 3 skipped it, so the arm looked like it was deciding and removing nothing. This test +// exists because that is precisely the failure mode the repo's own "acted: 0 is not diagnosable" +// warning describes, one layer further in. +func TestMergedCanDropUnstructuredOutput(t *testing.T) { + body := strings.Repeat("2026-08-22T10:00:00Z INFO handler finished request in 12ms\n", 500) + req := mergedReq(4, body) + vs := []extract.BulkVerdict{{Index: 2, Verdict: "drop"}} + raw, _ := json.Marshal(vs) + m := &countingModel{reply: string(raw)} + e, err := newExtractLLM([]byte("{\"selection_mode\":\"merged\",\"min_tokens\":300,\"allow_on_caching_backend\":true,\"economic_gate\":false}")) + if err != nil { + t.Fatalf("newExtractLLM: %v", err) + } + off := e.(components.Offload) + c := &components.Ctx{Ctx: context.Background(), Session: "merged-3", + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + Model: components.ModelSpec{Incoming: m, Static: m}} + before := schema.MessagesTokens(req) + rep := &components.Report{} + off.Offload(req, rep, c) + if schema.MessagesTokens(req) >= before { + t.Errorf("merged mode could not drop UNSTRUCTURED output: %d -> %d tokens, gates=%v. "+ + "corefStub only handles JSON, so a residue fallback is required or the arm silently "+ + "decides and removes nothing.", before, schema.MessagesTokens(req), rep.Gates) + } +} + +// A TRUNCATED reply and a MALFORMED one need different counters, because they need opposite fixes: +// raise the output budget versus fix the prompt. Folded together under merged_unparseable, a failure +// on ~70% of calls read as "the prompt is wrong" and sat unnoticed at 19 and 22 in two earlier arms -- +// where it looked negligible only because it was being compared against the DECISION count instead of +// the CALL count. +func TestMergedDistinguishesTruncatedReplyFromJunk(t *testing.T) { + body := strings.Repeat("{\"row\":\"real value here\"}\n", 400) + for _, tc := range []struct { + name, reply, wantGate string + }{ + // Opened the array, never closed it: ran out of output budget mid-verdict. + {"truncated", "[{\"i\":2,\"verdict\":\"drop\",\"needed_by\":\"none\",\"quote\":\"", "merged_reply_truncated"}, + // Never produced an array at all: a format failure. + {"junk", "I would rather explain my reasoning in prose, thank you.", "merged_unparseable"}, + } { + t.Run(tc.name, func(t *testing.T) { + req := mergedReq(3, body) + m := &countingModel{reply: tc.reply} + e, err := newExtractLLM([]byte("{\"selection_mode\":\"merged\",\"min_tokens\":300,\"allow_on_caching_backend\":true,\"economic_gate\":false}")) + if err != nil { + t.Fatalf("newExtractLLM: %v", err) + } + c := &components.Ctx{Ctx: context.Background(), Session: "trunc-" + tc.name, + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + Model: components.ModelSpec{Incoming: m, Static: m}} + rep := &components.Report{} + before := schema.MessagesTokens(req) + e.(components.Offload).Offload(req, rep, c) + if rep.Gates[tc.wantGate] == 0 { + t.Errorf("%s reply was not counted as %s; gates=%v", tc.name, tc.wantGate, rep.Gates) + } + if schema.MessagesTokens(req) != before { + t.Error("an unusable reply changed the request; it must fail open") + } + }) + } +} diff --git a/components/offload/extract_llm_prefix_test.go b/components/offload/extract_llm_prefix_test.go new file mode 100644 index 00000000..8e708c95 --- /dev/null +++ b/components/offload/extract_llm_prefix_test.go @@ -0,0 +1,166 @@ +package offload + +import ( + "context" + "testing" + + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/store" +) + +// The full-body ("allow_cached_prefix") path. Reaching the provider's cached prefix costs a +// cache-write, so it is guarded by two gates the tail path does not have: the co-reference +// index as a free eligibility pre-filter, and the S*T > 11.5*W break-even on the batch. +// +// The fixtures reuse coref's request (see corefReq): index 2 is a tool output whose novel +// identifier the NEXT assistant turn quotes (referenced => open), index 5 is a listing whose +// identifier nothing ever uses (unreferenced => spent). prefix_min_later_turns: 0 keeps the +// fixture short — the opportunity floor is coref's concern and is tested there. + +func newPrefixComponent(t *testing.T, model components.Model, extraYAML string) *ExtractLLM { + t.Helper() + return newCtxGuardComponent(t, model, "prefix_min_later_turns: 0\n"+extraYAML) +} + +// cache-aware Ctx whose ENTIRE transcript is already cached, so every candidate is prefix +// work and nothing qualifies as tail. +func prefixCtx(req int) *components.Ctx { + return &components.Ctx{Ctx: context.Background(), Session: "s", + Store: store.NewMemory(store.Options{}), + CacheAware: true, MaxCachedIdx: req - 1, CtxWindow: 200000} +} + +// Default OFF: prefix content is refused exactly as before, and the model is never called. +// This is the regression guard for every workload the published numbers came from. +func TestPrefixReachIsOffByDefault(t *testing.T) { + m := &silentModel{} + e := newPrefixComponent(t, m, "") + req := corefReq() + var rep components.Report + if _, err := e.Offload(req, &rep, prefixCtx(len(req.Input))); err != nil { + t.Fatal(err) + } + if rep.Gates["cached_prefix"] == 0 { + t.Fatalf("prefix content must be refused by default; gates=%v", rep.Gates) + } + if m.calls != 0 { + t.Fatalf("no model call may be made for prefix content by default, got %d", m.calls) + } +} + +// With prefix reach ON, an output whose identifiers a later turn carried forward must be +// refused by the INDEX — before any model call. This is the gate that makes the feature +// affordable: the expensive component never pays to look at content a free pass can clear. +func TestPrefixReachSkipsStillReferencedContentWithoutCallingTheModel(t *testing.T) { + m := &silentModel{} + e := newPrefixComponent(t, m, "allow_cached_prefix: true\n") + req := corefReq() + var rep components.Report + if _, err := e.Offload(req, &rep, prefixCtx(len(req.Input))); err != nil { + t.Fatal(err) + } + if rep.Gates["prefix_still_referenced"] == 0 { + t.Fatalf("a referenced prefix output must be gated by the index; gates=%v", rep.Gates) + } + if rep.Gates["cached_prefix"] != 0 { + t.Errorf("with prefix reach on, the blanket cached_prefix gate must not fire; gates=%v", rep.Gates) + } +} + +// The complement: an output nothing referred back to IS eligible, so the model gets +// consulted about how much of it to keep. Index decides WHETHER, model decides HOW MUCH. +func TestPrefixReachConsultsTheModelForSpentContent(t *testing.T) { + m := &silentModel{} + e := newPrefixComponent(t, m, "allow_cached_prefix: true\n") + req := corefReq() + var rep components.Report + if _, err := e.Offload(req, &rep, prefixCtx(len(req.Input))); err != nil { + t.Fatal(err) + } + if m.calls == 0 { + t.Fatalf("an unreferenced prefix output must reach the model; gates=%v", rep.Gates) + } +} + +// The break-even must be able to decline. A window barely above the request leaves no turns +// to amortize over (estimateTurnsRemaining -> ~0), so the rewrite cannot be repaid and the +// batch is dropped — with a reason, not silently. +func TestPrefixBatchDeclinesWhenTheRewriteCannotBeRepaid(t *testing.T) { + m := &silentModel{} + e := newPrefixComponent(t, m, + "allow_cached_prefix: true\nmodel_max_input_tokens: 400000\n") + req := corefReq() + c := prefixCtx(len(req.Input)) + c.CtxWindow = 900 // no headroom: T collapses, so 11.5*W can never be recovered + var rep components.Report + if _, err := e.Offload(req, &rep, c); err != nil { + t.Fatal(err) + } + if rep.Gates["prefix_rewrite_not_repaid"] == 0 { + t.Fatalf("an unrepayable prefix rewrite must be declined; gates=%v", rep.Gates) + } +} + +// A failed prefix batch must not suppress TAIL work. The tail costs no cache-write, so it +// was already free and profitable; dropping it because the prefix batch failed would make +// enabling the feature strictly worse than leaving it off. +func TestFailedPrefixBatchDoesNotSuppressTailWork(t *testing.T) { + m := &silentModel{} + e := newPrefixComponent(t, m, + "allow_cached_prefix: true\nmodel_max_input_tokens: 400000\n") + req := corefWithSecondListing() // index 7 is a fresh unreferenced output + // Everything except the last two messages is cached, so index 7 is TAIL. + c := &components.Ctx{Ctx: context.Background(), Session: "s", + Store: store.NewMemory(store.Options{}), + CacheAware: true, MaxCachedIdx: len(req.Input) - 3, CtxWindow: 900} + var rep components.Report + if _, err := e.Offload(req, &rep, c); err != nil { + t.Fatal(err) + } + if rep.Gates["prefix_rewrite_not_repaid"] == 0 { + t.Fatalf("expected the prefix batch to be declined at this window; gates=%v", rep.Gates) + } + if m.calls == 0 { + t.Fatalf("tail work must survive a declined prefix batch; gates=%v", rep.Gates) + } +} + +// prefix_classes decides what the model is even asked about, so the two classes it may name +// are asserted, and the two it may not are REFUSED at construction rather than silently +// widening the pre-filter to "consider everything" — which would destroy the only property +// that makes prefix reach affordable. +func TestPrefixClassesRejectsClassesThatAreNotEvidenceOfSpentContent(t *testing.T) { + for _, bad := range []string{"open", "opaque"} { + if _, err := newExtractLLM([]byte("allow_cached_prefix: true\nprefix_classes: [" + bad + "]\n")); err == nil { + t.Errorf("prefix_classes: [%s] must be refused: it is not evidence of spent content", bad) + } + } + if _, err := newExtractLLM([]byte("prefix_classes: [nonsense]\n")); err == nil { + t.Error("an unknown prefix_classes entry must be refused, not ignored") + } + for _, good := range []string{"unreferenced", "closed", "unreferenced, closed"} { + if _, err := newExtractLLM([]byte("prefix_classes: [" + good + "]\n")); err != nil { + t.Errorf("prefix_classes: [%s] must be accepted: %v", good, err) + } + } +} + +// Narrowing to `closed` alone must actually narrow: the fixture's unreferenced output stops +// being a candidate, so the model is not consulted about it. This is the knob the LOCA replay +// needs, where handing the model `unreferenced` content bought nothing. +func TestPrefixClassesClosedOnlyExcludesUnreferenced(t *testing.T) { + m := &silentModel{} + e := newPrefixComponent(t, m, + "allow_cached_prefix: true\nprefix_classes: [closed]\nmodel_max_input_tokens: 400000\n") + req := corefReq() // its only spent output is UNREFERENCED, never closed + var rep components.Report + if _, err := e.Offload(req, &rep, prefixCtx(len(req.Input))); err != nil { + t.Fatal(err) + } + if m.calls != 0 { + t.Fatalf("closed-only must not consult the model about unreferenced content, got %d calls", m.calls) + } + if rep.Gates["prefix_still_referenced"] == 0 { + t.Fatalf("the unreferenced output should now be filtered out; gates=%v", rep.Gates) + } +} diff --git a/components/offload/extract_llm_prefixask_test.go b/components/offload/extract_llm_prefixask_test.go new file mode 100644 index 00000000..7b9b110b --- /dev/null +++ b/components/offload/extract_llm_prefixask_test.go @@ -0,0 +1,165 @@ +package offload + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/internal/extract" + "github.com/rossoctl/context-guru/schema" + "github.com/rossoctl/context-guru/store" +) + +// fakeAsker stands in for the proxy's prefix asker. +type fakeAsker struct { + ask string + session string + reply string + cacheRead int + err error + calls int +} + +func (f *fakeAsker) Ask(_ context.Context, session, ask string) (string, components.PrefixUsage, error) { + f.calls++ + f.ask, f.session = ask, session + if f.err != nil { + return "", components.PrefixUsage{}, f.err + } + return f.reply, components.PrefixUsage{CacheRead: f.cacheRead}, nil +} + +func mergedWithAsker(t *testing.T, sess string, a components.PrefixAsker, m *countingModel) (*components.Report, int, int) { + t.Helper() + // THREE ZONES, because the two prompt shapes ship different amounts and the test has to tell them + // apart precisely: + // HEAD (offset 0) -- a bounded locator, expected in BOTH shapes (~90 chars). + // MID (offset ~900) -- inside mergedSampleChars, so expected ONLY in the fallback prompt. + // DEEP (past 4000) -- beyond truncation, so expected in NEITHER. + body := "HEAD_LOCATOR_LINE opening record\n" + + strings.Repeat("{\"row\":\"filler\"}\n", 40) + + "{\"row\":\"MID_SAMPLE_MARKER\"}\n" + + strings.Repeat("{\"row\":\"filler value here padding\"}\n", 400) + + "{\"row\":\"DEEP_SAMPLE_MARKER\"}\n" + req := mergedReq(4, body) + e, err := newExtractLLM([]byte("{\"selection_mode\":\"merged\",\"min_tokens\":300,\"allow_on_caching_backend\":true,\"economic_gate\":false}")) + if err != nil { + t.Fatalf("newExtractLLM: %v", err) + } + c := &components.Ctx{Ctx: context.Background(), Session: sess, + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + Model: components.ModelSpec{Incoming: m, Static: m}, PrefixAsk: a} + before := schema.MessagesTokens(req) + rep := &components.Report{} + e.(components.Offload).Offload(req, rep, c) + return rep, before, schema.MessagesTokens(req) +} + +// When a prefix ask is available it must be USED, and the question must NOT ship the outputs. Paying +// fresh tokens to send a truncated copy of content the model is already reading from cache defeats +// the entire mechanism -- and shows it an excerpt of something it could read in full. +func TestPrefixAskUsedAndCarriesNoSamples(t *testing.T) { + raw, _ := json.Marshal([]extract.BulkVerdict{{Index: 2, Verdict: "drop", NeededBy: "none"}}) + a := &fakeAsker{reply: string(raw), cacheRead: 19595} + m := &countingModel{reply: "[]"} + rep, before, after := mergedWithAsker(t, "pfx-1", a, m) + + if a.calls != 1 { + t.Fatalf("prefix asker called %d times, want 1", a.calls) + } + if m.calls != 0 { + t.Error("the plain completion was used even though a prefix ask was available") + } + if rep.Gates["prefix_ask_used"] == 0 { + t.Errorf("prefix_ask_used not counted; gates=%v", rep.Gates) + } + if strings.Contains(a.ask, "MID_SAMPLE_MARKER") { + t.Error("the prefix ask shipped the output BODY; the transcript is already above the " + + "question, so this pays fresh for content being read from cache -- and shows the model " + + "a truncated copy of something it could read in full") + } + if !strings.Contains(a.ask, "HEAD_LOCATOR_LINE") { + t.Error("the prefix ask carries no locator head, so the model cannot tell which output in " + + "the transcript a label refers to") + } + if !strings.Contains(a.ask, "Refer to them by these labels only") { + t.Error("the prefix ask carries no label inventory, so the model has no way to name a candidate") + } + if after >= before { + t.Errorf("verdicts from the prefix ask were not applied: %d -> %d", before, after) + } +} + +// A cache read of zero is the failure this mechanism cannot detect any other way: it costs ~10x and +// is otherwise indistinguishable from success. +func TestPrefixAskCountsZeroCacheRead(t *testing.T) { + raw, _ := json.Marshal([]extract.BulkVerdict{{Index: 2, Verdict: "keep"}}) + a := &fakeAsker{reply: string(raw), cacheRead: 0} + rep, _, _ := mergedWithAsker(t, "pfx-2", a, &countingModel{reply: "[]"}) + if rep.Gates["prefix_ask_cache_read_ZERO"] == 0 { + t.Errorf("a prefix ask that read NOTHING from cache was not counted; gates=%v", rep.Gates) + } +} + +// First turn of a session, feature off, or a transport failure: fall back to the plain completion. +// Treating "no prefix" as "no verdicts" would silently disable the component on every session's first +// turn and read as a model that declined to act. +func TestPrefixAskFallsBackToCompletion(t *testing.T) { + raw, _ := json.Marshal([]extract.BulkVerdict{{Index: 2, Verdict: "drop", NeededBy: "none"}}) + a := &fakeAsker{err: errors.New("no stashed prefix for this session")} + m := &countingModel{reply: string(raw)} + rep, before, after := mergedWithAsker(t, "pfx-3", a, m) + if m.calls != 1 { + t.Errorf("plain completion called %d times after the prefix ask failed, want 1", m.calls) + } + if rep.Gates["prefix_ask_failed"] == 0 { + t.Errorf("the failure was not counted; gates=%v", rep.Gates) + } + if after >= before { + t.Errorf("the fallback produced no removal: %d -> %d; a failed prefix ask must not disable "+ + "the component", before, after) + } + // And the fallback prompt DOES carry samples, because a bare completion has no other way to show them. + if !strings.Contains(m.lastAsk, "MID_SAMPLE_MARKER") { + t.Error("the fallback completion shipped no samples, so the model was shown nothing to judge") + } +} + +// OFFERED is not ANSWERED, and conflating them is what made three iterations read as "the model +// declines to act". Live, verdicts/calls read 2.80 while merged_batch_truncated fired 43 times in 162 +// calls -- impossible for offered batches. This asserts the two quantities are recorded separately, so +// a starved batch and a model that answers for a third of a full batch cannot look identical. +func TestMergedRecordsOfferedSeparatelyFromAnswered(t *testing.T) { + body := strings.Repeat("{\"row\":\"real value here padding\"}\n", 400) + req := mergedReq(6, body) // six tool outputs => six candidates offered + // The model answers for exactly ONE of them, which is the behaviour being made visible. + raw, _ := json.Marshal([]extract.BulkVerdict{{Index: 2, Verdict: "keep"}}) + m := &countingModel{reply: string(raw)} + e, err := newExtractLLM([]byte("{\"selection_mode\":\"merged\",\"min_tokens\":300,\"allow_on_caching_backend\":true,\"economic_gate\":false}")) + if err != nil { + t.Fatalf("newExtractLLM: %v", err) + } + c := &components.Ctx{Ctx: context.Background(), Session: "offered-1", + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + Model: components.ModelSpec{Incoming: m, Static: m}} + rep := &components.Report{} + e.(components.Offload).Offload(req, rep, c) + + offered := rep.Gates["merged_offered"] + answered := rep.Gates["merged_keep"] + rep.Gates["merged_drop"] + + rep.Gates["merged_drop_contradicts_obligation"] + if offered < 2 { + t.Fatalf("merged_offered = %d; the offered batch size must be recorded, not inferred from "+ + "verdicts; gates=%v", offered, rep.Gates) + } + if answered != 1 { + t.Fatalf("expected exactly 1 verdict answered, got %d; gates=%v", answered, rep.Gates) + } + if offered == answered { + t.Errorf("offered (%d) and answered (%d) are indistinguishable, which is precisely the "+ + "conflation that made a full batch look like a starved one", offered, answered) + } +} diff --git a/components/offload/parallel_pairing_test.go b/components/offload/parallel_pairing_test.go new file mode 100644 index 00000000..4fc027ed --- /dev/null +++ b/components/offload/parallel_pairing_test.go @@ -0,0 +1,46 @@ +package offload + +import ( + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/context-guru/schema" +) + +// PARALLEL TOOL CALLS. Live traffic that the provider rejected carried ONE assistant message with +// TWO tool_use blocks (see the capture digest in docs/experiments/loca/iter011/results.md). Anthropic +// requires every result for a parallel call to arrive in the message immediately after, and bifrost +// represents each result as its own ChatMessageRoleTool message. So the question is whether a +// perfectly ordinary parallel exchange is considered valid by our own validator -- and if not, +// whether that is the validator being wrong or the wire shape being wrong. +func TestValidateShapeOnParallelToolCalls(t *testing.T) { + a, b := "call_a", "call_b" + nm := "Read" + msgs := []bschemas.ChatMessage{ + {Role: bschemas.ChatMessageRoleUser, + Content: &bschemas.ChatMessageContent{ContentStr: sp("do two things")}}, + {Role: bschemas.ChatMessageRoleAssistant, + Content: &bschemas.ChatMessageContent{ContentStr: sp("calling both")}, + ChatAssistantMessage: &bschemas.ChatAssistantMessage{ + ToolCalls: []bschemas.ChatAssistantMessageToolCall{ + {ID: &a, Function: bschemas.ChatAssistantMessageToolCallFunction{Name: &nm, Arguments: `{}`}}, + {ID: &b, Function: bschemas.ChatAssistantMessageToolCallFunction{Name: &nm, Arguments: `{}`}}, + }}}, + {Role: bschemas.ChatMessageRoleTool, + Content: &bschemas.ChatMessageContent{ContentStr: sp("result a")}, + ChatToolMessage: &bschemas.ChatToolMessage{ToolCallID: &a}}, + {Role: bschemas.ChatMessageRoleTool, + Content: &bschemas.ChatMessageContent{ContentStr: sp("result b")}, + ChatToolMessage: &bschemas.ChatToolMessage{ToolCallID: &b}}, + } + v := schema.ValidateShape(msgs) + for _, x := range v { + t.Logf("violation: %s", x) + } + if len(v) != 0 { + t.Errorf("ValidateShape rejects an ordinary PARALLEL tool exchange (%d violations). "+ + "Either the rule only inspects msgs[i+1] for a single result -- so it cannot see the "+ + "second result one message later -- or the wire shape is genuinely wrong. This is the "+ + "shape live traffic was rejected on.", len(v)) + } +} diff --git a/components/offload/prefix_econ.go b/components/offload/prefix_econ.go new file mode 100644 index 00000000..9eda34c8 --- /dev/null +++ b/components/offload/prefix_econ.go @@ -0,0 +1,113 @@ +package offload + +import ( + "math" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/schema" +) + +// Economics of deliberately mutating the provider's CACHED PREFIX. +// +// Every other offloader refuses to touch the prefix (Ctx.TailOnly) because breaking the +// prefix hash at index i forces the provider to cache-WRITE everything from i onward. Two +// components choose to pay that on purpose — coref, and extract_llm when +// allow_cached_prefix is set — so the price lives here rather than being reimplemented, +// slightly differently, in each of them. +// +// cost = W x (2.50 - 0.20) = 11.5 x W (in cache-read-equivalents) +// benefit = S x T x 0.20 = S x T +// worth it when S x T > 11.5 x W +// +// S is the mass removed; W is the suffix the mutation forces the provider to re-write — +// counted from the shallowest mutated index to the CACHED boundary, because content past +// that boundary was never cached and would be written this turn regardless; T is how many +// turns remain to collect the saving on, which nobody has, so it is estimated from how +// fast the transcript has been growing. +// +// The consequence is counter-intuitive and worth restating wherever this is used: firing +// at 90% of the window means T is nearly zero — paying a rewrite for a saving collected +// once. The profitable moment to compact is EARLIER than the moment of maximum pressure. + +// cacheWriteX is one cache-write in cache-read-equivalents: ($2.50 - $0.20) / $0.20 on +// Anthropic's published per-MTok prices. Shared with deploy/harbor/coref.py. +const cacheWriteX = 11.5 + +// Co-reference classifier defaults, shared by coref and by extract_llm's prefix +// pre-filter. One definition on purpose: if the two components classified the same output +// differently, the "free deterministic pre-filter" would be answering a different question +// from the component whose measurements calibrated it. Mirrors deploy/harbor/coref.py. +const ( + corefClosedDistDefault = 12 + corefOpenRepsDefault = 3 + // corefMinLaterDefault is the opportunity floor: an output with fewer model turns after + // it has not yet HAD a chance to be referenced, so "unreferenced" says nothing about it. + corefMinLaterDefault = 8 +) + +// prefixRewriteWindow reports the last message index the provider is believed to already +// hold. An unknown boundary assumes the whole transcript is cached, which over-states the +// rewrite cost rather than under-stating it. +func prefixRewriteWindow(req *bschemas.BifrostChatRequest, c *components.Ctx) int { + end := len(req.Input) - 1 + if c != nil && c.CacheAware && c.MaxCachedIdx >= 0 && c.MaxCachedIdx < end { + end = c.MaxCachedIdx + } + return end +} + +// prefixRewritePays applies S*T > 11.5*W for a mutation of `saved` tokens whose shallowest +// touched index is `shallowest`. Returns (needed T, estimated T, whether it clears). +// +// Always clears when the context window is unknown — the same convention as every +// fraction-based threshold in this package: an unresolvable threshold imposes no +// constraint rather than silently disabling the pass. +func prefixRewritePays(req *bschemas.BifrostChatRequest, saved, shallowest int, c *components.Ctx) (need, have int, ok bool) { + if c == nil || c.CtxWindow <= 0 { + return 0, 0, true + } + if saved <= 0 { + return 0, 0, false + } + end := prefixRewriteWindow(req, c) + rewritten := 0 + for j := shallowest; j <= end && j < len(req.Input); j++ { + rewritten += schema.TextTokens(schema.MessageText(req.Input[j])) + } + rewritten -= saved // the removed mass is not part of what gets written back + if rewritten <= 0 { + return 0, estimateTurnsRemaining(schema.MessagesTokens(req), modelTurns(req), c.CtxWindow), true + } + need = int(math.Ceil(cacheWriteX * float64(rewritten) / float64(saved))) + have = estimateTurnsRemaining(schema.MessagesTokens(req), modelTurns(req), c.CtxWindow) + return need, have, need <= have +} + +// estimateTurnsRemaining projects how many more turns fit before the request reaches the +// model's window, assuming the transcript keeps growing at the average rate it has so +// far. Crude on purpose: T only has to be right to an order of magnitude to separate +// "this rewrite pays for itself" from "this rewrite is charity", and every cheaper proxy +// (elapsed turns, observed step rate) is the same shape of guess. +func estimateTurnsRemaining(reqTokens, turns, window int) int { + if window <= 0 || turns <= 0 || reqTokens <= 0 || reqTokens >= window { + return 0 + } + perTurn := reqTokens / turns + if perTurn <= 0 { + return 0 + } + return (window - reqTokens) / perTurn +} + +// modelTurns counts assistant messages — the closest thing in a request to "steps taken", +// which is the unit the growth rate is per. +func modelTurns(req *bschemas.BifrostChatRequest) int { + n := 0 + for i := range req.Input { + if req.Input[i].Role == bschemas.ChatMessageRoleAssistant { + n++ + } + } + return n +} diff --git a/components/offload/state.go b/components/offload/state.go index eba023b5..17253d22 100644 --- a/components/offload/state.go +++ b/components/offload/state.go @@ -270,17 +270,44 @@ func contentKey(s string) string { return extract.ContentKey(s) } // churn). The expand handler marks the restored content's key kept-verbatim so the // offloaders leave it alone thereafter. -func keptKey(ck string) string { return "cg:keep:" + ck } +// keptKey is scoped BY SESSION, and that scope is the whole point of the key's shape. +// +// The loop this guard prevents is intra-session by construction: the agent expands a marker, +// the next turn of THAT session re-sends the restored original, and re-compacting it sends +// the agent straight back to expand. A different session that happens to carry +// byte-identical content has never expanded anything and cannot be in a loop, so it needs +// no exemption. +// +// The key used to be the bare content hash, session-independent — deliberately, but the +// consequence was not the intended one. Because the hash is global, ONE expand in ONE +// session permanently exempted that byte-identical content in EVERY session thereafter. The +// content most likely to recur byte-identically across sessions is exactly the content most +// worth compacting: a config dump, a manifest, a schema, a file the agent re-reads every +// time. So the guard preferentially and permanently disabled compaction on the highest-value +// targets, nothing reported it, and the effect looked like yield decaying for no reason. +// +// Scoping by session is the minimal fix: it is the smallest scope that still prevents every +// loop the guard was built for. +func keptKey(session, ck string) string { return store.KeptPrefix + session + ":" + ck } -// MarkKeptVerbatim records that this original content was expanded and must not be -// re-compacted (keyed by content hash, session-independent). Exported for the proxy's -// expand loop, which has the restored original but not the offload Ctx. -func MarkKeptVerbatim(st store.Store, original string) { - st.Put(keptKey(contentKey(original)), []byte{1}) +// MarkKeptVerbatim records that this original content was expanded in this session and must +// not be re-compacted there. Exported for the proxy's expand loop, which has the restored +// original and the session id but not the offload Ctx. +// +// An empty session is a no-op rather than a global mark. It is unreachable on the live path +// — observe mode compacts nothing, so there is no marker to expand, and a request with no +// messages cannot carry one either — but if it ever becomes reachable, declining to record +// costs at most one expand bounce, while recording under an empty session segment would +// reinstate exactly the cross-session leak this scoping exists to remove. +func MarkKeptVerbatim(st store.Store, session, original string) { + if session == "" { + return + } + st.Put(keptKey(session, contentKey(original)), []byte{1}) } func isKeptVerbatim(c *components.Ctx, ck string) bool { - _, ok := c.Store.Get(keptKey(ck)) + _, ok := c.Store.Get(keptKey(c.Session, ck)) return ok } diff --git a/components/offload/summarize.go b/components/offload/summarize.go index a23615a3..cad7ea71 100644 --- a/components/offload/summarize.go +++ b/components/offload/summarize.go @@ -151,8 +151,9 @@ func (*Summarize) NeedsModel() bool { return true } func (s *Summarize) Offload(req *bschemas.BifrostChatRequest, rep *components.Report, c *components.Ctx) ([]string, error) { msgs := req.Input - // Keep msg0 (system/first) + the last keepLast; summarize the span between. - start, end := 1, len(msgs)-s.keepLast + // Keep msg0 (system/first) + the last keepLast; summarize the span between — with both + // boundaries aligned so neither cuts inside a tool exchange. See summarizeSpan. + headCount, start, end := summarizeSpan(msgs, s.keepLast) // Request-level trigger: don't summarize (an LLM call) until the transcript // is genuinely large / deep. Zero thresholds fire always (back-compat). if !s.trigger.Fires(req, c.CtxWindow) || end <= start { @@ -172,7 +173,7 @@ func (s *Summarize) Offload(req *bschemas.BifrostChatRequest, rep *components.Re // that checkpoint is still small — no LLM call, and the summary message stays // byte-identical (KV-cache stable). Roll the checkpoint forward only once the // tail grows past resummarize_tokens. - if out, keys, ok := s.tryReuse(c, msgs, start, end); ok { + if out, keys, ok := s.tryReuse(c, msgs, headCount, start, end); ok { if len(keys) == 0 { rep.Irreversible = true // reused a non-full checkpoint (nothing stashed) } @@ -224,7 +225,24 @@ func (s *Summarize) Offload(req *bschemas.BifrostChatRequest, rep *components.Re } summaryText := summaryWrapper(summary, key, mode) - summaryMsg := bschemas.ChatMessage{Role: bschemas.ChatMessageRoleSystem} + // USER, not system. The summary is injected context, and Anthropic will not accept a + // system-role message in the middle of `messages`: system content belongs in the + // top-level `system` field, and a system role inside the array must precede an + // assistant message or end it. This component emits [msgs[0], summary, tail...], so + // when msgs[0] is itself the system prompt — the normal case — a system-role summary + // lands at index 1 and the provider rejects the whole request: + // + // 400 messages.1: role 'system' must precede an 'assistant' message or end the array + // + // Measured on live LOCA-bench traffic: every task that triggered a summarization failed + // this way, including in an arm with NO other component enabled, so it is this + // component's own output and not a pipeline interaction. It went unnoticed because + // every prior measurement replayed through /compact, which never forwards upstream and + // therefore never has the body validated by a provider. + // + // A user-role message carrying the summary is both valid and conventional — it is what + // Claude Code's own compaction does. + summaryMsg := bschemas.ChatMessage{Role: bschemas.ChatMessageRoleUser} schema.SetMessageText(&summaryMsg, summaryText) // Checkpoint: this summary subsumes the leading span (len(span) messages from @@ -236,8 +254,14 @@ func (s *Summarize) Offload(req *bschemas.BifrostChatRequest, rep *components.Re // [msg0, summary, last-K] — reassign; apply.Body rebuilds losslessly. out := make([]bschemas.ChatMessage, 0, 2+s.keepLast) - out = append(out, msgs[0], summaryMsg) + out = append(out, msgs[:headCount]...) + out = append(out, summaryMsg) out = append(out, msgs[end:]...) + // Removing a span can orphan the tail's leading tool_result blocks; a provider rejects + // the whole request if it does. See dropOrphanedToolResults. + if repaired, n := dropOrphanedToolResults(out); n > 0 { + out = repaired + } req.Input = out if key != "" { return []string{key}, nil @@ -250,7 +274,7 @@ func (s *Summarize) Offload(req *bschemas.BifrostChatRequest, rep *components.Re // since that boundary is below resummarize_tokens. It returns the rebuilt // [msg0, priorSummary, msgs[boundary:]] and the (refreshed) stash key. No LLM // call. ok=false means "re-summarize fresh". -func (s *Summarize) tryReuse(c *components.Ctx, msgs []bschemas.ChatMessage, start, end int) ([]bschemas.ChatMessage, []string, bool) { +func (s *Summarize) tryReuse(c *components.Ctx, msgs []bschemas.ChatMessage, headCount, start, end int) ([]bschemas.ChatMessage, []string, bool) { if s.resummarizeTokens <= 0 { return nil, nil, false } @@ -277,11 +301,25 @@ func (s *Summarize) tryReuse(c *components.Ctx, msgs []bschemas.ChatMessage, sta c.Store.Put(cp.Key, b) } } - summaryMsg := bschemas.ChatMessage{Role: bschemas.ChatMessageRoleSystem} + // USER for the same reason as the fresh-summary path above: a system role at index 1 + // is rejected by the provider. The replayed checkpoint must match that shape exactly, + // or a replayed turn would emit different bytes from the turn that created it. + summaryMsg := bschemas.ChatMessage{Role: bschemas.ChatMessageRoleUser} schema.SetMessageText(&summaryMsg, cp.SummaryMsg) + // The replayed boundary must respect exchange atomicity exactly as the fresh path does, + // or a replayed turn emits different bytes from the turn that created it. + for boundary < len(msgs) && msgs[boundary].Role == bschemas.ChatMessageRoleTool { + boundary++ + } out := make([]bschemas.ChatMessage, 0, 2+(len(msgs)-boundary)) - out = append(out, msgs[0], summaryMsg) + out = append(out, msgs[:headCount]...) + out = append(out, summaryMsg) out = append(out, msgs[boundary:]...) + // Removing a span can orphan the tail's leading tool_result blocks; a provider rejects + // the whole request if it does. See dropOrphanedToolResults. + if repaired, n := dropOrphanedToolResults(out); n > 0 { + out = repaired + } if cp.Key != "" { return out, []string{cp.Key}, true } diff --git a/components/offload/summarize_pairing.go b/components/offload/summarize_pairing.go new file mode 100644 index 00000000..04987a13 --- /dev/null +++ b/components/offload/summarize_pairing.go @@ -0,0 +1,101 @@ +package offload + +import ( + bschemas "github.com/maximhq/bifrost/core/schemas" +) + +// Tool-pairing repair for a component that REMOVES messages. +// +// Anthropic (and every provider with the same shape) requires each `tool_result` to answer a +// `tool_use` that appeared earlier. summarize replaces a span of the transcript with one +// summary message — [msgs[0], summary, msgs[end:]...] — and the kept tail can begin part-way +// through a tool exchange, so its leading `tool_result` blocks refer to `tool_use` blocks that +// were just deleted. The provider then rejects the entire request: +// +// 400 messages.0.content.2: unexpected `tool_use_id` found in `tool_result` blocks +// +// Measured on live LOCA-bench traffic: 5 of 12 tasks failed this way once the earlier +// system-role defect was fixed and summarize could finally act. +// +// This is an invariant for any component that deletes messages, and the reason coref does not +// need it: coref rewrites a tool message's text IN PLACE and never removes a message, so +// pairing is preserved by construction. summarize removes, so summarize must repair. +// +// The repair is deliberately one-directional — DROP orphaned results, never synthesise +// placeholder ones. A synthetic "[tool result unavailable]" would be a second lie on top of +// the summary: the summary already claims to carry that content forward, so re-asserting a +// missing result invites the model to reason about an absence that the summary is supposed to +// have described. The rig-side shim used for LOCA's own trimmer synthesises because it must +// preserve a foreign agent's history; a component summarising its own span does not. +func dropOrphanedToolResults(msgs []bschemas.ChatMessage) ([]bschemas.ChatMessage, int) { + // Every tool_use id available to answer, accumulated as we walk forward. A result may + // answer any earlier call, not only the immediately preceding message, because a summary + // may sit between the call and its result. + seen := map[string]struct{}{} + out := make([]bschemas.ChatMessage, 0, len(msgs)) + dropped := 0 + for _, m := range msgs { + if m.ChatAssistantMessage != nil { + for _, tc := range m.ChatAssistantMessage.ToolCalls { + if tc.ID != nil { + seen[*tc.ID] = struct{}{} + } + } + } + if m.Role == bschemas.ChatMessageRoleTool && m.ChatToolMessage != nil && + m.ChatToolMessage.ToolCallID != nil { + if _, ok := seen[*m.ChatToolMessage.ToolCallID]; !ok { + dropped++ + continue // orphaned: its call is gone + } + } + out = append(out, m) + } + return out, dropped +} + +// summarizeSpan picks the span to summarize so that it NEVER cuts inside a tool exchange, +// and reports how many head messages to preserve. +// +// The naive boundaries — preserve msgs[0], summarize msgs[1 : len-keepLast] — are pure +// arithmetic and know nothing about tool pairing, which produced two separate provider +// rejections on live traffic: +// +// 400 messages.N.content.M: unexpected `tool_use_id` found in `tool_result` blocks +// — the kept tail began with a tool_result whose tool_use was inside the span +// 400 messages.N: `tool_use` ids were found without `tool_result` blocks immediately after +// — msgs[0] was an assistant tool_use whose result was inside the span +// +// Both are the same mistake seen from either side, so both are fixed by one rule: **a tool +// exchange is atomic**. Review put it best — an unanswered call means the agent is still +// waiting on that tool, so summarize after it completes, not through it. +// +// Two adjustments implement that: +// +// - END is advanced forward past any tool messages the kept tail would begin with, so the +// exchange is summarized WHOLE rather than split. Advancing (rather than retreating) +// keeps the call and its result on the same side of the boundary without ever keeping +// LESS context than asked for. +// - The HEAD is dropped when msgs[0] is an assistant message carrying tool calls, because +// its results necessarily lie inside the span. msgs[0] is preserved to retain the +// conversation's identity — its system prompt or opening user turn — and an assistant +// tool-call message is neither, so nothing is lost by folding it into the summary. +// +// Returns headCount (0 or 1), start, end. A caller that gets end <= start should skip. +func summarizeSpan(msgs []bschemas.ChatMessage, keepLast int) (headCount, start, end int) { + headCount = 1 + if len(msgs) > 0 && msgs[0].Role == bschemas.ChatMessageRoleAssistant && + msgs[0].ChatAssistantMessage != nil && len(msgs[0].ChatAssistantMessage.ToolCalls) > 0 { + headCount = 0 // preserving it would leave its calls unanswered + } + start = headCount + end = len(msgs) - keepLast + if end > len(msgs) { + end = len(msgs) + } + // Advance past a tail that would begin mid-exchange. + for end < len(msgs) && msgs[end].Role == bschemas.ChatMessageRoleTool { + end++ + } + return headCount, start, end +} diff --git a/components/offload/summarize_parallel_test.go b/components/offload/summarize_parallel_test.go new file mode 100644 index 00000000..53721caf --- /dev/null +++ b/components/offload/summarize_parallel_test.go @@ -0,0 +1,87 @@ +package offload + +import ( + "context" + "strings" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/schema" + "github.com/rossoctl/context-guru/store" +) + +// The shape live traffic actually carries, and which no existing fixture covered: assistant messages +// with TWO tool_use blocks (a parallel call), each answered by its own role=tool message. Live LOCA +// runs failed 28/75 with the provider reporting an unanswered call, and the capture hop showed +// summarize emitting a trailing assistant(tool_use, tool_use) with nothing after it. +// +// The rig-side shim's repair_tool_pairing synthesises results for unanswered calls and reported +// repairs=0 on that run, so the INPUT was well formed. That leaves summarize's restructuring. +func TestSummarizePreservesParallelToolExchanges(t *testing.T) { + big := strings.Repeat("tool output mentioning src/auth.py and TOKEN_GRACE_41ab\n", 80) + var msgs []bschemas.ChatMessage + msgs = append(msgs, bschemas.ChatMessage{Role: bschemas.ChatMessageRoleUser, + Content: &bschemas.ChatMessageContent{ContentStr: sp("do the thing")}}) + nm := "Read" + for i := 0; i < 10; i++ { + a := "pa_" + string(rune('a'+i)) + b := "pb_" + string(rune('a'+i)) + msgs = append(msgs, + bschemas.ChatMessage{Role: bschemas.ChatMessageRoleAssistant, + Content: &bschemas.ChatMessageContent{ContentStr: sp("calling two tools")}, + ChatAssistantMessage: &bschemas.ChatAssistantMessage{ + ToolCalls: []bschemas.ChatAssistantMessageToolCall{ + {ID: &a, Function: bschemas.ChatAssistantMessageToolCallFunction{Name: &nm, Arguments: `{}`}}, + {ID: &b, Function: bschemas.ChatAssistantMessageToolCallFunction{Name: &nm, Arguments: `{}`}}, + }}}, + bschemas.ChatMessage{Role: bschemas.ChatMessageRoleTool, + Content: &bschemas.ChatMessageContent{ContentStr: sp(big)}, + ChatToolMessage: &bschemas.ChatToolMessage{ToolCallID: &a}}, + bschemas.ChatMessage{Role: bschemas.ChatMessageRoleTool, + Content: &bschemas.ChatMessageContent{ContentStr: sp(big)}, + ChatToolMessage: &bschemas.ChatToolMessage{ToolCallID: &b}}, + ) + } + for _, keepLast := range []int{1, 2, 3, 4, 5} { + t.Run("keep_last="+string(rune('0'+keepLast)), func(t *testing.T) { + in := append([]bschemas.ChatMessage(nil), msgs...) + if v := schema.ValidateShape(in); len(v) != 0 { + t.Fatalf("fixture invalid, proves nothing: %v", v) + } + cfg := "{\"min_tokens\":1,\"start_from_message\":0,\"keep_last\":" + string(rune(0x30+keepLast)) + "}" + s, err := newSummarize([]byte(cfg)) + if err != nil { + t.Fatalf("newSummarize: %v", err) + } + req := &bschemas.BifrostChatRequest{Input: in} + c := &components.Ctx{Ctx: context.Background(), Session: "par" + cfg, + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + Model: components.ModelSpec{Incoming: stubMdl{}, Static: stubMdl{}}} + off, ok := s.(components.Offload) + if !ok { + t.Fatal("summarize is not an Offload") + } + if _, err := off.Offload(req, &components.Report{}, c); err != nil { + t.Fatalf("Offload: %v", err) + } + if v := schema.ValidateShape(req.Input); len(v) != 0 { + t.Errorf("keep_last=%d: summarize broke a valid PARALLEL transcript:", keepLast) + for _, x := range v { + t.Errorf(" %s", x) + } + for i, m := range req.Input { + n := 0 + if m.ChatAssistantMessage != nil { + n = len(m.ChatAssistantMessage.ToolCalls) + } + id := "" + if m.ChatToolMessage != nil && m.ChatToolMessage.ToolCallID != nil { + id = *m.ChatToolMessage.ToolCallID + } + t.Errorf(" [%d] %-9s calls=%d answers=%s", i, m.Role, n, id) + } + } + }) + } +} diff --git a/components/offload/summarize_role_test.go b/components/offload/summarize_role_test.go new file mode 100644 index 00000000..31081ae4 --- /dev/null +++ b/components/offload/summarize_role_test.go @@ -0,0 +1,200 @@ +package offload + +import ( + "context" + "strings" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/schema" + "github.com/rossoctl/context-guru/store" +) + +// fixedModel returns one canned summary, so the test asserts on the SHAPE of what +// summarize emits rather than on a model's wording. +type fixedModel struct{ out string } + +func (m *fixedModel) Complete(context.Context, string) (string, error) { return m.out, nil } + +// THE REGRESSION THIS GUARDS AGAINST +// +// summarize emitted its summary as a SYSTEM-role message and spliced it in as +// [msgs[0], summary, tail...]. When msgs[0] is itself the system prompt — the normal case — +// that puts a system role at index 1, and Anthropic rejects the entire request: +// +// 400 messages.1: role 'system' must precede an 'assistant' message or end the array +// +// It shipped because nothing asserted the summary's role, and because every measurement +// before this replayed through /compact, which never forwards upstream and so never had a +// body validated by a provider. It was found only when LOCA-bench ran against a real API: +// every task that triggered a summarization failed, including in an arm with no other +// component enabled. +// +// The contract is therefore: the summary must NOT be system-role, and no system-role +// message may appear anywhere except index 0. +func TestSummarizeEmitsNoSystemRoleAwayFromTheHead(t *testing.T) { + s := newSummarizeTestComponent(t, &fixedModel{out: "SUMMARY: explored the handler, 3 tests fail."}) + span := strings.Repeat("ran pytest tests/test_handler.py, 3 failures in src/mod/file.py\n", 40) + req := &bschemas.BifrostChatRequest{ + Input: []bschemas.ChatMessage{ + // index 0 is the system prompt, exactly as a real agent sends it + sysMsg("you are a coding agent"), + userMsg("Fix the failing handler in src/mod/file.py and run the tests."), + toolResultMsg(span), + toolResultMsg(span), + userMsg("keep going"), + }, + } + var rep components.Report + c := &components.Ctx{Ctx: context.Background(), Session: "s", Store: store.NewMemory(store.Options{})} + if _, err := s.Offload(req, &rep, c); err != nil { + t.Fatalf("Offload: %v", err) + } + if rep.Skipped { + t.Skip("summarize declined on this fixture; the role assertion needs it to act") + } + for i, m := range req.Input { + if m.Role == bschemas.ChatMessageRoleSystem && i != 0 { + t.Fatalf("system-role message at index %d — Anthropic rejects this "+ + "(400 messages.%d: role 'system' must precede an 'assistant' message or "+ + "end the array). Messages: %s", i, i, roleList(req.Input)) + } + } +} + +// sysMsg is a system-role message, the shape a real agent puts at index 0. +func sysMsg(text string) bschemas.ChatMessage { + m := bschemas.ChatMessage{Role: bschemas.ChatMessageRoleSystem} + schema.SetMessageText(&m, text) + return m +} + +func roleList(msgs []bschemas.ChatMessage) string { + var b strings.Builder + for i, m := range msgs { + if i > 0 { + b.WriteString(", ") + } + b.WriteString(string(m.Role)) + } + return b.String() +} + +// THE SECOND REGRESSION, found only after the first was fixed +// +// summarize replaces a span with one summary message, so the kept tail can begin part-way +// through a tool exchange — its leading `tool_result` blocks answer `tool_use` blocks that +// were just deleted. Anthropic rejects the whole request: +// +// 400 messages.0.content.2: unexpected `tool_use_id` found in `tool_result` blocks +// +// Measured on live LOCA-bench traffic: 5 of 12 tasks failed this way once the system-role +// defect was fixed and summarize could finally act at all. +// +// This is an invariant for any component that DELETES messages. coref does not need it — +// it rewrites a tool message's text in place and never removes a message, so pairing holds +// by construction. +func TestDropOrphanedToolResults(t *testing.T) { + call := func(id string) bschemas.ChatMessage { + m := bschemas.ChatMessage{Role: bschemas.ChatMessageRoleAssistant, + ChatAssistantMessage: &bschemas.ChatAssistantMessage{ + ToolCalls: []bschemas.ChatAssistantMessageToolCall{{ID: &id}}, + }} + return m + } + result := func(id string) bschemas.ChatMessage { + m := bschemas.ChatMessage{Role: bschemas.ChatMessageRoleTool, + ChatToolMessage: &bschemas.ChatToolMessage{ToolCallID: &id}} + schema.SetMessageText(&m, "output for "+id) + return m + } + + // A well-formed history must pass through untouched — the repair has to be idempotent + // and must never drop a result whose call is present, even at a distance (a summary can + // legitimately sit between a call and its result). + ok := []bschemas.ChatMessage{userMsg("go"), call("t1"), result("t1"), call("t2"), result("t2")} + got, n := dropOrphanedToolResults(ok) + if n != 0 || len(got) != len(ok) { + t.Errorf("well-formed history must be unchanged, dropped %d (%d -> %d)", n, len(ok), len(got)) + } + + // The summarize shape: the span holding call("t1") was replaced by a summary, so the + // tail's result("t1") is orphaned and must go, while result("t2") stays. + orphaned := []bschemas.ChatMessage{ + userMsg("go"), + userMsg("SUMMARY: earlier work, including a call whose result follows"), + result("t1"), // its call was deleted + call("t2"), result("t2"), + } + got, n = dropOrphanedToolResults(orphaned) + if n != 1 { + t.Fatalf("expected exactly 1 orphaned result dropped, got %d", n) + } + for _, m := range got { + if m.Role == bschemas.ChatMessageRoleTool && m.ChatToolMessage != nil && + m.ChatToolMessage.ToolCallID != nil && *m.ChatToolMessage.ToolCallID == "t1" { + t.Error("orphaned tool_result for t1 survived the repair") + } + } + if len(got) != len(orphaned)-1 { + t.Errorf("repair removed %d messages, want 1", len(orphaned)-len(got)) + } +} + +// THE THIRD REGRESSION, and the one that showed the first two shared a root cause +// +// Review's question was the right one: an unanswered `tool_use` means the agent is still +// waiting on that tool, so summarize AFTER the exchange completes rather than through it. +// Both earlier pairing defects were the same mistake seen from either side, caused by +// boundaries chosen arithmetically: +// +// msgs[0] preserved while its results sit in the span -> unanswered call +// tail starting on a tool_result whose call is in span -> orphaned result +// +// summarizeSpan makes a tool exchange atomic, which fixes both without inventing content. +func TestSummarizeSpanNeverCutsInsideAToolExchange(t *testing.T) { + asst := func(id string) bschemas.ChatMessage { + return bschemas.ChatMessage{Role: bschemas.ChatMessageRoleAssistant, + ChatAssistantMessage: &bschemas.ChatAssistantMessage{ + ToolCalls: []bschemas.ChatAssistantMessageToolCall{{ID: &id}}}} + } + res := func(id string) bschemas.ChatMessage { + m := bschemas.ChatMessage{Role: bschemas.ChatMessageRoleTool, + ChatToolMessage: &bschemas.ChatToolMessage{ToolCallID: &id}} + schema.SetMessageText(&m, "out "+id) + return m + } + + // The kept tail must never BEGIN on a tool message: end advances past the exchange. + msgs := []bschemas.ChatMessage{ + sysMsg("system"), userMsg("go"), + asst("t1"), res("t1"), asst("t2"), res("t2"), + } + for keep := 1; keep <= 4; keep++ { + headCount, start, end := summarizeSpan(msgs, keep) + if end < len(msgs) && msgs[end].Role == bschemas.ChatMessageRoleTool { + t.Errorf("keepLast=%d: tail begins on a tool message at %d — orphans its result", keep, end) + } + if start != headCount { + t.Errorf("keepLast=%d: start %d must equal headCount %d", keep, start, headCount) + } + } + + // A head that is an assistant tool-call message must NOT be preserved: its results are + // inside the span, so keeping it would leave the call unanswered. + headIsCall := []bschemas.ChatMessage{asst("t9"), res("t9"), userMsg("next"), userMsg("more")} + headCount, start, _ := summarizeSpan(headIsCall, 1) + if headCount != 0 { + t.Errorf("an assistant tool-call head must not be preserved, got headCount=%d", headCount) + } + if start != 0 { + t.Errorf("start must follow headCount, got %d", start) + } + + // A normal head (system prompt) IS preserved — the identity the head exists for. + headCount, _, _ = summarizeSpan(msgs, 2) + if headCount != 1 { + t.Errorf("a system-prompt head must be preserved, got headCount=%d", headCount) + } +} diff --git a/components/offload/summarize_trailing_test.go b/components/offload/summarize_trailing_test.go new file mode 100644 index 00000000..07c06270 --- /dev/null +++ b/components/offload/summarize_trailing_test.go @@ -0,0 +1,95 @@ +package offload + +import ( + "context" + "strings" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/schema" + "github.com/rossoctl/context-guru/store" +) + +// Live LOCA traffic produced 28 provider rejections in 75 runs with +// +// 400 messages.N: `tool_use` ids were found without `tool_result` blocks immediately after +// +// and a capture hop between the proxy and the gateway showed summarize emitting exactly three +// messages: [user, summary(user), assistant(thinking,tool_use,tool_use)] -- a request ENDING on an +// unanswered call. See docs/experiments/loca/iter011/results.md. +// +// The open question this test settles: does summarize turn a VALID transcript into that shape, or was +// the input already invalid? Those need opposite responses -- a component fix versus a rig/cascade +// investigation -- so guessing is not acceptable. +func TestSummarizeNeverEmitsATrailingUnansweredCall(t *testing.T) { + big := strings.Repeat("tool output line mentioning src/auth.py and TOKEN_GRACE_41ab\n", 80) + mk := func(tailKind string) []bschemas.ChatMessage { + msgs := []bschemas.ChatMessage{ + {Role: bschemas.ChatMessageRoleUser, + Content: &bschemas.ChatMessageContent{ContentStr: sp("Fix test_auth_expiry")}}, + } + for i := 0; i < 10; i++ { + id := "call_" + string(rune('a'+i)) + nm := "Read" + msgs = append(msgs, + bschemas.ChatMessage{Role: bschemas.ChatMessageRoleAssistant, + Content: &bschemas.ChatMessageContent{ContentStr: sp("reading")}, + ChatAssistantMessage: &bschemas.ChatAssistantMessage{ + ToolCalls: []bschemas.ChatAssistantMessageToolCall{ + {ID: &id, Function: bschemas.ChatAssistantMessageToolCallFunction{ + Name: &nm, Arguments: `{"p":"a.py"}`}}}}}, + bschemas.ChatMessage{Role: bschemas.ChatMessageRoleTool, + Content: &bschemas.ChatMessageContent{ContentStr: sp(big)}, + ChatToolMessage: &bschemas.ChatToolMessage{ToolCallID: &id}}) + } + if tailKind == "user" { + msgs = append(msgs, bschemas.ChatMessage{Role: bschemas.ChatMessageRoleUser, + Content: &bschemas.ChatMessageContent{ContentStr: sp("keep going")}}) + } + return msgs + } + for _, kind := range []string{"toolresult", "user"} { + t.Run("tail="+kind, func(t *testing.T) { + in := mk(kind) + if v := schema.ValidateShape(in); len(v) != 0 { + t.Fatalf("fixture invalid, test proves nothing: %v", v) + } + s, err := newSummarize([]byte("{\"min_tokens\":100,\"keep_last\":3}")) + if err != nil { + t.Fatalf("newSummarize: %v", err) + } + req := &bschemas.BifrostChatRequest{Input: in} + c := &components.Ctx{Ctx: context.Background(), Session: "trail-" + kind, + Store: store.NewMemory(store.Options{}), CtxWindow: 200000, + Model: components.ModelSpec{Incoming: stubMdl{}, Static: stubMdl{}}} + if off, ok := s.(components.Offload); ok { + if _, err := off.Offload(req, &components.Report{}, c); err != nil { + t.Fatalf("Offload: %v", err) + } + } + for i, m := range req.Input { + t.Logf(" [%d] role=%s calls=%d", i, m.Role, + func() int { if m.ChatAssistantMessage != nil { return len(m.ChatAssistantMessage.ToolCalls) }; return 0 }()) + } + if v := schema.ValidateShape(req.Input); len(v) != 0 { + t.Errorf("summarize turned a VALID transcript into one a provider rejects:") + for _, x := range v { + t.Errorf(" %s", x) + } + } + }) + } +} + +func sp(s string) *string { return &s } + +type stubMdl struct{} + +func (stubMdl) Complete(ctx context.Context, prompt string) (string, error) { + return "SUMMARY: read the auth module.", nil +} +func (stubMdl) CompleteSystem(ctx context.Context, sys, user string) (string, error) { + return "SUMMARY: read the auth module.", nil +} +func (stubMdl) Name() string { return "stub" } diff --git a/deploy/harbor/cache_opportunity.py b/deploy/harbor/cache_opportunity.py new file mode 100644 index 00000000..9a37991d --- /dev/null +++ b/deploy/harbor/cache_opportunity.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Was there a FREE moment to mutate the cached prefix? Measured, not inferred. + +The break-even inequality coref and extract_llm's prefix path both obey -- + + S x T > 11.5 x W + +-- prices a cache-WRITE that the mutation causes. But if the provider's cache has already +expired, the prefix is re-written on the next turn REGARDLESS. At that moment a prefix +mutation costs nothing incremental: W is already being paid. No break-even test is needed, +and a component that normally cannot justify a rewrite could act for free. + +Such moments arise whenever the gap between turns exceeds the cache TTL (Anthropic: +5 minutes for ephemeral_5m, 1 hour for ephemeral_1h) -- a slow tool, a user thinking, a +queue. The question this answers is whether they actually occur, and how much mass they +would have made free. + +DIRECT EVIDENCE, NOT TIMING. LOCA records per-step usage in trajectory.json, so we do not +have to guess from wall-clock: a step with cache_read_input_tokens == 0 while +cache_creation_input_tokens is large means the provider found nothing cached and wrote the +prefix from scratch. Step 1 is excluded -- nothing is cached before the first call, so it is +not an opportunity, it is the beginning. + +Usage: cache_opportunity.py [more-dirs...] +""" +import glob +import json +import os +import sys + + +def analyse(run_dir): + steps = cold = 0 + cold_mass = warm_mass = 0 + per_task = [] + for f in sorted(glob.glob(os.path.join(run_dir, "tasks/*/state0/trajectory.json"))): + try: + ut = json.load(open(f)).get("usage_tracking") or [] + except Exception: + continue + t_cold = t_steps = t_mass = 0 + for e in ut: + if not isinstance(e, dict): + continue + steps += 1 + t_steps += 1 + created = e.get("cache_creation_input_tokens") or 0 + read = e.get("cache_read_input_tokens") or 0 + if e.get("step", 0) <= 1: + warm_mass += read # first call: nothing to be cold about + continue + if read == 0 and created > 0: + cold += 1 + t_cold += 1 + cold_mass += created # W that was paid anyway => free to mutate + t_mass += created + else: + warm_mass += read + if t_steps: + per_task.append((os.path.basename(os.path.dirname(os.path.dirname(f))), + t_steps, t_cold, t_mass)) + return steps, cold, cold_mass, warm_mass, per_task + + +def main(): + if len(sys.argv) < 2: + print(__doc__) + return 2 + for d in sys.argv[1:]: + steps, cold, cold_mass, warm_mass, per_task = analyse(d) + if not steps: + print(f"{os.path.basename(d)}: no usage data") + continue + print(f"\n=== {os.path.basename(d)} ===") + print(f" steps {steps} cold-cache steps (beyond step 1): {cold} " + f"({100*cold/steps:.1f}%)") + print(f" prefix mass re-created at those steps: {cold_mass:,} tok") + print(f" prefix mass served from cache elsewhere: {warm_mass:,} tok") + if cold_mass + warm_mass: + print(f" => {100*cold_mass/(cold_mass+warm_mass):.1f}% of prefix traffic was " + f"ALREADY being re-written, i.e. free to mutate") + hot = [p for p in per_task if p[2]] + if hot: + print(f" tasks with at least one free moment: {len(hot)}/{len(per_task)}") + for name, ts, c, m in sorted(hot, key=lambda x: -x[3])[:5]: + print(f" {name:34s} {c}/{ts} steps cold, {m:,} tok") + print("\nCAVEAT that cuts the other way: a run using several parallel workers spaces one") + print("session's turns further apart than production would, so an expired cache here may") + print("be an artifact of the harness rather than of agent behaviour. Treat the rate as an") + print("UPPER bound; a single-worker run would give the honest one.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/harbor/capture_hop.py b/deploy/harbor/capture_hop.py new file mode 100644 index 00000000..c37ff474 --- /dev/null +++ b/deploy/harbor/capture_hop.py @@ -0,0 +1,177 @@ +"""A capture-only hop that records the request context-guru actually SENT, when the provider rejects it. + +WHY THIS EXISTS, AND WHY THE EXISTING SHIM COULD NOT DO IT. `loca_repair_shim.py` sits UPSTREAM of +context-guru (LOCA -> shim -> cg-proxy -> gateway), so it observes the request before compaction. When +the provider rejects a request because a component produced an invalid message list, the shim's copy is +the innocent one. It also *repairs* tool pairing, which would mask exactly the defect under +investigation. + +So this hop goes on the other side: + + LOCA -> repair shim -> cg-proxy -> [THIS HOP] -> gateway + +It repairs nothing and changes nothing. On a >=400 response it records a STRUCTURAL DIGEST of the +outgoing message list -- per message: index, role, the tool_use ids it declares, and the tool_result +ids it answers -- plus the provider's error. That digest is what identifies an orphaned pair; the raw +bodies are hundreds of kilobytes and mostly irrelevant, so only a bounded head is kept. + +Provoked by: `summarize` producing `messages.1: tool_use ids were found without tool_result blocks +immediately after` on 28 of 75 live runs, a FOURTH shape defect in a component that already had three +fixed (docs/experiments/loca/iter011/). Reasoning from the source did not converge -- both splice sites +appear to align the span boundary -- so the message list itself has to be read. +""" +import json +import os +import sys +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +UPSTREAM = os.environ["CAPTURE_UPSTREAM"].rstrip("/") +FAILLOG = os.environ.get("CAPTURE_FAILLOG", "/tmp/cg-loca/capture-failures.jsonl") +PORT = int(os.environ.get("CAPTURE_PORT", "4270")) + +# Same set as forever/http_utils.py REQUEST_STRIP, for the reason documented in +# loca_repair_shim.py: this hop re-frames with an explicit content-length, so forwarding +# transfer-encoding alongside it is a protocol violation. +_STRIP = {"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", + "trailer", "trailers", "transfer-encoding", "upgrade", "host", "content-length", + "accept-encoding"} + + +def digest(payload): + """Per-message structure: what each message declares and what it answers.""" + out = [] + for i, m in enumerate(payload.get("messages") or []): + uses, results, kinds = [], [], [] + c = m.get("content") + blocks = c if isinstance(c, list) else ([] if c is None else [{"type": "text"}]) + for b in blocks: + if not isinstance(b, dict): + continue + t = b.get("type") + kinds.append(t) + if t == "tool_use": + uses.append(b.get("id")) + elif t == "tool_result": + results.append(b.get("tool_use_id")) + out.append({"i": i, "role": m.get("role"), "types": kinds, + "tool_use": uses, "tool_result": results}) + return out + + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + if (self.headers.get("transfer-encoding") or "").lower() == "chunked": + chunks = [] + while True: + line = self.rfile.readline(65536).strip() + if not line: + continue + n = int(line.split(b";")[0], 16) + if n == 0: + while True: + if self.rfile.readline(65536) in (b"\r\n", b"\n", b""): + break + break + chunks.append(self.rfile.read(n)) + self.rfile.read(2) + raw = b"".join(chunks) + else: + raw = self.rfile.read(int(self.headers.get("content-length") or 0)) + + req = urllib.request.Request(UPSTREAM + self.path, data=raw, method="POST") + for k, v in self.headers.items(): + if k.lower() not in _STRIP: + req.add_header(k, v) + req.add_header("content-length", str(len(raw))) + try: + with urllib.request.urlopen(req, timeout=900) as r: + data, status, hdrs = r.read(), r.status, dict(r.headers) + except urllib.error.HTTPError as e: + data, status, hdrs = e.read(), e.code, dict(e.headers) + except Exception as e: + data, status, hdrs = json.dumps({"error": str(e)}).encode(), 502, {} + + # PER-REQUEST FLAP LOG. The expand tool sits at the FRONT of the prompt-cache hash (tools -> + # system -> messages), so every turn where it appears or disappears is a full prefix miss from + # position zero -- the most expensive invalidation available. Under INJECT_EXPAND=auto the tool + # is advertised only on turns whose outgoing body carries a marker, which is not every turn, so + # the array flaps. That was inferred from the advertise rule plus per-arm action rates, never + # measured per request. This makes it countable: one compact line per request recording how + # many tools were advertised, whether the expand tool was among them, and whether a marker was + # present. A flap is then a transition in has_expand between consecutive requests of a session. + if os.environ.get("CAPTURE_FLAPLOG"): + try: + pl = json.loads(raw) + tools = pl.get("tools") or [] + names = [t.get("name") for t in tools if isinstance(t, dict)] + body_s = raw.decode("utf-8", "replace") + with open(os.environ["CAPTURE_FLAPLOG"], "a") as fh: + fh.write(json.dumps({ + "seq": self.headers.get("x-cg-rig-seq"), + "n_tools": len(tools), + "has_expand": any("expand" in (n or "") for n in names), + "has_marker": "<= 400: + try: + payload = json.loads(raw) + snap = {"status": status, "body_bytes": len(raw), + # Stamped by loca_repair_shim.py so this record can be joined to the + # request the proxy was GIVEN, not just the one it produced. + "rig_seq": self.headers.get("x-cg-rig-seq"), + "n_messages": len(payload.get("messages") or []), + "system_present": "system" in payload, + "digest": digest(payload), + "error": data[:600].decode("utf-8", "replace")} + with open(FAILLOG, "a") as fh: + fh.write(json.dumps(snap) + "\n") + except Exception as e: + try: + with open(FAILLOG, "a") as fh: + fh.write(json.dumps({"status": status, + "capture_error": f"{type(e).__name__}: {e}"}) + "\n") + except Exception: + pass + self.send_response(status) + self.send_header("content-type", hdrs.get("Content-Type", "application/json")) + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + if self.path == "/capture-stats": + n = 0 + if os.path.exists(FAILLOG): + with open(FAILLOG) as fh: + n = sum(1 for _ in fh) + body = json.dumps({"captured": n}).encode() + self.send_response(200) + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + else: + self.send_response(404) + self.end_headers() + + def log_message(self, *a): + pass + + +if __name__ == "__main__": + print(f"[capture] :{PORT} -> {UPSTREAM} faillog={FAILLOG}", flush=True) + sys.stdout.flush() + ThreadingHTTPServer(("127.0.0.1", PORT), Handler).serve_forever() diff --git a/deploy/harbor/cc_capture.py b/deploy/harbor/cc_capture.py new file mode 100755 index 00000000..e74ac9f8 --- /dev/null +++ b/deploy/harbor/cc_capture.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Convert Claude Code session transcripts into CONTEXT_GURU_CAPTURE JSONL, so coref.py +can measure co-reference density on REAL agent traffic without an eval-box run. + +Why this exists. coref.py needs captured request bodies, and the corpus the improvement +plan cites (capture-tb / capture-swe / capture-swebench) lives on the eval box. But a +Claude Code transcript is the same traffic recorded one step earlier: the agent's own +append-only log of every message it sent. Converting it costs zero API dollars and yields +Tier-1-rich Read/Edit/Bash traffic — the substrate docs/proposals/coref-compaction.md §8 +names as the right one for a deterministic reference detector. + +Three things the transcript is NOT, and what this does about each: + + 1. It is one entry per content BLOCK, not per message. Consecutive same-role entries are + MERGED, because a real request body carries one assistant message with several blocks + (and one user message with the batched tool_results). Message COUNT is the axis + reference recency is measured on, so getting this wrong would rescale the whole A/B + distinction. + 2. It spans many context windows. These sessions reach 3,000+ model turns, so no single + request ever held the whole thing — the agent compacted repeatedly. Emitting it as one + transcript would measure a request that never existed. So it is SEGMENTED at + --segment-tokens, each segment treated as its own session. This approximates the real + compaction boundaries (which the transcript does not record) by the budget that forces + them. + 3. `thinking` blocks are model-authored, so they are mapped to text and count as a + reference-bearing surface. That is deliberate, not a shortcut: thinking is exactly + where a model restates the value it lifted out of a tool result, and the real wire + body carries it too. + +Output is standard {provider, model, body} JSONL, one record per model turn, as an actual +capture would be. One deviation, and it is the only reason coref.py needed a change: a +full prefix per turn is O(n^2) bytes (47 GB across this machine's transcripts), and +non-final records are read ONLY for their token size (session_turns, the T estimate). So +every record except a segment's last carries `turn_tokens` plus just the segment's first +user message — enough for coref.py to size the turn and to group the segment — and +coref.py uses that field when present. A real capture has no such field and is unaffected. + +Usage: + cc_capture.py [transcript.jsonl ...] [--segment-tokens N] [--min-turns N] + cc_capture.py --all [--segment-tokens N] # every local Claude Code session + + --segment-tokens 180000 split a transcript into window-sized sessions (0 = never split) + --min-turns 8 drop segments with fewer model turns than this + --all use ~/.claude/projects/*/*.jsonl +""" +import glob +import json +import os +import sys + +TOK = lambda s: max(1, len(s) // 4) # same proxy as coref.py / analyze_content.py + + +def load_entries(path): + """Main-conversation user/assistant entries, in file order. Sidechain (subagent) entries + are a DIFFERENT conversation with its own context, so mixing them in would invent + references across two transcripts that never saw each other.""" + out = [] + with open(path, errors="replace") as f: + for line in f: + if not line.strip(): + continue + try: + e = json.loads(line) + except ValueError: + continue + if e.get("type") not in ("user", "assistant") or e.get("isSidechain"): + continue + m = e.get("message") or {} + if m.get("content") is None: + continue + out.append(e) + return out + + +def blocks_of(entry): + """Anthropic content blocks for one entry, with thinking mapped to text and anything + non-textual dropped (an image carries no identifiers an exact matcher could use).""" + c = (entry.get("message") or {}).get("content") + if isinstance(c, str): + return [{"type": "text", "text": c}] if c else [] + if not isinstance(c, list): + return [] + out = [] + for b in c: + if not isinstance(b, dict): + continue + t = b.get("type") + if t == "text": + out.append({"type": "text", "text": b.get("text", "")}) + elif t == "thinking": + out.append({"type": "text", "text": b.get("thinking", "")}) + elif t == "tool_use": + out.append({"type": "tool_use", "id": b.get("id", ""), + "name": b.get("name", ""), "input": b.get("input", {})}) + elif t == "tool_result": + out.append({"type": "tool_result", "tool_use_id": b.get("tool_use_id", ""), + "content": b.get("content")}) + return out + + +def to_messages(entries): + """Merge consecutive same-role entries into one message each (see docstring note 1).""" + msgs = [] + for e in entries: + role = "assistant" if e.get("type") == "assistant" else "user" + bs = blocks_of(e) + if not bs: + continue + if msgs and msgs[-1]["role"] == role: + msgs[-1]["content"].extend(bs) + else: + msgs.append({"role": role, "content": bs}) + return msgs + + +def msg_tokens(m): + """Token size of one message, counted the way coref.py's session_turns counts it: the + reference-bearing surfaces and the tool-result bodies, not the JSON envelope.""" + texts, results = [], [] + c = m["content"] + for b in c: + t = b.get("type") + if t == "text": + texts.append(b.get("text", "")) + elif t == "tool_use": + texts.append(b.get("name", "") + " " + json.dumps(b.get("input", {}))) + elif t == "tool_result": + rc = b.get("content") + if isinstance(rc, str): + results.append(rc) + elif isinstance(rc, list): + results.append("".join(x.get("text", "") for x in rc if isinstance(x, dict))) + elif rc is not None: + results.append(json.dumps(rc)) + return TOK(" ".join(texts)) + sum(TOK(r) for r in results) + + +def segments(msgs, budget): + """Split into sessions no larger than budget tokens (see docstring note 2). Splits only + BEFORE a user turn, so a tool_result is never separated from the assistant turn that + requested it — an orphaned pair would look like an output nobody asked for.""" + if budget <= 0: + return [msgs] + out, cur, acc = [], [], 0 + for m in msgs: + n = msg_tokens(m) + if cur and acc + n > budget and m["role"] == "user": + out.append(cur) + cur, acc = [], 0 + cur.append(m) + acc += n + if cur: + out.append(cur) + return out + + +def first_user(msgs): + for m in msgs: + if m["role"] == "user": + return m + return msgs[0] if msgs else {"role": "user", "content": []} + + +def emit(seg, model, conv, out): + """One record per model turn, as an append-only capture would hold. Only a segment's + last record carries the full transcript; earlier ones carry their size (see docstring). + + `conv` is stamped explicitly because coref.py otherwise groups a session by the hash of + its first user message — correct for a real capture, where every request of a session + opens with the same task statement, but a mid-transcript segment opens on a tool_result + and those collide. Left inferred, 31 segments grouped down to 24 and the rest were + silently discarded, since only the largest member of a group is analyzed.""" + turns = [i for i, m in enumerate(seg) if m["role"] == "assistant"] + if not turns: + return 0 + head, acc, sizes = first_user(seg), 0, [] + for i, m in enumerate(seg): + acc += msg_tokens(m) + if m["role"] == "assistant": + sizes.append(acc) + for n in sizes[:-1]: + out.write(json.dumps({"provider": "anthropic", "model": model, "conv": conv, + "turn_tokens": n, + "body": {"model": model, "messages": [head]}}) + "\n") + last = turns[-1] + out.write(json.dumps({"provider": "anthropic", "model": model, "conv": conv, + "body": {"model": model, "messages": seg[:last + 1]}}) + "\n") + return len(sizes) + + +def model_of(entries): + for e in entries: + m = (e.get("message") or {}).get("model") + if m: + return m + return "claude-sonnet-5" + + +def main(): + args = sys.argv[1:] + if not args: + print(__doc__) + return 2 + opt = {"--segment-tokens": "180000", "--min-turns": "8"} + files, i, use_all, out_path = [], 0, False, None + while i < len(args): + a = args[i] + if a == "--all": + use_all = True + elif a in opt: + i += 1 + opt[a] = args[i] + elif a.startswith("--"): + print(f"unknown flag {a}") + return 2 + elif out_path is None: + out_path = a + else: + files.append(a) + i += 1 + if out_path is None: + print(__doc__) + return 2 + if use_all: + files += sorted(glob.glob(os.path.expanduser("~/.claude/projects/*/*.jsonl"))) + if not files: + print("no transcripts given") + return 2 + + budget, min_turns = int(opt["--segment-tokens"]), int(opt["--min-turns"]) + n_seg = n_turn = n_src = 0 + with open(out_path, "w") as out: + for path in files: + entries = load_entries(path) + msgs = to_messages(entries) + if not msgs: + continue + model = model_of(entries) + used = 0 + stem = os.path.basename(path)[:8] + for k, seg in enumerate(segments(msgs, budget)): + if sum(1 for m in seg if m["role"] == "assistant") < min_turns: + continue + t = emit(seg, model, f"{stem}#{k}", out) + if t: + n_seg += 1 + n_turn += t + used += 1 + if used: + n_src += 1 + print(f"wrote {out_path}: {n_src} transcripts -> {n_seg} sessions, {n_turn} turn records " + f"({os.path.getsize(out_path) / 1e6:.1f} MB)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/harbor/cfg-iter020-merged-v4.yaml b/deploy/harbor/cfg-iter020-merged-v4.yaml new file mode 100644 index 00000000..a8b0ea36 --- /dev/null +++ b/deploy/harbor/cfg-iter020-merged-v4.yaml @@ -0,0 +1,45 @@ +# iter020 attempt 4: act WHEN THERE IS ENOUGH CONTEXT, which attempts 1-3 did not. +# +# WHAT WENT WRONG IN ATTEMPT 3. extract_llm fired on 190 of 264 requests (72%), regardless of how +# large the context was, and `request_trigger_not_fired` never fired once. Three restraints were off +# simultaneously: +# +# 1. pinning `min_tokens` sets `explicit`, and shouldFire() then returns +# "explicit min_tokens/trigger configured" -- true unconditionally, bypassing the derived +# pressure trigger. Any explicit floor does this: min_tokens, trigger.min_request_tokens and +# trigger.min_output_tokens all mark `explicit`, so the floor and the when-to-act decision are +# coupled and cannot be set independently. +# 2. the explicit request trigger is only enforced when `!c.CacheAware` (extract_llm.go:761), so on +# a caching backend an operator's context threshold is silently ignored. +# 3. `economic_gate: false`, which removed the last thing that was refusing low-value work. +# +# Firing on a small context is not merely wasted spend. It removes outputs that have had no turns in +# which to be superseded -- exactly the case the contract says to keep -- so it is a harm mechanism, +# and it confounds the deferral number, since summarize firing less may just mean extract removed +# early and often. +# +# THE FIX IS TO PIN NOTHING. With no floor and no trigger configured, `explicit` is false and the +# derived trigger governs: fire above 0.60 context pressure (~77k of this 128k band), or above 0.25 +# with >10% growth, and otherwise stay out. That engages before summarize's 0.78 (~100k), which is +# the intended ordering -- selective removal first, blunt summary only if that was not enough. +# +# The cost is the floor: it reverts to the 300 default rather than 120. Measured, floor 300 offers +# smaller batches than 120 -- but a correctly-triggered arm at floor 300 answers the question, and an +# always-on arm at floor 120 does not. +pipeline: [format, extract_llm, summarize] +components: + extract_llm: + selection_mode: merged + strategy: code + model: {source: request} # the agent's own model: same cache namespace, and 0/59 bad quotes + # against haiku's 20.8% at the batch sizes bulk needs + allow_on_caching_backend: true + allow_cached_prefix: true + economic_gate: false # see docs/results/min-tokens-vs-economic-gate.md: it prices each + # candidate against a whole call, and merged makes one call per request + summarize: + trigger: {min_request_frac: 0.78} + resummarize_tokens: 20000 + min_tokens: 500 + keep_last: 3 + model: {source: config} diff --git a/deploy/harbor/cfg-iter020-merged.yaml b/deploy/harbor/cfg-iter020-merged.yaml new file mode 100644 index 00000000..c91d1042 --- /dev/null +++ b/deploy/harbor/cfg-iter020-merged.yaml @@ -0,0 +1,59 @@ +# iter020 arm: the merged design run as SPECIFIED for the first time. +# +# Three changes from iter018's s14, all from docs/experiments/loca/iter019/results.md: +# +# 1. min_tokens 3000 -> 800. iter018 offered the model 2.63 candidates per call. Bulk adjudication +# is a COMPARATIVE mechanism and was measured at ~15; at 2.63 it is close to the per-output +# design refuted at 6% live-kept. Measured offline: at batch 3-6 the model dropped a +# genuinely-spent output only 2/4 of the time, at batch 10 it cleared 100% of them. Small +# batches do not make it wrong, they make it unwilling to act -- which is what iter018's 94.6% +# keep rate looks like from the inside. +# +# 2. the contract now states the SPENT criterion (current step / unfinished user instruction / +# agent's own stated next step) and REQUIRES the model to name and quote the obligation. +# Stating the criterion alone was measured inert (4/4 false drops); requiring the evidence +# halved it. Code change, internal/extract/bulk.go. +# +# 3. trim is gone. Chosen zero times in 21 offline opportunities; in production accepted once +# against eight rejected as invented. +# +# model stays on the cheap client. Reading the real context via a cache-read on the request model +# is buildable and cheap (iter019 section 2) but the capability it was meant to enable -- the +# Tier-2 veto -- did not appear, so it is deliberately NOT bundled into this arm. One variable +# cluster at a time. +pipeline: [format, extract_llm, summarize] +components: + extract_llm: + selection_mode: merged + strategy: code + model: {source: config} + min_tokens: 800 + allow_on_caching_backend: true + allow_cached_prefix: true + # AMENDED MID-RUN, on the pre-registered abort criterion and before any outcome data. + # + # The first launch reached only 1.93 candidates per call at 213 requests with acted=0, and the + # counters named the cause: the economic gate SUPPRESSED 1,497 candidates against 224 that + # reached the model, top reason "cache-aware, saving below call cost". + # + # That gate prices each candidate's expected saving against the cost of A WHOLE MODEL CALL, + # which is right for the per-output loop where candidate == call. In merged mode there is ONE + # call regardless of batch size, so the marginal cost of adding a candidate is a few prompt + # tokens. The gate therefore suppresses exactly the candidates that would make the batch + # bulk-sized -- and it fights min_tokens directly: a lower floor yields smaller candidates, + # each of which looks even less worth "a call". + # + # This is the THIRD instance of one class of defect (prefix pre-filter, llm_max_per_request, + # now this): cost machinery written for per-output calls, applied to a one-call design. The + # correct fix is to amortise one call across the batch in merged mode; that is only worth + # building if bulk batches improve yield, which is what this run exists to answer. So the gate + # is disabled HERE, for a run that was pre-registered as mechanism-only and never going to make + # a cost claim. Calls do not multiply with candidates in merged mode, so the added spend is + # prompt tokens only. + economic_gate: false + summarize: + trigger: {min_request_frac: 0.78} + resummarize_tokens: 20000 + min_tokens: 500 + keep_last: 3 + model: {source: config} diff --git a/deploy/harbor/cfg-iter021-A-baseline.yaml b/deploy/harbor/cfg-iter021-A-baseline.yaml new file mode 100644 index 00000000..37726168 --- /dev/null +++ b/deploy/harbor/cfg-iter021-A-baseline.yaml @@ -0,0 +1,16 @@ +# iter021 ARM A -- the shipped product. The deterministic pipeline plus summarize, and nothing else. +# +# coref is deliberately ABSENT: it is new on the coref-compaction branch, so it is not part of "what we +# had beforehand". Its own value is a separate arm. +# +# summarize's configuration is IDENTICAL to arm B's, so any change in how often it fires is +# attributable to the one component that differs between them. +pipeline: [format, toon, dedup, failed_run, cmdfilter, extract, cachesplit, summarize] +components: + extract: {min_tokens: 400} + summarize: + trigger: {min_request_frac: 0.78} + resummarize_tokens: 20000 + min_tokens: 500 + keep_last: 3 + model: {source: config} diff --git a/deploy/harbor/cfg-iter021-B-merged.yaml b/deploy/harbor/cfg-iter021-B-merged.yaml new file mode 100644 index 00000000..7ccea10d --- /dev/null +++ b/deploy/harbor/cfg-iter021-B-merged.yaml @@ -0,0 +1,38 @@ +# iter021 ARM B -- arm A with ONE component inserted. Nothing else differs. +# +# extract_llm sits BEFORE extract, following ns-full.yaml's existing order. If the deterministic +# extractor runs first it shrinks outputs below the model pass's floor, starving the batch -- the exact +# failure iteration 020 spent five aborts diagnosing. +# +# NO min_tokens and NO trigger on extract_llm, on purpose. Pinning either sets `explicit`, and +# shouldFire() then returns true unconditionally, so the component fires on every request regardless of +# context size -- measured at 72% of requests with request_trigger_not_fired never firing once. Left +# unpinned, the derived trigger governs: fire above 0.60 context pressure (~77k of this 128k band) or +# above 0.25 with >10% growth, which engages before summarize's 0.78. +# +# economic_gate: false is NOT a shippable setting and the pre-registration says so. The gate prices each +# candidate against a whole model call, which is right for the per-output loop and wrong for a design +# making one call per request; with it on, 1,497 candidates were suppressed against 224 that got +# through and the arm does not run the design. Making it batch-aware is the real fix, and is +# deliberately not being done immediately before freezing the binary. +# +# model.source: request puts the adjudication on the agent's own model, which is required twice over -- +# prompt caches are per-model, so only that client can read the prefix the agent's request populated; +# and the cheap model's verbatim quoting degraded to 20.8% at the batch sizes this needs, against 0 of +# 59 on the request model. +pipeline: [format, toon, dedup, failed_run, cmdfilter, extract_llm, extract, cachesplit, summarize] +components: + extract: {min_tokens: 400} + extract_llm: + selection_mode: merged + strategy: code + model: {source: request} + allow_on_caching_backend: true + allow_cached_prefix: true + economic_gate: false + summarize: + trigger: {min_request_frac: 0.78} + resummarize_tokens: 20000 + min_tokens: 500 + keep_last: 3 + model: {source: config} diff --git a/deploy/harbor/coref.py b/deploy/harbor/coref.py new file mode 100755 index 00000000..2230508b --- /dev/null +++ b/deploy/harbor/coref.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +"""Measure CO-REFERENCE density in real captured requests: how much tool-output mass is +ever referenced again by a later model turn, how far back those references reach, and what +a co-reference-driven cut would COST in cache-writes vs save in cache-reads. + +Substrate for docs/proposals/coref-compaction.md. Answers, from traffic rather than +argument, the three questions that decide whether the `coref` component is worth building: + + 1. How much tool-output mass does no later model turn ever touch? (the free, safe cut) + 2. Of the mass that IS referenced: last referenced long ago and only once or twice (a + "closed" reference — the model lifted a value out, and that value survives in the + assistant turn that lifted it), or referenced recently / repeatedly (an "open" + reference — still load-bearing)? Recency is measured from the HEAD of the transcript, + which is the axis the A/B distinction actually rests on. + 3. Does the arithmetic work? A cut breaks the provider's cached prefix at its shallowest + mutated index, so the suffix is cache-WRITTEN once (11.5x a cache-read) and only then + starts saving reads. Prints the turns-remaining `T` each cut set would need to break + even, against the `T` the session actually had. + +METHOD, and its one non-negotiable guard. A reference is only counted for tokens the tool +output ITSELF INTRODUCED. If the agent calls Read(src/auth.py), the path is in the tool_use +argument, echoed by the tool_result, and used again by a later Edit(src/auth.py) — exact +matching would score that as a reference, but the value was in context before the output +existed. So every token already present at or before the producing turn is excluded as +echo, and only NOVEL tokens count. Without this the numbers are meaningless (they trend +toward "everything is referenced"). + +Reference-bearing surfaces are MODEL turns only — assistant text and tool_use arguments. +A later tool_result echoing a token is the environment repeating itself, not the model +using the value. + +Tier-1 (exact) matching ONLY. Mass reported as `unreferenced` means "no later EXACT use"; +it necessarily includes any use that arrived transformed (summed, unit-converted, reworded). +That gap is reported as `derived-value evidence`, never folded into `unreferenced`. + +Usage: + coref.py [more.jsonl ...] [key=value ...] + + min_later=8 an output with fewer model turns after it is never cut (no opportunity yet) + closed_dist=12 a reference is "closed" once its last reference is this many messages AGO + (measured from the head of the transcript, not from the output) + open_reps=3 ...unless referenced at least this many times (then it stays "open") + min_output=300 ignore tool outputs smaller than this (tokens) — below any cut floor + window=200000 model context window, for the trigger-crossing / agent-compact estimates + fire_frac=0.6 cut fires when a request reaches this fraction of the window + sweep=1 also print a threshold sensitivity sweep + json=1 emit machine-readable totals instead of the report +""" +import json, sys, re, hashlib +from collections import defaultdict + +TOK = lambda s: max(1, len(s) // 4) # cheap ~4 chars/token proxy (as analyze_content.py) +CACHE_WRITE_X = 11.5 # ($2.50 - $0.20) / $0.20 — one cache-write in cache-read-equivalents + +# An identifier-ish token: the things a model actually carries forward (paths, symbols, +# ids, hashes, error codes). Prose is filtered out by DISTINCTIVE below rather than by a +# stopword list, which does not survive a change of domain. +IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_./:\-]{2,63}|\b\d{3,}\b|\b[0-9a-f]{7,40}\b") +NUMERIC = re.compile(r"^\d[\d,._]*$") +CAMEL = re.compile(r"[a-z][A-Z]") + + +def distinctive(t): + """Keep tokens that look like an identifier rather than an English word: after trimming + surrounding punctuation they carry INTERIOR structure (_ . / : -), a digit, or CamelCase. + + Three rules here were measured, not guessed — an earlier version accepted any token of + 10+ characters, any token containing punctuation, and any number of 3+ digits, and on + real traffic the top "references" it found were `description`, `transparency`, + `efficiency`, `conditions`, `e.g.`, `try:`, `None:` and `2026`. Prose scored as + identifiers inflated referenced mass from 51% to 71%, and a spurious reference is the + expensive direction: it makes an output look load-bearing and suppresses the cut silently. + + - no bare length rule: long English words are still English words. A real identifier + almost always carries structure, a digit, or camelCase (`src/auth.py`, `session_id`, + `GraphStore`); one that carries none is indistinguishable from prose, so it is + dropped rather than guessed at. + - punctuation must be INTERIOR: `e.g.` / `try:` / `memory.` are prose plus a sentence + mark. Trimming the ends first leaves `e.g` / `try` / `memory`, which fail on their + own merits, while `src/auth.py` and `v1/messages` are untouched. + - a bare number needs 5+ digits or a separator: `2026` is a year and `447` is a line + number or a count, and both recur everywhere. Hashes, ids and versions survive. + """ + t = t.strip("._:-/") # trim first, so the test sees the token and not its punctuation + if len(t) < 4: + return False + if NUMERIC.match(t): + return sum(c.isdigit() for c in t) >= 5 or bool(set(",._") & set(t)) + return bool(set("_./:-") & set(t)) or any(c.isdigit() for c in t) or bool(CAMEL.search(t)) + + +def idents(s): + """Distinctive tokens, trimmed of surrounding punctuation — so `memory.` and `memory` + are one token rather than two that never match each other.""" + out = set() + for t in IDENT.findall(s or ""): + if distinctive(t): + out.add(t.strip("._:-/")) + return out + + +# --- normalize a provider body into one flat message list ----------------------------- +# +# Each entry: {"role", "texts": [model-authored text], "results": [(tool_use_id, text)]} +# `texts` is the reference-bearing surface (assistant prose + tool_use arguments); +# `results` is the mass under evaluation. + + +def normalize(body, provider): + out = [] + if provider == "anthropic": + for m in body.get("messages", []): + role, c = m.get("role"), m.get("content") + e = {"role": role, "texts": [], "results": []} + if isinstance(c, str): + e["texts"].append(c) + elif isinstance(c, list): + for b in c: + if not isinstance(b, dict): + continue + t = b.get("type") + if t == "text": + e["texts"].append(b.get("text", "")) + elif t == "tool_use": + e["texts"].append(b.get("name", "") + " " + json.dumps(b.get("input", {}))) + elif t == "tool_result": + rc = b.get("content") + if isinstance(rc, str): + txt = rc + elif isinstance(rc, list): + txt = "".join(x.get("text", "") for x in rc if isinstance(x, dict)) + else: + txt = json.dumps(rc) if rc is not None else "" + e["results"].append((b.get("tool_use_id", ""), txt)) + out.append(e) + else: # openai + for m in body.get("messages", []): + role = m.get("role") + e = {"role": role, "texts": [], "results": []} + c = m.get("content") + if role == "tool": + e["results"].append((m.get("tool_call_id", ""), c if isinstance(c, str) else json.dumps(c))) + else: + if isinstance(c, str): + e["texts"].append(c) + for tc in m.get("tool_calls") or []: + f = tc.get("function") or {} + e["texts"].append(f.get("name", "") + " " + str(f.get("arguments", ""))) + out.append(e) + return out + + +# --- the measurement ------------------------------------------------------------------ + + +def analyze_session(msgs, min_output): + """One record per tool output: its size, where it sits, and how the model referred + back to it afterwards. Only tokens the output INTRODUCED are eligible (see module + docstring), and only model turns count as referring.""" + ref_tokens = [idents(" ".join(m["texts"])) for m in msgs] # model-authored surfaces + res_tokens = [{} for _ in msgs] + for i, m in enumerate(msgs): + for tid, txt in m["results"]: + res_tokens[i][tid] = idents(txt) + + # Tokens present at or before message i, from ANY surface — the echo exclusion set. + # Built as a running union so a 150k-token transcript stays one pass. + prior = [set() for _ in msgs] + acc = set() + for i, m in enumerate(msgs): + prior[i] = set(acc) + acc |= ref_tokens[i] + for toks in res_tokens[i].values(): + acc |= toks + + # A token echoed by many outputs is boilerplate, not a carried value. + spread = defaultdict(int) + for i in range(len(msgs)): + for toks in res_tokens[i].values(): + for t in toks: + spread[t] += 1 + n_out = sum(len(m["results"]) for m in msgs) or 1 + common = {t for t, c in spread.items() if c > max(5, n_out // 4)} + + recs = [] + for i, m in enumerate(msgs): + for tid, txt in m["results"]: + size = TOK(txt) + if size < min_output: + continue + # Novel = introduced here: not in prior context, not in a sibling block of + # this same message (the producing tool_use lands in the turn before, but a + # batched turn can carry several), and not session boilerplate. + siblings = set() + for otid, otoks in res_tokens[i].items(): + if otid != tid: + siblings |= otoks + novel = res_tokens[i][tid] - prior[i] - siblings - common - ref_tokens[i] + hits = [] # (message index, how many novel tokens that turn reused) + used = set() + later = 0 + for j in range(i + 1, len(msgs)): + if not msgs[j]["texts"]: + continue + later += 1 # a model turn that COULD have referenced this output + inter = novel & ref_tokens[j] + if inter: + hits.append(j) + used |= inter + recs.append({ + "idx": i, + "size": size, + "novel": len(novel), + "refs": len(hits), + # ref_age: how many messages ago the LAST reference was, measured from the + # head of the transcript. This — not the output's own depth — is the A/B + # discriminator: B is "referenced recently", A is "referenced long ago". + "ref_age": (len(msgs) - hits[-1]) if hits else None, + # consume_lag: how long AFTER the output the model first/last used it. A + # separate signal: a long lag means the output stayed live for many turns. + "consume_lag": (hits[-1] - i) if hits else None, + "used_frac": (len(used) / len(novel)) if novel else 0.0, + # later_turns is the OPPORTUNITY to be referenced; near the tail it goes to + # zero and an output that never had a chance must not be scored as unused. + "later_turns": later, + }) + return recs + + +def classify(r, closed_dist, open_reps, min_later=8): + """OPAQUE — the output introduced NO trackable identifier, so there is no evidence + either way. Absence of evidence, not evidence of deadness: a tool + returning records of human-readable values + ([{"name":"david","id":123,"address":"foobarbaz"}]) yields no + distinctive tokens at all, because short lowercase words and 3-digit + numbers are what the precision rules in distinctive() exclude. Folded + into `unreferenced` it is a silent vote to delete. Never cut. + UNREFERENCED — introduced identifiers, and no later exact use (safest cut on Tier 1, + blind to Tier 2). + OPEN — referenced recently or repeatedly (still load-bearing), OR too new to + have had the chance: an output with fewer than min_later model turns + after it has no opportunity to be referenced, and scoring it as unused + would preferentially cut the most RECENT context. + CLOSED — referenced once/twice and not for a long time; whatever the model took + survives in the assistant turn that took it. The large-cut candidate.""" + if r["novel"] == 0: + return "opaque" + if min_later > 0 and r["later_turns"] < min_later: + return "open" + if r["refs"] == 0: + return "unreferenced" + if r["refs"] >= open_reps or r["ref_age"] < closed_dist: + return "open" + return "closed" + + +def derived_evidence(msgs): + """Tier-2 proxy: numeric tokens the model states that appear NOWHERE earlier. A value + it computed (summed, converted) rather than copied — a reference an exact matcher is + structurally unable to see. Reported as a caveat on `unreferenced`, never subtracted.""" + seen, derived, total = set(), 0, 0 + for m in msgs: + mine = idents(" ".join(m["texts"])) + nums = {t for t in mine if NUMERIC.match(t)} + if m["texts"]: + total += 1 + if nums - seen: + derived += 1 + seen |= mine + for _, txt in m["results"]: + seen |= idents(txt) + return derived, total + + +def session_turns(reqs, provider, window, fire_frac): + """Turns remaining after the cut would fire (T in the break-even inequality), plus the + session's peak request size. Append-only traffic means one capture per turn, so the + request whose message-count first crosses fire_frac*window dates the trigger.""" + sizes = [] + for r in reqs: + # A pre-sized record (cc_capture.py: a converted Claude Code transcript, where a full + # prefix per turn would be O(n^2) bytes). Only the SIZE of a non-final turn is ever + # read here, so carrying it directly is equivalent. A real capture never sets this + # field, so the capture path below is unchanged. + if "turn_tokens" in r: + sizes.append(int(r["turn_tokens"])) + continue + b = r["body"] + n = 0 + for m in normalize(b, provider): + n += TOK(" ".join(m["texts"])) + sum(TOK(t) for _, t in m["results"]) + sizes.append(n) + sizes.sort() + fire_at = fire_frac * window + crossed = [k for k, v in enumerate(sizes) if v >= fire_at] + t_remaining = (len(sizes) - crossed[0]) if crossed else 0 + return t_remaining, (sizes[-1] if sizes else 0) + + +def main(): + files = [a for a in sys.argv[1:] if "=" not in a] + opt = dict(a.split("=", 1) for a in sys.argv[1:] if "=" in a) + closed_dist = int(opt.get("closed_dist", 12)) + open_reps = int(opt.get("open_reps", 3)) + min_output = int(opt.get("min_output", 300)) + min_later = int(opt.get("min_later", 8)) + window = int(opt.get("window", 200000)) + fire_frac = float(opt.get("fire_frac", 0.6)) + if not files: + print(__doc__) + return 2 + + agg = {} + for f in files: + recs = [json.loads(l) for l in open(f) if l.strip()] + by_conv = defaultdict(list) + for r in recs: + # An explicit conversation id, when the producer knows one (cc_capture.py, which + # splits one long transcript into several window-sized sessions). Inferring it from + # the first user message is right for a real capture — every request of a session + # opens with the same task statement — but a mid-transcript segment opens on a + # tool_result, and those collide: 31 segments grouped down to 24, silently + # discarding the rest, because only the largest member of a group is analyzed. + key = r.get("conv") + # A REAL capture carries the agent's own session id in metadata.user_id (the + # Anthropic clients pack {device_id, account_uuid, session_id} in there). Prefer it: + # the first-user-message fallback below is catastrophically wrong on benchmark + # traffic, where every task's instruction opens with the SAME long preamble. On + # capture-swebench.jsonl the 200-char prefix has 19 distinct values and the top one + # covers 1,771 of 1,795 requests -- so 18 of 19 groups held nothing but stray + # single-message calls, and because only the largest member of a group is analyzed, + # the measurement silently reported ONE session's worth of data. The session id + # recovers 50 sessions from the same file. + if key is None: + md = (r["body"].get("metadata") or {}).get("user_id") + if md: + try: + key = (json.loads(md) or {}).get("session_id") or md + except (ValueError, TypeError): + key = md + if key is None: + for m in r["body"].get("messages", []): + if m.get("role") == "user": + key = hashlib.sha1(json.dumps(m.get("content"))[:200].encode()).hexdigest()[:8] + break + by_conv[key].append(r) + + mass = defaultdict(int) # bucket -> tokens + count = defaultdict(int) + dist_hist = defaultdict(int) # last-reference RECENCY bucket -> tokens + reps_hist = defaultdict(int) # reference count bucket -> tokens + used_fracs = [] + lags = [] + breakeven = [] # (needed_T, actual_T, cut_tokens, suffix_tokens) + derived_n = derived_d = 0 + peaks = [] + for conv, rs in by_conv.items(): + rs.sort(key=lambda r: len(r["body"].get("messages", []))) + top = rs[-1] # append-only: the largest request holds the whole transcript + prov = top.get("provider", "anthropic") + msgs = normalize(top["body"], prov) + recs_s = analyze_session(msgs, min_output) + if not recs_s: + continue + dn, dd = derived_evidence(msgs) + derived_n += dn + derived_d += dd + t_rem, peak = session_turns(rs, prov, window, fire_frac) + peaks.append(peak) + + cut_idx, cut_tok = [], 0 + for r in recs_s: + b = classify(r, closed_dist, open_reps, min_later) + mass[b] += r["size"] + count[b] += 1 + # `open` now also covers "too new to have been referenced yet", which has no + # last reference at all — so bucket on whether a reference EXISTS, not on the + # verdict, or the histogram indexes None. + if r["ref_age"] is None: + dist_hist["never"] += r["size"] + reps_hist["0"] += r["size"] + else: + d = r["ref_age"] + key = "1-3" if d <= 3 else "4-11" if d < 12 else "12-39" if d < 40 else "40+" + dist_hist[key] += r["size"] + reps_hist["1" if r["refs"] == 1 else "2" if r["refs"] == 2 else "3+"] += r["size"] + used_fracs.append(r["used_frac"]) + lags.append(r["consume_lag"]) + if b in ("unreferenced", "closed"): + cut_idx.append(r["idx"]) + cut_tok += r["size"] + + # Break-even for ONE batched pass: the rewrite starts at the shallowest cut. + if cut_idx and cut_tok: + start = min(cut_idx) + suffix = 0 + for m in msgs[start:]: + suffix += TOK(" ".join(m["texts"])) + sum(TOK(t) for _, t in m["results"]) + breakeven.append((CACHE_WRITE_X * suffix / cut_tok, t_rem, cut_tok, suffix)) + + total = sum(mass.values()) or 1 + if opt.get("json"): + agg[f] = {"mass": dict(mass), "count": dict(count), "total": total, + "dist": dict(dist_hist), "reps": dict(reps_hist), + "breakeven": breakeven, "derived": [derived_n, derived_d]} + continue + + print(f"\n===== {f} =====") + print(f"sessions {len(by_conv)} requests {len(recs)} " + f"tool outputs >={min_output} tok: {sum(count.values()):,} " + f"mass {total:,} tok (peak request: median {sorted(peaks)[len(peaks)//2]:,} tok)" + if peaks else f"sessions {len(by_conv)} requests {len(recs)}") + print("\n bucket outputs tokens share verdict") + verdict = { + "opaque": "introduced no trackable identifier -> NO EVIDENCE, never cut", + "unreferenced": "no later exact use -> free deterministic cut (Tier-1 blind spot applies)", + "closed": "value taken, survives in an assistant turn -> large-cut candidate", + "open": "recent or repeated -> KEEP", + } + for b in ("opaque", "unreferenced", "closed", "open"): + print(f" {b:13s} {count[b]:7,} {mass[b]:11,} {100*mass[b]//total:4d}% {verdict[b]}") + + print(f"\n last reference, messages AGO (recency from the head — the A/B axis)" + f" [open if <{closed_dist} ago or >={open_reps} refs]") + for k in ("1-3", "4-11", "12-39", "40+"): + print(f" {k:>6s} msgs ago {dist_hist[k]:11,} ({100*dist_hist[k]//total:2d}%)") + if lags: + sl = sorted(lags) + print(f" consume lag (output -> its last use), median {sl[len(sl)//2]} msgs " + f"— how long the output stayed live") + print("\n reference count") + for k in ("1", "2", "3+"): + print(f" {k:>6s}x {reps_hist[k]:11,} ({100*reps_hist[k]//total:2d}%)") + if used_fracs: + uf = sorted(used_fracs) + print(f"\n novel tokens actually reused, median {uf[len(uf)//2]:.1%} " + f"— how little of an output a reference consumes (low => 'took a value, " + f"dropped the rest' is the real pattern)") + + if breakeven: + need = sorted(b[0] for b in breakeven) + ok = sum(1 for n, t, _, _ in breakeven if t >= n) + med_cut = sorted(b[2] for b in breakeven)[len(breakeven)//2] + med_suf = sorted(b[3] for b in breakeven)[len(breakeven)//2] + print(f"\n break-even, one batched pass per session (S*T > {CACHE_WRITE_X}*W)") + print(f" median cut S {med_cut:,} tok median rewritten suffix W {med_suf:,} tok") + print(f" turns needed T: p25 {need[len(need)//4]:.0f} median " + f"{need[len(need)//2]:.0f} p75 {need[3*len(need)//4]:.0f}") + print(f" sessions where the observed T actually clears it: {ok}/{len(breakeven)}") + print(f" (a session whose T falls short is NOT a reason to skip the cut — it is a " + f"reason to justify it on steps / deferred agent-compaction, not on tokens)") + + if derived_d: + print(f"\n Tier-2 caveat: {100*derived_n//derived_d}% of model turns state a numeric " + f"value absent from all prior context (computed, not copied). Exact matching " + f"cannot see those references, so `unreferenced` is an UPPER bound on safe cuts.") + + if opt.get("sweep"): + print("\n sensitivity (closed_dist x open_reps -> closed mass share)") + hdr = " reps>= " + "".join(f"{d:>8d}" for d in (4, 8, 12, 24, 40)) + print(hdr + " <- closed_dist") + for reps in (2, 3, 4, 6): + row = f" {reps:>6d} " + for d in (4, 8, 12, 24, 40): + s = 0 + for conv, rs in by_conv.items(): + rs.sort(key=lambda r: len(r["body"].get("messages", []))) + top = rs[-1] + for r in analyze_session(normalize(top["body"], top.get("provider", "anthropic")), min_output): + if classify(r, d, reps, min_later) == "closed": + s += r["size"] + row += f"{100*s//total:7d}%" + print(row) + + if agg: + print(json.dumps(agg, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/harbor/coref_fixture.py b/deploy/harbor/coref_fixture.py new file mode 100755 index 00000000..80684e2b --- /dev/null +++ b/deploy/harbor/coref_fixture.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Ground-truth fixture for coref.py — four tool outputs whose correct classification is +known by construction, so the measurement can be checked rather than trusted. + +The load-bearing case is #2: it must come out UNREFERENCED even though a later turn +mentions `src/config.py`, because that path entered context as the tool_use ARGUMENT, not +as something the output introduced. An implementation without the echo-exclusion guard +scores it as referenced, and then reports near-total reference density on any traffic. + + #1 src/auth.py read -> CLOSED (novel TOKEN_GRACE_SECONDS lifted out once, early) + #2 src/config.py read -> UNREFERENCED (only overlap is the echoed path == the argument) + #3 ls -R listing -> UNREFERENCED (nothing ever comes back to it) + #4 pytest failure -> OPEN (novel error id reused 3x, most recently 1 msg ago) + +Emits append-only capture records (one per model turn, each a prefix of the final +transcript) matching CONTEXT_GURU_CAPTURE's {provider, model, body} JSONL shape. + +Usage: coref_fixture.py +""" +import json, sys + + +def filler(tag, n): + """Per-output distinct lines: shared filler would be dropped as session boilerplate, + which would mask whether the novel-token logic works.""" + return "\n".join(f"{i:4d}\t{tag}_line_{i} = compute_{tag}_{i}(arg_{i})" for i in range(n)) + + +def build(): + msgs = [] + + def user_text(t): + msgs.append({"role": "user", "content": t}) + + def asst(text=None, tool=None, tid=None, args=None): + c = [] + if text: + c.append({"type": "text", "text": text}) + if tool: + c.append({"type": "tool_use", "id": tid, "name": tool, "input": args}) + msgs.append({"role": "assistant", "content": c}) + + def result(tid, text): + msgs.append({"role": "user", "content": [{"type": "tool_result", "tool_use_id": tid, "content": text}]}) + + user_text("Fix the failing test test_auth_expiry in src/auth.py") + + # #1 CLOSED + asst(text="Reading the auth module.", tool="Read", tid="t1", args={"path": "src/auth.py"}) + result("t1", "src/auth.py\n" + filler("auth", 240) + "\nTOKEN_GRACE_SECONDS = 0 # novel\n") + asst(text="The bug is TOKEN_GRACE_SECONDS is 0; it must be 300.", tool="Edit", tid="t2", + args={"path": "src/auth.py", "old": "TOKEN_GRACE_SECONDS = 0", "new": "TOKEN_GRACE_SECONDS = 300"}) + result("t2", "ok") + + # #2 UNREFERENCED — the echo confound + asst(text="Checking config.", tool="Read", tid="t3", args={"path": "src/config.py"}) + result("t3", "src/config.py\n" + filler("config", 240)) + asst(text="Config is fine, adjusting anyway.", tool="Edit", tid="t4", + args={"path": "src/config.py", "old": "a", "new": "b"}) + result("t4", "ok") + + # #3 UNREFERENCED + asst(text="Surveying the tree.", tool="Bash", tid="t5", args={"cmd": "ls -R"}) + result("t5", filler("tree", 240)) + + # #4 OPEN + asst(text="Running the suite.", tool="Bash", tid="t6", args={"cmd": "pytest -q"}) + result("t6", "1 failed, 42 passed\n" + filler("pytest", 240) + "\nE AssertionError: XPIRE_DRIFT_7f3a\n") + for k in range(3): + asst(text=f"XPIRE_DRIFT_7f3a again; attempt {k}.", tool="Bash", tid=f"r{k}", args={"cmd": "pytest -q"}) + result(f"r{k}", "1 failed\nE AssertionError: XPIRE_DRIFT_7f3a\n") + return msgs + + +def main(): + if len(sys.argv) != 2: + print(__doc__) + return 2 + msgs = build() + with open(sys.argv[1], "w") as f: + for n in range(2, len(msgs) + 1): + if msgs[n - 1]["role"] != "assistant": + continue # capture fires on the request the agent sends, i.e. after a model turn + f.write(json.dumps({"provider": "anthropic", "model": "claude-sonnet-5", + "body": {"model": "claude-sonnet-5", "messages": msgs[:n]}}) + "\n") + print(f"wrote {sys.argv[1]}: {len(msgs)} messages, " + f"{sum(1 for m in msgs if m['role'] == 'assistant')} model turns") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/harbor/coref_reachability.py b/deploy/harbor/coref_reachability.py new file mode 100644 index 00000000..225bcc8d --- /dev/null +++ b/deploy/harbor/coref_reachability.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Is the deferral prize reachable? Reframed so it does not depend on measuring usage. + +A Claude Code transcript does NOT record request boundaries, and its parentUuid graph is too +fragmented here to reconstruct the active branch (chains collapse to 5-78 entries out of +thousands). So absolute pre-compaction request size is NOT measurable from this corpus, and +any figure derived from a linear read spans multiple context windows. Measured duplicate +tool-output mass is only 3% pooled / 2% median, so SHARE-based results are unaffected -- +but absolute sizes are not recoverable. + +The reframe removes the dependency. At the moment the agent compacts, usage IS the threshold +by definition -- that is what triggered it. So if the pass fires AT the crossing rather than +after overshooting it: + + required cut ~= growth_per_turn * headroom_turns (the deficit term is ~0) + +which needs only growth per turn, and that IS robustly measurable. The deficit term in +docs/results/coref-density.md (7.3% of the request at H=0) was an artifact of firing late -- +specifically of cc_capture.py segmenting at 180k, which puts the measurement point past the +crossing by construction. +""" +import json, glob, os, sys +TOK = lambda s: max(1, len(s) // 4) +WINDOW, MAXOUT = 200_000, 20_000 +THRESH = WINDOW - MAXOUT - 13_000 +# Measured available cut as a share of the request, from docs/results/coref-density.md +# (interactive corpus, opaque class and opportunity floor both in force). +AVAIL = {"unreferenced": 4.4, "unreferenced+closed": 9.6} + + +def load(path): + out = [] + for l in open(path, errors="replace"): + try: + e = json.loads(l) + except ValueError: + continue + if (e.get("type") in ("user", "assistant") and not e.get("isSidechain") + and (e.get("message") or {}).get("content") is not None): + out.append(e) + return out + + +def sz(e): + c = (e.get("message") or {}).get("content") + bs = ([{"type": "text", "text": c}] if isinstance(c, str) + else [b for b in c if isinstance(b, dict)] if isinstance(c, list) else []) + n = 0 + for b in bs: + t = b.get("type") + if t == "text": + n += TOK(b.get("text", "")) + elif t == "thinking": + n += TOK(b.get("thinking", "")) + elif t == "tool_use": + n += TOK(b.get("name", "") + json.dumps(b.get("input", {}))) + elif t == "tool_result": + rc = b.get("content") + s = (rc if isinstance(rc, str) else + "".join(x.get("text", "") for x in rc if isinstance(x, dict)) + if isinstance(rc, list) else (json.dumps(rc) if rc is not None else "")) + n += TOK(s) + return n + + +def main(): + rows = [] + for p in sorted(glob.glob(os.path.expanduser("~/.claude/projects/*/*.jsonl"))): + E = load(p) + if not E: + continue + turns = sum(1 for e in E if e.get("type") == "assistant") + rows.append(dict(name=os.path.basename(p)[:8], turns=turns, + tok=sum(sz(e) for e in E), + events=sum(1 for e in E if e.get("isCompactSummary") is True))) + comp = [r for r in rows if r["events"]] + print(f"main-conversation transcripts: {len(rows)} (subagent transcripts excluded: " + f"separate conversations with their own context)") + print(f"\nSESSION-LEVEL REACHABILITY -- did the agent ever compact itself?") + for lo, label in ((0, "all sessions"), (50, ">=50 model turns"), + (100, ">=100 model turns"), (200, ">=200 model turns")): + s = [r for r in rows if r["turns"] >= lo] + c = [r for r in s if r["events"]] + if s: + print(f" {label:22s} {len(c):3d}/{len(s):<3d} ({100*len(c)//len(s):3d}%) " + f"events={sum(r['events'] for r in c)}") + print(f"\n => the prize does not exist in the great majority of sessions. It is a") + print(f" long-session feature, and even among the longest it is under a third.") + + g = sorted(r["tok"] / max(1, r["turns"]) for r in comp if r["turns"] >= 10) + if not g: + return 1 + med = g[len(g) // 2] + print(f"\nGROWTH per model turn, in the {len(g)} sessions that actually compacted:") + print(f" min {g[0]:.0f} median {med:.0f} max {g[-1]:.0f} tok/turn") + print(f" (robust to the branch problem: it is a ratio of two quantities inflated alike)") + + print(f"\nREQUIRED CUT if the pass fires AT the crossing (deficit ~0), against a " + f"{THRESH:,}-token request:") + print(f" {'headroom':>10s} {'required':>10s} {'as share':>9s} " + f"{'unref 4.4%':>11s} {'+closed 9.6%':>13s}") + for H in (0, 20, 40, 60, 80): + req = med * H + share = 100 * req / THRESH + u = "yes" if AVAIL["unreferenced"] >= share else "no" + c = "yes" if AVAIL["unreferenced+closed"] >= share else "no" + print(f" {('H = '+str(H)):>10s} {req:9,.0f} {share:8.1f}% {u:>11s} {c:>13s}") + # SENSITIVITY, and it is load-bearing. docs/results/coref-density.md reports ~514 + # tok/turn from a different estimator (request-size deltas inside a 180k segment) where + # this uses total content / model turns. The verdict at H=40 FLIPS between them, so the + # honest answer is that it depends on an estimator not yet pinned down. + print(f"\nSENSITIVITY to the growth estimator (the conclusion depends on it):") + print(f" {'growth':>22s} {'H=20':>7s} {'H=40':>7s} {'H=60':>7s}") + for gv, label in ((med, f"{med:.0f} (this script)"), (514, "514 (density doc)")): + cells = [] + for H in (20, 40, 60): + share = 100 * gv * H / THRESH + cells.append(("u+c" if AVAIL["unreferenced+closed"] >= share + else "no") if AVAIL["unreferenced"] < share else "unref") + print(f" {label:>22s} " + " ".join(f"{c:>7s}" for c in cells)) + print(f" ('unref' = the shipped default suffices; 'u+c' = needs cut_closed, which ships") + print(f" OFF; 'no' = neither can supply it.)") + + print(f"\nWHAT IS ROBUST vs WHAT IS NOT:") + print(f" ROBUST -- the prize exists in only {100*len(comp)//len(rows)}% of sessions " + f"({len(comp)}/{len(rows)}), and under a third of the longest.") + print(f" ROBUST -- firing AT the crossing removes the deficit term entirely, worth") + print(f" 7.3 percentage points of required cut vs the density doc's late-fire") + print(f" measurement. That is arithmetic, not an estimate.") + print(f" NOT -- whether 40 turns of headroom is affordable. It flips on the growth") + print(f" estimator above. Pinning that down needs request-level data this") + print(f" corpus cannot give (no recorded request boundaries).") + print(f" UNCHANGED -- the 11% false-drop from the selection experiment applies to every") + print(f" 'yes' in these tables.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/harbor/loca_repair_shim.py b/deploy/harbor/loca_repair_shim.py new file mode 100644 index 00000000..b67b3ef1 --- /dev/null +++ b/deploy/harbor/loca_repair_shim.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +"""Rig-side tool-pairing repair, sitting between LOCA-bench and the context-guru proxy. + +WHY THIS EXISTS. LOCA's native context trimmer fires when a request outgrows +--max-context-size and drops whole messages. Anthropic requires every assistant `tool_use` +block to be answered by a `tool_result` in the immediately following user message, so the +trim orphans pairs and the provider returns 400 (observed at the 128k band: 42 orphaned ids +at messages.13). This was predicted in docs/proposals/coref-compaction.md §8, which also +says to port the existing fix rather than rediscover it. + +HISTORY -- READ BEFORE EDITING THE BODY-READING CODE. The first version of this shim read the +request body from `content-length` alone and forwarded every client header except a small +denylist. Both were wrong, and they compounded: a request sent with `Transfer-Encoding: +chunked` reached the gateway with an EMPTY body AND with `chunked` and `Content-Length: 0` +set simultaneously. The gateway answered `400 Bad request / Your browser sent an invalid +request` -- raw HTML, no Anthropic error body -- which was misattributed first to a +component under test and then to the benchmark, across two experiment iterations. A local +echo server accepted the contradictory framing with a 200, so a lenient stand-in would not +have caught it. See docs/experiments/loca/iter007/results.md. + +`repair_tool_pairing` below is lifted VERBATIM from forever's +forever/benchmarks/_anthropic_auth_hop.py so the two rigs cannot drift. + +WHERE IT SITS, AND WHY THAT MATTERS. + + LOCA -> [this shim] -> cg-proxy -> gateway + +Before cg-proxy, deliberately: compaction should be measured on WELL-FORMED traffic. If the +shim sat after, coref would index a malformed message list and extract_llm would be handed +requests the provider would reject anyway. + +WHAT IT IS NOT. This is not a context-guru feature and must not be mistaken for one. It +repairs the AGENT's malformed history. Note that coref structurally cannot cause this class +of bug: it rewrites a tool message's text in place and never removes a message, so pairing +is preserved. The repair count is reported so its rate is visible rather than silent. +""" +import json +import os +import sys +import threading +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +_TRIMMED = "[tool result unavailable: removed by context management]" + + +def repair_tool_pairing(messages: list) -> tuple[list, int]: + """Repair a message history to satisfy Anthropic's tool-block pairing rules, returning the + repaired list and the number of fixes applied. + + Anthropic requires every assistant ``tool_use`` block to be answered by a ``tool_result`` + (matching ``tool_use_id``) in the immediately following user message, and forbids a + ``tool_result`` with no preceding ``tool_use``. Naive context management (trimming messages + or clearing tool results) breaks both — it drops the results message while keeping the + ``tool_use`` block, or the reverse — yielding an API 400. This restores validity: + + * **orphaned results** (a ``tool_result`` whose ``tool_use`` was removed) are dropped, and a + message left empty by that removal is dropped too; + * **unanswered ``tool_use`` blocks** get a synthetic placeholder ``tool_result`` in the next + user message (created if the following message isn't a suitable user message). + + Pure and idempotent — a well-formed history returns unchanged with count 0. This is a + rig-level fix applied by the auth hop; Forever's core proxy is never involved.""" + repairs = 0 + + # Phase 1 — drop orphaned tool_result blocks and any message emptied by that. + phase1: list = [] + for idx, m in enumerate(messages): + if not isinstance(m, dict): + phase1.append(m) + continue + m = dict(m) + content = m.get("content") + if isinstance(content, list): + prior_use = set() + prev = messages[idx - 1] if idx > 0 else None + if isinstance(prev, dict) and isinstance(prev.get("content"), list): + prior_use = {b.get("id") for b in prev["content"] + if isinstance(b, dict) and b.get("type") == "tool_use"} + kept = [] + for b in content: + if (isinstance(b, dict) and b.get("type") == "tool_result" + and b.get("tool_use_id") not in prior_use): + repairs += 1 + continue + kept.append(b) + if content and not kept: # message held only orphaned results → drop it + continue + m["content"] = kept + phase1.append(m) + + # Phase 2 — ensure every assistant tool_use is answered in the next user message. + out: list = [] + i = 0 + while i < len(phase1): + m = phase1[i] + out.append(m) + content = m.get("content") if isinstance(m, dict) else None + use_ids = ([b["id"] for b in content + if isinstance(b, dict) and b.get("type") == "tool_use" and b.get("id")] + if isinstance(content, list) else []) + if use_ids: + nxt = phase1[i + 1] if i + 1 < len(phase1) else None + nxt_user = (isinstance(nxt, dict) and nxt.get("role") == "user" + and isinstance(nxt.get("content"), list)) + have = ({b.get("tool_use_id") for b in nxt["content"] + if isinstance(b, dict) and b.get("type") == "tool_result"} + if nxt_user else set()) + missing = [u for u in use_ids if u not in have] + if missing: + synth = [{"type": "tool_result", "tool_use_id": u, "content": _TRIMMED} + for u in missing] + repairs += len(missing) + if nxt_user: + nxt["content"] = synth + list(nxt["content"]) + else: + out.append({"role": "user", "content": synth}) + i += 1 + return out, repairs + + +# Lifted VERBATIM from forever/http_utils.py (HOP_BY_HOP | {host, content-length}), plus +# accept-encoding for the urllib reason documented at the use site. +_HOP_BY_HOP = frozenset({ + "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", + "te", "trailer", "trailers", "transfer-encoding", "upgrade", +}) +_REQUEST_STRIP = _HOP_BY_HOP | {"host", "content-length", "accept-encoding"} + +UPSTREAM = os.environ.get("SHIM_UPSTREAM", "http://localhost:4200/anthropic") +_repairs = 0 +_requests = 0 +_lock = threading.Lock() + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *a): # keep stdout clean; the counters are the output + pass + + def do_GET(self): + if self.path == "/shim-stats": + body = json.dumps({"requests": _requests, "repairs": _repairs}).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + self.send_error(404) + + def do_POST(self): + global _repairs, _requests + # Read the body under EITHER framing. Reading only content-length silently + # produced an EMPTY body for every chunked request, and the client (httpx) chooses + # chunked on its own terms -- which is why the resulting failures were intermittent. + if (self.headers.get("transfer-encoding") or "").lower() == "chunked": + chunks = [] + while True: + line = self.rfile.readline(65536).strip() + if not line: + continue + size = int(line.split(b";")[0], 16) + if size == 0: + while True: # consume trailers + t = self.rfile.readline(65536) + if t in (b"\r\n", b"\n", b""): + break + break + chunks.append(self.rfile.read(size)) + self.rfile.read(2) # trailing CRLF + raw = b"".join(chunks) + else: + raw = self.rfile.read(int(self.headers.get("content-length") or 0)) + try: + payload = json.loads(raw) + msgs = payload.get("messages") + if isinstance(msgs, list): + fixed, count = repair_tool_pairing(msgs) + if count: + payload["messages"] = fixed + raw = json.dumps(payload).encode() + with _lock: + _repairs += count + with _lock: + _requests += 1 + except Exception as e: # never break the run over a repair; forward untouched + print(f"[shim] passthrough after {type(e).__name__}: {e}", file=sys.stderr) + + req = urllib.request.Request(UPSTREAM + self.path.split("/anthropic", 1)[-1] + if "/anthropic" in self.path else UPSTREAM + self.path, + data=raw, method="POST") + for k, v in self.headers.items(): + # Hop-by-hop + transport-owned headers, lifted VERBATIM from forever's + # forever/http_utils.py REQUEST_STRIP (= HOP_BY_HOP | {host, content-length}) so the + # two rigs cannot drift -- the same reason repair_tool_pairing is copied verbatim. + # My original ad-hoc list had four entries and omitted transfer-encoding, which is + # what produced the empty-body 400s; see the HISTORY note above. + # + # DEVIATION, deliberate: accept-encoding is end-to-end and forever forwards it. This + # hop strips it because urllib does not transparently decompress, so forwarding + # `gzip` would relay compressed bytes as though they were plain. forever needs no + # such deviation because httpx handles content-coding for it. + if k.lower() not in _REQUEST_STRIP: + req.add_header(k, v) + req.add_header("content-length", str(len(raw))) + + # CORRELATION. The capture hop on the far side of context-guru records what the proxy SENT; + # this side records what it RECEIVED. Joining them by a stamped sequence number is what turns + # "summarize emitted an unanswered call" into "and here is the input it was given" -- the step + # that three rounds of reading the source failed to establish. copyHeaders in the proxy + # forwards unknown headers, so the id survives the hop. + with _lock: + globals()["_seq"] = globals().get("_seq", 0) + 1 + seq = globals()["_seq"] + req.add_header("x-cg-rig-seq", str(seq)) + if os.environ.get("SHIM_DIGEST"): + try: + pl = json.loads(raw) + dig = [] + for i, m in enumerate(pl.get("messages") or []): + c = m.get("content") + blocks = c if isinstance(c, list) else [] + uses = [b.get("id") for b in blocks + if isinstance(b, dict) and b.get("type") == "tool_use"] + res = [b.get("tool_use_id") for b in blocks + if isinstance(b, dict) and b.get("type") == "tool_result"] + dig.append({"i": i, "role": m.get("role"), + "n_use": len(uses), "n_res": len(res), + "use": uses[:4], "res": res[:4]}) + with open(os.environ["SHIM_DIGEST"], "a") as fh: + fh.write(json.dumps({"seq": seq, "n_messages": len(dig), + "body_bytes": len(raw), "digest": dig}) + "\n") + except Exception: + pass + try: + with urllib.request.urlopen(req, timeout=900) as r: + data, status, hdrs = r.read(), r.status, dict(r.headers) + except urllib.error.HTTPError as e: + data, status, hdrs = e.read(), e.code, dict(e.headers) + except Exception as e: + data, status, hdrs = json.dumps({"error": str(e)}).encode(), 502, {} + + # FAILURE CAPTURE. The `400 Bad request / Your browser sent an invalid request` HTML error + # survived the chunked-body fix: 1 occurrence in 75 runs with no proxy in the path, 6 in 75 + # with one. It is intermittent and rare, so the only way to attribute it is to record the + # request that caused it AT THE MOMENT it fails -- reconstructing afterwards is what made + # this take three iterations. Body goes to a side file, not stderr, because it is large and + # stderr is the run log. + if status >= 400: + try: + snap = {"status": status, "path": self.path, "body_bytes": len(raw), + "sent_content_length": str(len(raw)), + "req_headers": {k: v for k, v in self.headers.items() + if k.lower() not in ("authorization", "x-api-key")}, + "resp_head": data[:400].decode("utf-8", "replace"), + "resp_headers": {k: v for k, v in hdrs.items()}, + "upstream": UPSTREAM} + with open(os.environ.get("SHIM_FAILLOG", "/tmp/cg-loca/shim-failures.jsonl"), + "a") as fh: + fh.write(json.dumps(snap) + "\n") + except Exception: + pass # diagnostics must never break the run + self.send_response(status) + self.send_header("content-type", hdrs.get("Content-Type", "application/json")) + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +if __name__ == "__main__": + port = int(os.environ.get("SHIM_PORT", "4260")) + print(f"[shim] :{port} -> {UPSTREAM}", flush=True) + ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever() diff --git a/deploy/harbor/reward_pairs.py b/deploy/harbor/reward_pairs.py new file mode 100644 index 00000000..cf2d70c6 --- /dev/null +++ b/deploy/harbor/reward_pairs.py @@ -0,0 +1,201 @@ +"""Paired reward analysis for two LOCA arms, clustered by task. + +Written BEFORE iteration 010's arms finished, so the analysis cannot be shaped by the numbers. +Pre-registration: docs/experiments/loca/iter010/PREREGISTRATION.md (ecf4103, amended 4eecbdc). + +WHAT THIS GETS RIGHT THAT EARLIER READS DID NOT. + +Every result in iterations 007 and 008 was computed from `tasks/*/state0/eval.json`, which reads +ONE of the 5 seeds. `state0`..`state4` ARE the seeds, so that glob silently discarded 60 of 75 +completed runs and overstated per-run cost by 5x. This script globs `state*`, prints the eval count +against the config length, and refuses to summarise if they disagree -- the reconciliation, not the +glob, is what makes the n trustworthy. + +WHY CLUSTERING IS NOT OPTIONAL HERE. 75 runs are 5 seeds of 15 tasks. Seeds of one task share its +environment, its tools and its difficulty, so they are not independent observations: treating 75 +pairs as 75 independent ones would understate every interval by roughly sqrt(5). Extra seeds buy +precision WITHIN a task; they do not add tasks. So two figures are always printed together -- +pair-level (optimistic) and task-clustered (conservative) -- and the pre-registration commits in +advance to quoting the conservative one. Printing only the flattering end is the failure this +duplication exists to prevent. + +TWO-SIDED BY CONSTRUCTION. Where the baseline is itself lossy -- LOCA's native trimmer drops whole +messages, and a summariser paraphrases exact identifiers into prose -- selective removal can RAISE +reward. A one-sided harm bound cannot register that even when it happens, so gains and harms are +both counted and reported. +""" +import glob +import json +import os +import sys +from collections import defaultdict +from math import comb + +SOLVED = 1.0 + + +def load(run_dir): + """{(task, seed_state): (status, accuracy)} over ALL seeds.""" + out = {} + for f in glob.glob(os.path.join(run_dir, "tasks", "*", "state*", "eval.json")): + # ...//tasks///eval.json -- so task is [-3] and state is [-2]. + # An off-by-one here is silent and severe: keying on ("tasks", ) collapses all 5 + # seeds of a task onto one dict entry, so 75 runs read as 15 and the seed dimension + # vanishes exactly as it did in iterations 007 and 008. Caught by validating against a + # run whose true counts were already known. + parts = f.split(os.sep) + task, state = parts[-3], parts[-2] + try: + j = json.load(open(f)) + except (ValueError, OSError): + continue + out[(task, state)] = (j.get("status"), j.get("accuracy")) + return out + + +def cost_of(run_dir): + return sum(json.load(open(f)).get("total_cost_usd") or 0 + for f in glob.glob(os.path.join(run_dir, "summary-*.json"))) + + +def mcnemar_exact(b, c): + n = b + c + if n == 0: + return 1.0 + k = min(b, c) + return min(1.0, 2 * sum(comb(n, i) for i in range(k + 1)) / 2 ** n) + + +def cp_upper(k, n, alpha=0.05): + """Upper one-sided Clopper-Pearson bound on a rate given k events in n trials.""" + if n == 0: + return 1.0 + lo, hi = 0.0, 1.0 + for _ in range(200): + m = (lo + hi) / 2 + p = sum(comb(n, i) * m ** i * (1 - m) ** (n - i) for i in range(k + 1)) + if p > alpha: + lo = m + else: + hi = m + return (lo + hi) / 2 + + +def main(): + if len(sys.argv) < 3: + sys.exit("usage: reward_pairs.py [expected_n]") + a_dir, b_dir = sys.argv[1], sys.argv[2] + expected = int(sys.argv[3]) if len(sys.argv) > 3 else None + A, B = load(a_dir), load(b_dir) + print(f"baseline {os.path.basename(a_dir)}: {len(A)} evals ${cost_of(a_dir):.2f}") + print(f"treatment {os.path.basename(b_dir)}: {len(B)} evals ${cost_of(b_dir):.2f}") + if expected: + for nm, d in (("baseline", A), ("treatment", B)): + flag = "OK" if len(d) == expected else "!! MISMATCH" + print(f" {nm}: {len(d)}/{expected} expected runs -- {flag}") + if len(A) != expected or len(B) != expected: + print(" Incomplete or unexpected run count: n below is what was actually read, " + "not what was planned. Report it as such.") + + keys = sorted(set(A) & set(B)) + usable, errored = [], 0 + for k in keys: + if A[k][0] != "success" or B[k][0] != "success": + errored += 1 + continue + usable.append((k, A[k][1] == SOLVED, B[k][1] == SOLVED)) + n11 = sum(1 for _, x, y in usable if x and y) + n00 = sum(1 for _, x, y in usable if not x and not y) + harm = [k for k, x, y in usable if x and not y] # baseline solved, treatment did not + gain = [k for k, x, y in usable if y and not x] + + # Per-arm error counts, because ASYMMETRIC errors are a confound and not a nuisance. + # Pairwise exclusion is only unbiased if the two arms fail at the same rate for reasons + # unrelated to the treatment. At 32k the HTML-400 transport failure occurred 1/75 with no + # CG proxy in the path and 6/75 with one, so error rate is NOT independent of the pipeline + # and the symmetry has to be shown rather than assumed. + a_err = sum(1 for k in keys if A[k][0] != "success") + b_err = sum(1 for k in keys if B[k][0] != "success") + print(f"\ncommon={len(keys)} usable={len(usable)} errored={errored} " + f"(excluded pairwise, not counted as failures)") + print(f" errors by arm: baseline={a_err} treatment={b_err}" + + (" -- BALANCED, pairwise exclusion is defensible" + if abs(a_err - b_err) <= max(2, 0.25 * max(a_err, b_err, 1)) + else " -- !! ASYMMETRIC: exclusion may bias the comparison; treat as a confound")) + print(f"both-pass={n11} both-fail={n00} HARM={len(harm)} GAIN={len(gain)} " + f"discordant={len(harm)+len(gain)}") + print(f"solve rate: baseline={sum(1 for _,x,_ in usable if x)}/{len(usable)} " + f"treatment={sum(1 for _,_,y in usable if y)}/{len(usable)}") + + p = mcnemar_exact(len(harm), len(gain)) + print(f"\nMcNemar exact (two-sided), pair level: p={p:.3f}" + + (" -- not significant" if p >= 0.05 else " -- SIGNIFICANT")) + + # Task-clustered: a task counts as harmed only if it harms on net across its seeds. + per_task = defaultdict(lambda: [0, 0]) + for k, x, y in usable: + t = k[0] + if x and not y: + per_task[t][0] += 1 + elif y and not x: + per_task[t][1] += 1 + t_harm = sum(1 for h, g in per_task.values() if h > g) + t_gain = sum(1 for h, g in per_task.values() if g > h) + n_tasks = len({k[0] for k, _, _ in usable}) + print(f"task-clustered ({n_tasks} tasks): net-harmed={t_harm} net-gained={t_gain} " + f"p={mcnemar_exact(t_harm, t_gain):.3f}") + + # ---- INTENT-TO-TREAT ---------------------------------------------------------------- + # Pairwise exclusion of transport failures is NOT neutral here, and the direction matters. + # Measured: the format-only arm errors more than format+coref (6/75 vs 0/46 at 32k; 10 vs 4 + # at 64k). The mechanism is plausible and points one way -- coref shrinks the largest requests, + # so it avoids whatever fails at size. If the treatment PREVENTS the failure, then excluding + # those pairs discards exactly the cases where the treatment helped most, biasing the + # comparison AGAINST it. + # + # So an intent-to-treat reading is reported alongside: a run that errored did not solve its + # task, from the user's point of view, whichever layer failed. ITT is the conservative choice + # for a harm claim and the honest one for a benefit claim, which is why both appear rather + # than whichever flatters. + # + # This DEVIATES from the pre-registration, which said errors are "excluded, not counted as + # failures". The deviation is recorded rather than quietly applied: the reason is that errors + # were assumed independent of the arm and are now measured not to be. + itt = [] + for k in keys: + itt.append((k, A[k][0] == "success" and A[k][1] == SOLVED, + B[k][0] == "success" and B[k][1] == SOLVED)) + i_harm = sum(1 for _, x, y in itt if x and not y) + i_gain = sum(1 for _, x, y in itt if y and not x) + print(f"\nINTENT-TO-TREAT (errors count as unsolved, n={len(itt)}):") + print(f" solved: baseline={sum(1 for _,x,_ in itt if x)} treatment={sum(1 for _,_,y in itt if y)}" + f" harm={i_harm} gain={i_gain} McNemar p={mcnemar_exact(i_harm, i_gain):.3f}") + i_per_task = defaultdict(lambda: [0, 0]) + for k, x, y in itt: + if x and not y: + i_per_task[k[0]][0] += 1 + elif y and not x: + i_per_task[k[0]][1] += 1 + ith = sum(1 for h, g in i_per_task.values() if h > g) + itg = sum(1 for h, g in i_per_task.values() if g > h) + n_t_itt = len({k[0] for k, _, _ in itt}) + print(f" task-clustered ({n_t_itt} tasks): net-harmed={ith} net-gained={itg} " + f"p={mcnemar_exact(ith, itg):.3f} harm bound <= {100*cp_upper(ith, n_t_itt):.0f}%") + + print("\nNON-INFERIORITY -- upper 95% bound on the harm rate:") + print(f" pair level (optimistic, n={len(usable)}): " + f"{100*cp_upper(len(harm), len(usable)):.0f}% [{len(harm)} harm events]") + print(f" task level (CONSERVATIVE, QUOTE THIS, n={n_tasks}): " + f"{100*cp_upper(t_harm, n_tasks):.0f}% [{t_harm} net-harmed tasks]") + print(" The pre-registration commits to the task-level figure. Seeds of one task share its" + "\n environment and difficulty, so pair-level treats correlated observations as" + "\n independent and reports an interval that is too narrow.") + + if harm: + print(f"\nharmed pairs: {[(t, s) for t, s in harm][:10]}") + if gain: + print(f"gained pairs: {[(t, s) for t, s in gain][:10]}") + + +if __name__ == "__main__": + main() diff --git a/deploy/harbor/runlog_capture.py b/deploy/harbor/runlog_capture.py new file mode 100644 index 00000000..f94c460a --- /dev/null +++ b/deploy/harbor/runlog_capture.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Convert benchmark-harness run logs into CONTEXT_GURU_CAPTURE JSONL, so coref.py can +measure co-reference density on real LONG-HORIZON BENCHMARK traffic. + +Companion to cc_capture.py, which does the same for Claude Code transcripts. That corpus is +real but it is interactive research work; this one is what the acceptance criteria in +docs/proposals/coref-compaction.md §8 actually care about — agent trajectories from +UltraHorizon, LOCA-bench and the Momento/travel harnesses, at 200+ turns with hard context +wipes. Still zero API dollars: the runs already happened and left their logs on disk. + +Three input shapes, auto-detected: + + llm_calls.jsonl (loopb / UltraHorizon) {model, messages, tools, response} per line. + Already a request body per model turn, append-only — a capture in all + but name. OpenAI dialect (role "tool" + tool_calls). + trace.jsonl (litellm) {request: {messages, model}, ...} per line. + Same, one line per upstream call. + all_trajectories.json (LOCA-bench) {env: {state: {claude_messages: [...]}}}. + A finished conversation rather than per-turn bodies, so the turn + prefixes are reconstructed from it. Anthropic dialect. + +Two behaviours worth knowing, because both change what gets measured: + + CONTEXT WIPES ARE SESSION BOUNDARIES. UltraHorizon clears the agent's context mid-run + ("The context has been cleared. Please check your notes"), so the message count DROPS. A + drop starts a new session here. Measuring across a wipe would look for references from + before a boundary the model cannot see across, and would score every one of them as + absent — inventing cuttable mass out of the harness's own reset. + + ONLY THE LARGEST BODY IS EMITTED IN FULL. coref.py indexes the largest request per session + and reads the others only for their token size (the T estimate), so every other turn is + emitted as a `turn_tokens` record. That is what keeps this tractable: the UltraHorizon logs + alone are 9 GB of growing prefixes. A real capture sets no such field and is unaffected. + +Usage: + runlog_capture.py [log ...] [--min-turns N] [--label NAME] + + --min-turns 8 drop sessions with fewer model turns than this + --label NAME prefix for the conversation ids (defaults to the file's parent dirs) +""" +import json +import os +import sys + +TOK = lambda s: max(1, len(s) // 4) # same proxy as coref.py / analyze_content.py + + +# --- sizing (must match coref.py's normalize + session_turns) ----------------------------- + +def _openai_tokens(m): + texts, results = [], [] + c = m.get("content") + if m.get("role") == "tool": + results.append(c if isinstance(c, str) else json.dumps(c)) + else: + if isinstance(c, str): + texts.append(c) + for tc in m.get("tool_calls") or []: + f = tc.get("function") or {} + texts.append(f.get("name", "") + " " + str(f.get("arguments", ""))) + return TOK(" ".join(texts)) + sum(TOK(r) for r in results) + + +def _anthropic_tokens(m): + texts, results = [], [] + c = m.get("content") + if isinstance(c, str): + texts.append(c) + elif isinstance(c, list): + for b in c: + if not isinstance(b, dict): + continue + t = b.get("type") + if t == "text": + texts.append(b.get("text", "")) + elif t == "thinking": + texts.append(b.get("thinking", "")) + elif t == "tool_use": + texts.append(b.get("name", "") + " " + json.dumps(b.get("input", {}))) + elif t == "tool_result": + rc = b.get("content") + if isinstance(rc, str): + results.append(rc) + elif isinstance(rc, list): + results.append("".join(x.get("text", "") for x in rc if isinstance(x, dict))) + elif rc is not None: + results.append(json.dumps(rc)) + return TOK(" ".join(texts)) + sum(TOK(r) for r in results) + + +def body_tokens(msgs, provider): + f = _anthropic_tokens if provider == "anthropic" else _openai_tokens + return sum(f(m) for m in msgs) + + +# --- readers: each yields a list of per-turn message lists ------------------------------ + +def read_llm_calls(path): + """loopb / UltraHorizon: one request body per line, already append-only.""" + out, model = [], "unknown" + with open(path, errors="replace") as f: + for line in f: + if not line.strip(): + continue + try: + d = json.loads(line) + except ValueError: + continue + msgs = d.get("messages") + if isinstance(msgs, list) and msgs: + out.append(msgs) + model = d.get("model") or model + return out, model, "openai" + + +def read_litellm_trace(path): + """litellm trace: one upstream call per line, body under `request`.""" + out, model = [], "unknown" + with open(path, errors="replace") as f: + for line in f: + if not line.strip(): + continue + try: + d = json.loads(line) + except ValueError: + continue + req = d.get("request") or {} + msgs = req.get("messages") + if isinstance(msgs, list) and msgs: + out.append(msgs) + model = d.get("model") or req.get("model") or model + return out, model, "openai" + + +def read_loca(path): + """LOCA-bench all_trajectories.json: a FINISHED conversation per env/state, so the turn + prefixes are reconstructed by cutting it after each model turn. Uses claude_messages + (the wire conversation) over full_messages_history, which omits the opening task.""" + with open(path, errors="replace") as f: + doc = json.load(f) + runs, model = [], "claude-sonnet-5" + for env, states in (doc or {}).items(): + if not isinstance(states, dict): + continue + for state, node in states.items(): + if not isinstance(node, dict): + continue + msgs = node.get("claude_messages") or node.get("full_messages_history") + if not isinstance(msgs, list) or not msgs: + continue + prefixes = [msgs[: i + 1] for i, m in enumerate(msgs) if m.get("role") == "assistant"] + if prefixes: + runs.append((f"{env}/{state}", prefixes)) + return runs, model, "anthropic" + + +def detect(path): + base = os.path.basename(path) + if base == "all_trajectories.json": + return "loca" + if base.endswith(".jsonl"): + with open(path, errors="replace") as f: + for line in f: + if not line.strip(): + continue + try: + d = json.loads(line) + except ValueError: + continue + if "messages" in d: + return "llm_calls" + if isinstance(d.get("request"), dict) and "messages" in d["request"]: + return "trace" + return None + return None + + +# --- emit ------------------------------------------------------------------------------- + +def wipe_split(prefixes): + """Split where the message count DROPS — a harness context wipe (see docstring).""" + segs, cur = [], [] + for msgs in prefixes: + if cur and len(msgs) < len(cur[-1]): + segs.append(cur) + cur = [] + cur.append(msgs) + if cur: + segs.append(cur) + return segs + + +def emit(prefixes, provider, model, conv, out, min_turns): + if len(prefixes) < min_turns: + return 0 + sizes = [body_tokens(m, provider) for m in prefixes] + top = max(range(len(prefixes)), key=lambda i: (len(prefixes[i]), sizes[i])) + for i, n in enumerate(sizes): + if i == top: + continue + out.write(json.dumps({"provider": provider, "model": model, "conv": conv, + "turn_tokens": n, + "body": {"model": model, "messages": prefixes[top][:1]}}) + "\n") + out.write(json.dumps({"provider": provider, "model": model, "conv": conv, + "body": {"model": model, "messages": prefixes[top]}}) + "\n") + return len(prefixes) + + +def main(): + args = sys.argv[1:] + if not args: + print(__doc__) + return 2 + min_turns, label, files, out_path = 8, None, [], None + i = 0 + while i < len(args): + a = args[i] + if a == "--min-turns": + i += 1 + min_turns = int(args[i]) + elif a == "--label": + i += 1 + label = args[i] + elif a.startswith("--"): + print(f"unknown flag {a}") + return 2 + elif out_path is None: + out_path = a + else: + files.append(a) + i += 1 + if out_path is None or not files: + print(__doc__) + return 2 + + n_sess = n_turn = n_src = 0 + skipped = 0 + with open(out_path, "w") as out: + for path in files: + kind = detect(path) + if kind is None: + skipped += 1 + continue + stem = label or "/".join(path.rstrip("/").split("/")[-4:-1]) + tag = f"{stem}:{os.path.basename(path)[:12]}" + try: + if kind == "loca": + runs, model, provider = read_loca(path) + groups = [(f"{tag}#{name}", pref) for name, pref in runs] + else: + prefixes, model, provider = (read_llm_calls if kind == "llm_calls" + else read_litellm_trace)(path) + groups = [(f"{tag}#{k}", seg) for k, seg in enumerate(wipe_split(prefixes))] + except (ValueError, OSError): + skipped += 1 + continue + used = 0 + for conv, pref in groups: + t = emit(pref, provider, model, conv, out, min_turns) + if t: + n_sess += 1 + n_turn += t + used += 1 + if used: + n_src += 1 + print(f"wrote {out_path}: {n_src} logs -> {n_sess} sessions, {n_turn} turn records " + f"({os.path.getsize(out_path) / 1e6:.1f} MB)" + + (f"; {skipped} unrecognized" if skipped else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/harbor/selection_rescore.py b/deploy/harbor/selection_rescore.py new file mode 100644 index 00000000..3e5ae0d4 --- /dev/null +++ b/deploy/harbor/selection_rescore.py @@ -0,0 +1,372 @@ +"""Re-score the held-out selection experiment under two corrections it documented but never made. + +The experiment (docs/results/coref-selection-experiment.md) concluded that the deterministic +co-reference index beats every model arm, and that "no combination of index and model beat it". +Its own limitations section names two reasons that conclusion is not final, and this script +addresses both. **It makes no model calls** -- all 8,105 decisions were recorded, so every number +here is a re-score of data already on disk. Cost: $0. + +CORRECTION 1 -- FLOOR SYMMETRY. From the experiment's limitations: "An asymmetry that flatters the +deterministic arms: min_later_turns is a hard structural guard present only in them. Model arms +received later_turns as information with no enforced floor. A floor-symmetric re-run is cheap and +has not been done." Confirmed in the code: score.py's deterministic() returns "keep" outright when +later_turns < min_later, while runner.py passes later_turns to the model arms as prompt text only. +So the index gets a guarantee and the models get a suggestion. Corrected in BOTH directions -- +floor added to the models, floor removed from the index -- because fixing only the flattering +direction would be its own bias. + +CORRECTION 2 -- TIER-2 GROUND TRUTH, AND WHY IT IS REPORTED AS A BRACKET. `future_referenced` is +exact identifier matching, so it sees Tier-1 reuse only. But the argument for spending a model call +is that a model catches Tier-2/3 reuse an exact matcher structurally cannot; when a model correctly +keeps an output the agent later restated in transformed form, the scorer records "not referenced" +and credits it nothing. The metric is blind to the capability under test. + +Widening it deterministically turns out to be mostly impossible, and finding that out is a result: + + STRICT (numeric reformatting, case, substring for structured/long tokens) is defensible and + barely moves the ground truth -- the deterministic slice of Tier-2 is nearly empty on this + corpus. + + LOOSE additionally matches path BASENAMES. Reported separately as an UPPER bound because + basename equality across different directories is genuinely ambiguous reuse. + +An earlier version folded a third rule -- the basename's STEM -- into a single "Tier-2" number, and +it was wrong in the expensive direction: the stem key `yaml` matched 42 distinct identifiers, so a +later mention of ANY yaml file scored as reuse of a DIFFERENT one, and hyphenated tokens leaked +plain English (`Append-only` -> `only`, `Cost-based` -> `based`, plus `agent`, `memory`, `context`, +`session`, `task`). That one rule supplied 237 of 250 gains and drove the index's measured +false-drop from 11% to 53%. A false MATCH indicts the deterministic arms for reuse that never +happened, so it is the error that most needs guarding against here. See struct_keys. + +Tier-3 (semantic paraphrase) is deliberately NOT addressed: it needs a judge, and an LLM-judged +ground truth reintroduces the noise that ruled out UltraHorizon in +docs/results/measurement-limits.md section 6. **So even the loose column is a lower bound on true +reuse, and any residual gap between index and model arms stays a bounded claim, not a clean one.** + +VERIFICATION IS THE POINT. Two hard gates, because a silent mismatch would produce +authoritative-looking nonsense: + 1. the as-published scoring must reproduce the published table row for row -- which requires + score.py's exact conventions, not merely similar ones (see metrics); + 2. the candidate re-derivation must reproduce candidates.json field for field before its added + Tier-2 columns are trusted at all. +Gate 2 exists because this script recomputes novel-identifier sets prep.py already computed; the +assertion, not the similarity of the code, is what makes that safe. A silent partial join is how +deploy/harbor/coref.py's session-key collision discarded 98% of requests. +""" +import importlib.util +import json +import os +import re +import sys +from collections import defaultdict + +EXP = os.environ.get("CG_EXP_DIR", "/tmp/cg-exp") +CAPTURES = [("/tmp/cg-loca.jsonl", "LOCA"), ("/tmp/cg-uh.jsonl", "UltraHorizon"), + ("/tmp/cg-cc-all.jsonl", "ClaudeCode")] +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +cf = _load("cf", os.path.join(HERE, "coref.py")) # idents/normalize/distinctive/TOK +score = _load("score", os.path.join(EXP, "score.py")) # deterministic() + PRICE + TOK +MIN_OUTPUT = 300 +MIN_LATER = 8 + +_NUM = re.compile(r"^[\d,._]*\d[\d,._]*$") +_SUFFIXED = re.compile(r"^([\d.,]+)\s*([kKmMbB])$") + + +def num_key(t): + """Canonical numeric value, or None. Collapses 1200 / 1,200 / 1200.0 / 1.2k onto one key, so a + count restated in a different format still counts as reuse.""" + s = t.strip() + m = _SUFFIXED.match(s) + mult = 1 + if m: + s, suf = m.group(1), m.group(2).lower() + mult = {"k": 1_000, "m": 1_000_000, "b": 1_000_000_000}[suf] + bare = s.replace(",", "").replace("_", "") + if not _NUM.match(bare or "x"): + return None + try: + v = float(bare) * mult + except ValueError: + return None + return ("num", int(v)) if v == int(v) else ("num", round(v, 6)) + + +def struct_keys(t): + """Path BASENAME only, and only when the basename is itself distinctive by coref.py's own rule. + + Split on path separators alone -- never on `.`, `:` or `-`. Splitting on those is what produced + the stem rule described in the module docstring, which manufactured reuse at scale. Reusing + cf.distinctive() rather than inventing a length threshold is deliberate: it is the repo's own + measured filter for "identifier, not prose", and `yaml` / `only` / `auth` all fail it for + carrying no interior structure, digit, or CamelCase. + """ + out = set() + last = re.split(r"[/\\]", t)[-1] + if last != t and cf.distinctive(last): + out.add(("seg", last.lower())) + return out + + +def norm_keys(t, loose=False): + """Normalized keys a token may be matched by. `loose` adds basename matching, kept opt-in so + the two ground truths bracket the answer instead of one of them deciding it by fiat.""" + keys = {("lc", t.lower())} + n = num_key(t) + if n: + keys.add(n) + if loose: + keys |= struct_keys(t) + return keys + + +def substr_eligible(t): + """Only structured or long tokens may match by substring. A short bare token would hit + incidentally -- `1200` inside `112009` -- and manufacture reuse.""" + return len(t) >= 8 or any(ch in t for ch in "._:-/") + + +SUBSTR_BUDGET = 200 # per candidate; reported when it binds rather than silently truncating + + +def future_ref_t2(novel, idx_strict, idx_loose, fut_text_lc): + """(strict, loose, rule) for one candidate's novel identifier set.""" + rule = "" + strict = False + for t in novel: + for k in norm_keys(t): + if k in idx_strict: + return True, True, k[0] + checked = 0 + for t in novel: + if not substr_eligible(t): + continue + checked += 1 + if checked > SUBSTR_BUDGET: + rule = "budget" + break + if t.lower() in fut_text_lc: + return True, True, "substr" + for t in novel: + if struct_keys(t) & idx_loose: + return strict, True, rule or "seg" + return strict, False, rule + + +def rederive(path, label): + """Recompute prep.py's candidates for one capture, adding both Tier-2 columns. + + The exact-match field is recomputed identically so it can be checked against candidates.json; + that assertion is what licenses trusting the added columns. + """ + recs = [json.loads(l) for l in open(path) if l.strip()] + by = defaultdict(list) + for r in recs: + by[r.get("conv")].append(r) + out = [] + for conv, rows in by.items(): + rows.sort(key=lambda r: len(r["body"].get("messages", []))) + top = rows[-1] + msgs = cf.normalize(top["body"], top.get("provider", "anthropic")) + turns = [j for j, m in enumerate(msgs) if m["texts"]] + if len(turns) < 10: + continue + F = turns[int(len(turns) * 0.6)] + + ref = [cf.idents(" ".join(m["texts"])) for m in msgs] + res = [{t: cf.idents(x) for t, x in ((b[0], b[1]) for b in m["results"])} for m in msgs] + first = {} + for i, m in enumerate(msgs): + for s in [ref[i]] + list(res[i].values()): + for t in s: + first.setdefault(t, i) + spread = defaultdict(int) + for i in range(len(msgs)): + for tk in res[i].values(): + for t in tk: + spread[t] += 1 + n_out = sum(len(m["results"]) for m in msgs) or 1 + common = {t for t, c in spread.items() if c > max(5, n_out // 4)} + + # Index the held-out future once per conversation rather than once per candidate. + idx_strict, idx_loose, fut_chunks = set(), set(), [] + for j in range(F + 1, len(msgs)): + if not msgs[j]["texts"]: + continue + for t in ref[j]: + idx_strict |= norm_keys(t) + idx_loose |= struct_keys(t) + fut_chunks.append(" ".join(msgs[j]["texts"]).lower()) + fut_text_lc = " ".join(fut_chunks) + future_msgs = [j for j in range(F + 1, len(msgs)) if msgs[j]["texts"]] + + for i, m in enumerate(msgs): + if i >= F: + break + for tid, txt in m["results"]: + if cf.TOK(txt) < MIN_OUTPUT: + continue + sib = set() + for o, ot in res[i].items(): + if o != tid: + sib |= ot + novel = {t for t in res[i][tid] + if first.get(t, i) >= i and t not in sib and t not in common + and t not in ref[i]} + exact = any(novel & ref[j] for j in future_msgs) + strict, loose, rule = future_ref_t2(novel, idx_strict, idx_loose, fut_text_lc) + out.append({"key": f"{label}|{conv}|{i}|{tid}", + "future_referenced": exact, + "t2_strict": exact or strict, + "t2_loose": exact or loose, + "rule": "" if exact else rule}) + return out + + +def verdicts_from(path): + """(verdict, kept) per key. `kept` is needed because score.py credits a trim with partial + removal, and omitting that is what made an earlier version of this script fail gate 1.""" + out = {} + for l in open(path): + try: + r = json.loads(l) + except ValueError: + continue + p = r.get("parsed") or {} + v = p.get("verdict") + out[r["key"]] = (v if v in ("drop", "trim", "keep") else None, p.get("kept")) + return out + + +def metrics(cands, verdict_of, gt="future_referenced", floor=False): + """removed% / false-drop% / live-kept%, using score.py's conventions EXACTLY: + + - mass counts every joined candidate, including ones whose verdict failed to parse, so an + arm is charged for the mass it declined to decide about; + - a `trim` removes max(0, size - TOK(kept)), not the whole output; + - `live-kept` credits only a literal `keep`, never a trim; + - percentages floor-divide. + + Matching these is what makes gate 1 meaningful. Approximating them produced numbers that + looked plausible and disagreed with the published table by up to 17 points. + """ + mass = removed = dropped = fdrop = 0 + live = live_kept = 0 + overridden = 0 + for c in cands: + v, kept = verdict_of(c) + mass += c["size"] + ref = c[gt] + # The live denominator counts every joined candidate, INCLUDING ones whose verdict failed + # to parse. score.py does this, and it is defensible: an arm that returned unparseable + # output for a live output did not keep it. Skipping them instead inflated sonnet/bulk's + # live-kept from 57% to 68% -- it has 149 parse failures -- which is precisely the kind of + # plausible-looking disagreement gate 1 exists to catch. + if ref: + live += 1 + if v is None: + continue + if floor and c["later_turns"] < MIN_LATER and v != "keep": + v, kept = "keep", None + overridden += 1 + if v == "drop": + dropped += 1 + removed += c["size"] + if ref: + fdrop += 1 + elif v == "trim": + removed += max(0, c["size"] - score.TOK(kept or "")) + elif ref: + live_kept += 1 + return {"removed": 100.0 * removed / mass if mass else 0.0, + "fdrop": 100 * fdrop // max(1, dropped), + "live_kept": 100 * live_kept // max(1, live), + "dropped": dropped, "live": live, "overridden": overridden} + + +def main(): + corpus = sys.argv[1] if len(sys.argv) > 1 else "ClaudeCode" + cands_all = json.load(open(os.path.join(EXP, "candidates.json"))) + for c in cands_all: + c["key"] = f'{c["corpus"]}|{c["conv"]}|{c["idx"]}|{c["id"]}' + cands = [c for c in cands_all if c["corpus"] == corpus] + print(f"=== {corpus}: {len(cands)} candidates (of {len(cands_all)} total), " + f"exact-referenced-after-F={sum(c['future_referenced'] for c in cands)}") + + print("\n-- GATE 2: re-derive candidates from the captures --") + t2 = {} + for path, label in CAPTURES: + if label == corpus and os.path.exists(path): + for r in rederive(path, label): + t2[r["key"]] = r + missing = [c["key"] for c in cands if c["key"] not in t2] + mismatch = [c["key"] for c in cands + if c["key"] in t2 and t2[c["key"]]["future_referenced"] != c["future_referenced"]] + print(f" re-derived={len(t2)} joined={len(cands)-len(missing)}/{len(cands)} " + f"exact-mismatches={len(mismatch)}") + have_t2 = not (missing or mismatch) + if not have_t2: + print(f" !! DOES NOT REPRODUCE candidates.json -- Tier-2 columns WITHHELD " + f"(missing={len(missing)} mismatch={len(mismatch)})") + else: + print(" OK: reproduces candidates.json exactly.") + for c in cands: + c["t2_strict"] = t2[c["key"]]["t2_strict"] + c["t2_loose"] = t2[c["key"]]["t2_loose"] + rules = defaultdict(int) + for c in cands: + if c["t2_loose"] and not c["future_referenced"]: + rules[t2[c["key"]]["rule"] or "seg"] += 1 + e = sum(c["future_referenced"] for c in cands) + print(f" referenced: exact={e} strict={sum(c['t2_strict'] for c in cands)} " + f"loose={sum(c['t2_loose'] for c in cands)} gains by rule: {dict(rules)}") + if rules.get("budget"): + print(f" NOTE: substring budget bound on {rules['budget']} candidates -- " + f"under-counted, not over-counted.") + + arms = [] + for cc in (False, True): + arms.append((f"deterministic (cut_closed={cc})", + lambda c, k=cc: (score.deterministic(c, k), None), "internal")) + arms.append((f" same, NO opportunity floor", + lambda c, k=cc: (score.deterministic(c, k, min_later=0), None), "none")) + for f in sorted(os.listdir(EXP)): + if not f.endswith(".jsonl") or f.startswith("smoke"): + continue + model = f.rsplit("-", 1)[0] + if model not in score.PRICE: + continue + vs = verdicts_from(os.path.join(EXP, f)) + arms.append((f"{model} / {f.rsplit('-',1)[1][:-6]}", + lambda c, v=vs: v.get(c["key"], (None, None)), "none")) + + gts = [("EXACT (as published)", "future_referenced")] + if have_t2: + gts += [("TIER-2 STRICT (num/case/substr)", "t2_strict"), + ("TIER-2 LOOSE (+basename; UPPER bound)", "t2_loose")] + for gtname, gtfield in gts: + print(f"\n{'='*100}\nGROUND TRUTH: {gtname}\n{'='*100}") + print(f"{'arm':40s} {'removed':>8s} {'f-drop':>7s} {'live-kept':>10s} " + f"{'FLOOR-SYMMETRIC':>16s} {'f-drop':>7s} {'live-kept':>10s} {'ovr':>4s}") + for label, vf, floorkind in arms: + a = metrics(cands, vf, gtfield) + row = (f"{label:40s} {a['removed']:7.1f}% {a['fdrop']:6d}% {a['live_kept']:9d}%") + if floorkind == "internal": + print(row + " (floor already enforced internally)") + else: + b = metrics(cands, vf, gtfield, floor=True) + print(row + f" {b['removed']:15.1f}% {b['fdrop']:6d}% " + f"{b['live_kept']:9d}% {b['overridden']:4d}") + + +if __name__ == "__main__": + main() diff --git a/docs/components.md b/docs/components.md index 4bb8460b..11992a21 100644 --- a/docs/components.md +++ b/docs/components.md @@ -22,6 +22,7 @@ messages (`role:"tool"`; for Anthropic, `tool_result` blocks normalized to that | `extract_llm` | Offload (LLM) | query-irrelevant content via an LLM-written sandboxed filter | via expand | large output in a large request | `strategy` (code), `model.source`, `trigger`, `rewrite`, `skip_file_reads` | | `smartcrush` | Offload | middle items of a JSON array | via expand | JSON-array tool output | `min_items` (5), `min_tokens` (200), `keep_first` (3), `keep_last` (2) | | `mask` | Offload | older tool outputs (age-based) | via expand | more than `keep_recent` outputs | `keep_recent` (3), `min_tokens` (100), `keep_head_chars` (96) | +| `coref` | Offload | tool outputs no later turn referred back to (co-reference-based) | via expand | a threshold crossing, once per budgeted pass; **opt-in, in no preset** — [measured; yield is workload-dependent](results/coref-density.md) | `min_tokens` (300), `cut_closed` (false), `min_later_turns` (8), `min_batch_frac` (0.05), `rewrite_budget` (3), `trigger` | | `summarize` | Offload (LLM) | the middle of the transcript → one summary | via expand | long trajectories | `summary_level` (regular), `keep_last` (3), `min_tokens` (500), `resummarize_tokens` (6000), `model.source`, `trigger` | Presets (`config/config.go`), verbatim: **`codesmart`** (the proxy default) @@ -32,8 +33,8 @@ Presets (`config/config.go`), verbatim: **`codesmart`** (the proxy default) `[format, dedup, failed_run, cmdfilter, smartcrush, extract, extract_llm, cachesplit]` · `coding` `[format, skeleton, cmdfilter, cachesplit]` · `mcp` `[format, smartcrush, cachesplit]` · **`agent`** `[format, dedup, failed_run, mask, extract, extract_llm, cachesplit]` — for long agentic -sessions; `mask` is the biggest lever there (~27–30% content-token savings, no reward loss — see -[RESULTS.md](RESULTS.md)) · +sessions; **`extract_llm`** is the biggest lever there (~27% content-token savings, no reward loss — +see [RESULTS.md](RESULTS.md)); that figure was long miscredited to `mask` · **`general`** `[format, toon, dedup, failed_run, cmdfilter, mask, extract, extract_llm, collapse, cachesplit]` — the recommended all-round pipeline: the reward-neutral levers of `agent` plus the situational shrinkers (`toon`/`cmdfilter`/`collapse`) that cost nothing when they don't fire. `balanced` is @@ -309,13 +310,48 @@ after (older): [older tool output masked; starts: 700 701 def __rmul__(self, m) ``` - **Config:** `keep_recent` (3), `min_tokens` (100), `keep_head_chars` (96). **Shines:** long agent - trajectories where old tool results are unlikely to matter (top lever on terminal/code traffic: 27.5% - on Terminal-Bench, 12.5% on SWE-bench; scales down to ~4% on small structured customer-service outputs). - **Inert:** ≤ `keep_recent` tool outputs, small outputs. + trajectories where old tool results are unlikely to matter — **on non-caching traffic only**. + **Inert:** ≤ `keep_recent` tool outputs, small outputs, and *structurally on every caching + request* — see [the geometry](components/mask.md#when-its-inert). + **The 27.5% / 12.5% figures previously credited here were never `mask`'s**: they belong to + `extract_llm`, whose LLM trimming of large file reads some of the team call the "programming + masker". `mask` was not in the arm that produced them (`codesmart` has no `mask`), and it could + not have been — behind the tail gate its candidate and permitted sets are disjoint. - **`keep_head_chars`** leaves a one-line head-peek of the hidden output inside the marker (see above) so the model knows *what* was masked without a blind `expand` round-trip — evidence showed a bare marker on a masked source-file read forces needless expands. Set `0` for the opaque marker (≈2pp more savings). +### `coref` +Co-reference-aware compaction: cut the tool outputs that no later model turn ever carried anything +forward from. Decides from **back-references** rather than from content or age — for each output, which +identifiers it *introduced* (tokens already in context before it existed are echoes, not references), +and whether a later turn used them. See [components/coref.md](components/coref.md), the +[cheat sheet](reference/coref-glossary.md) for every term, and the +[proposal](proposals/coref-compaction.md) for the derivation. + +``` +after: [tool output compacted: no later turn referred back to it; starts: 0 tree_line_0 = compute_tree_0(arg_0) …] <> [full output: call context_guru_expand] +``` + +- **Config:** `min_tokens` (300), `cut_unreferenced` (true), `cut_closed` (**false**), `closed_dist` (12), + `open_reps` (3), `min_later_turns` (8), `min_batch_frac` (0.05), `rewrite_budget` (3), `break_even` + (true), `keep_head_chars` (96), `trigger`. **Shines:** long sessions with a lot of survey-and-discard traffic (listings, wide + searches, exploratory reads never returned to) — complementary to `mask`, which drops the *old* where + this drops the *never-used*, and an old-but-hot span is the case `mask` gets wrong. **Inert:** below the + trigger, everything referenced, batch below `min_batch_frac`, budget spent, or break-even unmet. +- **It is the one component that mutates the cached prefix on purpose**, so the cut is **batched** (one + cache-write serves the whole pass — a single early cut can never repay its own rewrite), **budgeted** + (`rewrite_budget` per session), and **latched** (the decision is stored and replayed byte-for-byte, + never re-derived; unlike `mask` it must not use `repairLostFreeze`, because a history-dependent + decision re-derived at depth can emit different bytes). +- **Opt-in and in no preset.** The measurement pass has run, but on Claude Code workstation transcripts + rather than the eval-box captures — [unreferenced mass runs 13% on interactive traffic, + 51% on UltraHorizon and 22% on LOCA-bench; recency is measured to be nearly inert while reference count + does all the work; and an output the index cannot see into is `opaque` — never cut — which is 40% of + LOCA's mass](results/coref-density.md). + So `cut_unreferenced` (default on) is justified, while `cut_closed` — the large case-A cut — stays off + until those thresholds are re-measured on the right corpus. + ### `summarize` (LLM) Compresses the **middle of the trajectory** into one LLM-written summary (ported from CE-Manager's ReSum-style summarizer). Restructures the message list to `[msg0, , last-K]`; diff --git a/docs/components/coref.md b/docs/components/coref.md new file mode 100644 index 00000000..03d78713 --- /dev/null +++ b/docs/components/coref.md @@ -0,0 +1,171 @@ +# coref + +!!! info "Offload — lossy, reversible" + Co-reference-aware compaction: at a threshold crossing, cut the tool outputs that no later turn ever carried anything forward from — in one batched pass, under an explicit cache-write budget. + +!!! warning "Opt-in, in no preset, not yet measured on the eval-box corpus" + `coref` is the implementation of [a proposal](../proposals/coref-compaction.md). Its mechanism is + tested, and the measurement pass has run on three corpora — Claude Code sessions, UltraHorizon and + LOCA-bench — but **not on the `capture-swe`/`capture-tb` captures** the acceptance criteria are written + against ([results and caveats](../results/coref-density.md)). The `closed` cut stays **off by default** + because its yield ranges from 15% of mass on interactive traffic to **0% on LOCA**, and a knob that + varies that much by workload has no defensible default. `cut_unreferenced` (the default) + needs no threshold and is justified **on yield** — 13% of tool-output mass on interactive traffic, + 51% on UltraHorizon, 22% on LOCA-bench. + + **On accuracy it is not free.** Against held-out ground truth over 885 real tool outputs, + `cut_unreferenced` removes content the agent later used **11% of the time** — not a boundary + artifact (57% of those references land 51+ turns past the firing point) and not reducible with + the features the index has. Since ground truth is Tier-1 exact matching, 11% is a **lower + bound**. Budget for it: [the selection + experiment](../results/coref-selection-experiment.md). + + Two things that measurement already settles, before you tune anything: **`closed_dist` is nearly + inert** (a 10× sweep moves the answer 2–3 points) and **`open_reps` is the dial** (2 → 6 moves it 18). + Tuning the recency threshold is wasted effort. + +## How it works + +Every other offloader here decides from *content* (is this a duplicate, is it noise, is it superseded) +or from *age*. `coref` decides from **back-references**: for each tool output, which identifiers it +*introduced*, and whether any later model turn used them. + +1. **Index (Tier 1, exact, no LLM).** For each tool output, collect the identifier-ish tokens + it introduced — paths, symbols, ids, hashes, error codes. "Introduced" is the load-bearing word: + a token already present at or before the producing turn is an **echo**, not something taken from + the output. If the agent calls `Read(src/auth.py)`, the path is in the tool-call argument, echoed + by the result, and used again by a later `Edit(src/auth.py)` — an exact matcher scores that as a + reference, but nothing was ever lifted out. Tokens spread across many outputs are dropped as + session furniture. +2. **Classify**, into one of four: `opaque` (introduced nothing trackable — **no evidence**, never + cut) · `unreferenced` (introduced identifiers, no later turn used any) · `closed` (used once or + twice, and not for a long time) · `open` (used recently, repeatedly, or **too new to have had the + chance**). +3. **Cut, once, in a batch.** Only the classes in the cut set, and only when the batch is large + enough to be worth the cache-write it costs (below). +4. **Latch.** The decision is stored per session and replayed byte-for-byte thereafter. It is never + re-derived. + +Recency is measured **from the head of the transcript**, not from the output's own position: "referenced +recently" is a statement about *now*. A span referenced three times forty turns ago is a hot span that +happens to be old, which is why repetition (`open_reps`) overrides age. + +## Why it is batched, budgeted, and rare + +`coref` is the only component that **mutates the already-cached prefix on purpose**. Every age-based +offloader refuses to (`Ctx.TailOnly`) because breaking the prefix hash at index *i* forces the provider +to cache-**write** the suffix, at 11.5× the price of a cache-read. Cutting *S* tokens with *W* tokens of +transcript after the cut and *T* turns left in the session: + +``` +cost = 11.5 x W (cache-read-equivalents) +benefit = S x T +worth it when S x T > 11.5 x W +``` + +A single early cut cannot clear that: 5k cut from 20% depth of a 150k transcript needs *T* > 276 turns. +The same transcript cut by 60k in **one pass** needs *T* > 23, which a long session reaches. Hence: + +- `min_batch_frac` — one rewrite has to serve the whole pass. +- `rewrite_budget` — the spend is capped per session and answerable, not incidental. +- `break_even` — the inequality itself, with *T* estimated from how fast the transcript has been + growing. Its consequence is counter-intuitive and deliberate: **firing at 90% of the window means + *T* ≈ 0**, so the profitable moment to compact is *earlier* than the moment of maximum pressure. + +## Before → After + +``` +after: [tool output compacted: no later turn referred back to it; starts: 0 tree_line_0 = compute_tree_0(arg_0) …] <> [full output: call context_guru_expand] +after: [tool output compacted: the value taken from it survives in a later turn; starts: …] <> [full output: call context_guru_expand] +``` + +The second form is the `closed` cut (`cut_closed: true`). Its claim about a surviving witness is free +rather than asserted: `coref` only ever cuts **tool outputs**, and references live in **model turns**, +which it never cuts — so "a later turn referred back to this" and "the value taken from it is still in +the request" are the same fact. + +## Lossiness + +Lossy but reversible — cut outputs are stashed and recovered via `context_guru_expand` / `GET /expand`, +and an expanded output is marked kept-verbatim so it is never re-cut. + +!!! warning "Reversibility is a capability, not a guarantee" + Expansion is **model-initiated**: the tool is advertised and the host loop only answers a call. + Nothing detects a bad cut. So a wrong cut costs one `expand` round-trip **only when the model + notices** — and if it doesn't, it answers from less than it had, silently. + + That makes `expand` rate a precision metric for *noticed* errors only. It is still the right inner + loop (cheap, no scoring, any traffic) but it is blind to the silent case by construction, so a + falling expand rate is ambiguous. **Reward is the only instrument that sees it**, which is why it + is a gate here rather than one number among several. Tier 3 is where the silent case lives: a + missing semantic reference gives the model nothing to look up, so nothing prompts recovery. + +### What the marker leaves behind + +Two rules, both aimed at the gap between "notices and expands the right thing" and "notices but +cannot tell which marker": + +- **It describes the shape of structured content**, not just its first line — + `200 records, fields: address, id, name`. That is *addressable*: an agent hunting for an address + can tell this is the output to expand. A head peek of one arbitrary row cannot do that, though it + works well for a file read or a traceback, which is what the peek is still used for. +- **It never claims the cut was safe.** An earlier version wrote *"no later turn referred back to + it"* — precisely the claim that is false whenever the reference was transformed or semantic — which + read as reassurance and discouraged the expand call that would have repaired the mistake. A marker + that talks the model out of recovering is worse than an opaque one. A test enforces this. + +## Configuration + +| Key | Default | Meaning | +|---|---|---| +| `trigger` | fires always | Gates **new** cuts only (`min_request_frac` against the resolved context window is the natural dial). Replay of latched decisions is never gated — a latched cut that stops being replayed flips the prefix, which is the churn the whole design avoids. | +| `min_tokens` | 300 | Per-output floor. Matches `coref.py`'s `min_output` so the component and the measurement consider the same population. | +| `cut_unreferenced` | `true` | Cut outputs that introduced trackable identifiers which nothing later referred back to. The honest ceiling of a zero-LLM implementation; needs no calibrated threshold. Note this excludes `opaque` outputs by construction — an output the index cannot see into is never cut, at any setting. | +| `min_later_turns` | 8 | Opportunity floor: an output with fewer model turns after it is treated as `open`. Near the tail "no references yet" and "recent" are the same thing, so without this a batched pass would preferentially cut the newest context. `mask`'s `keep_recent`, expressed in turns. | +| `cut_closed` | `false` | Cut the `closed` class — the large, early, case-A cut. **Off by default**: measured yield is 15% of mass on interactive traffic, 8% on UltraHorizon and 0% on LOCA, so enable it per config for a measured arm rather than globally. | +| `closed_dist` | 12 | A reference is `closed` once its last use is this many messages ago (from the head). **Measured to be nearly inert** — leave it alone. | +| `open_reps` | 3 | Used at least this many times ⇒ `open` regardless of age. The dial that matters; 3 is the conservative setting and each step up trades ~5 points of cuttable mass for reclassifying genuinely repeated spans. | +| `min_batch_frac` | 0.05 | The pass must cut at least this fraction of the request, or it declines and leaves the request byte-identical. It implements the *token* break-even argument and is a **poor proxy for the deferral argument** — it cannot ask whether this cut is the decisive one ([design, unbuilt](../proposals/coref-compaction.md#the-deferral-gate-designed-unquantified)). Was 0.15, which [measurement showed admits 1 of 19 real sessions](../results/coref-density.md) — a gate no traffic can clear is an off switch, not a conservative default. The right value is an experimental result; this is a starting point. | +| `rewrite_budget` | 3 | Prefix-rewrite passes allowed per session. `0` disables new cuts entirely (replay continues). An unreadable counter reads as **exhausted**, never as zero. | +| `break_even` | `true` | Apply `S × T > 11.5 × W` with an estimated *T*. Ignored when the context window is unknown, like every other fraction-based threshold. | +| `keep_head_chars` | 96 | Head-peek left inside the marker so the model knows what was cut without a blind `expand`. `0` for the opaque marker. | +| `marker_mode` | `full` | `full` (stash + resolvable marker) / `summary` / `off`. | + +## What it deliberately does not do + +- **It does not consult `repairLostFreeze`.** `mask` and `failed_run` may re-derive a lost decision at + depth, because their replacement is a pure function of `(content, config)` and so reproduces the bytes + the provider already cached. A co-reference decision is **history-dependent by construction** — + re-deriving it against a longer transcript can yield a different class and different bytes, which is + exactly the prefix flip the repair exists to prevent. A lost `coref` freeze declines. +- **It never resurrects a span.** New evidence cannot un-cut, because un-cutting is a second rewrite. + Monotonicity here is a cache-cost requirement, not tidiness. +- **It does not cut what it cannot see.** An output that introduced no trackable identifier is `opaque`, + not `unreferenced`, and is never cut at any setting. Measured, that is 8% of tool-output mass on + interactive traffic and **40% on LOCA-bench** — bulk record dumps of plain values like + `[{"name":"david","id":123,"address":"foobarbaz"}]`, where short lowercase words and 3-digit numbers + are exactly what the identifier rules exclude. Absence of evidence is not evidence of deadness. +- **It cannot tell an anchor reference from a payload reference.** If the model says *"remember david 123 + address"*, the reference is real but the value needed (`foobarbaz`) was never copied into a model turn. + So a *low* `used_frac` is ambiguous, and this is the substantive reason `cut_closed` ships off. +- **It does not see Tier 2.** A value that was summed, unit-converted or reworded before being restated + leaves no exact match. `unreferenced` means "no later *exact* use" and must never be read as "unused"; + the LLM escalation for that case is an open question in the proposal, not shipped. + +## When it shines + +Long sessions that cross a context threshold with a lot of survey-and-discard traffic — directory +listings, wide searches, exploratory reads that the agent never returns to. It is complementary to +`mask`: `mask` drops the *old*, `coref` drops the *never-used*, and an old-but-hot span is exactly the +case `mask` gets wrong and `coref` protects. + +## When it's inert + +Below the `trigger`; no output above `min_tokens`; every large output referenced (`open`); a batch below +`min_batch_frac`; the rewrite budget spent; or the break-even inequality unmet — which, near the window +edge, is the common and correct outcome. + +See also: [cheat sheet: every term on one page](../reference/coref-glossary.md) · +[the held-out selection experiment](../results/coref-selection-experiment.md) · +[the proposal and its derivation](../proposals/coref-compaction.md) · +[Components overview](../components.md) · [mask](mask.md) · [dedup](dedup.md) diff --git a/docs/components/extract_llm.md b/docs/components/extract_llm.md index fc890ff9..4bd00137 100644 --- a/docs/components/extract_llm.md +++ b/docs/components/extract_llm.md @@ -35,28 +35,55 @@ understated. way — neither is within two orders of magnitude of paying for $3.26 — and the gate reasons with the *more generous* rate, so the shipped decline is the conservative one. -The reason is arithmetic, not implementation quality. A request to a caching backend is -~99.95% cached, so a token removed from a cached region saves the **cache-read** rate -(`$0.30/MTok`), not the fresh-input rate (`$3/MTok`) — a **10× haircut**. An extraction call -costing ~$0.012 must therefore remove a *lot* of tokens to break even: - -| Backend | Content | Break-even output size | -|---|---|---| -| Caching | seen once | **~42,600 tokens** | -| Caching | recurring (amortized over replays) | **~30,500 tokens** | -| Non-caching | seen once | ~3,400 tokens | -| Non-caching | recurring | **~1,800 tokens** | - -The caching figures are why the component is now **off by default on caching backends**: no -realistic tool output reaches 30,500 tokens, so the gate would only ever be declining. +The reason is arithmetic, not implementation quality. A token removed from a **cached region** +saves the cache-read rate (`$0.30/MTok`), not the fresh-input rate (`$3/MTok`) — a **10× +haircut** — and an extraction call costing ~$0.012 must remove a lot of tokens to break even. + +!!! warning "Corrected: the applied turn is not a cache-read turn" + The figures below were recomputed. The original table priced *every* turn at the cache-read + rate, which is right for a **replay** turn and wrong for the turn the cut is actually made. + When cache-aware, `extract_llm` is confined to the **tail** — content that arrived this turn + and has never been cached. On that turn it is billed as a cache-**write** (`$3.75/MTok`, i.e. + *dearer* than fresh input) or as plain fresh input if it falls past the last `cache_control` + breakpoint. Either way it is 10–12.5× the rate it was assigned. + + Confirmed on live traffic rather than argued: a real SWE-bench trial reported **52,561 + `cache_creation` tokens** against 746,047 cache-read across 18 turns. New tail content is + cache-*created* every turn. + + So the saving is `removed × (first-turn rate + reuses × cache-read rate)`, not + `removed × (1 + reuses) × cache-read rate`. + +| Backend | Content | Break-even, as first published | **Corrected** | +|---|---|---|---| +| Caching | seen once | ~42,600 tokens | **~12,900** | +| Caching | recurring (amortized over replays) | ~30,500 tokens | **~11,600** | +| Non-caching | seen once | ~3,400 tokens | ~3,400 *(unchanged)* | +| Non-caching | recurring | ~1,800 tokens | ~1,800 *(unchanged)* | + +The non-caching rows are unchanged by construction — with one rate, `first + r×rate` is exactly +`(1+r)×rate`. + +**The verdict for small-output workloads survives the correction**, which is why the component +stays off by default on caching backends: SWE-bench's largest measured tool output is **2,760 +tokens**, still ~4× below even the corrected break-even, so the gate would still only ever be +declining. What changes is large-output workloads — on LOCA-bench captures the eligible set goes +from **7 to 31** of 1,639 outputs. + +Two consequences worth knowing before tuning. The cached/non-caching break-even ratio falls from +~20× to **~6.4×**, because the applied turn is billed at nearly the fresh rate in both regimes and +only replays take the haircut. And **recurrence is a much weaker lever** than the original +arithmetic implied — it multiplies the expected saving by **1.12×, not 1.40×**, because the applied +turn now dominates the sum. These use the **measured** compression ratio, and that measurement is the uncomfortable part: on real captures an accepted extraction removed only **31–254 tokens per call** on outputs of 400–2,000 tokens — an actual ratio around **0.10–0.12**, not the ~0.45 one might assume. The model declines to cut aggressively, and correctly so: its contract is recall-first. -Most tool outputs are nowhere near 30,500 tokens — in one measured Terminal-Bench capture the -**largest** tool output was 2,053 tokens, ~15× below the cached break-even. That is why the same +Most tool outputs are nowhere near the break-even — in one measured Terminal-Bench capture the +**largest** tool output was 2,053 tokens, ~5.7× below the corrected cached figure (~15× below the +figure as originally published). That is why the same component **wins on a non-caching backend and loses on a caching one**, and why the fix is not "compress harder" but "decide per call". Since #28 the [economic gate](#economics) makes that decision automatically, so the component is safe to leave enabled — it simply declines to spend @@ -259,6 +286,51 @@ own prefix** (11.5× a read — exactly the mistake this workstream exists to av the compaction model to the agent model**. Re-open only if a provider prices in-context follow-up questions at a flat rate. +## Reaching the cached prefix (`allow_cached_prefix`) + +When cache-aware, this component is normally confined to the **uncached tail**: mutating the cached +prefix breaks the provider's prefix hash and forces a cache-write of everything after it. That +restriction is a cache-**cost** property, not a safety property of the model call — and on measured +traffic the mass sits where it cannot reach. On LOCA captures, `cached_prefix_above_floor` showed +large outputs skipped for no reason other than being in the prefix. + +`allow_cached_prefix` (default **false**) lifts it. Because the cost is real, enabling it switches on +two gates the tail path does not have: + +1. **The co-reference index as a free eligibility pre-filter.** A prefix output is a candidate only + if it introduced identifiers and no later model turn carried any forward. It runs *first*, ahead + of the model and economic gates, so no call is ever paid to look at content a deterministic pass + can already clear. +2. **The `S × T > 11.5 × W` break-even**, applied to the prefix **batch** — one cache-write serves all + of it, so it cannot be decided per candidate. + +| Key | Default | Meaning | +|---|---|---| +| `allow_cached_prefix` | `false` | Permit extraction on content the provider has already cached, paying a cache-write for it. | +| `prefix_classes` | `[unreferenced, closed]` | Which co-reference verdicts become prefix candidates. `open`/`opaque` are **refused at construction** — neither is evidence of spent content, and admitting them would turn the pre-filter into "consider everything". | +| `prefix_min_later_turns` | 8 | Opportunity floor for prefix candidates, mirroring `coref`'s: an output with fewer model turns after it has not yet *had* a chance to be referenced. | + +**Why `prefix_classes` is a list rather than a constant** — the two classes ask the model different +questions, and which is worth paying for is an experimental question: + +- **`closed`** (referenced once or twice, long ago) is where **trimming** earns its call. An exact + matcher cannot tell "took the value, rest is chaff" from "took an **anchor** and still needs the + payload it points at" — the ambiguity that keeps `coref`'s `cut_closed` off by default. A model can + read the output, see the reference was a name or id, and keep the payload a blind cut would lose. +- **`unreferenced`** is where a **veto** might. The index found no later *exact* reuse, which is not + "unused": a value the model summed, converted or reworded leaves no substring behind (tiers 2 and + 3). Handing this class to the model asks it to notice an implicit reference the index structurally + cannot see. It yields little when the index was right and is the only mechanism for catching when + it was wrong. + +!!! note "This is not the *merged* design" + The above uses the index as a **gate** and leaves the deadness judgement deterministic and + Tier-1-blind. Review's proposal is different — put the co-reference reasoning **inside this + component's prompt**, so one already-paid call does both jobs. That design is **untested**, and + is *not* what [the selection experiment](../results/coref-selection-experiment.md) refuted (that + refuted model-*as-decider*). See + [the proposal's design notes](../proposals/coref-compaction.md#two-design-notes-from-review-neither-implemented). + ## Metrics `/stats` gains an `extract` block (purely additive — every pre-existing field keeps its name, so diff --git a/docs/components/mask.md b/docs/components/mask.md index 60c3177f..800eea4c 100644 --- a/docs/components/mask.md +++ b/docs/components/mask.md @@ -34,12 +34,52 @@ Lossy but reversible — masked outputs are stashed and recovered via `context_g ## When it shines -Long agent trajectories where old tool results are unlikely to matter. In the `agent` preset it is -the biggest lever (~27% content-token savings, no reward loss — see -[RESULTS.md](../RESULTS.md)). +Long agent trajectories where old tool results are unlikely to matter, **on non-caching traffic**. + +!!! danger "`mask` is not the `agent` preset's biggest lever, and never was" + This page used to claim ~27% content-token savings for `mask`. That number belongs to + **`extract_llm`** — some of the team call its LLM trimming of large file reads the "programming + masker", and the name collision put the figure here. Three things settle it: the arm that + produced the number (`codesmart`) contains **no `mask`**; + [comparison.md](../results/comparison.md#per-component--per-compressor) attributes the savings + to `extract_llm` + `extract` + `cmdfilter`/`dedup` and never mentions `mask`; and `mask` is + structurally incapable of it on caching traffic (below). ## When it's inert -≤ `keep_recent` tool outputs, small outputs. +≤ `keep_recent` tool outputs, small outputs — **and, structurally, every request on a +caching backend.** That last one is not a tuning caveat; it is geometry, and it was found while +measuring [`coref`](coref.md). + +!!! danger "On sequential caching traffic `mask` masks nothing, for any `keep_recent ≥ 1`" + `mask` consults `TailOnly` before modifying a message, because editing content the provider + has already cached breaks the prefix hash and forces a cache-write. `TailOnly` permits index + `i` only when `i > MaxCachedIdx`, and on a sequential conversation `MaxCachedIdx = prevLen − 1` + — everything the *previous* request contained. + + But `mask`'s candidates are, by definition, the outputs **older** than the newest + `keep_recent`. On turn *N* every one of those was present in turn *N−1*, so every candidate + sits at `i ≤ prevLen − 1` and every candidate is refused. **The candidate set and the + permitted set are disjoint by construction**, and no value of `keep_recent ≥ 1` separates + them. A probe over captured sequential traffic masked **0 of 8** eligible outputs. + + `repairLostFreeze` is what makes it *appear* to work in production: once a mask has been + frozen, the replacement is a pure function of `(content, config)`, so it can be re-derived + byte-identically at any depth and replayed. That covers **maintaining** existing masks — it + cannot **create** the first one at depth. So `mask` earns its savings on the turns before the + prefix is cached, then coasts. + + An earlier version of this box said the published 12.5% / 27.5% figures "straddle a behaviour + change". That was too generous to `mask`: **those figures were never `mask`'s at all** (see the + box above). The geometry here is the mechanism that makes the misattribution impossible to + sustain — a component whose candidate and permitted sets are disjoint on every caching request + cannot be any preset's biggest lever on caching traffic. What `mask` actually saves there has + never been measured, and the honest number is currently unknown. + + This is also the cleanest statement of why [`coref`](coref.md) is shaped the way it is: it is + the only offloader that *deliberately ignores* `TailOnly`, because on a long session all the + cuttable mass is behind `MaxCachedIdx`, and an offloader that refuses to reach there can only + ever work on traffic that is not being cached. See also: [Components overview](../components.md) · [Choose a preset](../how-to/choose-a-preset.md) +· [coref](coref.md) diff --git a/docs/components/summarize.md b/docs/components/summarize.md index 212b08a3..0ac144f7 100644 --- a/docs/components/summarize.md +++ b/docs/components/summarize.md @@ -50,6 +50,41 @@ recovered via `context_guru_expand` / `GET /expand`. | `include_tool_calls` | `false` | `false` → tool outputs masked in the summarized trajectory. | | `resummarize_tokens` | 6000 | Tail growth that triggers rolling the checkpoint forward. | | `model.source` | `incoming` | LLM source: `incoming` (proxied model+key) or `config` (cheap model). | + +!!! danger "This component could not send a valid Anthropic request until 2026-08-21" + `summarize` had **three independent message-shape defects**, each masking the next, all found by + running [LOCA-bench](../experiments/loca/iter005/results.md) against a real API — never by tests + or by replay, because `/compact` returns the rewritten body **without forwarding upstream**, so + no provider ever validated it. + + | defect | provider response | fixed by | + |---|---|---| + | summary emitted as a **`system`-role** message at index 1 | `400 messages.1: role 'system' must precede an 'assistant' message or end the array` | emit it as a **user** message, as Claude Code's own compaction does | + | `tool_result` blocks kept whose `tool_use` was deleted | `400 …unexpected tool_use_id found in tool_result blocks` | `dropOrphanedToolResults` | + | `tool_use` blocks kept whose `tool_result` was deleted | `400 …tool_use ids were found without tool_result blocks immediately after` | **`summarizeSpan`** — see below | + + The last two are the same mistake from either side, and both came from boundaries chosen by + arithmetic (`msgs[1 : len−keepLast]`) that knew nothing about tool pairing. The root fix is one + rule: **a tool exchange is atomic.** `summarizeSpan` advances `end` past any tool messages the + kept tail would begin with, so an exchange is summarized whole rather than split; and it drops + the preserved head when `msgs[0]` is an assistant tool-call message, because that message's + results necessarily lie inside the span and the head exists to carry the conversation's + *identity* (its system prompt or opening user turn), which a tool call is not. + + Review framed it best: an unanswered `tool_use` means the agent is still **waiting** on that + tool, so summarize *after* the exchange completes, not through it. That needs no synthetic + `[tool result unavailable]` placeholder — which would be a second fiction on top of the summary. + + Guarded by `schema.ValidateShape` plus an all-presets test that fails on the pre-fix code in + 0.07s. **Any component that deletes messages needs this invariant**; `coref` does not, because it + rewrites a tool message's text in place and never removes a message. + +!!! warning "Run it alone" + `summarize` restructures the transcript and changes the message count, so another component's + in-place edits can race `apply`'s rebuild. Two experiment iterations + ([004](../experiments/loca/iter004/results.md), 005) were invalidated by ignoring this. If a + pipeline needs both compaction and summarization, chain **two proxies** — a compaction pipeline + in front of a summarize-only one — which is also the deployable shape. | `trigger` | — | Gates the first summary: `min_request_tokens`, `min_messages`. | ## When it shines diff --git a/docs/experiments/README.md b/docs/experiments/README.md new file mode 100644 index 00000000..199c7cd2 --- /dev/null +++ b/docs/experiments/README.md @@ -0,0 +1,117 @@ +# Experiment log + +Chronological, per-run record of every measurement, in the `iterNNN` notation used by the +`forever` project so the two are readable side by side. One directory per run, never edited +after the fact except to add a retraction banner. + +**These are the runs. The [results](../RESULTS.md) pages are the *arguments*** — they synthesise +across runs, get rewritten as understanding changes, and are where a reader should start. An +iteration page is the opposite: what was executed, what came back, what it does and does not +prove. If the two ever disagree, the iteration page is the record of fact. + +Each page carries the same sections: **Result** (a table), **What this proves**, **What this does +NOT prove** (the caveats, in full), **Next levers**, **Artifacts** (paths, so a number can be +traced back to the bytes that produced it). + +## Index + +| Iteration | Date | What | Headline | Status | +|---|---|---|---|---| +| [captures/iter001](captures/iter001/results.md) | 2026-08-20 | Component gating on `capture-swebench`, tail vs non-tail | The tail gate costs `mask` 93% of its effect and `failed_run` all of it; `extract_llm` cannot fire on SWE-bench at all | ✅ | +| [loca/iter001](loca/iter001/results.md) | 2026-08-20 | First LOCA replay, one request per conversation | `mask` 52.3% vs `coref` 6.5% | ⚠️ **retracted** — cold-start artifact | +| [loca/iter002](loca/iter002/results.md) | 2026-08-20 | Sequential LOCA replay, 197 turns, 5 arms, summarizer at a context max | Deferral: 72% fewer summarizations, delivered by `coref` | ⚠️ **config invalid** — the pipeline 400s in production (iter004) | +| [loca/iter003](loca/iter003/results.md) | 2026-08-20/21 | LOCA reward integration; band characterisation | 8K saturates at 1.0, 128k collapses to 0.0, 64k partial. Three rig bugs, two mine | ✅ | +| [loca/iter004](loca/iter004/results.md) | 2026-08-21 | Reward, 3 arms × 12 tasks @64k | **Invalid as a reward test** — every error tracked a `summarize` firing. But **`extract_llm` acted for the first time** (prefix reach) | ⚠️ | +| [loca/iter004b](loca/iter004b/results.md) | 2026-08-21 | Same, `summarize` removed | **Reward parity: per-task outcomes byte-identical to baseline**, 17.1% removed, 31% cheaper, 0 model calls | ✅ | +| [loca/iter005](loca/iter005/results.md) | 2026-08-21 | Deferral on live traffic; `summarize` chained in its own proxy | **Blocked — three shape defects in `summarize`, each masking the next; it 400s on every use** | ⚠️ → fixes `80e95d5`, `0971a32`, `2d6902d` | +| [loca/iter006](loca/iter006/results.md) | 2026-08-21 | Stage 1 ablation: `off` / `format` / `+coref`, 75 tasks @64k | Launched; see `iter007` for its outcome | → iter007 | +| [loca/iter007](loca/iter007/results.md) | 2026-08-21 | Stage 1 checkpoint: `format` (n=**75**) + `coref` (n=**62**), then stopped | **Stopped** — correctly, for the shim bug: HTML 400s root-caused to my *own* replay shim (chunked bodies dropped), not `format`/LOCA. `coref` acted on 4.2% of requests (~981k tokens). ⚠️ Its power argument is **retracted** by iter010: n and cost were wrong 5× (state0-only reads); re-paired it gives 48 pairs, 11 discordant, 7:4 for `coref` | ~$215, shim fixed | +| [loca/iter008](loca/iter008/results.md) | 2026-08-21 | 32k band headroom probe, matched 15 tasks × 5 seeds, no CG in path | **The band was the problem.** Over all 75 runs 32k solves **52.7%** vs 64k's **33.3%**, with **0 errors** (confirms the shim fix live) at **$1.13/run**. Still 45-56k peak contexts, so pressure remains | $85.10 | +| [loca/iter009](loca/iter009/results.md) | 2026-08-21 | Re-score the selection experiment: floor symmetry + deterministic Tier-2 ground truth | **Merged stays refuted.** Floor symmetry moves live-kept 0-2pts (overrides 6-23/885); Tier-2 widening (408→473 referenced) raises every arm's false-drop and does not close the 36pt gap. `cut_unreferenced`'s error floor revised **11% → 21-24%** | **$0** | +| [loca/iter010](loca/iter010/results.md) | 2026-08-21 | First live reward measurement: `format` vs `format`+`coref`, 32k, n=75/arm, pre-registered | **No reward effect either way** — task-clustered 4 harm / 4 gain, p=1.000, harm bound ≤51% (the ≤10% target needed zero harm events). **23% of pairs flipped** with no direction. `coref` adds ~1pp removal (6.8% of requests) and cost **rose** $93→$98. Errors 6→0 | $191.03 | +| [loca/iter011](loca/iter011/results.md) | 2026-08-21/22 | Deferral: `summarize` vs `+coref` vs `+fold`, 32k, n=75/arm | **Deferral confirmed and monotonic** — `summarize` fires on 56.1% → 42.3% → **36.9%** of requests, and cost tracks it: $152 → $125 → **$92**. Fold arm best on all axes (35 solved, cheapest, most deferral). No reward difference (32–35/75, p≥0.75). ⚠️ First attempt aborted at 28/75 invalid requests → root-caused **two real `apply` defects** (`62126f4`, `caf32d7`) | ~$380 | +| [loca/iter012](loca/iter012/results.md) | 2026-08-21 | The fold (`+extract_llm` full-body) vs lossless, 32k, n=75 | **`savings_pct` is inflated 3–8×.** Unique-token ordering *inverts*: fold removes **5.26M** vs lossless **6.98M**. Components cannibalise each other. Apparent $ saving is trajectory noise, not compaction. Reward 3 harm/6 gain, p=0.51 | $89.52 | +| [loca/iter014](loca/iter014/results.md) | 2026-08-23 | **The MERGED design** — coref's criterion inside `extract_llm`'s own bulk call, vs separate components, 128k, n=75/arm | **It declines to act: `merged_keep` 88%, `merged_trim` 1 of 2,074.** Reward indistinguishable (ITT 16 vs 16, p=1.00). Wins on efficiency: **−25% calls, −58% CG spend**. Overcount ratio **31.7×**, the largest measured | ~$460 | + +## Before designing an arm + +[**What can and cannot be measured**](../results/measurement-limits.md) is the prerequisite read: the +statistical power actually available (reward at n=12 detects only 50pp effects; realistic ones need +~200–350 tasks), the measured cost model, the benchmark-suitability matrix, why replay cannot catch +schema defects, why a reported yield measures `coref`'s economic *throttle* rather than its +capability, and a table of rig traps that each produced a valid-looking wrong number. + +## Terminology: "fold" is ambiguous in this log — read it carefully + +Two different things have both been called *the fold*, and conflating them has already caused one +misreport: + +| term | meaning | status | +|---|---|---| +| **fold (pipeline sense)** — used in `loca/iter002`–`iter004`, `iter012`, `iter011` arm 3 | `extract_llm` **added as its own component** alongside `coref`, each acting separately in sequence | **measured** | +| **merged / fold (single-call sense)** — the design proposed in discussion | the co-reference criterion carried **inside `extract_llm`'s own prompt**, so ONE model call makes both judgements | **NOT measured.** `loca/iter009` refuted only its *per-output* variant; **bulk adjudication is the surviving shape** and has never been run for reward | + +Prefer **"`+extract_llm` arm"** for the first and **"merged"** for the second. Any bare use of "fold" +below predates this note and means the *pipeline* sense. + +## Conventions + +- **Retractions stay.** A wrong run is deleted from the argument, never from the log — the + retraction and its cause are usually more instructive than the number was. +- **Cost is always stated**, even when it is $0, because "free" is a property worth knowing. +- **Every arm names its binary.** A stale binary silently produced a null result in + `loca/iter002`; recording the build is the cheapest guard against repeating that. +- **Replay is not validation.** `/compact` runs the pipeline and returns the body *without + forwarding upstream*, so no provider ever checks it. Replay can tell you what a component + removes; it **cannot** tell you the result is a valid request. `loca/iter005` found a shipped + component that 400s on every use, invisible to every replay-based measurement here. +- **Pre-register the reading.** For anything with arms, commit the design *and* how each outcome will + be interpreted **before** the numbers exist (`loca/iter004`, `loca/iter006`). Cheap insurance, and + it is what stopped a +1-task difference at n=12 being written up as an improvement. +- **A green test suite only covers the dialect its fixtures use.** Every unit test passed while 37% of + live Anthropic runs failed, because the fixtures hand-build messages in the OpenAI shape + (`ToolCalls` populated) and the wire dialect carries tool calls as `tool_use` content blocks that + bifrost cannot represent at all (`loca/iter011`). Assert on wire bodies in the dialect that actually + runs, not on the internal representation. +- **Ask "would this check have failed?", not "did it come back clean?"** Two checks in one night could + not have detected what they were written for: a baseline-reuse validation that compared data to + itself (`loca/iter012`), and a fix verification that reported zero failures while the component under + test never fired (`loca/iter011`). +- **Quote unique savings, never `savings_pct`, for anything non-deterministic.** The same removed + content is re-credited on every turn that replays a frozen rewrite, inflating `coref` by 4–8× and + `extract_llm` by 2.8× while `format` (deterministic, in place) stays at exactly 1× (`loca/iter012`). + The two numbers even *order the arms differently*. +- **Per-arm benchmark cost cannot price a component.** Two arms differing by one component came out + $4.57 more expensive and $3.71 cheaper in successive iterations, both dominated by how long the + agent's own path happened to run. Attribute cost from the token counters, never from the bill. +- **Report three yield numbers, not one** — eligible, acted, and refused-for-economics. A single + figure cannot distinguish "nothing left to remove" from "economically throttled", and those call + for opposite responses. +- **State power in the design, not the caveats.** An arm that cannot detect the effect it is looking + for should say so before it runs. +- **Check what a results glob actually matches before quoting an n.** `tasks/*/state0/eval.json` + read 15 of 75 completed runs across two iterations, because `state0`…`state4` are seeds. It + understated n by 5× and overstated cost per run by 5×, and nothing about the output looked wrong + (`loca/iter010` amendment 1). Count the files, compare against the config length, and reconcile the + two before drawing an economic conclusion from either. +- **Count the headroom before counting n.** Harm can only show on tasks that currently pass; + improvement only on tasks that currently fail. At 64k the entire harm signal had to come from 3 + tasks, so no n was going to help. Binary-outcome sensitivity peaks at a 50% base rate, which is + why 32k (53%) measures for less money than 64k (25%) — see + [measurement-limits §1](../results/measurement-limits.md). +- **A metric change that makes the story more interesting deserves an audit before it is believed.** + A Tier-2 matcher that collapsed paths onto their stem reported the index's false-drop tripling + (11% → 53%). It was an artifact: the key `yaml` matched 42 distinct identifiers, so any later YAML + mention scored as reuse of a different file (`loca/iter009`). Audit which inputs a new rule actually + fires on, and put the conservative default on the side of the expensive error. +- **"The benchmark cannot do X" is a hypothesis about your configuration.** It was stated as a fact + about LOCA, then disproved by a matched run on the *same 15 tasks* at a different data volume: + 25% → 53% (`loca/iter008`). That $85 run saved several hundred dollars of underpowered arms. +- **Suspect your own harness before the thing under test.** Three HTML 400s were attributed to a + component, then to the benchmark, and were caused by a body-framing bug in the replay shim + (`loca/iter007`). The tell was there in the data: the arm with *more* components had *fewer* + errors. When error counts move opposite to the amount of machinery, look at the transport. +- **Choose the endpoint the budget can afford.** Savings are continuous and cheap to measure to high + precision; reward is binary and expensive. Superiority on a binary outcome at a 20% base rate costs + 4–20× what bounding harm costs, and bounding harm is usually the actual claim — so state a + non-inferiority margin up front and buy that (`loca/iter007`). diff --git a/docs/experiments/captures/iter001/results.md b/docs/experiments/captures/iter001/results.md new file mode 100644 index 00000000..b4497863 --- /dev/null +++ b/docs/experiments/captures/iter001/results.md @@ -0,0 +1,68 @@ +# Captures — iteration 001 (what actually fires) + +**Date:** 2026-08-20 · **Goal:** before spending on any scored benchmark, find out which +components actually act on captured traffic and which are silently declining. Replay only — no +agent, no task, no reward. **Cost: ~$3.** + +Run: `deploy/harbor/replay2.py /tmp/cg-runs/capture-swebench.jsonl --configs ...`, once with +`CACHE_MODE=off` and once with `CACHE_MODE=on`. 200 requests. Binary: `/tmp/cg-runs/cg-proxy-rp`. + +## Result + +The delta between the two cache modes **is** the tail gate's cost: + +| component | non-tail (`off`) | tail-gated (`on`) | effect | +|---|---|---|---| +| `mask` | **50.67%** | **3.33%** | loses **93%** | +| `failed_run` | 1.29% | **0%** | **fully inert** | +| `extract` | 0.84% | 0.84% | unaffected (not tail-gated) | +| `cmdfilter` | 0.23% | 0.23% | unaffected | +| `dedup` | 0% | 0% | never fires on this corpus | + +`extract_llm`, separately: **0% saved in every configuration** — gate on and off, cache-aware and +not. Recorded reason `below_output_floor`. + +Tool-output sizes, the quantity that explains it: + +| corpus | p50 | max | ≥3,000 (its floor) | ≥30,500 (its break-even as then published) | +|---|---|---|---|---| +| SWE-bench, a fresh run made for this pass | 71 | **2,760** | **0** | **0** | +| `capture-swebench` | 106 | 5,674 | a handful | **0** | +| `capture-tb` | ~0 | 1,906 | **0** | **0** | +| LOCA-bench | 185 | **59,857** | **54** | **7** | + +## What this proves + +- **The tail gate is expensive and undocumented.** `mask` keeps only 7% of its effect on caching + traffic and `failed_run` none — and both ship in presets. +- **`extract_llm`'s output floor exceeds the largest tool output SWE-bench produces.** It is + structurally incapable of acting there, and the same holds for Terminal-Bench. +- **`codesmart` therefore saves ~1%** on caching traffic — `extract` 0.84% + `cmdfilter` 0.23%. +- **The binding constraint is tool-output SIZE, not context length**, which is where the argument + had been focused. That makes LOCA the only benchmark in the set where these components can act. +- A mispricing found here was fixed: the gate charged every saved token at the cache-READ rate, + including the turn the cut is applied on, which is a cache-WRITE. See + [`extract_llm`'s doc](../../../components/extract_llm.md). + +## What this does NOT prove + +- **Replay is not reward.** Nothing here says any cut is safe. +- **`capture-tb` and `capture-swe` are smoke captures** (6 and 2 outputs above 300 tokens). Only + `capture-swebench` supports a claim, and it is 50 shallow sessions. +- **The ~27% figure could not be reproduced** in any configuration tried. Consistent with + `config.go`'s own note that the published numbers describe an ancestor of the preset, but not an + explanation of where it came from. +- **One observation is unexplained and stays that way:** `extract_llm` burned 12.8 s across 20 + requests while `acted=0`, `discarded_changes=0`, nothing above INFO in the log. + +## Next levers + +1. LOCA, as the only corpus where anything can fire → [loca/iter001](../../loca/iter001/results.md). +2. Re-measure what `mask` actually saves on caching traffic — currently unknown, and the docs no + longer claim a number. +3. Diagnose the 12.8 s. + +## Artifacts + +`/tmp/cg-coref/phase0-{on,off}.json`, `/tmp/cg-coref/final-{on,off}.json` on the eval box; +analysis in [component gating](../../../results/component-gating.md). diff --git a/docs/experiments/loca/iter001/results.md b/docs/experiments/loca/iter001/results.md new file mode 100644 index 00000000..bf73ce94 --- /dev/null +++ b/docs/experiments/loca/iter001/results.md @@ -0,0 +1,54 @@ +# LOCA-bench — iteration 001 (first replay) · ⚠️ RETRACTED + +!!! danger "Retracted — the setup made every request a cold first turn" + This run sent **one request per conversation** (the deepest). Each was therefore turn 1 with + `MaxCachedIdx = -1`, so **the tail gate never engaged**. That inflated `mask` to essentially + its non-tail figure and produced a "`mask` removes 8.3× more than `coref`" comparison that is + an artifact of the harness, not a property of the components. + + It also reported **zero `closed`** outputs on LOCA, which is false — `closed` requires a + reference that has since gone stale, which cannot exist when every request is turn 1. + [iteration 002](../iter002/results.md) surfaces 25 of them. + + Kept for the record. The cause is more instructive than the numbers were. + +**Date:** 2026-08-20 · **Goal:** get `coref` to act on real traffic for the first time. +**Cost: $0** (deterministic arms). Binary: `/tmp/cg-coref/cg-proxy-fix`. + +Substrate: the 9 deepest real request bodies from the LOCA capture, 3.34 MB, tool-output mass +4,940–232,505 tokens each, replayed cache-aware. + +## Result (as measured — see the retraction) + +| arm | acted on | tokens saved | shrink | +|---|---|---|---| +| `mask` | 9/9 | 402,135 | **52.3%** ← inflated | +| `coref` | 2/9 | 48,532 | 6.5% | +| `coref` + `cut_closed` | 2/9 | 48,532 | 6.5% — identical | + +Classification over the 148 outputs above the 300-token floor: **142 (96%) `open`**, 6 `opaque`, +**0 `closed`**, ~17 cut as `unreferenced`. + +## What survived the retraction + +- **`coref` does act on real traffic** — 48,532 tokens. That much is real and was the point. +- **96% of eligible outputs are `open`.** LOCA's agents reference tool results immediately and + repeatedly, so the detector correctly reports little is safe to remove. Reproduced in iter002. +- **A live observation of the `skipReduce` first-refusal interaction:** an earlier probe ran + `[mask, coref, extract]` together and reported `coref` doing nothing. `mask` ran first and + replaced every output with a short marker, so `coref` saw only sub-floor content. **Arms must be + run one component at a time**, which is how iter002 was built. + +## What this does NOT prove + +Everything comparative. The `mask`-vs-`coref` ratio is a harness artifact; the `closed` count is +wrong; no reward was measured. + +## Next levers + +1. **Sequential replay** so the cache boundary advances → [iteration 002](../iter002/results.md). +2. Reward, which replay cannot give. + +## Artifacts + +`/tmp/cg-coref/loca-replay.jsonl` (9 bodies), `/tmp/cg-coref/arm-*.log` on the eval box. diff --git a/docs/experiments/loca/iter002/results.md b/docs/experiments/loca/iter002/results.md new file mode 100644 index 00000000..a155e1d1 --- /dev/null +++ b/docs/experiments/loca/iter002/results.md @@ -0,0 +1,91 @@ +# LOCA-bench — iteration 002 (deferring summarization) + +**Date:** 2026-08-20 · **Goal:** the proposal's largest claimed win, measured end to end — does +compacting the full request body **defer the summarization** an agent runs when its context fills? +**Cost: ~$0.16** across five arms. Binaries: `cg-proxy-fix` (S1–S4), `cg-proxy-fold` (S4b). + +**Setup.** LOCA traffic is append-only, so turn *k* is reconstructed as `messages[0:k]` of the +final request, cut at assistant boundaries: **197 turns across 9 conversations**, replayed in order +under a stable per-conversation session id so `MaxCachedIdx` advances turn by turn. This is the fix +for [iteration 001](../iter001/results.md)'s cold-start artifact. + +**`summarize` is wired at a 60,000-token context max and runs LAST**, mimicking an agent compacting +when full. It fires only when compaction failed to keep the turn under the max, so **firings are the +deferral measurement**. + +Run: `/tmp/cg-coref/seqrun.sh `, arms strictly sequential. + +## Result + +| arm | pipeline | **summarizer firings** | vs baseline | shrink | +|---|---|---|---|---| +| **S1** | `summarize` alone | **71** | — | 64.6% | +| **S2** | `codesmart` − `extract_llm` | **46** | **−35%** | 58.2% | +| **S3** | + **tail** `extract_llm` | **46** | **+0** | 58.2% | +| **S4** | + `coref` | **20** | **−72%** | 45.6% | +| **S4b** | + `extract_llm` prefix reach | **20** | −72% | 45.6% | + +Shrink *falls* as firings fall — correct, since `summarize` is the largest single shrinker, so +deferring it removes its contribution. Firings are the objective, not shrink. + +Per-component in S4b: + +| component | firings | tokens saved | +|---|---|---| +| **`format`** (lossless JSON repack) | 119 | **1,266,088** | +| `coref` | 28 | 1,008,646 | +| `dedup` | 54 | 15,546 | +| `extract_llm` | **0** | **0** | + +`extract_llm` gates, S4 vs S4b — the proof that prefix reach engaged: + +| gate | S4 | S4b | +|---|---|---| +| `cached_prefix` | 6,597 | **gone** | +| `cached_prefix_above_floor` | 279 | **gone** | +| `prefix_still_referenced` | — | **6,519** | +| `economic_gate` | 25 | **103** | + +## What this proves + +- **Deferral works and is large: 72% fewer summarizations**, and **`coref` delivers it** (28 + firings, 1,008,646 tokens, taking firings 46 → 20). First clearly positive end-to-end result for + this line of work. +- **The deterministic pipeline gets a third of the way for free** (71 → 46). +- **The tail `extract_llm` lever adds exactly nothing** — S2 and S3 byte-identical. +- **`allow_cached_prefix` engages correctly and contributes nothing.** 98.8% of prefix candidates + are still referenced; what survives cannot clear break-even. +- **The pre-filter was pointed at the wrong class.** For `unreferenced` content, dropping dominates + trimming — a call can at best preserve part of what is already spent, where `coref` removes it + free. Fixed by making the class selection configurable (`prefix_classes`). +- **`format` is the largest single lever** and is lossless. Every lossy component competes for the + remainder. + +## What this does NOT prove + +- **No reward.** Nothing says the 20-vs-71 trade preserved task success. This is the gate and replay + cannot answer it. +- **9 conversations, 197 turns.** Direction and mechanism, not calibration. +- **Turns are reconstructed prefixes.** Valid because LOCA is append-only, but each `/compact` call + is independent — a real session carries the *compacted* transcript forward, so effects that + compound across turns are invisible here. +- **The 60,000 max is a choice**, made because it fires often enough to measure (66 of 197 turns + exceed it). Claude Code's real threshold is ~167,000, which only 2 of 197 turns reach. +- **The summarizer is cheap here because of freeze/replay** — 71 firings cost 17 model calls. An + agent whose transcript actually changes after each summary pays more. +- **A stale binary produced a null result.** S4 ran on a binary built before `allow_cached_prefix` + existed, so its "fold" arm was really `coref` alone. Caught only because the gate counters were + byte-identical to S3. Every arm now records its binary. + +## Next levers + +1. **Reward on LOCA** — LOCA-bench's own ReAct agent with the deterministic GEM scorer, pointed at + cg-proxy via `LOCA_ANTHROPIC_BASE_URL`. This is the gate. → iteration 003. +2. `prefix_classes: [closed]` / `[unreferenced, closed]` variants, now that the knob exists. +3. A second benchmark on the long-horizon axis. + +## Artifacts + +`/tmp/cg-coref/loca-seq.jsonl` (197 turns, 37 MB), `out-S1.log`, `out-rest.log`, `out-S4b.log`, +`st-*.json`, `s{1,2,3,4}-*.yaml` on the eval box. Analysis: +[deferring summarization on LOCA](../../../results/coref-loca.md). diff --git a/docs/experiments/loca/iter003/results.md b/docs/experiments/loca/iter003/results.md new file mode 100644 index 00000000..932d0a8a --- /dev/null +++ b/docs/experiments/loca/iter003/results.md @@ -0,0 +1,139 @@ +# LOCA-bench — iteration 003 (reward, at last) + +**Date:** 2026-08-20/21 · **Goal:** the gate. Every measurement before this one was replay — +tokens removed, components fired — and none of it says whether a cut costs the agent the task. +LOCA-bench's own ReAct agent with its **deterministic GEM scorer** answers that. + +**Integration:** LOCA's native Anthropic `/v1/messages` calls point straight at a context-guru +proxy via `LOCA_ANTHROPIC_BASE_URL`, with `LOCA_ANTHROPIC_API_KEY` a dummy — the proxy holds the +real credential. No `forever`, no auth hop. Copied from `forever`'s wiring rather than run inside +it. Per the requester's `CLAUDE.md` the proxy's upstream is the plain litellm gateway and +`ANTHROPIC_CUSTOM_HEADERS` is cleared, so benchmark traffic never transits Context Guru's own +service. + +Run: `/tmp/cg-loca/reward.sh ` → +`loca run-claude-api -c task-configs/debug.json -m aws/claude-sonnet-5 --max-workers 1`. +Binary: `/tmp/cg-coref/cg-proxy-fold`. + +## Result — baseline + +| arm | accuracy | success | steps | cg requests | cg saved | cost | +|---|:--:|:--:|--:|--:|--:|--:| +| **`off`** (passthrough) | **1.0000** | 1/1 | 9 | 9 | **0** | $0.4303 | + +Matches `forever`'s own iter001 (`raw` = 1.0) — the integration is correct. `saved = 0` with no +component acting is the transparency check: on `off` the proxy is verifiably inert. + +Arms against this baseline are in progress; this page records the baseline and the three-stage +diagnosis that produced it. + +## What a 1.0 baseline means here (and why it differs from forever's read) + +`forever` recorded a saturated `debug` baseline as a **problem** — a task where `raw` scores 1.0 +cannot show a memory *lift*. For this work the polarity is reversed: the question is whether +compaction **loses** reward while deferring summarization, so a saturated baseline is the ideal +control for a **regression** test. Any arm below 1.0 is a real loss, with no ceiling effect to hide +behind. Headroom becomes necessary only later, if the goal changes from "does not hurt" to "helps". + +## The diagnosis: three silent failures, one honest signal + +The first two attempts returned **`accuracy 0.0` with `Success: 1/1`** — an episode that ran cleanly +and scored zero. Worth recording in full, because each layer failed differently and only one +number told the truth. + +1. **`anthropic` SDK not installed.** Loud, immediate, easy. +2. **A macOS `.venv` shipped inside `mcp_convert/`**, its binaries stripped in transit. Rebuilding + it changed nothing — which was itself useful: it *eliminated* the venv rather than confirming it. +3. **The real cause: `python` on `PATH`.** LOCA launches its Python MCP servers with the bare + command `python` (`gem/tools/mcp_server/config_loader.py`). Invoking `.venv/bin/loca` directly + does **not** put `.venv/bin` on `PATH`, so those subprocesses got `/usr/bin/python` — 3.9 here, + without `mcp`/`fastmcp`, and below LOCA's own 3.10 floor. The servers died **silently**, their + tools never registered, and the agent — reasonably — reported the workspace as empty and wrote + 1 CSV row instead of 4. + +**The tell.** The agent's tool list contained *only* `filesystem_*` and `memory_*`. Those two are +exactly the **npm**-backed servers; `canvas`, `python_execute` and `claim_done` are the **Python** +ones. A partition that clean points at one shared cause, not three coincidences. + +**The signal was `tool_success_counter: 0`** — 8 tool calls, none successful, visible only in +per-task `eval.json` and not in the run summary, while everything at the top level (`Success: 1/1`, +a plausible transcript, a real cost) looked healthy. + +!!! warning "But that counter is NOT a general health signal — an earlier version of this page + over-claimed it" + Runs that *pass* emit a different feedback shape entirely + (`'evaluation': 'passed', 'parsed_action': 'claim_done_claim_done'`) with no such counter, and a + later 128k run showed `tool_success_counter: 0` when the tools were working perfectly well and + the real cause was the tool-use **budget**. So the counter appears on failure and does not + distinguish *why*. In the case above the tools genuinely were broken, but the counter was + incidental rather than diagnostic. The durable lesson is narrower: **read per-task `eval.json`, + not the summary** — and confirm a cause before naming one. + +!!! danger "Third instance of the same meta-pattern in this project" + A broken tool environment produces a plausible transcript and a real-looking score: + + - Harbor's Claude Code segfault surfaced as `NonZeroAgentExitCodeError` → reads as `reward=0` + ([REPRODUCE.md](../../../results/REPRODUCE.md)) + - a stale proxy binary nulled iter002's fold arm, caught only because gate counters were + byte-identical to the previous arm + - this + + In all three the failure was **upstream of the model** and the score was **numerically valid**. + The lesson is procedural: before believing any benchmark number, check that the tools worked — + `tool_success_counter`, gate counters, a transparency assertion on the `off` arm. + +## The 128k band: three more layers, two of them mine + +Escalating to a band where compaction would actually engage produced three further failures, and +the sequence is the useful part. + +1. **`[Errno 11] write could not complete without blocking` (EAGAIN) on every band above 8K.** + Chased through a full band bisect (32k/64k/96k/128k all failed; a *small* task at 128k failed + too, proving it was the band not the task) and attributed to LOCA's MCP stdio transport. + **It was my own runner:** `| tail -25` on LOCA's stdout made it a pipe, and Rich writing a + band-scaled environment description overflowed it. `debug` printed little and survived. The + message named the writer — *stdout* — and I read "write" and reached for the transport twice + before checking my own harness. **Four runs and a bisect spent on a self-inflicted bug.** + +2. **HTTP 400 — 42 orphaned `tool_use` ids.** With the pipe gone, the real blocker appeared: LOCA's + native trimmer drops messages and orphans `tool_use`/`tool_result` pairs. This is predicted + verbatim in [the proposal §8](../../../proposals/coref-compaction.md#8-consequences-for-benchmark-selection), + *including the instruction to port `repair_tool_pairing()` from `forever` rather than rediscover + it*. I rediscovered it first and read my own note second. The fix is now a rig-side shim + (`/tmp/cg-loca/repair_shim.py`) that lifts the function **verbatim** so the two rigs cannot + drift, sits **before** cg-proxy so compaction is measured on well-formed traffic, and counts + repairs so the rate is visible. `coref` structurally cannot cause this class of bug — it + rewrites a tool message's text in place and never removes a message. + +3. **`--max-tool-uses 100` is too small for this band.** With the shim in place the run completed + (`Success: 1/1`, no 400) but scored 0.0 with `tool_use_counter: 105` — it hit the cap. At 128k + the environment holds **49 quizzes and 30 assignments** to enumerate; the agent wrote 1 row + before running out of calls. `trim_events: 0` rules out both context rot and the shim. Cost was + **$5.23** for the one task, 12× the 8K band. + +**What the 128k band did establish:** the shim fired **354 repairs across 42 requests**, so pair +orphaning is constant there — and the band produces genuine scale (105 tool calls, 42 steps) rather +than the saturated triviality of `debug`. It is the right band; it needs a real tool budget. + +## What this does NOT prove + +- **n = 1 task, 1 trial.** `debug` is a single Canvas task. Nothing here is a distribution. +- **8K band, so almost no context pressure.** The point of the higher bands + (`final_64k/128k/256k`) is that compaction has something to do; at `debug` it may not fire at all, + in which case reward parity is trivially satisfied and says little. +- **Cost is not "cheap" as the dive suggested.** $0.43 for one 8K task on sonnet, ~4× `forever`'s + own $0.10 estimate and ~1.4× their measured $0.30. Budget the higher bands accordingly. + +## Next levers + +1. Arms at `debug` — confirm no reward regression and observe whether components fire at all. +2. **Escalate the dial** to a band where compaction genuinely engages, which is the only place + deferral can be shown to be reward-neutral rather than vacuous. +3. `--use-clear-tool-uses`, LOCA's native context editing, as the baseline to beat. +4. A second benchmark on the long-horizon axis (UltraHorizon). + +## Artifacts + +`/tmp/cg-loca/out-baseline{,2,3}.log`, `/tmp/cg-loca/st-baseline*.json`, and LOCA's own run dirs +`/tmp/cg-loca/outputs/inf_claude_api_debug_*/` (per-task `eval.json`, `trajectory.json`) on the +eval box. diff --git a/docs/experiments/loca/iter004/results.md b/docs/experiments/loca/iter004/results.md new file mode 100644 index 00000000..f32f4fb1 --- /dev/null +++ b/docs/experiments/loca/iter004/results.md @@ -0,0 +1,150 @@ +# LOCA-bench — iteration 004 (reward under real pressure) + +**Date:** 2026-08-21 · **Goal:** the reward gate, at a band that can actually answer it. +**Status:** running — this page states the design and the pre-registered reading before the +numbers land, so the interpretation cannot be fitted to them afterwards. + +## Why the 64k band, and why not the others + +[iteration 003](../iter003/results.md) established the instrument's shape the hard way. Neither +of the obvious bands works: + +| band | runs? | baseline accuracy | verdict | +|---|:--:|:--:|---| +| 8K (`debug`) | yes | **1.0** (saturated) | good regression control, but only `format` fires — says nothing about `coref` | +| 128k | yes (post-shim) | **0.0** | genuine context-rot collapse; a zero floor at n=1 measures nothing | +| **64k** | **yes** | **1/3 partial** | **the only band with pressure *and* headroom** | + +64k baseline probe (3 tasks): `ExcelMarketResearch` 1.0, `CanvasListTest` 0.0, +`CourseAssistant` 0.0 — all completing cleanly, 0 pairing repairs needed. + +## Design + +**12 tasks**, one per distinct environment (variety, so one flaky environment cannot dominate), +excluding `NhlB2bAnalysis` whose 232k-token tool output makes runs slow without adding signal. + +| arm | pipeline | +|---|---| +| `a64-det` | `codesmart` − `extract_llm`, + `summarize` at a context max | +| `a64-full` | + `coref` + `extract_llm` with `allow_cached_prefix` | +| `a64-off` | passthrough baseline | + +Chain: `LOCA → repair shim → cg-proxy → litellm gateway`. Deterministic GEM scorer, no LLM judge. + +**Arm order puts the baseline LAST, deliberately.** iteration 003 found that back-to-back arms +share the provider's prompt cache — `cache_read` identical to the byte, `cache_write` falling to 0 +after the first arm. Whichever arm runs first pays the prefix write and the rest ride free. Running +`off` last means the compaction arms cannot get a free ride from it. This does not remove the +confound; it stops it flattering the arms under advocacy. **Cost is therefore reported with that +caveat and never as a clean saving.** + +## Pre-registered reading + +Decided before the numbers, so the conclusion is not fitted to them: + +- **Arms ≥ baseline** → compaction is reward-neutral-or-better under real pressure. Combined with + [iteration 002](../iter002/results.md)'s 72% fewer summarizations, that is the first genuinely + positive case for the component. +- **Arms < baseline** → the cuts cost tasks. `coref` fails its own gate, and the 11% false-drop + measured in [the selection experiment](../../../results/coref-selection-experiment.md) is doing + visible damage. +- **All three identical** → the pipeline is not engaging even at 64k, LOCA cannot test this at any + feasible band, and the reward question moves to UltraHorizon. + +**Power:** n=12 detects a gross effect only. A 1–2 task difference is noise at this size and will be +reported as noise, not read as a result. The corpus-level guard from +[§8](../../../proposals/coref-compaction.md#8-consequences-for-benchmark-selection) applies: do not +stop at first significance. + +## Result — INVALID as a reward test, and the reason is the finding + +| arm | solved | errors | **solved/12** | cost | +|---|--:|--:|--:|--:| +| **`a64-off`** (baseline) | **4** | **0** | **0.333** | $21.34 | +| `a64-det` | 2 | 1 | 0.167 | $20.39 | +| `a64-full` | 3 | **3** | 0.250 | $16.81 | + +Per task, aligned (`1` solved, `0` failed, `E` errored): + +``` +off: 0 1 0 0 0 0 1 0 1 0 1 0 +det: 0 1 0 0 0 0 0 0 1 E 0 0 +full: 0 1 0 0 0 1 E 0 1 E 0 E +``` + +Read naively this says compaction costs reward (4 → 2 and 4 → 3). **It does not, because the +losses are a configuration bug of mine.** + +### Every error coincided with a `summarize` firing + +| arm | `summarize` firings | task errors | +|---|--:|--:| +| `a64-det` | **1** | **1** | +| `a64-full` | **3** | **3** | +| `a64-off` | 0 | 0 | + +All three errors are HTTP 400 **schema** violations, not model failures: + +- `messages: Unexpected role "tool"` (×2) — a `tool`-role message reached the provider. Anthropic + accepts tool results only as **user** messages carrying `tool_result` blocks; `role: tool` is the + OpenAI shape. +- `messages.1: role 'system' must precede an 'assistant' message` — misplaced `system`. + +[`components.md`](../../../components.md) states the cause plainly: `summarize` *"restructures the +whole transcript (changes the message count) — **run it alone** so no other component's in-place +edits race `apply`'s rebuild."* I ran it alongside nine other components. The count correlation is +exact, so the mechanism is not in doubt. + +**So the arms measure my misconfiguration, not compaction.** Rerun without `summarize` is +iteration 004b; the baseline needs no rerun (no `summarize`, no errors). + +!!! danger "This also undermines [iteration 002](../iter002/results.md)" + iter002 used the same `summarize`-in-a-pipeline configs and reported **72% fewer + summarizations**. But it replayed through `/compact`, which never forwards upstream — so the + malformed bodies were **never validated by a provider**. The same pipeline 400s in production. + + The deferral number is therefore measured on requests that could not have been sent. What + survives is narrower: the *mechanism* (compaction reduces how often a context max is reached) + is still sound, but the specific pipeline that produced 71 → 20 is not a shippable + configuration, and the figure must be re-earned with `summarize` isolated. + + A replay harness that does not forward upstream cannot catch schema violations. That is a + structural blind spot in every `/compact`-based measurement in this project. + +### What did work, and it is the fold + +`extract_llm` **acted for the first time in this entire investigation**: 27 firings, **584,125 +tokens**, in the `allow_cached_prefix` arm. + +| component | `a64-det` | `a64-full` | +|---|--:|--:| +| total saved | 20.0% (1,202,964) | **31.8% (1,680,426)** | +| `format` (lossless) | 1,103,713 (**92%** of that arm's saving) | 722,190 | +| **`extract_llm`** | — | **584,125** (27) | +| `dedup` | 54,171 | 239,408 | +| `coref` | — | 45,454 (4) | +| model calls | 1 | 18 | + +The fold adds **~12 points of saving**, and here it is `extract_llm` doing the work while `coref` +contributes little — the **reverse** of iter002, where `coref` did everything and the fold added +nothing. The difference is the band: at 64k there is prefix content large enough to clear both the +output floor and the break-even. That is the first evidence the fold does something no other +configuration achieves. + +And once again `format` — a lossless JSON repack — is the single largest lever, at 92% of the +deterministic arm's total. + +## Corrections to this page's own design + +- The `mean` accuracy I first computed **excluded errored tasks from the denominator**, which + flattered the compaction arms (0.182/0.333/0.333). An errored task is a failed task; the table + above uses `/12` throughout. +- Cost came out *lower* for the arms ($16.81–20.39 vs $21.34) but this is **not** a saving: the arms + errored out of tasks early, and the prompt-cache confound described in the design section applies. + Cost is not interpretable here. + +## Artifacts + +`/tmp/cg-loca/out-arms64.log`, `loca-a64-*.log`, `st-a64-*.json`, `shim-a64-*.log`, and LOCA's run +dirs `/tmp/cg-loca/outputs/inf_claude_api_cg_64k_12_*` on the eval box. +Task set: `task-configs/cg_64k_12.json`. diff --git a/docs/experiments/loca/iter004b/results.md b/docs/experiments/loca/iter004b/results.md new file mode 100644 index 00000000..fb4301b3 --- /dev/null +++ b/docs/experiments/loca/iter004b/results.md @@ -0,0 +1,73 @@ +# LOCA-bench — iteration 004b (reward, cleanly) + +**Date:** 2026-08-21 · **Goal:** re-ask [iteration 004](../iter004/results.md)'s question with +`summarize` removed from the pipelines, since every task error there coincided exactly with a +`summarize` firing. **Cost: ~$37** for two arms (baseline reused from 004 — it had no `summarize` +and no errors). + +Same 12 tasks, same 64k band, same chain (`LOCA → repair shim → cg-proxy → gateway`), same +deterministic GEM scorer. The only change is dropping `summarize`, which +[`components.md`](../../../components.md) says must **run alone**. + +## Result + +| arm | solved | errors | **/12** | cost | tokens removed | per-task | +|---|--:|--:|--:|--:|--:|---| +| **`off` — baseline** | **4** | 0 | **0.333** | $21.34 | 0% | `010000101010` | +| **`ns-det`** — `codesmart` − `extract_llm` | **4** | **0** | **0.333** | **$14.67** | 17.1% | `010000101010` | +| **`ns-full`** — + `coref` + prefix-reach `extract_llm` | **5** | **0** | 0.417 | $22.64 | 20.3% | `010001101100` | + +For contrast, the same arms **with** `summarize` (iteration 004): 2/12 with 1 error, and 3/12 with +3 errors. Removing it took errors to **zero in both arms**, confirming the diagnosis. + +Per component: + +| component | `ns-det` | `ns-full` | +|---|--:|--:| +| **`format`** (lossless) | 672,240 (46) — **99%** of the arm's saving | 851,961 (81) | +| **`extract_llm`** | — | **494,180 (38)** | +| `coref` | — | 92,449 (17) | +| `dedup` | 7,734 (6) | 82,683 (13) | +| model calls | **0** | 11 | + +## What this proves + +**Reward parity, demonstrated more strongly than a matching average.** `ns-det`'s per-task outcome +string is **byte-identical to the baseline** — `010000101010` — the same four tasks solved and the +same eight failed, not merely the same mean. Removing 17.1% of content changed **nothing** about +which tasks succeeded, at **31% lower cost**, with **zero model calls**. + +**The lossy components can act at this band without degrading reward.** `ns-full` ran +`extract_llm` 38 times and `coref` 17 times, removed 20.3%, and did not lose tasks. + +**`format` remains the dominant lever** — 99% of `ns-det`'s saving from a lossless JSON repack. That +has now held in every configuration measured, replay and live, at every band. + +## What this does NOT prove + +- **`ns-full`'s +1 task is noise, and is reported as noise.** The [pre-registered + reading](../iter004/results.md#pre-registered-reading) said a 1–2 task difference at n=12 would be + treated as sampling variation, and it is: `ns-full` gained tasks 6 and 10 and **lost task 11**. Net + +1 is not evidence that compaction helps. +- **n = 12, single trial, one band.** Enough for a gross regression check, not for a distribution. + §8's guard applies: do not stop at first significance. +- **Cost is only partly interpretable.** `ns-det` ran *after* the baseline, so it inherits some of + the prompt-cache confound described in [iteration 004's design](../iter004/results.md#design). + Note the direction that is safe to read: **`ns-full` cost MORE than baseline** ($22.64 vs $21.34) + despite removing 20.3% of tokens — 11 model calls plus pipeline overhead outweighed the saving. + Removing tokens is not the same as saving money. +- **`summarize` remains untested on live traffic.** Dropping it made these arms valid and left the + deferral question open — see the warning on [iteration 002](../iter002/results.md). + +## Next levers + +1. **Re-earn the deferral number with `summarize` isolated**, on live traffic that a provider + validates. iter002's 71 → 20 came from a pipeline that 400s in production. +2. **More tasks / trials** — n=12 cannot separate `ns-full` from `ns-det`. +3. **A band sweep** — 64k is the only band tested where reward is partial; 32k and 96k are unknown. +4. A second benchmark on the long-horizon axis (UltraHorizon). + +## Artifacts + +`/tmp/cg-loca/out-arms64ns.log`, `loca-ns-*.log`, `st-ns-*.json`, `ns-det.yaml`, `ns-full.yaml`, and +LOCA's run dirs `/tmp/cg-loca/outputs/inf_claude_api_cg_64k_12_*` on the eval box. diff --git a/docs/experiments/loca/iter005/results.md b/docs/experiments/loca/iter005/results.md new file mode 100644 index 00000000..35ec7e40 --- /dev/null +++ b/docs/experiments/loca/iter005/results.md @@ -0,0 +1,179 @@ +# LOCA-bench — iteration 005 (the deferral number, and a shipped bug) + +**Date:** 2026-08-21 · **Goal:** re-earn [iteration 002](../iter002/results.md)'s deferral figure on +**live, provider-validated** traffic, after [iteration 004](../iter004/results.md) showed the +pipeline that produced it returns HTTP 400 in production. + +**Outcome: the experiment was blocked by a defect in `summarize`, and the defect is worth more than +the measurement.** Fixed in `80e95d5`; the re-run is iteration 005b. + +## Design — chaining, so `summarize` runs alone + +[`components.md`](../../../components.md) says `summarize` must **run alone**, because it +restructures the transcript and changes the message count. iteration 002 and 004 both violated that. +Rather than drop it, give it its own proxy and chain them — which is also the deployable shape, a +compaction service in front of a summarizer: + +``` +LOCA → repair shim → cg-proxy A (compaction) → cg-proxy B (summarize alone) → gateway +``` + +Deferral then reads directly off proxy B: does A reduce how often B fires? Both proxies report +`/stats` independently, and every request reaches a real provider. + +| arm | proxy A | proxy B | +|---|---|---| +| `d-sum` | `off` | `summarize` alone — baseline firing rate | +| `d-det` | `codesmart` − `extract_llm` | `summarize` alone | +| `d-full` | + `coref` + prefix-reach `extract_llm` | `summarize` alone | + +## What happened + +The chain worked mechanically — both stages reporting, **0 pairing repairs needed**: + +| arm | proxy A saved | proxy A components | proxy B `summarize` firings | +|---|--:|---|--:| +| `d-det` | 1,137,902 (12.7%) | `format` 132, `dedup` 14 | **4** | +| `d-full` | 4,279,701 (**34.6%**) | `format` 165, **`coref` 61**, `extract_llm` 35, `dedup` 26 | **3** | +| `d-sum` | 0 | — | *(stopped early)* | + +But **every arm errored tasks**, including the baseline — which has no compaction at all: + +``` +d-det solved 3, errors 3 01000EE01E10 +d-full solved 2, errors 3 010001E00E0E +d-sum errors 2 of first 3 EE0 ← no compaction, still failing +``` + +All 400s, all the same shape: + +``` +messages.1: role 'system' must precede an 'assistant' message or end the array +``` + +**The baseline arm erroring is what settles it.** With `A=off`, the only component in the path is +`summarize`. So this is not a pipeline interaction and not `coref` or `extract_llm` — it is +`summarize`'s own output. + +## The defect + +`summarize.go`, both the fresh-summary and checkpoint-replay paths: + +```go +summaryMsg := bschemas.ChatMessage{Role: bschemas.ChatMessageRoleSystem} +... +out = append(out, msgs[0], summaryMsg) // [msg0, summary, last-K] +``` + +The summary is a **`system`-role message at index 1**. Anthropic wants system content in the +top-level `system` field; a system role inside `messages` must precede an assistant message or end +the array. At index 1, followed by the kept tail, it does neither — so the provider rejects the whole +request. When `msgs[0]` is itself the system prompt, which is the normal case, this fires every time +`summarize` acts. + +**`summarize` therefore cannot work on live Anthropic traffic at all**, and the shipped `summarize` +preset would fail immediately in production. + +Fixed by emitting the summary as a **user** message — valid, and what Claude Code's own compaction +does. Both paths changed together, since a replayed checkpoint must match the shape of the turn that +created it or the bytes differ. + +## Why it shipped — the part worth keeping + +Two independent gaps, and the second is structural: + +1. **Nothing asserted the summary's role.** The existing tests mention `ChatMessageRoleSystem` only + for the *input* system prompt at index 0. Now guarded by `summarize_role_test.go`, verified as a + real guard by temporarily restoring the old role and confirming it fails. + +2. **Every prior measurement replayed through `/compact`, which never forwards upstream.** A body no + provider ever validates cannot fail schema validation. This is not a one-off oversight but a + **structural blind spot in replay-based measurement**, and it silently affected every + `/compact`-based result in this branch — [density](../../../results/coref-density.md), + [the eval-box pass](../../../results/coref-evalbox.md), + [component gating](../../../results/component-gating.md) and + [iteration 002](../iter002/results.md). Replay measures *what components remove*; it is + incapable of telling you whether the result is a **valid request**. Both halves matter. + +## What this does NOT prove + +- **No deferral number.** The comparison is void: every arm lost tasks to the defect, and the + baseline lost them too, so firing counts cannot be compared. iteration 005b re-runs it fixed. +- **`coref` and `extract_llm` are not implicated.** They ran clean here — `coref` 61 firings for + 637,825 tokens, `extract_llm` 35 for 680,166, 0 pairing repairs, and the compaction-free arm + failed identically. +- **The firing counts observed (4 and 3) are not evidence of deferral** even between themselves: the + arms took different trajectories (260 vs 286 requests), so raw counts are not comparable. Per + request they are 1.54% and 1.05%, and with a broken baseline neither means anything. + +## Iterations 005b and 005c: two fixes, a third defect, and a deliberate stop + +Fixing the system-role defect did not unblock the experiment. It let `summarize` act more often, +which surfaced the next violation — and then the next. + +| attempt | fix applied | task errors | new failure | +|---|---|--:|---| +| 005 | — | 3 | `messages.1: role 'system' must precede…` | +| 005b | user-role summary (`80e95d5`) | **5** | `messages.0.content.2: unexpected tool_use_id in tool_result blocks` | +| 005c | + drop orphaned results (`0971a32`) | 3 | `messages.1: tool_use ids without tool_result blocks immediately after` | + +Each defect **masked the next**. While the system-role bug fired, `summarize` barely got to act; +fixing it raised the error count to 5, which looked like a regression and was really the component +finally running far enough to break differently. + +The three are one family — **`summarize` does not maintain the provider's message-shape +invariants**: + +1. a `system`-role message spliced mid-array; +2. `tool_result` blocks kept whose `tool_use` was deleted (orphaned results); +3. `tool_use` blocks kept whose `tool_result` was deleted (unanswered calls) — the converse of (2), + arising because `msgs[0]` is preserved verbatim and its results may sit inside the removed span. + +Both fixes are real, tested, and worth keeping. **They do not make `summarize` usable.** (3) needs +phase 2 of a pairing repair — synthesising a placeholder result, or declining to preserve an +unanswered call — and I explicitly argued *against* synthesising when writing (2), on the grounds +that a fake result is "a second lie on top of the summary". That reasoning was wrong on the +decisive point: **an invalid request is worse than an imperfect one.** It is still a design decision +rather than a patch, and it needs to be made deliberately. + +**So I stopped rather than attempt a third fix in the same session.** Three defects in one component, +each revealed only by fixing the last, is a signal about the component's readiness, not a queue of +chores. A fourth patch written at speed would more likely add a fourth defect than reach a working +state. + +One further error in 005c was *not* of this family: a raw +`

400 Bad request

` with no Anthropic error body, so it did not come from the +model API. **Diagnosed in [iteration 007](../iter007/results.md): it was a bug in my own replay +shim, not in any component.** The shim read request bodies from `content-length` only, so any +request the client sent with `Transfer-Encoding: chunked` was forwarded with an *empty* body and +with both framing headers set at once. Recorded here as closed. + +## Status of the deferral claim + +**Unmeasurable until `summarize` maintains message-shape invariants.** That is the honest position. +The claim itself remains plausible — [iteration 002](../iter002/results.md) showed the *mechanism* +(compaction reduces how often a context max is reached) on replayed traffic — but no valid live +measurement exists, and the component the claim depends on cannot currently send a request a +provider will accept. + +`coref` and `extract_llm` are **not** implicated in any of this. They ran clean in every arm, and the +arms containing neither of them failed identically. + +## Next levers + +1. **Give `summarize` a real pairing-repair pass**, both directions, as a deliberate design step: + decide what an unanswered preserved call becomes. Until then the component should arguably be + marked unusable on providers that enforce pairing. +2. **A schema-validity test that does not need a provider.** All three defects are checkable + statically against Anthropic's documented rules — system position, results answered, calls + answered. A validator run over pipeline output in tests would have caught every one, and would + close the replay blind spot without requiring live traffic for every measurement. +3. **Audit the other `/compact`-only results** for schema validity, now that replay is known to be + blind to it. +4. `summarize`'s interaction with `cachesplit` (which edits the top-level `system` field) is + untested and adjacent to defect (1). + +## Artifacts + +`/tmp/cg-loca/out-defer2.log`, `proxyA-d-*.log`, `proxyB-d-*.log`, `st-d-*-{A,B}.json`, +`chain.sh`, `sum-only.yaml` on the eval box. Fix and guard: `80e95d5`. diff --git a/docs/experiments/loca/iter006/results.md b/docs/experiments/loca/iter006/results.md new file mode 100644 index 00000000..d4dc554f --- /dev/null +++ b/docs/experiments/loca/iter006/results.md @@ -0,0 +1,92 @@ +# LOCA-bench — iteration 006 (stage 1: what does `coref` add over lossless?) + +**Date:** 2026-08-21 · **Status:** running. Design and reading pre-registered before results, as in +[iteration 004](../iter004/results.md), and for the same reason: I have made several wrong +diagnoses on this instrument, so committing the interpretation first is cheap insurance. + +## The question changed, and that matters more than the design + +Every measurement so far says the same thing: **`format` — a lossless JSON repack — does 92–99% of +all token saving**, in replay and live, at every band. The lossy components compete for what is left. + +So the decision-relevant question is no longer *"does `coref` cost reward"* but: + +> **Does any lossy component add enough over the *lossless* baseline to justify its risk and cost?** + +That makes this an **ablation against `format`-alone**, not against passthrough. Measured against +passthrough, `codesmart` appears to save 17%; measured against `format`-alone it may add 1–3%. The +second number is the one that decides whether this line of work ships. + +## Arms + +75 tasks (the full `final_64k_set_config.json`), 64k band — the only band with both context pressure +and headroom ([why](../../../results/measurement-limits.md#6-benchmark-suitability)). 6 workers. +Binary: `cg-proxy-v5` (all fixes: cache-write pricing, `allow_cached_prefix`, `prefix_classes`, both +`summarize` shape fixes, atomic exchange boundaries). + +| arm | pipeline | model calls | +|---|---|--:| +| `s1-format` | `[format]` — **the real baseline** | 0 | +| `s1-coref` | `[format, coref]` | 0 | +| `s1-off` | passthrough — the floor | 0 | + +All three deterministic, so stage 1 costs ~**$220** and carries no LLM spend. Baseline **last**, so +the compaction arms cannot inherit its prompt cache +([why](../../../results/measurement-limits.md#5-cost-model-measured)). + +Stage 2 (~$284, `+extract_llm` tail and prefix) runs **only if** stage 1 justifies it. + +## Pre-registered reading + +**Reward** (the gate). n=75 detects only ~20pp effects, so: + +- **Zero discordant pairs** vs `format` → the strongest available evidence of safety, as in + iteration 004b. +- **Any discordance** → report the count and p, and treat <6 discordant as **noise**, not signal. +- **`format` vs `off` is a first-class safety question**, not a formality: `format` rewrites *every* + JSON tool result and is on by default in every preset. If it loses tasks, that is the most + important finding available here. + +**Yield** — three numbers, never one, or the result is uninterpretable +([why](../../../results/measurement-limits.md#3-eligible--acted--yield-measures-the-throttle-not-the-capability)): + +| number | source | +|---|---| +| **eligible mass** | `coref`'s `class_*` gates — what the index judged dead | +| **acted mass** | what survived all four economic gates | +| **refused for economics** | `batch_too_small` + `break_even` + `rewrite_budget` | + +**The decision rule, and it is not symmetric:** + +- acted-yield over `format` is **material** and reward is undamaged → stage 2 is justified. +- acted-yield is **negligible and** refused-for-economics is **small** → `coref` genuinely has little + to remove after a lossless pass. **Rethink, do not spend stage 2.** +- acted-yield is **negligible but** refused-for-economics is **large** → `coref` is *economically + throttled*, not incapable. Different problem, and the lever is the + [TTL observation](../../../proposals/coref-compaction.md#two-design-notes-from-review-neither-implemented), + not more arms. + +**Also collected free:** `cache_opportunity.py` over all three arms, to see whether any prefix +rewrite was already free. Expected ~0 on LOCA (local mock tools, turns seconds apart), which is a +property of the benchmark rather than an answer. + +## What this cannot show + +- **Neither merged design.** This tests `coref`'s *deterministic* dropping. It does not test the + [merged call](../../../proposals/coref-compaction.md#two-design-notes-from-review-neither-implemented) + — co-reference reasoning inside `extract_llm`'s prompt — which is untested and **not** what the + selection experiment refuted. It also does not test the implemented prefix-reach-with-index-gate. +- **Generalisation.** LOCA is Tier-2/3-heavy, adverse to an exact-match detector. A null here cannot + be generalised to Tier-1-rich long-horizon traffic, which **no available benchmark provides**. +- **Small effects.** ~20pp is the floor at n=75. If `coref`'s marginal contribution is a few percent + of tokens, **no affordable experiment can resolve its reward impact** — which is itself a finding, + and a reason to rethink rather than scale. + +## Result + +_Pending — checkpoint report to follow._ + +## Artifacts + +`/tmp/cg-loca/out-s1.log`, `loca-s1-*.log`, `st-s1-*.json`, `ab-format.yaml`, `ab-coref.yaml`, +`task-configs/cg_64k_75.json`, `stage1.sh`, `s1arms.sh` on the eval box. diff --git a/docs/experiments/loca/iter007/results.md b/docs/experiments/loca/iter007/results.md new file mode 100644 index 00000000..51dcb55e --- /dev/null +++ b/docs/experiments/loca/iter007/results.md @@ -0,0 +1,170 @@ +# LOCA iteration 007 — stage 1 of the selection experiment, stopped at checkpoint + +**Date:** 2026-08-21 +**Config:** `cg_64k_75` (LOCA, 64k context dial, `aws-claude-sonnet-5`), 12 workers +**Arms planned:** `s1-off` (no CG) · `s1-format` (deterministic only) · `s1-coref` (`format`+`coref`) +**Arms run:** `s1-format` (complete, **n=75**, $113.83) · `s1-coref` (killed at checkpoint, **n=62**) + +!!! warning "Corrected by [iteration 010](../iter010/PREREGISTRATION.md) — every n and cost below was wrong by 5×" + `state0`…`state4` are the **5 seeds**, and this iteration read `state0` only. So each arm ran + **75** configs, not 15, and cost **$1.52 per run**, not $7.59. Re-paired across all seeds, stage 1 + gives **48 usable pairs with 11 discordant (7 favouring `coref`, 4 favouring `format`, + McNemar p≈0.55)** — not the "1 discordant in 10, p=1.00" stated below — and the true `format` + solve rate is **33.3%**, not 20%. The `group_by_seed` claim below is also false: it groups for + *reporting*, and all 75 configs execute either way. **The stopping decision still stands** (the + shim bug was real and the arms were contaminated), but the power argument that accompanied it does + not. Corrected numbers are in iteration 010's amendment; the wrong ones are left visible below + rather than edited away, per this log's retraction convention. +**Spend:** ~$215 (arm 2's cost is real but unrecorded — killed before its summary was written) +**Status:** **stopped deliberately at the checkpoint**, per the standing instruction to stop and +re-evaluate rather than run all arms to completion. + +## Why it was stopped + +Two independent reasons, one a defect of mine and one a property of the benchmark. + +### 1. My replay shim corrupted an unknown fraction of requests + +The three `400 Bad request` HTML errors in `s1-format` were **not** component failures. Root cause, +confirmed by an isolated two-process test (`echo_srv.py` + shim + `shimtest.py`): + +| request framing | body delivered upstream | headers forwarded | +|---|---|---| +| `Content-Length` | 49 / 49 bytes ✅ | correct | +| `Transfer-Encoding: chunked` | **0 / 49 bytes** ❌ | **`chunked` *and* `Content-Length: 0`** | + +`repair_shim.py` read the body from `content-length` only, and copied all client headers through +except a small denylist that did not include `transfer-encoding`. So for any chunked request it +forwarded an empty body under two mutually contradictory framing headers. A permissive server +accepts this (the local echo server returned 200); a real gateway answers exactly what we saw — +`Your browser sent an invalid request.` `httpx` selects chunked on its own terms, which is why the +failures looked random and intermittent. + +**Fixed:** the shim now decodes chunked bodies properly and strips `transfer-encoding` before +forwarding. Both framings verified to deliver the full body under a single correct `content-length`. + +This exonerates `format`, exonerates LOCA, and closes the matching unexplained error in +[iteration 005](../iter005/results.md). + +**It also means the discriminating experiment was unnecessary.** The plan was to finish the `off` +arm to decide "LOCA problem or `format` problem." The mechanism lives in the transport, which no +component touches, so the answer was neither, and the `off` arm could not have distinguished them. + +Supporting evidence from the data already collected: `s1-coref` — which *also* runs `format` — had +**1** such error against `s1-format`'s **3**, and succeeded on all three tasks that `format` errored +on. A component-caused failure would not behave that way. + +### 2. The benchmark cannot power the reward comparison at this budget + +Measured, not assumed: + +- **Cost:** ~~$7.59 per task per arm~~ → **$1.52 per run** (75 runs, not 15). *Superseded.* +- **Base solve rate:** `format` solved **2 of 10** clean tasks (20%). Accuracy is **binary** (every + value is exactly 0.0 or 1.0) — there is no continuous score to recover extra power from. +- ~~**Grouping:** `group_by_seed` collapsed 75 tasks to 15 runs; reaching n=75 requires patching + LOCA's source and would cost 5× per arm.~~ **FALSE.** It is indeed not exposed as a CLI flag, but it + groups for *reporting* only — all 75 configs execute regardless, as iteration 008 demonstrates + (unpatched, 75 configs, 75 evals). No patch is needed and there is no 5× cost. +- **Discordance:** 1 discordant pair in 10 clean pairs, in coref's favour. Exact McNemar **p = 1.00**. + +Required pairs for a *superiority* claim at the observed ~10% discordance rate: + +| discordance rate | if 100% favour one arm | 80% split | 70% split | +|---|---|---|---| +| 10% | 55 | 119 | 279 | +| 20% | 28 | 60 | 140 | +| 30% | 19 | 40 | 93 | + +The 100% column is fiction; 80% is optimistic. So a superiority claim costs **$911–$4,200** for a +single two-arm comparison. That is not a good use of the remaining budget, and superiority was never +the claim of interest. + +## What the data does establish + +**The components act, and coref acts non-trivially.** From CG's own counters (LOCA's +`context_management_events` field is empty in both arms, but that field tracks *LOCA's* native +clear-tool-uses mechanism, not a proxy — it is not evidence about CG): + +| arm | component | calls | acted | tokens removed | +|---|---|---|---|---| +| `s1-format` | `format` | 1244 | most | up to 29,399 in one request | +| `s1-coref` | `format` | 1271 | most | up to 56,215 in one request | +| `s1-coref` | `coref` | 1271 | **54 (4.2%)** | **~981,000 total**, incl. single hits of 60,080 and 57,118 | + +So coref no-ops on 95.8% of requests and acts rarely but large — consistent with its economic gate, +and the first LOCA run in which coref's action rate is measured rather than inferred. + +**Trajectories are short and not cap-limited.** Tool-call counts ran 9–53, far below the ~106 ceiling +observed earlier, and the agent terminated on its own. Task failures are genuine task failures, not +truncation artifacts. This removes a hypothesis but also explains the low headroom: most LOCA tasks +at 64k fail for reasons context management cannot affect. + +## The reframe this forces + +Superiority on reward was always the wrong primary endpoint. Two endpoints, correctly ordered: + +1. **Primary — savings.** Continuous, measured per request, thousands of observations per arm, + already precise. coref's ~981k tokens over 1271 calls needs no additional n. +2. **Secondary — non-inferiority on reward**, with a pre-declared margin. Ties are *informative* + here, which is why this is affordable where superiority is not: + +| pairs | cost (2 arms) | if 0 harmed | with harm at the observed rate | +|---|---|---|---| +| 20 | $304 | ≤ 14% harmed | ≤ 22% | +| 40 | $607 | ≤ 7% harmed | ≤ 11% | +| 60 | $911 | ≤ 5% harmed | ≤ 8% | +| 100 | $1,518 | ≤ 3% harmed | ≤ 6% | + +The honest statement of what stage 1 bought: **n=10 pairs bounds harm at ≤26%** — too wide to be +worth anything. The margin has to be declared *before* the run, and the budget chosen to buy it. + +## Addendum — the model knobs, and the band table + +Two things checked after the arms were stopped, both of which change the next move. + +**The agent is already Sonnet 5.** Run directories read `aws-claude-sonnet-5`, so the 20% base solve +rate *is* Sonnet's. There is no Haiku→Sonnet upgrade available; the only step up is Opus, which +raises the per-run cost (corrected: $1.52, not $7.59). Separately, `extract_llm` is configured `model: {source: config}` — +it inherits the request's model, so it is Sonnet 5 too, and Haiku is the *untried, cheaper* option +there rather than a starting point. Stage 1's arms were `[format, coref]` with no `extract_llm` at +all, so no model knob could have affected these numbers. + +**The 32k and 96k bands were never measured.** Both probes failed with +`[Errno 11] write could not complete without blocking` — EAGAIN from my own pipe handling — with +`messages: 0`, before the agent ran. The 128k band's reported zero floor was collected during the +broken-shim window and is equally unestablished. So "64k is the only viable band" rested on one +measured point between a saturated 8k and two rows that are rig artifacts. + +Since solve rate falls from 1.0 at 8k to 0.20 at 64k, an intermediate band plausibly has both +pressure and headroom, and **the thin-headroom problem may be an artifact of band choice rather than +a property of LOCA.** Measuring 32k on the fixed rig is the cheapest next step and should come before +buying more pairs at 64k or moving to a larger agent model. + +## Checked and dismissed: the broken terminal MCP is not the cause of the 20% + +`cli-mcp-server` fails to start in every run here — `AttributeError: 'Server' object has no +attribute 'list_tools'` — and LOCA logs `Failed to list tools from mounted server +'FastMCPProxy-MCP_terminal-*': Connection closed` (30 occurrences in `s1-format`, 29 in `s1-coref`, +92 of 181 terminal mounts in the 32k probe). Given three prior rig artifacts in one day, the obvious +hypothesis was a fourth: tasks failing because a tool was missing. + +**It does not hold, and the check is worth recording so it is not repeated.** In the 64k arm the +agent called `canvas_*`, `google_cloud_bigquery_run_query`, `python_execute`, `filesystem_*`, +`email_get_emails`, `google_sheet_get_sheet_data`, `pdf_tools_read_pdf_pages`, `snowflake_write_query` +and `woocommerce_*` — and issued **zero** terminal calls across every task, with **no** step +observation referencing the terminal failure. These tasks do not use the terminal server; its failure +to register is startup noise. + +So the 20% base solve rate stands as a real property of the 64k band, not a rig artifact. Recorded as +a caveat worth fixing for tasks that *would* need a shell, not as an explanation of these numbers. + +## Carried forward + +- Stage-1 numbers are **not** reportable as a reward comparison: contaminated by the shim bug, and + underpowered regardless. The *component-activity* numbers above are unaffected (they come from CG's + own counters on requests that reached it) and are reportable. +- **Do not relaunch the same cut.** Repeating an underpowered design with cleaner errors buys a + cleaner p = 1.00. +- Open, unchanged by this iteration: the **merged** design (co-reference criterion inside + `extract_llm`'s prompt) is still untested; the deferral claim still needs `summarize` on + provider-validated traffic; `summarize` + `cachesplit` still untested. diff --git a/docs/experiments/loca/iter008/results.md b/docs/experiments/loca/iter008/results.md new file mode 100644 index 00000000..7d194731 --- /dev/null +++ b/docs/experiments/loca/iter008/results.md @@ -0,0 +1,113 @@ +# LOCA iteration 008 — the 32k band headroom probe + +**Date:** 2026-08-21 +**Question:** is the ~20% base solve rate at 64k a property of LOCA, or an artifact of the band? +**Config:** upstream `final_32k_set_config.json` — the **same 15 tasks × 5 seeds** as the 64k set, so +this is a matched comparison: identical tasks, smaller data volume. +**Path:** LOCA → `repair_shim.py` (**fixed**) → benchmark gateway. **No context-guru in the path**, +deliberately — this measures the band, not a component. +**Cost:** $85.10 (**$1.13 per run**, 75 runs) · **Result: the band was the problem.** + +!!! warning "n and cost corrected by [iteration 010](../iter010/PREREGISTRATION.md)" + This ran **75** configs (15 tasks × 5 seeds), not 15 — `state0`…`state4` are the seeds and only + `state0` was read. The headline conclusion is unchanged and in fact firmer: over all 75 runs the + 32k solve rate is **52.7%** (vs the 53% measured on `state0` alone), against **33.3%** at 64k. + But per-run cost is **$1.13**, not $5.67, so every dollar figure below overstates by ~5×, and the + `group_by_seed` caveat is wrong — no patch is needed to reach n=75. + +## Result + +| | 32k | 64k (matched tasks) | +|---|---|---| +| **solve rate** | **53%** (8/15) | 25% (3/12) | +| errors | **0/15** | 3/15 | +| cost/**run** | **$1.13** | $1.52 | + +Per task, 32k solved four that 64k did not (`CanvasArrangeExam`, `CourseAssistant`, `SetConfCrDdl`, +`UpdateMaterialInventory`), lost one (`ExcelMarketResearch`), and cleanly ran all three that 64k +errored on. + +**Three findings:** + +1. **The thin-headroom problem was an artifact of band choice.** 53% is not merely better than 25%, + it is near the *optimum* for detecting change: sensitivity to both improvement and degradation is + maximal at a 50% base rate. The 64k band was starving every reward arm of headroom in both + directions. +2. ~~**The shim fix is confirmed on live traffic.** 0 errors in 15 tasks…~~ **RETRACTED.** That read + `state0` only. Over all 75 runs this arm had **1** HTML-400 error, not 0, so the fix **reduced but + did not eliminate** the failure. [Iteration 010](../iter010/PREREGISTRATION.md)'s first arm — same + band, same fixed shim, but **with the CG proxy in the path** — had **6/75**. Current state of the + evidence: + + | run | shim | CG in path | HTML-400 errors | + |---|---|---|---| + | 64k `s1-format` | buggy | yes | 12/75 | + | 32k, this run | fixed | **no** | **1/75** | + | 32k, iter010 arm 1 | fixed | yes | **6/75** | + + So a second cause survives the shim fix, and the CG proxy's presence multiplies its rate ~6×. + Under investigation; it is not yet attributed. +3. **It is also cheaper** — $1.13 vs $1.52 per run, because the smaller data volume means shorter + trajectories. + +## The band still has compaction pressure + +The obvious worry: if 32k contexts never grow, coref has nothing to act on and the band is useless +for our purpose. Measured (approximate, ~4 chars/token, from cumulative step payloads): + +| band | peak context, median | p75 | max | steps, median | +|---|---|---|---|---| +| 32k | **~45k tokens** | ~52k | ~56k | 12 | +| 64k | ~64k tokens | ~83k | ~213k | 10 | + +So 32k still builds 45–56k-token contexts — ample material. **The tradeoff is real and must be +stated:** less context means less to remove, so absolute savings will be smaller at 32k than at 64k. + +**That argues for splitting the endpoints across bands, which is legitimate because savings do not +need reward headroom:** + +- **Savings** — measure at 64k, where it is already precisely measured (`coref` acted on 54/1271 + requests, ~981k tokens, in [iteration 007](../iter007/results.md)). Thousands of observations, + no significance test required. +- **Reward** — measure at 32k, the only band with headroom in both directions. + +## What each budget now buys + +Two-sided (per the two-sided-effect argument in +[measurement-limits §1](../../../results/measurement-limits.md)), at $5.67/task, 2 arms: + +| pairs | cost | harm bound if 0 harmed | requires | +|---|---|---|---| +| 15 | $170 | ≤ 18% | nothing — grouped as-is | +| 30 | $340 | ≤ 10% | `group_by_seed=False` | +| 45 | $510 | ≤ 6% | `group_by_seed=False` | +| 75 | $850 | ≤ 4% | full set, all 5 seeds | + +Same money goes measurably further at 32k than at 64k: + +| budget | 32k | 64k | +|---|---|---| +| $400 | n=35, harm ≤ 8% | n=26, harm ≤ 11% | +| $600 | n=52, harm ≤ 6% | n=39, harm ≤ 7% | + +**Caveat on n from seeds:** reaching n>15 means patching `group_by_seed`, and the extra pairs are +5 seeds of the same 15 tasks — correlated, not independent. The effective n is therefore below the +nominal count, and any analysis should cluster by task rather than treat 75 as 75 free observations. + +## Recommendation for the re-cut + +Two arms at 32k — `format` (lossless baseline) vs `format`+`coref` — with the margin declared in +advance, which is what [iteration 007](../iter007/results.md) failed to do. Approximately $340–$510 +buys a ≤10%–≤6% bound, against the ≤26% that stage 1 bought for $215. + +Not yet addressed by this probe, and still open: the **merged** design (co-reference criterion inside +`extract_llm`'s prompt), `summarize` on provider-validated traffic, and `summarize` + `cachesplit`. + +## Rig notes + +- `cli-mcp-server` fails to start here as everywhere (`'Server' object has no attribute + 'list_tools'`), so the terminal MCP never registers. Checked and dismissed as a cause in + [iteration 007](../iter007/results.md): these tasks issue zero terminal calls. +- The tracebacks in the run log are overwhelmingly the *agent's own* generated Python failing + (`NameError: name 'students' is not defined`), which is normal agent behaviour and not a rig fault. + A monitor filter matching `Traceback` is therefore useless here. diff --git a/docs/experiments/loca/iter009/results.md b/docs/experiments/loca/iter009/results.md new file mode 100644 index 00000000..57c8ac1c --- /dev/null +++ b/docs/experiments/loca/iter009/results.md @@ -0,0 +1,109 @@ +# Iteration 009 — re-scoring the selection experiment for $0: the merged design stays refuted + +**Date:** 2026-08-21 · **Cost: $0** (no model calls — all 8,105 decisions were already recorded) +**Script:** `deploy/harbor/selection_rescore.py` · **Corpus:** Claude Code, n=885 candidates + +## Why this ran + +You approved building the **merged** design — folding the co-reference criterion into the +`extract_llm` call. Reading the code to build it turned up that +[the selection experiment](../../../results/coref-selection-experiment.md) had already measured the +per-output version of exactly that design and **refuted** it (6% live-kept, haiku; 14%, sonnet — both +inside the null model's error bar), and had found the deterministic index beating every model arm with +"no combination of index and model beat it". + +But that comparison carried two documented weaknesses, and both were free to test: + +1. **A floor asymmetry the doc itself flags as unfixed** — `min_later_turns` is a hard guard in the + deterministic arms (`score.py:11`) and merely prompt text for the model arms (`runner.py:47,53`). + The index got a guarantee, the models got a suggestion. Listed under *not settled*. +2. **Ground truth blind to the capability under test** — `future_referenced` is Tier-1 exact + matching, while the entire argument for a model call is catching Tier-2/3 reuse an exact matcher + cannot see. A model that correctly keeps a transformed-reuse output is credited nothing. + +## Result: both corrections applied, neither rescues the model arms + +| ground truth | index (unref only) | haiku bulk | sonnet bulk | sonnet strict | +|---|---|---|---|---| +| **exact** (as published) | 11% fd / **95%** lk | 36% / 58% | 29% / 57% | 30% / 58% | +| **Tier-2 strict** | 21% fd / **92%** lk | 44% / 56% | 39% / 53% | 39% / 53% | +| **Tier-2 loose** (upper bound) | 24% fd / **91%** lk | 46% / 55% | 41% / 52% | 41% / 52% | + +*fd = false-drop, lk = live-kept.* + +**1. Floor symmetry changes essentially nothing.** The floor overrides only **6–23 of 885** decisions +per arm and moves live-kept by 0–2 points (haiku bulk 58% → 59%; sonnet bulk 57% → 57%). Removing it +from the deterministic arms costs them one point (95% → 94%). **The documented asymmetry is not the +explanation for the gap** — it was a reasonable suspicion and it is now closed. + +**2. Widening the ground truth hurts every arm and does not close the gap.** Referenced candidates go +408 → **473** (strict) → 484 (loose), i.e. +16%. Every arm's false-drop rises. The live-kept gap +between the index and the best model arm does not narrow — it slightly *widens* (95 vs 58 → 92 vs 56). + +**3. A genuine update to a shipped default, in the unhelpful direction.** `cut_unreferenced`'s +false-drop floor is **21% (strict) to 24% (loose)**, not the published 11%. The "free safe cut" is +roughly twice as lossy as recorded. That revises `coref-selection-experiment.md` finding 4 and matters +independently of the merged question. + +**4. Tier-2 blindness cannot be fixed deterministically — which is itself the answer to your +suggestion.** You chose the deterministic option specifically to avoid judge noise, and that was the +right call on noise. But the attempt shows the deterministic slice of Tier-2 is nearly empty: +65 of +885 candidates, from substring (50), case (15) and basename (11). Real Tier-2/3 reuse is semantic, so +**closing this bias needs a judge, and a judge reintroduces the noise that ruled out UltraHorizon** +([measurement-limits §6](../../../results/measurement-limits.md)). The residual blindness means the +gap is a *bounded* claim rather than a clean one — but a 36-point live-kept gap is not one a modest +correction closes. + +## Pre-registered reading → Phase 2B + +The reading was fixed before the numbers existed: *"bulk unchanged, or corrections touch few decisions +→ merged is settled negatively."* Both happened. **Do not implement bulk adjudication.** Spend the +budget on the 32k reward arm instead. + +This does not make the merged idea silly — the mechanism you described (Tier-2/3 and anchor-vs-payload +are judgement calls an exact matcher structurally cannot make) is real, and it is written into the +code's own comments. What the evidence says is that a model asked to make those calls performs *worse* +overall than the blind matcher, and that this survives correcting both known biases against it. + +## A fifth rig artifact, caught by audit rather than by luck + +My first Tier-2 implementation also matched a path's **stem** (`src/auth.py` → `auth`). It reported +the index's false-drop rising from 11% to **53%** — a dramatic, publishable-looking result, and +entirely an artifact. An audit of which keys the rule fired on: + +| collapsed key | distinct identifiers it matched | examples | +|---|---|---| +| `yaml` | **42** | `config.yaml`, `capture.yaml`, `base_model_config.yaml` | +| `json` | 25 | three unrelated output paths | +| `only`, `based`, `agent`, `memory`, `context`, `session`, `task` | 10–22 each | leaked from `Append-only`, `Cost-based` | + +A later mention of *any* YAML file was scoring as reuse of a *different* one. That single rule supplied +**237 of 250** gains. The fix reuses `coref.py`'s own `distinctive()` filter rather than inventing a +length threshold — `yaml`, `only` and `auth` all fail it for carrying no interior structure, digit, or +CamelCase — and basename matching is now reported as a separate **upper bound** rather than folded in. + +**The lesson is the day's recurring one, now five for five:** the error was in my instrument, not in +the thing measured, and it pointed in the direction that would have made the story more interesting. +A false *match* indicts the deterministic arms for reuse that never happened — the expensive +direction — which is why the conservative default belongs there. + +Two smaller instrument corrections, both caught by the reproduce-first gate: `removed%` must count +trims as partial removal and charge arms for candidates they failed to parse, and the live-kept +*denominator* must include parse failures. Approximating those disagreed with the published table by +up to 17 points while looking entirely plausible — sonnet/bulk's 149 parse failures alone moved its +live-kept from 57% to 68%. + +## Verification + +- **Gate 1** — the exact column reproduces the published table row for row (index 11.8%/11%/95%, + haiku bulk 40.2%/36%/58%, sonnet bulk 33.6%/29%/57%, haiku digest 90.9%/40%/6%). No re-scored number + was read until this passed. +- **Gate 2** — candidate re-derivation from the raw captures reproduces `candidates.json` exactly: + 885/885 joined, **0** exact-match mismatches. The Tier-2 columns are only reported when this holds. + +## Next + +Phase 2B: two arms at 32k, `format` vs `format`+`coref`, two-sided, **margin declared before the run** +(iteration 007's failure). ~$340 for n=30 (harm ≤10%) or ~$510 for n=45 (≤6%). Reward is the axis +nothing has measured, and [iteration 008](../iter008/results.md) established 32k as the only band with +the headroom to measure it. diff --git a/docs/experiments/loca/iter010/PREREGISTRATION.md b/docs/experiments/loca/iter010/PREREGISTRATION.md new file mode 100644 index 00000000..6381eef6 --- /dev/null +++ b/docs/experiments/loca/iter010/PREREGISTRATION.md @@ -0,0 +1,226 @@ +# Iteration 010 — pre-registration (written BEFORE the run) + +**Status at time of writing: nothing has been run.** This file exists because +[iteration 007](../iter007/results.md) declared its margin *after* seeing results, and so bought a +≤26% harm bound it could not use. The convention in [the index](../../README.md) is to commit the +design *and the reading of each outcome* before the numbers exist. + +**Approved budget: ~$340.** Approved margin: **bound harm at ≤10%.** + +## Question + +Does adding `coref` to a lossless deterministic pipeline change task reward? + +Reward is the axis **nothing in this repo has measured**. The selection experiment +([iter009](../iter009/results.md)) settled decision quality on captured traffic and says so in its own +warning box; [measurement-limits §1](../../../results/measurement-limits.md) prices reward and explains +why 64k could not deliver it. [Iteration 008](../iter008/results.md) established 32k as the only band +with headroom in both directions (53% base solve rate vs 25% at 64k). + +## Arms + +| arm | pipeline | role | +|---|---|---| +| `s2-format` | `[format]` | **lossless** baseline — reformatting only, removes no information | +| `s2-coref` | `[format, coref]` | adds the co-reference cut under test | + +`format` is the baseline rather than passthrough deliberately: the question is what `coref` adds over +lossless, not over nothing. Same binary, same config, same task set, same seeds, same order. + +## Design + +- **Band:** 32k (upstream `final_32k_set_config.json` task set). +- **n = 30 pairs**: the **15 tasks × the first 2 seeds** of each. Requires defeating LOCA's + `group_by_seed` (default `True`, not exposed as a CLI flag), which otherwise collapses the set to 15. +- **Paired** on `(task, seed)`; identical pairs in both arms. +- **Two-sided**, per the argument in [measurement-limits §1](../../../results/measurement-limits.md): + where the baseline is itself lossy, selective removal can *raise* reward, so a one-sided harm bound + would be unable to register a gain even if one occurred. +- **Path:** LOCA → fixed `repair_shim.py` → cg-proxy → benchmark gateway + (`ANTHROPIC_BASE_URL=$ANTHROPIC_BENCHMARK_BASE_URL`, `ANTHROPIC_CUSTOM_HEADERS=` cleared). + +## Endpoints, declared now + +**Primary — non-inferiority on reward.** Harm event = a pair where `s2-format` solved and `s2-coref` +did not. Report the upper 95% (Clopper–Pearson) bound on the harm rate. + +**The bound is bracketed, and both ends are stated up front**, because the 30 pairs are 2 seeds of 15 +tasks and are therefore correlated, not independent: + +| assumption | effective n | bound if 0 harm events | +|---|---|---| +| pairs independent (optimistic) | 30 | ≤ 10% | +| task is the unit (conservative) | 15 | ≤ 18% | + +The truth is between. **The conservative figure is the one to quote in any claim**; the optimistic one +is reported only for comparability with the cost table in measurement-limits §1. Analysis clusters by +task — seeds averaged within task — with the pair-level count shown alongside. + +**Secondary — superiority.** McNemar exact on discordant pairs. Given ~1 discordant pair in 10 at 64k, +this is expected to be non-significant at n=30 and is reported for its point estimate and direction, +**not** as a test we are powered to pass. + +**Tertiary — savings.** `coref` acted-count and tokens removed from CG's own `/stats` counters. This +needs no significance test (thousands of per-request observations) and is reported as the yield triple: +eligible / acted / refused-for-economics. + +## How each outcome will be read + +| outcome | reading | consequence | +|---|---|---| +| 0–1 harm events, no gain | non-inferior within a wide bound; `coref`'s savings are ~free on reward | ship-able at this band; extend to n=45 only if a tighter bound is wanted | +| 0 harm events **and** ≥3 gains one-way | the two-sided argument is supported — the lossy baseline is being beaten | strong result; extend to n=45 to firm it up | +| ≥4 harm events | real degradation | `coref` needs its gates tightened before any reward claim | +| ≥3 errors of any kind | rig failure, not a result | fix and re-run; do not interpret | + +**Stopping rule:** arms run to completion. No interim peeking at reward to decide whether to continue — +interim looks inflate false positives, and iteration 007 was stopped on *rig* grounds, which is a +different and legitimate reason. + +## Pre-declared threats + +- **Correlated seeds** — handled by the bracket above; the conservative end is the quotable one. +- **20% of pairs may be unusable** if any task errors; n falls accordingly and the bound widens. Report + actual n, never planned n. +- **Tier-3 blindness does not apply here.** Reward is blind to nothing, which is the entire reason for + running it. This experiment is not subject to iter009's ground-truth limitation. +- **`cut_unreferenced` carries a 21–24% false-drop floor** ([iter009](../iter009/results.md)). If the + primary shows no harm despite that, the most likely explanation is `expand` recovery or the agent + not needing the dropped content — both worth stating rather than claiming the floor is wrong. + +--- + +# AMENDMENT 1 — n raised from 30 to 75 (written before the run; nothing had completed) + +**Trigger:** a verification check on the first 5 evals of arm 1, before the arm was allowed to spend. +The arm was stopped at 5 evals (~$0 recorded) and this amendment written before relaunching. + +**What the check found — three of this experiment's premises were wrong.** + +`state0`…`state4` in a LOCA output directory are **the 5 seeds**. Every run reported in iterations +007 and 008 read `state0` only, so every "n=15" was 15 observations out of **75 that were actually +executed and paid for**: + +| run | as reported | corrected | +|---|---|---| +| 32k baseline (iter008) | n=15, 53%, $5.67/task | **n=75, 52.7%, $1.13/run** | +| 64k `s1-format` (iter007) | n=15, 25%, $7.59/task | **n=75, 33.3%, $1.52/run** | +| 64k `s1-coref` (iter007) | n=14 | **n=62** | + +And the premise this experiment's sizing rested on — *"`group_by_seed` collapses 75 configs into 15 +runs"* — **is false**. It groups for *reporting*; all 75 configs execute either way. iteration 008 +proves it: unpatched, 75 configs, 75 evals. The `run_claude_api.py` patch was therefore unnecessary +and **has been reverted**, so this run uses the stock tool. + +**Consequences for the plan as pre-registered above:** + +1. **Cost was overstated ~5×.** The full 5-seed, 2-arm experiment costs **~$170**, not $850. The + approved $340 covers it twice. +2. **n=30 would discard 60% of the data for no saving.** Raising to the full 75 configs per arm is + strictly more information at less than the approved budget. +3. **Discordance is ~23%, not the ~10% assumed.** Re-pairing stage 1 across all seeds gives **48 + usable pairs, 11 discordant (7 favouring `coref`, 4 favouring `format`, McNemar p≈0.55)** — not + the "1 discordant in 10, p=1.00" reported in iteration 007. Direction favours `coref`; the test is + not significant and is not claimed to be. + +**Amended design:** both arms on the full `final_32k_set_config.json` (75 configs = 15 tasks × 5 +seeds). Everything else above — arms, band, endpoints, two-sidedness, stopping rule, threats — stands +unchanged. + +**Amended bound:** + +| assumption | effective n | bound if 0 harm events | +|---|---|---| +| pairs independent (optimistic) | 75 | ≤ 4% | +| **task is the unit (conservative, quotable)** | **15** | **≤ 18%** | + +**The conservative end barely moves, and that is the honest headline.** It is driven by the number of +independent *tasks* (15), not the number of seeds. Extra seeds buy precision *within* a task, not more +independent tasks. So this amendment buys a better point estimate and a better-powered McNemar, **not +a materially tighter harm bound.** Tightening that end needs more distinct tasks, which LOCA's 32k set +does not have. + +**Why this is an amendment and not a post-hoc choice:** it is committed before the run, the trigger +was a verification check rather than an outcome, no reward comparison from the amended design existed +when it was written, and it moves n in the direction that makes the pre-registered test *harder* to +pass by luck. + +--- + +# AMENDMENT 2 — intent-to-treat added, because errors are NOT independent of the arm + +**Written while arm 2 was still running, before any reward comparison had been computed.** + +The pre-registration above says errored runs are *"excluded, not counted as failures"*. That assumed +transport errors are independent of the pipeline. **They are measurably not.** + +| run | pipeline | HTML-400 errors | +|---|---|---| +| 64k `s1-format` | `[format]` | **10** | +| 64k `s1-coref` | `[format, coref]` | 4 | +| 32k `s2-format` | `[format]` | **6/75** | +| 32k `s2-coref` | `[format, coref]` | **0/46** (in progress) | + +The arm with *more* compaction errors *less*, consistently, in both bands. The mechanism is plausible: +`coref` shrinks the largest requests, so it avoids whatever fails at size. + +**Why this breaks pairwise exclusion.** If the treatment prevents the failure, excluding those pairs +discards exactly the cases where the treatment helped most — biasing the comparison **against** +`coref`. Exclusion is not the neutral choice here; it is the anti-treatment choice. + +**So both readings are now reported**, and neither is chosen after the fact: + +- **per-protocol** (errored pairs excluded) — as originally pre-registered; +- **intent-to-treat** (an errored run counts as unsolved) — because from the user's point of view a + request that fails did not solve the task, whichever layer failed. + +ITT is the conservative reading for a *harm* claim and the honest one for a *benefit* claim. Both +appear in `deploy/harbor/reward_pairs.py` output, always, so whichever is more flattering cannot be +quoted alone. + +**This is a deviation from the pre-registration and is logged as one.** The trigger was a measurement +(error counts by arm), not an outcome; no reward comparison for iteration 010 existed when it was +written. + +## The HTML-400 cause: three hypotheses tested, three refuted + +Recorded so they are not retested. The error is `400 Bad request / Your browser sent an invalid +request` — raw HTML, no Anthropic error body. + +| hypothesis | test | verdict | +|---|---|---| +| chunked bodies dropped by the shim | isolated echo-server test; fixed and verified both framings | **real, fixed — but the error survives it** (1/75 with no proxy, 6/75 with one) | +| a hard request-size limit returning 400 | 50KB / 200KB / 400KB through direct, proxy-only, and shim+proxy | **refuted.** 50KB and 200KB pass everywhere; 400KB **times out** on all three paths *including direct to the gateway* — size causes timeouts, not 400s | +| a timeout poisons a pooled keep-alive connection, so the next request looks malformed | time out a 500KB request through the proxy, then send 4 small ones | **refuted.** All four returned 200 | + +The CG proxy's own `copyHeaders` was also read and correctly strips `Transfer-Encoding`, +`Content-Length`, `Connection`, `Keep-Alive` and `Host`, and it sets the body via +`strings.NewReader`, so Go computes a correct length. Framing is not obviously wrong there. + +**Stopped guessing and instrumented instead:** the shim now records the exact body length, the +content-length it set, the forwarded headers (credentials stripped), and the response head for any +≥400 (`65f5962`). Armed for iteration 011's 225 runs. Attribution will come from the captured +request, not from further argument. + +## Where the extra errors come from — localised to the proxy hop, cause still unknown + +Comparing [iteration 008](../iter008/results.md) (shim → gateway, **no CG**) against iteration 010's +arm 1 (shim → **CG `[format]`** → gateway) on the identical 75 task+seed runs: + +| | no CG | CG + `format` | +|---|---|---| +| HTML-400 errors | **1**/75 | **6**/75 | +| solved (of 68 usable pairs) | 36 | 33 | + +So the proxy hop costs roughly **5 extra failures per 75 runs**, which localises the second cause to +CG rather than the shim. That sits awkwardly with the code reading: `copyHeaders` correctly strips +`Transfer-Encoding`, `Content-Length`, `Connection`, `Keep-Alive` and `Host`, and the body is set via +`strings.NewReader` so Go computes a correct length. Something not yet found is responsible. + +Note this comparison is **not controlled** — different runs at different times, and the only +difference is the presence of an entire hop — so it locates the problem without explaining it. Left +to the armed capture rather than to further argument. + +**If it is a genuine CG defect it matters beyond this rig**, because it would mean the proxy +occasionally makes a request the provider's front end rejects. That would be a product bug, not a +measurement artifact, and is the reason it is being chased rather than worked around. diff --git a/docs/experiments/loca/iter010/results.md b/docs/experiments/loca/iter010/results.md new file mode 100644 index 00000000..f9ad81be --- /dev/null +++ b/docs/experiments/loca/iter010/results.md @@ -0,0 +1,88 @@ +# Iteration 010 — `coref` vs lossless at 32k: the first live reward measurement + +**Date:** 2026-08-21 · **Pre-registered:** `ecf4103`, amended `4eecbdc` (n) and `535de68` (ITT), all +committed **before** the arms ran. **Cost:** $191.03 ($93.23 + $97.80), against $340 approved. +**n = 75 runs per arm** (15 tasks × 5 seeds), paired on `(task, seed)`, two-sided. + +## Headline: no measurable reward effect, in either direction — and the bound is too wide to call it safe + +| | `format` (lossless) | `format`+`coref` | +|---|---|---| +| solved, per-protocol (n=69 pairs) | 33 | 35 | +| solved, intent-to-treat (n=75) | 33 | **38** | +| HTML-400 errors | **6** | **0** | +| tokens removed | 24.2% (6.98M) | **25.1%** (6.95M) | +| `coref` acted | — | **86 / 1266 requests (6.8%)**, 511,721 tokens | +| cost | $93.23 | **$97.80** | + +**Reward, as pre-registered on the task-clustered end:** + +| reading | harm | gain | p | harm bound | +|---|---|---|---|---| +| per-protocol, pair level (n=69) | 7 | 9 | 0.804 | ≤18% | +| **per-protocol, task-clustered (n=15)** | **4** | **4** | **1.000** | **≤51%** | +| intent-to-treat, pair level (n=75) | 7 | 12 | 0.359 | — | +| **intent-to-treat, task-clustered (n=15)** | **4** | **5** | **1.000** | **≤51%** | + +**The pre-registered ≤10% harm bound was not achieved, and could not have been.** That figure assumed +*zero* harm events. Four tasks were net-harmed, so the bound is ≤51% — effectively no constraint. This +is worth stating plainly: **the experiment was bought at a size that only pays off under a clean +result, and the result was not clean.** + +## The most informative number is the churn, not the totals + +**16 of 69 pairs (23%) flipped outcome**, nearly evenly split (7 harm / 9 gain). `coref` is not +quietly inert here — it changes a quarter of outcomes — but the changes have **no consistent +direction**. Per-task, four tasks net-worse and four net-better. + +That pattern is more consistent with `coref` perturbing trajectories than with it systematically +helping or hurting. Two consequences: + +1. **A directional claim needs far more independent tasks than LOCA has.** At ~23% discordance split + near 50/50, detecting a real direction needs hundreds of pairs; LOCA offers **15 independent + tasks**, verified against its env registry — the shipped roster *is* the whole universe. +2. **Churn is itself a cost.** An agent whose outcome flips on a quarter of runs is less predictable, + even when the average is unchanged. No prior measurement here could see that, because none paired + individual runs. + +## `coref` removed the transport failures — 6 → 0 + +The arm with *more* compaction had *zero* HTML-400s against the baseline's six, matching the direction +seen at 64k (10 vs 4). The mechanism is plausible: `coref` shrinks the largest requests below whatever +fails at size. This is why intent-to-treat matters — under ITT `coref` solves **38 vs 33**, and five of +those five extra solves are runs where the baseline's request failed outright. + +**But this is a benefit against a rig defect, not against the provider.** The underlying 400 is still +unattributed and may be a CG bug (see the pre-registration's localisation section: identical runs +scored 1 error without the proxy and 6 with it). If the defect is fixed, this benefit disappears. **It +must not be quoted as a reason to run `coref`.** + +## Yield: `coref` adds about one point on top of lossless + +`format` alone removes 24.2%; adding `coref` reaches 25.1%. `coref`'s own contribution is **511,721 +tokens over 1,266 requests**, acting on **6.8%** of them — consistent with its economic gate, and +close to the 4.2% seen at 64k. + +**Removing more tokens did not cost less.** The `coref` arm cost **$97.80 against $93.23** — 4.9% +*more* while removing 0.9pp more. Two candidate explanations, not separated here: cache-write costs +from invalidating prefixes, and trajectory divergence (a changed context changes what the agent does, +so per-arm cost is not a controlled comparison). This is the same "removing tokens is not saving +money" result seen on replay, now on live traffic. + +## Provisional status, per the pre-registration's own rule + +The pre-registration says **≥3 errors of any kind is a rig failure, not a result**. Arm 1 had 6. So +**these conclusions are provisional** and stand only if the transport defect is independent of what is +being compared. Two facts argue it is not fully independent: errors are treatment-dependent (6 vs 0), +and they concentrate in the baseline. Both readings are therefore reported, and the ITT figures are +the ones that survive the confound. + +## What this settles and what it does not + +**Settled:** `coref` at 32k over 15 tasks × 5 seeds produces **no detectable reward difference** in +either direction, adds **~1pp** of token removal over lossless, and does **not** reduce cost. + +**Not settled:** whether the churn reflects a real effect too small for 15 tasks; whether the cost +increase is cache-write economics or trajectory divergence; and whether `coref` helps where the +baseline is genuinely lossy — which is exactly what +[iteration 011](iter011/PREREGISTRATION.md), already running, tests against `summarize`. diff --git a/docs/experiments/loca/iter011/PREREGISTRATION.md b/docs/experiments/loca/iter011/PREREGISTRATION.md new file mode 100644 index 00000000..5027559c --- /dev/null +++ b/docs/experiments/loca/iter011/PREREGISTRATION.md @@ -0,0 +1,93 @@ +# Iteration 011 — pre-registration: does selective compaction DEFER summarization? + +**Written before the run. Nothing had been launched when this was committed.** + +This is the question the work started from: *"we want to test if it's worthwhile deferring the context +summarization, when using extract_llm+coref on full body and pay the write cache"*, and *"we have +summarizer as well, so we can use it before the reset if the context max is reached, to mimic Claude +Code behavior."* + +It has never been measured on live, provider-validated traffic. [Iteration 002](../iter002/results.md) +showed the *mechanism* on replayed traffic; [iteration 005](../iter005/results.md) tried it live and +was blocked by three message-shape defects in `summarize`, each masking the next. Those are fixed +(`80e95d5`, `0971a32`, `2d6902d`) and guarded by `schema.ValidateShape` plus a test over all 11 +presets. The binary in use (`/tmp/cg-coref/cg-proxy-v5`, 2026-08-21 08:54) loads +`pipeline="[format summarize]"` cleanly. + +## Why deferral should matter at all — the argument being tested + +A summary is **lossy in a way that is worse for co-reference than a trim**. A trim removes whole +messages, so what is gone is at least knowable; a summary paraphrases exact identifiers into prose, +silently corrupting the literal tokens Tier-1 matching depends on. So summarising *early* destroys +precisely what selective removal would have kept. If selective compaction can postpone or avoid a +summary, it should preserve accuracy **and** save money — the effect is two-sided, per +[measurement-limits §1](../../../results/measurement-limits.md). + +## Arms + +| arm | pipeline | role | +|---|---|---| +| `s3-sum` | `[format, summarize]` | **blunt baseline** — summarise when large, à la Claude Code `/compact` | +| `s3-coref-sum` | `[format, coref, summarize]` | selective removal first, then summarise if still needed | +| `s3-fold-sum` | `[format, coref, extract_llm, summarize]` | the **fold**: `extract_llm` full-body (`allow_cached_prefix: true`) + `coref` + summarise | + +`summarize` config is **identical in all three arms** — only what precedes it differs. Any difference +in its firing rate is therefore attributable to upstream components, which is the whole design. + +## The trigger, chosen from measured data rather than guessed + +`trigger.min_request_tokens: 30000`, from the real request-size distribution at 32k (2,362 requests, +arm 1 of [iteration 010](../iter010/PREREGISTRATION.md)): + +| percentile | request tokens | +|---|---| +| p50 | 14,154 | +| p75 | 33,603 | +| p90 | 43,728 | +| max | 110,163 | + +**35% of requests exceed 30,000**, so the trigger fires mid-trajectory on most sessions without firing +constantly. 40,000 would reach only 12% of requests — too rare to measure deferral; 20,000 reaches 46% +and would fire almost immediately, leaving nothing to defer. + +## Endpoints, declared now + +**Primary — deferral, measured directly.** `summarize`'s firing count and model-call count per arm, +from CG's `/stats`. This is a count over thousands of requests, not a sampled estimate, so it needs no +significance test. **Deferral is established if `s3-coref-sum` and `s3-fold-sum` fire `summarize` +strictly less often than `s3-sum`.** + +Reported as the yield triple — eligible / acted / refused — because a single number cannot separate +"nothing left to summarise" from "economically throttled". + +**Secondary — reward.** Paired on `(task, seed)`, two-sided, via `deploy/harbor/reward_pairs.py`. +**Quoted on the task-clustered end**, as pre-registered in iteration 010: LOCA has exactly **15** +`*_s2l` environments — verified, the shipped roster is the entire task universe — so 75 runs are 5 +seeds of 15 tasks and no seed count makes them independent. The conservative bound floor is ~≤18% and +**cannot be improved on this benchmark**. + +**Tertiary — cost.** Total $/arm and tokens removed per component. Note the sign convention that keeps +tripping this work up: removing tokens is not the same as saving money once cache-writes are priced. + +## How each outcome will be read + +| outcome | reading | +|---|---| +| fewer `summarize` firings in B/C **and** no reward harm | **deferral works** — the founding claim is supported live for the first time | +| fewer firings but reward harm at task level | deferral trades accuracy for cost; the trade needs pricing, not celebration | +| no reduction in firings | selective removal does not shift the trigger at this band; the claim fails **here**, and the band is a stated limit, not a refutation everywhere | +| `summarize` errors or emits invalid requests | the iteration-005 failure recurred; fix, do not interpret | + +## Pre-declared threats + +- **`summarize` is documented as "run it alone (its own preset) — it restructures the whole + transcript."** These arms deliberately violate that, because the deferral question *is* the + interaction. Shape validity is therefore a monitored outcome, not an assumption: any 400 or + shape violation invalidates the arm rather than counting as a result. +- **`summarize` + `cachesplit` remains untested** and is excluded from all three arms. +- **`extract_llm` full-body pays a cache-write.** Arm C is expected to be *more* expensive; the + question is whether deferral repays it, and cost is reported per arm rather than assumed. +- **Trigger choice is a free parameter.** It is fixed here, in advance, from measured percentiles, and + is identical across arms. It must not be retuned after seeing results. +- **`coref` acted on only 4.2% of requests at 64k**, and 32k contexts are smaller, so the deferral + effect may be small. A null result is a real possible outcome and is pre-committed above. diff --git a/docs/experiments/loca/iter011/results.md b/docs/experiments/loca/iter011/results.md new file mode 100644 index 00000000..ffa50f72 --- /dev/null +++ b/docs/experiments/loca/iter011/results.md @@ -0,0 +1,339 @@ +# Iteration 011 — ABORTED at arm 1: `summarize` has a fourth shape defect + +**Date:** 2026-08-21 · **Pre-registered:** `77f36e3` · **Arms run:** `s3-sum` only (75 runs, ~$8) +**Arms 2 and 3 cancelled before spending**, per the pre-registration's own rule. + +## What happened + +Arm 1 (`[format, summarize]`) completed 75 runs. **28 of them — 37% — failed**, all with the same +provider rejection: + +``` +400 messages.1: `tool_use` ids were found without `tool_result` blocks immediately after: toolu_… + Each `tool_use` block must have a corresponding `tool_result`… +``` + +The pre-registration named this exact case: *"`summarize` errors or emits invalid requests → the +iteration-005 failure recurred; fix, do not interpret."* So arms 2 and 3 were stopped rather than run +at a 37% failure rate, saving roughly $180. + +~~**This is a fourth message-shape defect in `summarize`.**~~ **WRONG ATTRIBUTION — see the root cause +below. `summarize` is not at fault**; the defect is in `apply`, and it is one cause with two faces. + +## Two things the aborted arm did establish + +**1. `summarize` works mechanically, and the trigger was chosen correctly.** It fired **48 times** +across 1,201 requests with **35 model calls**, removing **1,668,795 tokens** and lifting total removal +to **29.6%** (against `format`-alone's 24.2% in [iteration 010](../iter010/results.md)). The +data-driven trigger — `min_request_tokens: 30000`, from measured p50/p75/p90 of 14k/34k/44k — put +firing where it was intended. Nothing about the *gating* needs revisiting. + +**2. The HTML-400 from iteration 010 is a DIFFERENT error, and remains unattributed.** The 29 captured +failures are all proper JSON Anthropic errors (28 pairing + 1 timeout), served by `Server: uvicorn`. +The iteration-010 failures were raw HTML with no Anthropic body. Two distinct faults were being +conflated; this iteration separates them and solves neither. + +## Why reasoning from the source did not converge + +Both splice sites in `summarize.go` (the fresh path at 257–259 and the checkpoint-replay path at +315–317) *do* advance their boundary past leading tool messages, and `summarizeSpan` already drops the +head when `msgs[0]` is an assistant tool-call message. With `headCount = 1` the emitted layout is +`[msgs[0], summary, tail…]`, so index 1 is the *summary* — which cannot carry tool calls. The reported +orphan is at `messages.1` in **all 28** cases, so the emitted list is not the list the code appears to +produce. + +**Diagnosis by reading was therefore abandoned in favour of instrumentation**, which is the lesson +from the day's earlier failures rather than a new one. + +## Instrumentation: the capture had to move + +`loca_repair_shim.py` sits **upstream** of context-guru, so it records the request *before* compaction — +the innocent copy — and it *repairs* tool pairing, which would mask this very defect. Its capture also +stored headers but not the body. + +`deploy/harbor/capture_hop.py` (new) sits on the other side: + +``` +LOCA → repair shim → cg-proxy → [capture hop] → gateway +``` + +It repairs nothing and, on any ≥400, records a **structural digest** of the outgoing message list — +per message: index, role, block types, `tool_use` ids declared, `tool_result` ids answered — plus the +provider's error. That digest is what identifies an orphaned pair; the bodies run to hundreds of +kilobytes and are mostly irrelevant. + +## Status + +- Arm 1's reward numbers are **not reported**: a 37% invalid-request rate is a broken arm, not a result. +- The **deferral question remains unmeasured on live traffic**, for the second time, and for the same + component's shape handling. +- Next: read the captured digest, fix the defect, extend `ValidateShape` and its test to cover the + shape that escaped, rebuild, then re-run all three arms. + +## ROOT CAUSE (found, fixed, `62126f4`): bifrost cannot represent an Anthropic tool call + +Found by instrumenting the rebuild rather than reading it, after three rounds of source-reading failed +to converge. + +**`bschemas.ChatContentBlock` has no `tool_use` type.** Its enum is `text` / `image_url` / +`input_audio` / `file` / `refusal`. An Anthropic assistant turn carries its calls as `tool_use` +**content blocks**, so after unmarshaling, the ids are simply *absent*. Everything that reasons about +tool pairing saw an assistant message with **zero calls** on every Anthropic request. + +One cause, two defects: + +| defect | consequence | +|---|---| +| `dropOrphanedToolResults` builds its answerable set from `ToolCalls` alone | that set was **always empty** on Anthropic traffic, so **every** `tool_result` looked orphaned and the "repair" **deleted all of them**. The provider then rejected the request for the unanswered calls left behind. | +| `schema.ValidateShape` was blind to the same ids | it could not see the breakage — which is why a test asserting **all 11 presets** emit shape-valid requests stayed green while 37% of live runs failed | + +The instrumented rebuild made it plain — `summarize`'s output contained **no tool messages at all**: + +``` +out[0] user bi=0 EMIT (head) +out[1] user NO-MATCH (the summary) +out[2] assistant bi=1 EMIT ← declares pa_a, pb_a +out[3] user bi=17 EMIT ("final question") +``` + +**Why every unit test passed while live traffic failed 28/75:** the offload tests hand-build +`ChatMessage`s with `ToolCalls` populated — the *OpenAI*-shaped representation — so they never +exercised the Anthropic path. The validator I had built to prevent exactly this class of bug was +validating a dialect the rig does not use. + +**Fix:** `normalize` recovers the ids where the dialect is still known, rather than teaching every +consumer about Anthropic. It deliberately marks such messages non-lossless — honest, since bifrost +genuinely cannot round-trip them — which only makes the write-back guard more conservative. Full suite: +24 packages, 0 failures. + +Also fixed on the way (`2ec2445`): `ValidateShape` rejected every ordinary **parallel** exchange, +because it inspected only `msgs[i+1]` while bifrost splits one Anthropic results message into a run of +`tool` messages. It emitted the *same error text* as the real defect, making it useless precisely where +it was needed. + +## A verification that proved nothing, caught before it was believed + +The first live check of the fix reported **`captured failures: 0`** and looked like a clean pass. It +was not: the same output showed `llm_calls=0` and no `[summarize]` line — **`summarize` never fired**, +so zero failures said nothing about the fixed path. Trajectory length varies, and the earlier +reproduction had happened to fire it twice. + +Re-verified with a diagnostic-only trigger (`min_request_tokens: 5000` instead of 30000) so the +component is forced to act. **This is the same failure as iteration 012's vacuous baseline-reuse +check**: a check that cannot fail is not evidence, and both were written by the same reasoning — asking +"did the run come back clean?" instead of "would this have detected the problem?" + +## Re-run with both fixes: verified on live traffic + +The fix landed in two stages, because the first unmasked the second. + +| stage | arm-1 failure rate | failure mode | +|---|---|---| +| pre-fix | **28/75 (37%)** | `tool_use` ids without `tool_result` — results all deleted | +| after fix 1 (`62126f4`, tool-call ids) | 13/39 (33%) | **different**: `messages: Unexpected role "tool"` | +| after fix 2 (`caf32d7`, role leak) | **0/16 (0%)** | none | + +Fix 1 stopped `dropOrphanedToolResults` deleting every result. Those surviving synthetic `role=tool` +messages then reached the rebuild, which had never had to serialize one — and while the results were +being deleted, nothing could. **One defect had been masking the other.** + +**The verification is not vacuous, and was checked for that specifically.** In the clean run +`summarize` is firing hard — **316 component log entries**, up to **39,313 tokens removed in a single +request** — with **zero** `Unexpected role` or pairing errors reaching the wire. That check exists +because two earlier "clean" verifications this session proved nothing: one where `summarize` never +fired, and one comparing a reused baseline against itself. + +A 5-task probe **cannot** verify this fix: its trajectories are too shallow to meet `summarize`'s +`min_messages` and span floor, so it never fires even with the trigger lowered to 5,000 tokens. Arm 1 +of the real experiment is therefore the verification, run behind a hard gate — more than 5 failures +and arms 2 and 3 are cancelled — so a failed fix costs one arm rather than three. + +## A THIRD defect, same family, found in the clean run — documented, not fixed + +The re-run's arm 1 shows 2 failures in 38 runs, and **neither is the pairing bug**: + +| count | error | +|---|---| +| 1 | the still-unattributed `400 Bad request` transport fault (6/75 in [iteration 010](../iter010/results.md)) | +| 1 | **new:** `messages.7.content.1.thinking: each thinking block must contain thinking` | + +**Root cause hypothesis, same as the tool-call defect:** `bschemas.ChatContentBlock`'s type enum is +`text` / `image_url` / `input_audio` / `file` / `refusal`. It cannot represent a `thinking` block any +more than it can a `tool_use` one. So **any path that re-marshals an assistant message instead of +emitting its original raw bytes silently drops the thinking content**, and the provider rejects the +empty block. + +In `rebuildCountChanged` that path is reached whenever a body-derived message fails to byte-match its +pre-pipeline form — the `matched < 0` branch, which exists for genuinely new messages (the summary) +and cannot currently tell them apart from a modified survivor. + +**The general shape of all three defects is one fact:** bifrost's schema is a lossy model of an +Anthropic request, so *correctness depends on never re-serialising a message that came from the body*. +Two of the three defects are instances of that rule being broken; the third (`ValidateShape` blindness) +is the same gap in the checker. + +**Deliberately not fixed now.** It is rare (1 in 38), it does not block the experiment, and two +substantive changes to this package already landed tonight without review. Stacking a third unreviewed +change to the byte-losslessness machinery raises the risk of a fourth defect more than it lowers the +risk of this one. A likely fix is to match assistant messages by their tool-call id set — as tool +messages now are — and to **decline the rebuild** rather than fresh-marshal anything body-derived. + +## Arm 1 (`[format, summarize]`) — 94.7% "savings", 63% MORE money, same solves + +Gate passed: **4 failures of 75**, against 28 before the fixes. + +| | `[format]` (iter010) | `[format, summarize]` | +|---|---|---| +| solved | 33 / 69 clean (47.8%) | **33 / 71 clean (46.5%)** | +| **total cost** | **$93.23** | **~$152.09** ($148.89 LOCA + $3.20 CG calls) — **+63%** | +| requests | 2,362 | 2,613 | +| mean pre-compaction request | **~12.2k tokens** | **~141k tokens** — **11.5×** | +| reported `savings_pct` | 24.2% | **94.7%** (349.6M of 369.0M) | +| `summarize` unique saved | — | **129.0M** of 288.2M reported (**2.23× overcount**) | +| `compaction_resets` | — | **777** | + +**This is the sharpest demonstration yet that removing tokens is not saving money.** `summarize` +reports removing 94.7% of all tokens and costs **63% more** for the **same number of solved tasks**. + +**The mechanism is not the expand loop.** That was the obvious hypothesis — the marker invites the +model to call `cg_expand`, restoring the span and regrowing context — and it is **not supported**: +`expand` appears just **twice** in the whole run log. + +What the counters do show is that the *trajectories diverged enormously*: mean pre-compaction request +size is **11.5× larger** than the lossless arm's on the identical tasks and band. A plausible reading, +**not established here**, is that lossy summarisation makes the agent redo work — re-reading files and +re-querying tools it can no longer see — so total context grows even as each individual request is +smaller. `cache_write=2.3M` against `fresh_input=59.4M` and `cache_read=53.9M` is consistent with the +prefix being invalidated repeatedly, billing fresh where the lossless arm billed cache reads. + +**Two cautions on the 94.7%.** First, `tokens_before` = 369M is a *counterfactual* — it assumes the +same trajectory would have occurred without compaction, and the trajectory is exactly what changed. +Second, `summarize`'s own overcount ratio is **2.23×**, so its unique contribution is 129M, not 288M. +Both push the honest figure well below the headline. + +**Bearing on the deferral question:** if this holds in arms 2 and 3, it strengthens rather than weakens +the case for *deferring* summarisation — a summary here is not merely lossy, it appears to be actively +expensive. Whether `coref` and the fold reduce `summarize`'s firing rate, and whether cost falls with +it, is precisely what the remaining arms measure. + +## ALL THREE ARMS — deferral is real, monotonic, and tracks cost + +**Cost: ~$372** for the three arms ($361.12 LOCA + $8.40 CG model calls). + +| arm | `summarize` fired | **per request** | mean request | total cost | solved | errors | +|---|---|---|---|---|---|---| +| `[format]` *(iteration 010)* | – | – | 12.2k | **$93.23** | 33 / 69 | 6 | +| 1 `[format, summarize]` | 1,466 | **56.1%** | 141k | **$152.08** | 33 / 71 | 4 | +| 2 `[format, coref, summarize]` | 805 | **42.3%** | 72.9k | **$125.00** | 32 / 75 | **0** | +| 3 `[format, coref, extract_llm, summarize]` | 647 | **36.9%** | 99.3k | **$92.44** | **35 / 71** | 4 | + +### The deferral claim is supported — this is the headline + +**Each layer of selective compaction makes the blunt summariser fire less often, monotonically: +56.1% → 42.3% → 36.9% of requests.** That is the mechanism the whole project was premised on, measured +live for the first time, and it is not subtle — the fold cuts summarisation by **a third** relative to +summarising alone. + +**Cost tracks deferral almost exactly:** $152.08 → $125.00 → $92.44. Less summarising, less money. + +**And the fold arm is best on every axis at once** — most tasks solved (35), lowest cost ($92.44), +most deferral (36.9%). It is the only configuration in this entire body of work that beats the lossless +baseline on cost *and* solves while removing substantially more content. + +### Four things that must be said against the headline + +**1. The cost margins over lossless are inside the noise.** $92.44 vs $93.23 is 0.8%, and +[iteration 012](../iter012/results.md) established that per-arm LOCA cost cannot price a component — +two arms differing by one component came out 4.9% apart in *opposite directions* across successive +iterations, driven by trajectory length. The $152 → $92 *ordering* is large enough to believe; the +final 0.8% is not. + +**2. No reward difference is detectable.** Paired, task-clustered: + +| comparison | harm | gain | p | bound | +|---|---|---|---|---| +| arm 1 → arm 2 | 3 | 2 | 1.000 | ≤44% | +| arm 1 → arm 3 | 4 | 6 | 0.754 | ≤51% | + +Solve counts across all four configurations are 32–35 of 75 — indistinguishable. **Deferral buys cost, +not accuracy**, on this evidence. Note arm 1 → arm 2 is confounded (4 errors vs 0, asymmetric), while +arm 1 → arm 3 is balanced (4 vs 4) and therefore the cleaner comparison. + +**3. `summarize` is never NECESSARY at this band, which limits what this can mean.** Contexts run +12–44k against Sonnet-5's 1M window, so nothing here is close to exhausting. The trigger +(`min_request_tokens: 30000`) was chosen *specifically to make `summarize` fire*, so these arms measure +**summarising when it is not needed** — which is why arm 1 is so much worse than lossless. The question +originally asked — *does deferral pay when summarisation is genuinely forced?* — **cannot be answered +here**, because these tasks never run out of context. That needs a band where context truly exhausts, +and iteration 008 showed the 128k data is unreliable. **This is a limitation of the design chosen, not +of the result.** + +**4. Mean request size is not monotonic** — 141k → 72.9k → 99.3k. The fold defers *most* while carrying +*larger* requests than arm 2, so "defers more" and "smaller contexts" are not the same axis. Unexplained. + +### The overcounting caution, again + +Reported `savings_pct` reaches **94.7% / 88.1% / 92.1%**, and it should not be quoted. `summarize`'s +overcount ratio is **2.23×**, and `tokens_before` (369M in arm 1) is a *counterfactual* assuming an +unchanged trajectory — when the trajectory is exactly what changed. Unique contributions are far +smaller, and `coref`'s is tiny throughout: **414,724** tokens in arm 2, **233,424** in arm 3. + +That last figure deserves emphasis. **`coref`'s own unique removal is ~0.2–0.4M tokens, yet its presence +cut summarisation by 25% per request and cost by $27.** If that is causal, the value of `coref` is not +the tokens it removes but the *summarisation it prevents* — a leverage effect no yield metric in this +repo would ever have shown. It could equally be trajectory divergence; distinguishing them needs a +design that holds trajectory fixed, which no arm here does. + +## RETRACTION of the economic magnitude: the summarize config was unrealistic + +Raised in review, and correct: **`resummarize_tokens: 6000` is far too small for this workload, and the +first-summary trigger was ~30× too eager.** + +Measured, from this arm's own traffic: + +| single tool output | tokens | +|---|---| +| p50 | 192 | +| **p90** | **6,419** | +| max | 20,720 | + +**12% of individual tool outputs exceed 6,000 on their own**, and p90 sits *at* the threshold. So the +un-summarised tail crosses it within a turn or two — which is why arm 1 produced **10 fresh summaries +per run** (751 across 75 runs). It was not summarising once near a limit; it was re-summarising +continuously. + +And the first trigger, `min_request_tokens: 30000`, was set against Sonnet-5's **1M** window — **3% of +it**. Claude Code compacts near ~90%. **Arm 1 is therefore NOT the `/compact` analogue this document +called it**; it is "summarise at 3% of the window, then re-summarise every 6k". The trigger was chosen +from the observed size distribution *specifically to make `summarize` fire*, because at the 32k band +contexts (12–44k) never approach a 1M window — and in forcing it to fire, a configuration nobody would +deploy was built. + +**What is retracted:** the economic magnitude. Summarisation's **+63% cost penalty is largely an +artifact of this configuration**, not a property of summarisation. The $152 → $125 → $92 ordering +measures the cost of *over-eager resummarisation* and how much `coref` relieves it, and **does not +transfer to a realistic deployment.** + +**What stands:** the deferral *mechanism* and its quantitative fit. `coref` removes ~2,490 tokens per +request against a 6,000-token threshold — 41% of the rollforward budget — and produced **−42% fresh +summaries** against a predicted −41%. Threshold leverage is real. But what it defers is over-eager +resummarisation. + +Also under-weighted at first: in the summarize arm requests reached **769,874 tokens**, genuinely +approaching the 1M window, against ~110k in the lossless arm. Summarisation did not merely cost more — +it drove contexts *toward* exhaustion. + +### What a sound re-run requires (→ iteration 013) + +| knob | this run | corrected | +|---|---|---| +| declared context window | 1M (Sonnet-5's real window) | **128k**, via `MODEL_INFO_URL` (key `max_input_tokens`) | +| enforcement of that window | **none** — requests reached 770k | LOCA `--clear-trigger-tokens 128000` + `--use-clear-tool-uses` | +| first summary trigger | 30k = **3%** of window | **~100k = 78%** of 128k | +| `resummarize_tokens` | **6,000** (= p90 of ONE tool output) | **~20,000**, so the tail must accumulate several outputs | +| band | 32k (contexts 12–44k, never pressured) | **128k**, where contexts naturally approach the limit | + +The enforcement row matters most and was missing entirely here: with LOCA's clearing active the baseline +becomes **genuinely lossy**, which is the case iteration 010 could not test at all — its shim reported +`repairs=0`, meaning LOCA's trimmer never fired and the baseline kept everything. Only against a lossy +baseline can selective compaction show a *gain* rather than at best breaking even. diff --git a/docs/experiments/loca/iter012/PREREGISTRATION.md b/docs/experiments/loca/iter012/PREREGISTRATION.md new file mode 100644 index 00000000..0450d419 --- /dev/null +++ b/docs/experiments/loca/iter012/PREREGISTRATION.md @@ -0,0 +1,74 @@ +# Iteration 012 — pre-registration: what does the FOLD add, without `summarize`? + +**Written before the run.** Nothing had been launched when this was committed. + +[Iteration 011](../iter011/results.md) is blocked: any arm containing `summarize` fails ~37% of runs +because `apply.rebuildCountChanged` drops the body message holding parallel `tool_result`s +(reproduced in `apply/parallel_wire_test.go`, `2ec2445`). Rather than idle on that, this iteration +tests a question that does **not** touch the broken path. + +## Question + +`coref` alone added ~1pp of removal over lossless and no reward effect +([iteration 010](../iter010/results.md)). Does adding **`extract_llm` with full-body reach** — the +"fold", where one LLM call both trims content and reaches past the cached prefix — add yield worth +paying for, and at what reward cost? + +This is the arm the original selection experiment could not speak to: it measured *decision quality* +on captured traffic and explicitly could not speak to reward. + +## Arms + +| arm | pipeline | source | +|---|---|---| +| baseline | `[format]` | **reused** from iteration 010 (`…_144315`, n=75, $93.23) | +| `s4-fold` | `[format, coref, extract_llm]` | new, this iteration | + +`extract_llm` config: `allow_cached_prefix: true` (full-body reach), `allow_on_caching_backend: true` +(required — these runs are `CACHE_MODE=on`, where the component is disabled by default), +`strategy: code`, `min_tokens: 3000`, `llm_max_per_request: 4`. + +**The baseline is reused rather than re-run**, and that is a deliberate trade recorded up front: it +saves ~$93 and the config is byte-identical (`ab-format.yaml`, same 32k task set, same binary, same +`CACHE_MODE`/`INJECT_EXPAND`). The cost is that the arms ran at different times, so any drift in +gateway behaviour is not controlled. Pairing is still per `(task, seed)`, and the tasks are +deterministic given their seed, so the comparison remains paired. **If the baseline's error count or +solve rate differs materially from iteration 010's recorded 6 and 33/69, the reuse is invalid and a +fresh baseline is required** — that check runs before any conclusion. + +## Endpoints, declared now + +**Primary — yield, and whether it is repaid.** `extract_llm`'s acted count, tokens removed, model +calls, and **$/arm**. Iteration 010's central negative result was that removing *more* tokens cost +*more* money ($93.23 → $97.80 for +0.9pp). The fold spends real model calls on top, so the +pre-registered question is not "does it remove more" — it will — but **"is the removal repaid?"** +Reported as the yield triple: eligible / acted / refused-for-economics. + +**Secondary — reward.** Paired, two-sided, via `deploy/harbor/reward_pairs.py`, quoted on the +**task-clustered** end. LOCA has 15 independent tasks, so the harm bound floors near ≤18% at zero +events and cannot be improved on this benchmark. + +**Tertiary — churn.** Iteration 010's most informative number was that 23% of pairs flipped outcome +with no direction. The fold does strictly more rewriting, so churn is expected to rise; whether it +does is worth knowing independently of the totals, because an unpredictable agent is a cost even at +constant mean. + +## How each outcome will be read + +| outcome | reading | +|---|---| +| removal up **and** $/arm down | the first configuration in this work that actually saves money | +| removal up, $/arm up | consistent with iteration 010 — token removal is not cost saving under caching; the fold is not worth its calls at this band | +| reward harm at task level beyond the bound | the fold trades accuracy for tokens; requires pricing, not shipping | +| `extract_llm` never acts | the gate suppressed it; report the refusal reasons, and the arm says nothing about the fold itself | + +## Pre-declared threats + +- **The HTML-400 transport fault is still unattributed** (1/75 without a proxy, 6/75 with one). Errors + are treatment-dependent, so **both per-protocol and intent-to-treat are reported**, per iteration + 010's amendment 2. +- **`extract_llm` pays a cache-write** when it reaches past the cached prefix. That is the cost the + break-even in `prefix_econ.go` exists to price, and this arm is a live test of whether it holds. +- **Reused baseline** — validity check stated above, run before conclusions. +- A **null yield** result is a real possible outcome: `extract_llm` fired only 38 times at 64k, and 32k + contexts are smaller. diff --git a/docs/experiments/loca/iter012/results.md b/docs/experiments/loca/iter012/results.md new file mode 100644 index 00000000..71f96c08 --- /dev/null +++ b/docs/experiments/loca/iter012/results.md @@ -0,0 +1,95 @@ +# Iteration 012 — the fold: the headline savings number is inflated 3–8×, and components cannibalise each other + +**Date:** 2026-08-21 · **Pre-registered:** `ded9636`, before the run · **Cost:** $89.52 total +($88.56 LOCA + $0.96 of CG's own model calls) · **n = 75**, baseline reused from +[iteration 010](../iter010/results.md). + +## The result that matters: `savings_pct` is not a saving + +| arm | `savings_pct` | **unique** tokens saved | cache writes | CG model $ | LOCA $ | total $ | +|---|---|---|---|---|---|---| +| `[format]` | 24.2% | **6,977,633** | 2,113,551 | – | 93.23 | **93.23** | +| `[format, coref]` | 25.1% | 6,567,928 | 3,118,809 | – | 97.80 | **97.80** | +| `[format, coref, extract_llm]` | **26.0%** | **5,259,955** | 3,040,796 | 0.96 | 88.56 | **89.52** | + +**The ordering inverts.** By `savings_pct` the fold looks best (26.0% > 25.1% > 24.2%). By tokens +actually removed once each, it is **worst** — 5.26M against the lossless baseline's 6.98M. + +Two mechanisms, both measured, neither previously visible: + +### 1. Overcounting, by a factor of up to 8 + +CG's own `/stats` reports both figures, and they diverge sharply for the non-deterministic components: + +| arm | component | acted | reported saved | **unique** | overcount | +|---|---|---|---|---|---| +| `+coref` | `coref` | 86 | 511,721 | **125,724** | **4.07×** | +| fold | `coref` | 56 | 601,621 | **71,612** | **8.40×** | +| fold | `extract_llm` | 63 | 625,702 | **223,279** | **2.80×** | +| all | `format` | 599–626 | 4.97–6.98M | same | 1.00× | + +The same removed content is counted again on every later turn that replays the frozen rewrite. +`format`, which rewrites in place deterministically, has a ratio of exactly 1. **So every headline +`coref` yield figure in this log — including iteration 010's "511,721 tokens" — is inflated, and the +honest number is 4–8× smaller.** In the fold arm `coref`'s real contribution is **71,612 tokens of +23,775,292**: 0.3%. + +### 2. The components cannibalise each other + +`format`'s own unique saving **falls** as components are added ahead of it: 6,977,633 → 6,442,204 → +4,965,064. `coref` and `extract_llm` remove content `format` would otherwise have reformatted more +cheaply, so **total unique removal goes DOWN as the pipeline grows**. That is an architectural +result, not a tuning detail, and no measurement in this work could see it before, because +`savings_pct` sums per-component credit and therefore rewards double-counting. + +## The apparent cost saving is NOT attributable to compaction + +The fold's $89.52 against the baseline's $93.23 looked like the first money-saving configuration in +this work. **On inspection it is trajectory variance, and the pre-registered reading should not be +applied.** The token economics point the other way: + +- the fold pays **927,245 more cache-write tokens** than the baseline (≈ +$2.32 at the 1.25× write rate); +- it removes **1,717,678 fewer unique tokens** (≈ +$0.34 at cache-read rate, +$3.44 at fresh); +- plus **$0.96** of its own model calls. + +So on compaction-attributable terms the fold is break-even at best. The $3.71 difference is the +agent's own path — it also *solved more tasks* (36 vs 33), and a run that succeeds differs in length +from one that flails. [Iteration 010](../iter010/results.md) showed the same magnitude in the +**opposite** direction ($97.80 vs $93.23), which is the tell: **per-arm LOCA cost is dominated by +trajectory divergence and cannot price a component.** Only the token counters can. + +## Reward: direction favours the fold, still not significant + +| reading | harm | gain | p | harm bound | +|---|---|---|---|---| +| per-protocol, pair level (n=69) | 6 | 9 | 0.607 | ≤16% | +| **per-protocol, task-clustered (n=15)** | **3** | **6** | **0.508** | **≤44%** | +| intent-to-treat, pair level (n=75) | 6 | 12 | 0.238 | — | +| intent-to-treat, task-clustered | 3 | 6 | 0.508 | ≤44% | + +3 harm / 6 gain is the most favourable direction measured so far, and under ITT the fold solves +**39 vs 33**. But p=0.508 at 15 independent tasks, and the bound is ≤44%. **This is not evidence the +fold helps reward; it is an absence of evidence that it hurts, at a resolution too coarse to be worth +much.** Churn is 15/69 = 22%, essentially unchanged from iteration 010's 23% — the fold does more +rewriting without flipping more outcomes. + +Errors: **0 in the fold arm** against 6 in the baseline, consistent with both earlier arms. + +## A pre-registered check that could not work + +The pre-registration promised to validate the reused baseline by confirming its error count and solve +rate still matched iteration 010's. **That check is vacuous**: reusing the same run directory compares +the data to itself, so it matched trivially and could not detect the risk it was written for +(gateway drift between arms run 3.5 hours apart). Recorded as a design error in the pre-registration, +not as a passed check. A real check needs a fresh baseline, which is what the $93 was saved by +avoiding — so the saving bought an unverifiable comparison. + +## What this settles + +**Settled:** `savings_pct` overstates non-deterministic components by 3–8× and must not be quoted +again without its unique counterpart; adding components can *reduce* total unique removal; per-arm +LOCA cost cannot attribute cost to a component; `format` alone does ~95% of the real removal work. + +**Not settled:** whether the fold's reward direction is real (needs more independent tasks than LOCA +has); and the deferral question, still blocked on the `apply` parallel-tool-result defect +(`2ec2445`). diff --git a/docs/experiments/loca/iter013/results.md b/docs/experiments/loca/iter013/results.md new file mode 100644 index 00000000..c0c6d5ee --- /dev/null +++ b/docs/experiments/loca/iter013/results.md @@ -0,0 +1,248 @@ +# Iteration 013 — 128k band, realistic summarize config (IN PROGRESS) + +**Pre-registered:** the corrections in [iteration 011's retraction](../iter011/results.md). +Four arms, `cg-proxy-v7`, 128k declared window, `min_request_frac 0.78` (~100k), +`resummarize_tokens 20000`, LOCA clearing at 128k, baseline first. + +## Correction: `shim repairs` is the WRONG indicator for "the baseline is lossy" + +The pre-run gate used the shim's `repairs` count to check that LOCA's clearing was firing, on the +reasoning that clearing orphans `tool_use`/`tool_result` pairs. **That indicator does not work**, and +it reported `repairs=0` in every arm, which nearly led to recording the baseline as lossless for a +second time. + +The right counter is **`compaction_resets`**, which counts turns whose cached-prefix boundary restarted +because *the transcript shrank under a stable session id* — i.e. the agent compacted its own +transcript. In the blunt-reset arm: + +| signal | value | meaning | +|---|---|---| +| `compaction_resets` | **319** | LOCA's clearing fired ~4.3× per run | +| `llm_calls` | **0** | no CG model call — so the shrinking is not CG's doing | +| components acted | `format` only (lossless) | CG removed no information | +| shim `repairs` | 0 | LOCA's `clear_tool_uses` evidently rewrites tool-output CONTENT rather than deleting messages, so no pair is ever orphaned and the shim has nothing to repair | + +**So the baseline in this iteration IS genuinely lossy** — 319 agent-side compactions — which is +exactly what [iteration 010](../iter010/results.md) could never achieve and what the whole move to +128k was for. The design works; the instrumentation choice was wrong. + +Recorded because it is the same class of error as the three vacuous checks already logged: a signal +was chosen for its plausibility rather than verified to move when the thing it measures moves. + +## Arm 1 — `[format]`, blunt reset (complete) + +| | | +|---|---| +| requests | 1,171 (mean arriving **86,125** tokens) | +| `format` acted | 481 (41.1%/req), **14,500,733** unique tokens | +| `compaction_resets` | **319** | +| cache write / fresh input | 7,403,630 / 37,666,936 | +| CG model calls | 0 | + +Arms 2–4 and the merged arm ([iteration 014](../iter014/PREREGISTRATION.md)) are queued. + +## The 128k band exposes a NEGATIVE INTERACTION between the proxy and the agent's own context management + +| arm | solved | errors | mean arriving request | LOCA $ | CG model $ | total | +|---|---|---|---|---|---|---| +| `[format]` blunt reset | 15 / 72 (20.8%) | **3** | 86,125 | 189.95 | 0 | **189.95** | +| `[format, coref, extract_llm, summarize]` | 16 / 60 (26.7%) | **15** | **692,613** | 204.94 | 22.66 | **227.60** | + +The extra twelve errors are not random. Five of them are: + +``` +prompt is too long: 1,584,405 … 6,035,894 tokens > 1,000,000 max +``` + +**The agent's transcript reached 6 MILLION tokens — 6× the model's real window — and compaction could +not bring it back under.** One further error is the known `thinking`-block defect +([iteration 011](../iter011/results.md)). + +### RETRACTED: the "proxy suppresses the agent's clearing" explanation + +The first explanation offered here was that CG's compaction keeps requests under the provider's +`clear_tool_uses` trigger, so the agent's own clearing never fires and its raw history grows unchecked. +**That is wrong, and the check that refutes it is one line:** + +| arm | trajectories | with `context_management_events` | total events | +|---|---|---|---| +| `[format]` blunt reset | 75 | **0** | **0** | +| separate components | 75 | **0** | **0** | +| merged | 75 | **0** | **0** | + +**Server-side context management never ran in ANY arm, including the baseline.** So the baseline was +never "protected by the provider's unconditional trigger" — there was no trigger firing anywhere. The +retracted claim had the shape of a good explanation and was asserted without checking the one counter +that records the mechanism. + +LOCA does request it (`context_management` edits plus `betas: +["context-management-2025-06-27"]`), and CG passes the parameter through byte-identically (verified by +`apply.TestContextManagementParamSurvivesCountChange`). So the feature is asked for and forwarded, yet +never applied. Whether the gateway strips the beta, or the Bedrock-routed `aws/claude-sonnet-5` does not +implement it, is under direct test. + +**What still stands, and what does not.** The measurements stand: the baseline had **0** `prompt is too +long` errors against the CG arms' **5**, and mean arriving requests of **86,125** against **692,613** on +identical tasks. What does not stand is the *causal* story. With no server-side clearing anywhere, the +difference must come from the trajectories themselves — the CG arms ran 3× more requests, each ~8× +larger — which is the same trajectory-divergence confound +[iteration 012](../iter012/results.md) established makes per-arm behaviour hard to attribute. + +So the honest position is narrower: **at this band, pipelines including CG compaction reached contexts +that exceeded the model's window while the lossless baseline did not, and why is not established.** + +**Caveats, because this is one arm.** `compaction_resets` cannot separate CG's own shrinking from +LOCA's clearing in the CG arms (summarize acted 2,511 times, and each shrink registers), so the reset +counts are not comparable across arms — only the baseline's 319 is unambiguous. And the two arms +differ by four components at once. Isolating this needs an arm where CG compacts but LOCA's clearing +threshold is set from the *pre*-compaction size, which no current knob exposes. + +## Yield at this band, in unique tokens + +| component | acted | reported saved | **unique** | overcount | +|---|---|---|---|---| +| `summarize` | 2,511 (70.8%/req) | 2,184,615,650 | **1,592,800,663** | 1.37× | +| `format` | 1,118 (31.5%/req) | 177,283,065 | 177,283,065 | 1.00× | +| `extract_llm` | 223 (6.3%/req) | 23,131,709 | **1,953,083** | **11.8×** | +| `coref` | 513 (14.5%/req) | 16,559,942 | **874,969** | **18.9×** | + +`coref`'s unique contribution is **874,969 of 2.46 billion tokens before — 0.04%** — while its reported +figure overstates it **19-fold**. `extract_llm`'s is 0.08%. On unique tokens at this band the two +components under study are rounding error against `format` and `summarize`. + +## ANSWERED: why CG arms carry 8× the context — the agent re-runs tools instead of expanding + +With server-side clearing inert everywhere, the size difference cannot be a reset. Measured directly: + +| arm | steps/run | duplicate tool calls/run | **% of calls that are exact repeats** | `cg_expand` calls | +|---|---|---|---|---| +| `[format]` lossless baseline | 15.5 | 0.1 | **0.4%** | **0** | +| merged (keeps 94% of candidates) | 39.2 | 2.8 | **9.9%** | **0** | +| separate (4 components, most removal) | 47.0 | 8.2 | **25.3%** | **0** | + +**The repeat rate is dose-dependent in removal.** ~~And `expand` was called zero times in 225 runs.~~ +**That was wrong — it was grepped for `cg_expand`, and the tool is named `context_guru_expand`.** The +model calls it constantly and is REFUSED; see the section below. + +The loop: CG removes content and leaves a `<>` marker plus an `expand` tool. The agent does +not expand — it **re-issues the same tool call with the same arguments**. That returns *fresh* output, +so the transcript grows rather than shrinking, which invites more compaction, which removes more, which +provokes more repeats. Hence 3× the steps and 8× the mean request, ending in five requests that exceeded +the model's window outright. + +The baseline stays small for an unremarkable reason: nothing is removed, so nothing is lost, so nothing +is repeated (0.4%). + +### This is a fourth outcome the design did not enumerate + +`corefstub.go` reasons carefully about a wrong cut having three outcomes: + +1. the model notices and expands the right marker — one round-trip plus a cache-write; +2. it notices something is missing but cannot tell WHICH marker holds it — several expands, or it gives up; +3. it never notices, and answers from less information than it had. + +The measured outcome is none of these. **The model notices, does not expand, and re-runs the tool.** +That is *worse than (1)* — a full tool execution plus fresh output rather than a cached round-trip — and +unlike (3) it **compounds**, because each repeat enlarges the transcript that provoked it. + +It also undercuts the reversibility argument that justifies lossy compaction here — though for a +different reason than first recorded. Recovery is not declined; it is **broken**. See below. + +### Caveats + +- Duplicate detection is regex-based over each step's action payload. The **monotone ordering** across + arms (same regex, same corpus) is the load-bearing part; the absolute percentages are approximate. +- Longer trajectories offer more opportunity for repeats, but the figure quoted is repeats as a **share + of calls**, so it is not merely a length artifact. +- **Not established:** that the specific repeated calls are the ones whose output was compacted. That + needs per-marker correlation, which the current capture does not record. +- `INJECT_EXPAND=auto` was set in every CG arm, so the tool was present and advertised. Whether a + different prompt or a more legible residue would get it used is untested. + +## WHY the agent re-runs tools: expand is CALLED and REFUSED + +An earlier draft of this document claimed `expand` was never called. That was a grep error — it searched +for `cg_expand`, and the injected tool is named **`context_guru_expand`**. Corrected: + +| arm | `context_guru_expand` mentions | **`Tool '…' not found` rejections** | +|---|---|---| +| `[format]` lossless baseline | 0 | 0 | +| separate components | 38 | **17** | +| merged | **108** | **48** | + +**The model reaches for recovery constantly and roughly half those calls are refused outright.** The +refusal text is the client's: `"content": "Tool 'context_guru_expand' not found"`. + +### The mechanism, and it is architectural rather than a prompt problem + +1. `expand.Inject` in `InjectAuto` advertises the tool **only when the outgoing request carries a + marker**, and `serve` ties interception to the same test — `advertised := expand.HasTool(provider, + body)` on the OUTGOING body. One condition, deliberately, so a tool is never advertised without + being intercepted *on that request*. +2. **The client's history never carries the markers.** LOCA stores its own original content and resends + it every turn — confirmed: **0 of 75** trajectories contain `< **CORRECTION (iteration 019).** The conclusion below — that the model "declines to act on +> evidence", read as a negative answer on the merged design — is **not safe**. This arm returned +> **2,078 decisions across 2,030 calls = 1.02 candidates per call**, and `merged_kept_whole_batch` +> never fired, which rules out empty-array replies as the explanation. Bulk adjudication is a +> *comparative* mechanism measured at ~15 candidates; at 1.02 this arm was running the **per-output** +> design already refuted at 6% live-kept, not the bulk design it is labelled as. The cause was the +> prefix eligibility pre-filter removing 149,681 candidates (fixed at `extract_llm.go:695`). The +> efficiency and deferral findings are unaffected — they do not depend on batch size. See +> [iteration 019](../iter019/results.md). + +## The result + +| | separate components | **merged** | +|---|---|---| +| pipeline | `[format, coref, extract_llm, summarize]` | `[format, extract_llm(merged), summarize]` | +| solved (per-protocol) | 15 / 56 | 13 / 56 | +| **solved (intent-to-treat, n=75)** | **16** | **16** | +| errors | **15** | **8** | +| `extract_llm` acted | 223 (6.3%/req) | **1,827 (61.3%/req)** | +| `summarize` acted | 2,511 (70.8%/req) | **1,754 (58.8%/req)** | +| **CG model calls** | 2,700 | **2,030 (−25%)** | +| **CG model spend** | **$22.66** | **$9.50 (−58%)** | +| LOCA spend | $204.94 | $223.22 | +| total | **$227.60** | **$232.72** | + +**Reward is indistinguishable.** Intent-to-treat: **16 vs 16 solved**, 6 harm / 6 gain, p=1.000. +Task-clustered per-protocol: 2 net-harmed, 0 net-gained, p=0.500, harm bound ≤39%. There is no reward +signal here in either direction, at a resolution too coarse to be worth much — and the error counts are +asymmetric (15 vs 8), so per-protocol exclusion is confounded and ITT is the reading that survives. + +## What the gate counters say, and this is the real answer + +The merged counters exist to separate "the model kept everything" from "the model was never asked" from +"the model returned junk". They are unambiguous: + +| verdict | count | share | +|---|---|---| +| **`merged_keep`** | **1,824** | **88%** | +| `merged_drop` | 249 | 12% | +| `merged_trim` | **1** | 0.05% | +| `merged_trim_not_contained` | 4 | rejected as invented | +| `merged_unparseable` | 19 | 0.9% | +| `merged_call_failed` | 9 | 0.4% | + +**Shown fifteen outputs with their co-reference evidence and told the true cost of a wrong removal, the +model keeps 88% of them and trims essentially never (1 of 2,074).** The pre-registered reading for this +outcome was written before the run: *"`merged_keep` dominates → the model declines to act on evidence; +consistent with iteration 009 and a negative answer."* + +That is what happened. And it is the *opposite* failure from the one the per-output design showed — +there, a model shown one output at a time dropped nearly everything (6% live-kept). Shown many at once +and told removals are not recoverable, it becomes conservative instead. Both are the same underlying +fact from opposite sides: **the model is not discriminating between outputs; it is responding to how the +question is framed.** + +The single trim in 2,074 decisions is worth its own note. Trimming — returning a subset of records +verbatim — is where the merged design's value was supposed to lie, since it is the judgement an exact +matcher cannot make. The model essentially never did it. + +## Where merged genuinely wins + +**Efficiency, which was the second pre-registered endpoint.** One call per request instead of up to +`llm_max_per_request`: **2,030 calls against 2,700 (−25%)** and **$9.50 of CG model spend against +$22.66 (−58%)**, for the same reward. It also deferred `summarize` more (58.8% vs 70.8% of requests) +and had **half the errors** (8 vs 15). + +Total cost is $232.72 vs $227.60 — 2% apart, and [iteration 012](../iter012/results.md) established +that per-arm LOCA cost cannot price a component, so that difference should not be read as anything. +**The CG-spend comparison is the one that is attributable, because it is a direct count of this +component's own calls.** + +## Yield, in unique tokens + +| arm | component | acted | reported | **unique** | overcount | +|---|---|---|---|---|---| +| merged | `extract_llm` | 1,827 | 573,265,890 | **18,065,171** | **31.7×** | +| separate | `extract_llm` | 223 | 23,131,709 | 1,953,083 | 11.8× | +| separate | `coref` | 513 | 16,559,942 | 874,969 | 18.9× | + +Merged `extract_llm` removes **9× more unique tokens** than separate `extract_llm` + `coref` combined +(18.1M vs 2.8M) — so the fold does more work per call, which is consistent with acting on 61.3% of +requests instead of 6.3%. **It also carries the largest overcount ratio measured anywhere in this work, +31.7×**, which is what happens when a component acts often and its rewrites are replayed on every later +turn. Anyone quoting its 573M "saved" would be overstating by a factor of thirty. + +## Verdict + +**The merged design is cheaper per decision and no worse on reward, but it does not do the thing it was +built to do.** It was meant to catch what the deterministic index structurally cannot — Tier-2/3 reuse +and the anchor-vs-payload call. Instead it keeps 88% and trims once in two thousand. + +Combined with [iteration 009](../iter009/results.md) — where the free index beat every model arm at 95% +live-kept and 11% false-drop, and neither floor-symmetry nor a widened Tier-2 ground truth closed the +gap — the honest position is that **model-based selection has now failed in both directions**: reckless +when shown one output, inert when shown many. The remaining case for merged is purely economic (fewer +calls for the same outcome), not qualitative. + +**Not settled:** whether a middle framing exists between "drops everything" and "keeps everything". The +two arms differ by ~26 points of prompt framing in iteration 009's measurements, so framing clearly +dominates the model's behaviour — which is itself an argument that the decision is not really being made +on the evidence. + +## A rig bug that invalidated two arms and nearly took the headline with it + +Found while reading the third arm's counters: they listed `coref` and `extract_llm` as having acted, +but that arm's pipeline was `[format, summarize]`, which contains neither. + +**Cause.** `stage6.sh` was generated with `sed 's|cg-proxy-v7|cg-proxy-v8|g' stage5.sh`, but the kill +line reads `pkill -f "cg-proxy-v[7]"`. The brackets mean sed's literal pattern never matched it, so the +generated script **killed v7 while launching v8** — and v8 proxies were never cleaned up between arms. +When a later arm reused a port, the previous arm's proxy was still bound to it, answered `/healthz` +identically, and the new proxy silently failed to bind. That arm then ran the **previous arm's +pipeline** and reported that proxy's **cumulative** counters. + +| arm | port | components that acted | matches its config? | +|---|---|---|---| +| `s7-xllm-sum` | 6700 | coref, extract_llm, format, summarize | **yes** — valid | +| `s7-merged` | 6800 | extract_llm, format, summarize (**no coref**) | **yes** — valid | +| `s7-sum` | 6700 (**reused**) | coref, extract_llm, format, summarize | **NO** — invalid | + +`s7-sum` reported 5,779 requests ≈ 3,548 + 2,231, i.e. the earlier arm's total plus its own. + +**The headline comparison survives, and not by design.** Arms 1 and 2 alternated ports 6700/6800, so +each got a genuinely fresh proxy, and their component sets match their configs exactly — `s7-merged` +records **no `coref`**, which is the one thing that would betray a stale-proxy mix-up, since every +other arm has it. Had arms 1 and 2 shared a port, the merged result would have been garbage and would +have looked entirely plausible. + +**Two fixes, both at the source rather than in the copy:** the kill pattern is now +`cg-proxy-v[0-9]` so it covers every version, and the generated `stage6.sh` is deleted. A `/healthz` +200 is also no longer accepted as proof the intended proxy answered — the arm now requires its OWN +proxy log to show a `pipeline=` line and echoes which pipeline bound the port, so a stale proxy cannot +be mistaken for a fresh one. + +**Not re-run:** the two invalidated arms were the expendable deferral context, ~$460 for the pair, and +the question they served is already answered at 32k by [iteration 011](../iter011/results.md). Re-running +them is a budget decision, not a correctness one. + +This is the same failure family as the three vacuous checks and the wrong lossy-baseline signal already +logged: **a check that cannot distinguish success from a plausible-looking substitute.** `/healthz` +returning 200 answered "is something listening" when the question was "is MY process listening". diff --git a/docs/experiments/loca/iter016/PREREGISTRATION.md b/docs/experiments/loca/iter016/PREREGISTRATION.md new file mode 100644 index 00000000..8cca8dff --- /dev/null +++ b/docs/experiments/loca/iter016/PREREGISTRATION.md @@ -0,0 +1,55 @@ +# Iteration 016 — pre-registration: merged arm with the expand tool always available + +**Written before the run.** Merged arm only, by direction; a matched baseline follows **only if** this +one shows improvement and no new defects. + +## What changed and why + +`INJECT_EXPAND=always` instead of `auto`. Under `auto` the expand tool is advertised only on turns whose +outgoing body carries a `<>` marker, and interception is deliberately tied to the same +per-request test (`expand.HasTool(body)`). A marker appears only when a component actually acted that +turn — `summarize` 70.8%, `coref` 14.5%, `extract_llm` 6.3% — so roughly a third of turns had no marker, +no tool and no interception. The model treats the tool as a persistent capability (its own history shows +it calling it) and called it on those turns; the raw `tool_use` was relayed to LOCA, which has no such +tool, and answered **`Tool 'context_guru_expand' not found` — 48 of 108 attempts.** Recovery refused, +the model re-ran the original tool instead. + +Verified before running: the advertise rule (`expand.TestWhenIsExpandAdvertised`) and end-to-end, that a +**marker-free** request now leaves the proxy with `n_tools=2, has_expand=true`. + +## Pre-declared success criteria + +| # | measurement | current | expected if the diagnosis is right | +|---|---|---|---| +| 1 | `Tool '…' not found` in the LOCA log | **48** | **0** | +| 2 | `has_expand` transitions (flap log) | inferred, uncounted | **0** | +| 3 | cache-read share of input | **47.5%** | **> 47.5%**, toward the lossless arm's 69.2% | +| 4 | exact-repeat tool calls | **9.9%** | **< 9.9%** | +| 5 | solved (ITT, n=75) | 16 | ≥ 16 | +| 6 | total cost | $232.72 | < $232.72 | + +**1 and 2 are the fix working. 3 and 4 are the mechanism it was supposed to break. 5 and 6 are whether +any of it matters.** If 1 and 2 pass but 3–6 do not move, the diagnosis was right and the consequence +was small — which is a real answer, not a failure. + +## New instrumentation + +The capture hop now records, per request, how many tools were advertised, whether expand was among them, +and whether a marker was present. **This turns the flap from an inference into a count** — it had been +argued from the advertise rule plus per-arm action rates, never observed per request. + +Stated limit: transitions are counted in arrival order across interleaved sessions, so the figure is an +**upper bound** on per-session flapping. Under `always` it should be 0 regardless, which is why it is +still a usable check. + +## Threats + +- **`always` has its own hazard**, named in `expand/inject.go`: an unresolvable expand call on a + marker-free turn gets replayed to the client and "reads as the summary came back empty". That hazard is + already occurring in its worst form, so this trades a certain failure for a possible one — but if + criterion 1 does not reach 0, this is why. +- **One arm, no matched baseline yet.** Comparisons are against iteration 014's merged arm, which ran on + `v8` with `auto`. Two things differ (binary and inject mode), so a difference cannot be attributed to + the inject mode alone. The baseline arm that follows will fix that. +- **The merged design itself still declines to act** (`merged_keep` 94%). This iteration tests the + recovery path, not the selection quality. diff --git a/docs/experiments/loca/iter016/results.md b/docs/experiments/loca/iter016/results.md new file mode 100644 index 00000000..87768cbb --- /dev/null +++ b/docs/experiments/loca/iter016/results.md @@ -0,0 +1,83 @@ +# Iteration 016 — always-advertise expand: STOPPED at 70/75, and it corrects two earlier claims + +**Pre-registered:** `459e532`. **Stopped deliberately** once the pre-registered criteria were answered; +everything remaining would have characterised a configuration already shown to be wrong. + +## Criterion 1 FAILED, in the direction the pre-registration warned about + +| criterion | before (`auto`) | with `always` | +|---|---|---| +| 1. `Tool 'context_guru_expand' not found` | 48 | **180 at 62/75 (~215 projected)** | +| expand calls attempted | 108 | **371** | +| 2. tools-array flap transitions | uncounted | **0** ✅ | +| tool advertised | ~70% of turns | **100%** (2,299/2,299) | +| **marker present** | — | **1.2%** | + +The flap is genuinely fixed. But the tool is now advertised on every request while **only 1.2% carry +anything to expand**, so the model calls it 3.4× more often, CG cannot resolve it, and the raw `tool_use` +is relayed to a client that has no such tool. + +**This is exactly the hazard `expand/inject.go` documents**, and which the pre-registration named as the +reason criterion 1 might fail: *"Advertising the tool on a request with nothing to expand invites a call +that resolves nothing, and the host then has to replay the model's raw tool_use to the client."* The +change traded a certain failure for a **more frequent** one. + +**The inject mode was never the root cause.** Both modes fail identically: under `auto` the tool appears +rarely and the model calls it from memory on other turns; under `always` it appears constantly and the +model calls it constantly. Either way **CG replays an unresolvable expand call to a client that will +always reject it.** The fix is in the proxy's expand loop — answer every call itself, resolving from the +session stash or replying that the content is still present — not in a mode flag. `always` remains the +right end state once that lands, because it is what removes the cache flap; it is simply unsafe before it. + +## CORRECTION 1: "the merged design declines to act" was misleading about YIELD + +| | acted | **unique tokens removed** | +|---|---|---| +| separate: `coref` + `extract_llm` | 513 + 223 | **2,828,052** | +| merged (iteration 014) | 1,827 | **18,065,171 — 6.4× MORE** | + +Both statements are true and were conflated: + +- **by verdict count** the merged design keeps ~94% (`keep` 4,181, `drop` 259, `trim` 1); +- **by tokens** its few drops are the large ones, so it removes **6.4× more mass** than deterministic + `coref` plus separate `extract_llm` combined. + +"Declines to act" described the verdict distribution and gave the wrong impression about yield. The +accurate statement: **it removes a small fraction of candidates but the biggest ones, so it out-removes +the deterministic index by mass — while almost never trimming.** `merged_trim` = **1 of 4,441 decisions**. +That last figure is the real gap: trimming is the judgement an exact matcher cannot make, and in practice +the design is all-or-nothing. + +It also explains the 1.2% marker rate: drops touch ~10% of requests but concentrate on the largest, so a +handful of requests carry several markers each. + +## CORRECTION 2: repeated tool calls are NOT the main driver of context growth + +| arm | steps/run | repeats/run | repeats as share of the EXTRA steps | +|---|---|---|---| +| lossless baseline | 15.5 | 0.1 | — | +| merged | 39.2 (+23.7) | 2.8 (+2.7) | **11%** | +| separate | 47.0 (+31.5) | 8.2 (+8.1) | **26%** | + +An earlier section presented the repeat loop as the explanation for the CG arms carrying 8× the context. +**It explains 11–26% of the extra steps at most.** The dominant term is simply that the agent takes +2.5–3× more steps, and since every request carries all prior outputs, context grows superlinearly — 3× +the steps gives roughly 7–8× the mean request, which accounts for 86,125 → 647,795 with no repeat loop +required. + +The repeat rate is real and dose-dependent, which is why it looked causal. In absolute terms it is ~3 +extra calls per run out of ~24 extra steps. + +**What is ruled out:** differential success (15 vs 16 solved). **What is not established:** whether +compaction makes the agent explore more with *non-identical* calls — the repeat metric only catches exact +duplicates — or whether the lossless baseline simply terminates early for its own reasons. Distinguishing +them needs the sequence of distinct calls per run, not duplicate counts. + +## Status + +- `INJECT_EXPAND` should return to `auto` until the proxy-side interception fix lands; `always` is + measurably worse in the meantime. +- The baseline arm is **not** run: comparing against an arm whose recovery path is broken in a new way + would not be informative. +- The flap instrumentation stays — it converted an inference into a count, and it is what showed the + fix worked on its own terms while failing on the one that mattered. diff --git a/docs/experiments/loca/iter017/PREREGISTRATION.md b/docs/experiments/loca/iter017/PREREGISTRATION.md new file mode 100644 index 00000000..405357fb --- /dev/null +++ b/docs/experiments/loca/iter017/PREREGISTRATION.md @@ -0,0 +1,61 @@ +# Iteration 017 — pre-registration: the expand loop, fixed + +**Written before the run.** Merged arm only, per direction; a matched baseline follows only if this one +improves and shows no new defects. + +## What was wrong + +The proxy replayed the model's own `tool_use` to the client whenever **no** expand id resolved +(`proxy.go`, the `got == 0` branch). LOCA has no `context_guru_expand` tool — the proxy injects it — so +LOCA answered `Tool 'context_guru_expand' not found`. The model lost its recovery path and **re-ran the +original tool** instead: a full tool execution plus fresh output, enlarging the transcript that provoked +the cut. + +Measured: **17 of 38** attempts refused in the separate arm, **48 of 108** in the merged arm, with +exact-repeat tool calls at **9.9%** and **25.3%** against **0.4%** in the lossless baseline. + +## The three changes (`0afa393`) + +1. **`got == 0` sends the continuation with placeholders** rather than relaying. The placeholder text + already existed per call id, so the model is told the content is gone — actionable, unlike a missing + tool. Capped at one such round. +2. **Interception no longer requires this request to have advertised the tool.** A session that has ever + been offered it is remembered, and every **non-streaming** response is inspected — free, since a JSON + body is read in full either way. The advertise gate is kept for SSE alone, where buffering has a real + cost. +3. **Unresolved calls are counted**, split into `expand_unresolved_malformed` (the model invented an id) + and `expand_unresolved_missing` (**a marker was issued and its stash is gone — reversibility silently + failed**). Both in `/stats`. Neither existed before, which is why this went unnoticed for three + iterations. + +`INJECT_EXPAND` returns to **`auto`**: with interception decoupled, `always` is no longer needed to make +recovery work, and `auto` avoids advertising a tool on the ~99% of turns with nothing to expand. +**This costs the tools-array flap again** — a known trade, and criterion 5 measures it. + +## Pre-declared criteria + +| # | measurement | before | expected | +|---|---|---|---| +| 1 | `Tool '…' not found` in the client log | 48 | **0** | +| 2 | `expand_unresolved_missing` | invisible | **any value is new information**; >0 means reversibility is failing | +| 3 | exact-repeat tool calls | 9.9% | **< 9.9%** | +| 4 | steps/run | 39.2 | ≤ 39.2 | +| 5 | cache-read share | 47.5% | may **fall** vs iteration 016's `always`; compare to iteration 014's 47.5% under `auto` | +| 6 | solved (ITT) / total cost | 16 / $232.72 | ≥ 16 / < $232.72 | + +**1 is the fix working. 2 is new visibility. 3–4 are the loop it was supposed to break. 5 is the trade +being accepted. 6 is whether it matters.** + +Declared in advance: **if 1 passes and 3–4 do not move, the diagnosis was right and the consequence was +small.** Iteration 016 established that repeats explain only 11–26% of the extra steps, so 3–4 moving a +little is the expected case, not a triumph — and step count, whose cause is still unestablished, is the +dominant term. + +## Threats + +- **The volume story is already withdrawn** ([iteration 013](../iter013/results.md)): CG arms send 32–35% + *fewer* tokens upstream than the baseline and are billed only 12–14% more, because the cost is a + cache-tier shift, not volume. So criterion 6 is about the tier mix, and no large cost win is expected. +- **One arm, no matched baseline**, and it differs from iteration 014 by binary and by these fixes. +- **The merged design still barely trims** (`merged_trim` 1 of 4,441). This iteration tests recovery, not + selection quality. diff --git a/docs/experiments/loca/iter019/results.md b/docs/experiments/loca/iter019/results.md new file mode 100644 index 00000000..882fc3c5 --- /dev/null +++ b/docs/experiments/loca/iter019/results.md @@ -0,0 +1,168 @@ +# Iteration 019 — offline probes: what is actually wrong with the merged design + +**Type:** offline probe series, no LOCA run. **Cost:** under $2 of gateway spend. +**Instrument:** direct calls to `aws/claude-sonnet-5` on the benchmark gateway (never through +Context Guru), one synthetic 14.6k-token transcript, 4 trials per arm. + +Iteration 018 left three things unexplained: the merged design kept ~94% of outputs, `merged_trim` +was ~1 across every arm, and solves fell 16 → 13 → 8. This iteration probes the decision itself +offline instead of buying another $234 LOCA run, and the answer is that **merged has never been +tested in the configuration it was measured good in.** + +## 1. Corrections to earlier claims in this repo + +Recorded first, because decisions were taken on them. + +* **`iter014/results.md`'s "negative answer" is unsafe.** It concluded from `merged_keep` 88% that + "the model declines to act on evidence". But that arm ran **2,078 decisions across 2,030 calls = + 1.02 candidates per call**, and `merged_kept_whole_batch` **never fired**, which rules out the + alternative explanation (calls returning well-formed empty arrays contribute zero counted + decisions). Bulk adjudication is a *comparative* mechanism measured at ~15 candidates; at 1.02 the + arm was running the **per-output** design already refuted at 6% live-kept. The iteration measured a + starved variant and attributed the result to the design. +* **The cause was the coref pre-filter, not a per-call cap.** During this work I attributed the + starvation to `llmMaxPerReq` at `extract_llm.go:818`. That is wrong: `s6-merged.yaml` sets no + `llm_max_per_request`, so the guard `if e.llmMaxPerReq > 0` never fired. The cause was the prefix + eligibility pre-filter (149,681 candidates removed), already fixed at `extract_llm.go:695`. +* **Current batch size is 2.63, not 15.** iter018: 2,873 decisions / 1,092 calls. The binding + constraint is now `min_tokens: 3000` — few outputs in this workload are that large. +* **`AllowCachedPrefix`'s comment is wrong on one point.** It states "the tail restriction is not a + safety property of the model call, it is a cache-cost property". It is also an *information* + property. See §4. + +## 2. The model can read a cached prefix — the cache idea works + +Riding the already-cached context instead of re-sending excerpts was proposed to give the model the +evidence it lacks. Measured on the live route: + +| call | tools | `tool_choice` | cache_write | cache_read | +|---|---|---|---|---| +| prefix + appended ask | yes | — | 0 | **19,595** | +| same | yes | `none` | 0 | **19,595** | +| same | **omitted** | — | 0 | 19,129 *(separate entry)* | + +* Appending a trailing user message to an identical prefix lands a **full cache read, no write** — + ~10% of fresh input, ≈$0.03 per call at 100k, ≈$61 across a 2,000-request run. +* `tool_choice: none` is **not** part of the cache key, so tool_use replies can be suppressed free. +* `tools` **are** part of the key; omitting them creates a separate entry. +* This route **rejects assistant prefill** ("the conversation must end with a user message"), which + the appended-ask shape satisfies naturally. +* Requires `model: {source: request}` — caches are per-model, so a haiku call cannot read sonnet's + cache. `components.go:91` already supports this. + +## 3. Transport versus judgment — the model must never carry text + +Two failure classes were conflated. **Judgment** ("is this spent") is what a model is for. +**Transport** (naming an item, reproducing text) is what it is worst at, and the design asks for it. + +| what the model must emit | outcome | +|---|---| +| opaque `tool_use_id` values | **hallucinated** — answered `toolu_01…07` for `toolu_probe_00…07` | +| small integer labels | **0 bad labels in 40+ trials** | +| short verbatim quotes | **0 non-verbatim of 59 quotes** | +| `trim` retained records, verbatim | **8 of 9 invented** (production: `merged_trim` 1, `merged_trim_not_contained` 8) | + +**Rules:** short labels for identity, short verifiable quotes for evidence, and never ask the model +to reproduce content. A positional selector (`keep_records: [0,5,9]`) is *not* a fix — it requires +annotating the content, which is impossible when the content is read from the cache, and breaks when +`format` has reformatted the JSON. A value-based selector (`keep_where: {field, values}`) survives +both. + +## 4. Relevance is local; need is global + +The forward-looking question ("will this be needed") was assumed answerable from the goal plus the +output. It is not. **Need = relevance minus what has already been captured elsewhere**, and the +second term lives in the later turns. So *both* halves of the decision require the same window: + +| | needs the later turns for | +|---|---| +| backward (coref / veto) | reuse an exact matcher missed | +| forward (extract_llm) | whether the information was already extracted and recorded | + +This is why the tail restriction mattered: on tail content there are few later turns, so relevance ≈ +need, and the local prompt was sound. `allow_cached_prefix: true` moves the model onto old outputs — +where the gap is widest — while still asking the local question. The design compensated with the +index as a pre-filter; **merged bypasses that pre-filter**, so merged-on-full-body has *neither* the +index veto nor the window. Both safeguards absent, and that coincides with solves 16 → 13 → 8. + +## 5. Trim is dead weight + +Offered across 21 opportunities in these probes, `trim` was chosen **zero** times, and the +keep/drop/trim and keep/drop arms were **identical on every metric**. Production agrees: 1 accepted +in ~5,000 decisions. What trim was for is already covered twice — `mergedResidue` leaves a shape +descriptor, and `tryMark` stashes the original, verified recoverable end-to-end (120/120 rows). + +A speculation of mine was **refuted**: I predicted a binary contract would improve drop quality by +simplifying the question. It changed nothing. + +## 6. The false drop is a policy error, not a comprehension error + +The trap: batch 3's value is restated later only in transformed form (`1200` → `"1.2k"`), so the +index sees `refs=0`, while the user's standing instruction ("reconcile ledger batches 0-**3**") is +incomplete for it. Correct answer: **keep**. + +Asked the factual question with no removal decision attached, the model was right **3/3**: +`{"batch3_reconciled": false, "batches_with_filed_totals": [0,1,2]}`. It has the facts and drops +anyway. The cause is our own instruction — `BuildBulkPrompt` says *"WHAT THE AGENT IS DOING NOW +(judge relevance toward this)"* — and batch 3 is genuinely irrelevant to the current step while +still required by an outstanding obligation. **The criterion was never stated: "spent" as +"not needed for the current step" versus "not needed by any unfinished obligation".** + +This is a production harm mechanism, not a synthetic curiosity: LOCA tasks are multi-step single +instructions, so a current-step criterion systematically drops data needed by later steps of the +same instruction — silently. + +### What fixed it, and what did not + +| arm | Tier-2 false drop | found all 3 truly-spent | +|---|---|---| +| control — relevance to current step | 4/4 | 4/4 | +| **criterion restated** (obligations count) | **4/4** | 4/4 | +| criterion **+ must name and quote the obligation** | **2/4** | 4/4 | +| solo, batch = 1 | 4/4 | n/a | + +**Stating a better rule is inert. Requiring the model to emit the evidence is what moves it.** Arms 2 +and 3 carry identical criterion text; only the required output field differs. + +Pooled over four independent runs (**exploratory, not pre-registered**): controls **14/14** false +drops, forced-evidence **9/14**, Fisher exact ≈ p 0.04. Yield never suffered — all 28 trials found +the genuinely-spent outputs, and none became over-cautious. + +**9/14 is 64%.** The forcing function moved the veto from *always wrong* to *usually wrong*. That is +a detectable improvement and an unusable capability: single-digit false-drop is the bar for removing +things silently. This is **not** a population false-drop rate — the case was selected for difficulty, +n is 14, and the index's documented 11% comes from 1,119 decisions on real traffic. What it +establishes is that the capability is unreliable exactly where it was supposed to add value, and the +deterministic index fails the same case for free. + +Batch = 1 dropped the output **4/4**, including when keep was correct — `coref-selection-experiment.md` +finding 1 (6% live-kept when shown one output) replicating in a different harness. + +## 7. Methodology note: self-reports are reliable about beliefs, not about causes + +Asked to critique its own instructions, the model named `"Filed. Continue."` as a false-closure cue +that tripped it. Removing that cue made the false drop **worse** (3/4 against 2/4). The *factual* +half of its self-report was accurate and independently confirmed; the *causal* half was wrong. + +Self-reporting earned its place here by generating a cheap testable hypothesis that took two minutes +to eliminate. Use it that way — as a hypothesis generator requiring validation like any other +method — not as evidence. + +## 8. Limits + +Synthetic single transcript, adversarially constructed by the author, n=4 per arm, ground truth for +the trap is the author's judgment (defensible — no total was ever filed for batch 3 — but a +judgment). This is decision quality, which `docs/results/measurement-limits.md` establishes cannot +speak to reward. Nothing here measures reward. + +## 9. What follows + +1. **Cut `trim`.** Never chosen, only verdict requiring transport, fully covered by residue + expand. +2. **Make the criterion explicit and force the evidence** — obligations (a)/(b)/(c), named and quoted + per candidate. Free in yield, and the only thing measured to help. +3. **Do not build on the veto.** It is the justification for the model call and it fails ~2/3 of the + time on the case it exists for. +4. **Merged has still never run bulk-sized.** Batch is 2.63; the 58% result came from ~15. Lowering + `min_tokens` is a config change, not a code change. Any decision to close merged should come after + that, otherwise it closes a design that was never once run as specified. +5. Correct `iter014/results.md` in place, and fix the `AllowCachedPrefix` comment. diff --git a/docs/experiments/loca/iter020/PREREGISTRATION.md b/docs/experiments/loca/iter020/PREREGISTRATION.md new file mode 100644 index 00000000..4ffecb4e --- /dev/null +++ b/docs/experiments/loca/iter020/PREREGISTRATION.md @@ -0,0 +1,139 @@ +# Iteration 020 — pre-registration: the merged design, run as specified + +**Written before the run.** Iterations 014, 016 and 018 all measured a merged arm that was never +configured the way the design was measured good: the model was shown **1.02, then 2.63 candidates per +call** against the ~15 that produced 58% live-kept. This iteration fixes that and two other defects +found offline in [iteration 019](../iter019/results.md), then asks whether the mechanism behaves. + +## Why the earlier arms did not test the design + +| iteration | candidates/call | verdict on merged | why it does not stand | +|---|---|---|---| +| 014 | **1.02** | "declines to act", read as negative | prefix pre-filter removed 149,681 candidates; this is the per-output design refuted at 6% | +| 016 / 018 | 2.63 (018) | 94.6% keep | still far below bulk size; `merged_trim` 0–1 | + +Offline, batch size proved to be a **yield/safety trade-off**, not a detail: at batch 3–6 the model +dropped a genuinely-spent output only **2/4** of the time; at batch 10 it dropped it **4/4** and +cleared **100%** of genuinely-spent candidates. Small batches do not make the model wrong, they make +it **unwilling to act** — which is exactly what a 94.6% keep rate looks like from the inside. + +## The three changes + +1. **`min_tokens` 3000 → 800** (`deploy/harbor/cfg-iter020-merged.yaml`) — config only, to reach a bulk-sized batch. +2. **The contract states the SPENT criterion and forces the evidence** (`internal/extract/bulk.go`) — + spent only if needed by none of (a) the current step, (b) an unfinished user instruction, (c) a + next step the agent itself stated; and the model must **name which** and **quote it verbatim**. + Measured: stating the criterion alone was **inert** (4/4 false drops); requiring the evidence + halved it. A `drop` that names an obligation is now **refused**. +3. **`trim` removed** — chosen 0 times in 21 offline opportunities; in production accepted once + against eight rejected as invented. It was the only verdict requiring the model to transport text. + +`mergedMaxItems` 15 → **12**: quote fidelity degraded with batch size (4 of 37 non-verbatim at 16 +against 0 of 16 at 10), so the ceiling sits between them and this takes the conservative end. + +**Deliberately NOT bundled:** reading the real context via a cache-read on the request model. It is +buildable and cheap (iter019 §2, full cache read, no write, ≈$0.03/call at 100k) but the capability it +was meant to enable — the Tier-2 veto — did not appear even with the window. One variable cluster at +a time. + +## Scope: this is a MECHANISM run, not a reward claim + +One arm cannot support a reward conclusion, and this pre-registration does not attempt one. Solves +will be reported as context only, and comparisons to iterations 014/016/018 are **indicative at +best** — different binaries, different expand behaviour. If the mechanism behaves, the reward pair +(baseline + merged, both on this binary) is iteration 021. + +**Config:** `deploy/harbor/cfg-iter020-merged.yaml` (committed — the merged configs for +iterations 014/016/018 were not, which made them unreproducible), 128k band, `aws/claude-sonnet-5`, cheap model `aws/claude-haiku-4-5`, +LOCA clearing at 128k, `INJECT_EXPAND=always`, n=75. Expected ~$230. + +## AMENDMENT 1 — economic gate disabled (written before any outcome data) + +The first launch hit the pre-registered abort criterion: **1.93 candidates/call at 213 requests, with +`extract_llm.acted` = 0**. Acting on it is following this plan, not deviating from it. + +The counters identified the cause, and it was not `min_tokens`: **`economic_gate` suppressed 1,497 +candidates against 224 that reached the model**, top reason *"cache-aware, saving below call cost"*. +The gate prices each candidate against the cost of a whole model call — correct for the per-output +loop, wrong for a design with one call per request, where the marginal cost of a candidate is a few +prompt tokens. It also fights `min_tokens`: a lower floor produces smaller candidates, each of which +looks even less worth "a call", which is why 3000 → 800 did not help. + +`below_output_floor` (11,036) is **not** evidence about the floor — every request re-scans the whole +transcript, so small outputs are re-counted each turn. That is an occurrence count, not an event +count, and it was nearly misread as one. + +**Amendment:** `economic_gate: false` for this arm. Calls do not multiply with candidates in merged +mode, so the added spend is prompt tokens (~$25 of haiku). This run was pre-registered as +mechanism-only and makes no cost claim; the proper fix — amortising one call across the batch — is +worth building only if bulk batches improve yield, which is the question here. + +**No endpoint, threshold or pre-registered reading is changed.** Primary endpoint remains candidates +per call ≥ 8. + +**A finding already banked, independent of the outcome:** this is the third instance of one defect +class — the prefix pre-filter (iter014), `llm_max_per_request`, and now the economic gate — cost +machinery written for per-output calls, silently applied to a one-call design. Each one starved the +batch, and each was invisible until the batch size was measured directly. + +### Additional observation from the aborted launch + +`merged_quote_not_verbatim` ran **19 of 224 verdicts (8.5%)** on **haiku**, against **0 of 59** on +sonnet in the iter019 probes. At a batch of ~2 that cannot be a batch-size effect, so the forced- +evidence mechanism may not survive the cheap model. If it holds at bulk size, that is a material +limit on the fix, since every probe behind it was run on sonnet. + +## Primary endpoint (mechanism) + +**Candidates per call ≥ 8**, computed as (sum of verdict gates) / `extract.calls`. Below that the arm +has again not run the design, and no other number from it means anything. + +## Pre-registered readings + +Written before the numbers exist. + +| outcome | conclusion | next | +|---|---|---| +| batch ≥8 **and** unique-token yield rises materially over iter018 | small batches were the cause of "declines to act"; the design does act when asked properly | iteration 021: the reward pair | +| batch ≥8 but yield flat | willingness was not the constraint; the negative answer survives, now fairly earned | close merged; keep the deterministic index | +| batch <8 | `min_tokens` still too high, or the economic gate is suppressing small candidates | diagnose, do not interpret anything else | +| `merged_drop_contradicts_obligation` non-trivial | the model asserts an obligation then drops anyway; the coherence guard is load-bearing and the prompt needs work | report the rate; it is a safety finding either way | +| `merged_quote_not_verbatim` > 10% of quotes | batch is above this model's transport ceiling | lower `mergedMaxItems` and re-run | +| `merged_drop_unjustified` dominant | the model is skipping the criterion field, so the forcing function is not running | prompt defect, not a design result | + +## Checkpoints during the run + +Per the working pattern established earlier: **stop and look, do not wait for the end.** + +* **~200 requests:** candidates/call, `merged_batch_truncated`, `merged_drop_unjustified`. If + candidates/call < 6, **abort** and lower `min_tokens` again — finishing a starved arm wastes ~$230 + and produces another uninterpretable iteration. +* **~600 requests:** `merged_quote_not_verbatim` share, drop rate, `summarize` firing rate. +* **Throughout:** one proxy only, bound port verified — the stale-proxy bug in iteration 013 + invalidated two arms by running a previous arm's pipeline on a reused port. + +## Secondary endpoints + +* **Deferral:** `summarize.acted / requests`, against iter018's 56.1%. Deferral follows removed mass, + so it should move with yield — and it is **not** evidence that the removals were correct. +* **Yield in unique tokens**, not reported savings: frozen replays re-credit the same removal, and the + overcount ran 31.7× in iteration 014. +* **Expand:** `expand_restored`, `expand_unresolved_missing`, repeat rate. More removal should mean + more recovery traffic; unresolved must stay 0. +* **CG spend**, which is attributable in a way total LOCA cost is not (iteration 012). + +## Verification before launch + +* `CGO_ENABLED=1 go build ./...` and the full test suite green; each new guard confirmed to FAIL when + its subject is reverted (done: coherence check, quote check, trim degradation). +* Benchmark traffic on `$ANTHROPIC_BENCHMARK_BASE_URL` with `ANTHROPIC_CUSTOM_HEADERS=` — never + through Context Guru. +* Shim `repairs` and HTML 400 count checked in the first minutes. + +## Known limits, stated up front + +Everything motivating this iteration is **decision quality on one synthetic transcript**, n=4 per arm, +with ground truth the author judged — see iter019 §8. It cannot speak to reward, which is why this run +is scoped to mechanism. The offline probes also say the Tier-2 veto remains unreliable (kept 50–100% +of the time depending on batch size and signal clarity, against 0% for the old prompt), so a rise in +yield here should be read as *more willingness to act*, not as *better discrimination*. diff --git a/docs/experiments/loca/iter020/results.md b/docs/experiments/loca/iter020/results.md new file mode 100644 index 00000000..6d796ab4 --- /dev/null +++ b/docs/experiments/loca/iter020/results.md @@ -0,0 +1,121 @@ +# Iteration 020 — the merged design, finally run as specified + +**Six launches, five aborted.** Each abort was triggered by the pre-registered criterion and each +exposed a defect the previous configuration had hidden. The sixth ran to completion and produced this +iteration's only instrument-independent numbers. + +**Headline: 21/75 solves at $240 total, against iteration 018's 8/75 at $243.** That is *not* an effect +estimate for the merged design — see §5 before quoting it. + +> **CORRECTION.** This iteration first reported **22/75**. The correct figure is **21/75**. The original +> method multiplied each task's `avg_accuracy` by its 5 seeds, and `avg_accuracy` averages only the runs +> that COMPLETED — so any task with an errored seed was over-credited. Recomputed per seed from +> `tasks//state/eval.json`, which is what intent-to-treat requires and what iteration 021's +> amendment 1 now mandates for both arms. + +## 1. Read this before quoting any number from the LOCA output + +LOCA prints **`Overall Success: 70/75`**. That is the count of runs that completed **without erroring**, +not the count of tasks solved. The comparable metric is accuracy-weighted: + +| metric | value | +|---|---| +| `Overall Success` (ran without error) | 70/75 | +| **accuracy-weighted solves** | **21/75** | + +Per task: two at 1.0, two at 0.8, one at 0.6, one at 0.2, nine at 0.0. Anyone reading the raw output +will see 70/75 first, and it is off by a factor of three. + +## 2. What the run did + +| | iter018 | **iter020** | +|---|---|---| +| accuracy-weighted solves | 8/75 | **21/75** | +| runs completed without error | 67/75 | 70/75 | +| LOCA cost | $234.09 | **$207.02** | +| CG model spend | $9.21 | $33.07 | +| **total** | ~$243 | **~$240** | +| `summarize` fired on | 56.1% | **41.3%** | +| `extract_llm` acted on | — | 1,465 / 2,303 requests (63.6%) | +| unique tokens removed by `extract_llm` | 318,955 | **8,140,204** | +| drops / keeps | 151 / 2,718 | **791 / 1,386** | +| expand restored / unresolved | 176 / 0 | **866 / 5** | + +The prefix-ask mechanism carried **37,336,778 tokens at cache-read price with zero cache writes**, over +778 asks with 4 failures (first turn of a session). `adjudicate_stray` was **0** — the model never +called the injected maintenance tool on its own. + +**Do not quote `savings_pct` (94.3%).** Iteration 012 established it is inflated 3-8× by frozen replays +re-crediting the same removal; `saved_tokens_unique` is the honest figure. + +## 3. Where the mechanism still falls short + +* **Verdict coverage 65%** (2,178 answered of 3,360 offered). A third of offered candidates get no + verdict and are silently kept, so the design is not running at full strength even here. +* **59 batches truncated** at the 12-item cap, so candidates were discarded on those calls. Visible only + because `merged_offered` now exists. +* **`merged_unparseable` 133** — replies that were neither a tool call nor parseable JSON. +* **`merged_quote_not_verbatim` 149 of 2,178 verdicts (6.8%)** — fabricated obligation quotes. Harmless + in direction (a false obligation argues for keeping) but it is the batch-size ceiling signal. +* **`merged_drop_contradicts_obligation` 1** — one drop that named an outstanding obligation, refused. + Rare, and the guard is load-bearing. +* **`refusal_reached_model` 48 (1.6%)** — expand refusals that survived to the model. + +## 4. Seven defects in one component's measurement path + +Every one of these produced the *same* misleading signal — "the model declines to act" — and each was +invisible until the layer above it was removed. + +| # | defect | effect | origin | +|---|---|---|---| +| 1 | coref prefix pre-filter | removed 149,681 candidates, leaving ~1 per call | pre-existing | +| 2 | `llm_max_per_request` | caps candidates in a design with one call | pre-existing | +| 3 | economic gate | prices each candidate against a whole call; suppressed 1,497 vs 224 through | pre-existing | +| 4 | any pinned floor disables the pressure trigger | fired on 72% of requests regardless of context | pre-existing | +| 5 | `verdicts ÷ calls` used as batch size | counts what was ANSWERED, not OFFERED | mine | +| 6 | 2048 output-token ceiling | reply cut mid-array on ~70% of calls | mine | +| 7 | `tool_choice: none` | drove the model into PROSE — 0 of 6 labels | mine | + +Numbers 1-4 predate this branch and are documented in +[`docs/results/min-tokens-vs-economic-gate.md`](../../../results/min-tokens-vs-economic-gate.md). +5-7 are mine, introduced while trying to measure or improve the thing. + +**The judgement machinery was never the problem.** A sampled reply shows the model reasoning correctly +under the criterion — *"the task is not yet complete, and no summary of this raw data has been recorded +elsewhere"* — and simply saying so in prose, which the parser scored as a failure. The contract already +calls "keep everything" a valid and often correct answer. + +### An eighth, in the rig rather than the product + +`capture_hop.py` tested for markers with `"<>` exists on the wire only +as `</state/eval.json`, so the comparison is +**paired**: 75 (task, seed) pairs, each arm scored on the same seed. + +Accuracy is not binary on this benchmark (observed values 0.0, 0.2, 0.6, 0.8, 1.0), so: + +* **Primary — task-clustered.** Mean accuracy per task (15 clusters), paired Wilcoxon signed-rank, + two-sided, α = 0.05. **The clustered test governs.** Five seeds of one task are correlated, not five + free observations, and treating 75 as independent is the mistake this file exists to avoid. +* **Secondary — per-pair.** All 75 pairs, paired Wilcoxon signed-rank, plus explicit counts of + **improved / worsened / unchanged** pairs. Reported as a sensitivity check, never as the headline. +* **Harm.** Clopper-Pearson upper bound on the proportion of worsened pairs. Declared **before** the + run, per iteration 007's failure: **a harm upper bound above 25% blocks any positive claim**, whatever + the point estimate does. + +**Margin:** no minimum effect size is claimed as a win. The honest reading of a null result at this n is +"underpowered", not "no effect" — previous harm bounds reached ±39%. + +## AMENDMENT 1 — errored runs score ZERO (written before any solve count was read) + +Arm A produced **16 upstream 400s in 3,347 requests (0.5%)**, all `"prompt is too long"`, on bodies of +2.6 MB to 14.8 MB. Diagnosed while arm A was still running and before its results existed. + +**Cause, and it is mine.** A single tool output arrives larger than any component can reduce: `extract` +matched no noise pattern (`acted` = 0, `no_obvious_noise` 16,891), `cmdfilter` matched nothing +(`acted` = 0), `toon` needs a uniform object array, `dedup` needs an exact duplicate — so the only real +compactor is `summarize`, which protects `keep_last: 3`, and a fresh oversized output sits in exactly +that protected tail. `extract_llm` would decline it too, by design: `over_model_context` leaves any +output that exceeds the compaction model's context verbatim. + +The product has a component for precisely this — `collapse`, "the content-agnostic fallback for an +oversized tool output that no more specific component handled", which keeps a head/tail window and +stashes the original behind a marker. It is in the `general` and `codesafe` presets and **not** in +`codesmart`, which is what these arm configs descend from. So this is a rig-configuration error, not a +product defect — with the caveat that a user of the shipped `codesmart` preset has the same gap. + +**Why the arms are NOT being restarted:** both lack `collapse`, so both take the same class of failure, +and the comparison stays fair in expectation. Restarting costs both arms. + +**What is being fixed instead — the analysis plan, stated before any outcome is known.** Errors are the +one place where this omission could bias the result: arm B removes more, so it may error *less*, and +excluding errored runs would then compare arm B's survivors against arm A's. Iteration 014 hit this +exactly, with 15 errors against 8, and concluded that intent-to-treat is the reading that survives. + + * **Primary analysis is INTENT-TO-TREAT.** A run that errored, or has no `eval.json`, scores + **accuracy 0**. Every one of the 75 pairs is scored. + * **Per-protocol** (errored pairs dropped) is reported as a sensitivity check only, alongside the + error count per arm, and never as the headline. + * **The error counts themselves are a reported endpoint.** A large asymmetry is a finding about + oversized-output handling, whichever direction it points. + +Nothing else changes: the clustered test still governs, the 25% harm bound still blocks a positive +claim, and no minimum effect size is claimed as a win. + +## AMENDMENT 2 — arm B ran BELOW its own coverage gate, and was continued anyway + +At 477 requests, arm B's verdict coverage was **34%**. The checkpoint above requires **≥50%** and says +to abort below it. Recording the decision here rather than in the results, so it cannot be +retrospectively smoothed over. + +**Continued, on the operator's call.** The gate was written to catch "the design never ran", and by every +other measure it is running: `prefix_ask_cache_read_ZERO` = 0, `merged_reply_truncated` = 0, +`adjudicate_stray` = 0, 44 real drops and 686k unique tokens removed by that point. What is happening is +PARTIAL ANSWERING — the model returns verdicts for about a third of a full 12-item batch — not a dead +mechanism. Coverage also rose through iteration 020 (46% → 51% → 65%), so 34% at 20% of the run may be +an early reading rather than the final one. + +**The cost of continuing is stated, not hidden:** arm B's numbers are a **FLOOR** on what the design can +do, because it acted on roughly a third of the candidates it identified. A null result therefore cannot +be read as "merged does not help" — only as "merged at ~34% coverage does not help detectably". + +**The honest alternative was to abort**, which would have preserved the gate's authority at the cost of +re-running arm B, and there is no tested coverage fix to re-run it *with*: the leading candidate is a +SMALLER batch, since a probe answered 6 of 6 on six items against ~34% on twelve, and that is untested. + +Nothing else changes. The clustered test governs, ITT scores errored runs zero, and the 25% harm bound +still blocks a positive claim. + +## Pre-registered readings + +| outcome | conclusion | next | +|---|---|---| +| clustered p < 0.05, direction positive, harm bound ≤25% | merged adds value over the shipped pipeline | make the economic gate batch-aware, re-run to confirm on a shippable config, then decompose (coref alone; per-output alone) | +| clustered null, per-pair positive | suggestive and underpowered; seeds are carrying it | do not claim; decide whether more seeds are worth the money | +| clustered null, both directions flat | no detectable marginal value at this n and this cost | close merged; keep the deterministic pipeline; report the ceiling honestly | +| clustered negative, or harm bound > 25% | merged harms the product | close it, and record which removals caused the harm | + +## Secondary endpoints + +* **Cost.** Reported, but iteration 012 established that per-arm LOCA cost cannot price a component. + **CG model spend is the only attributable figure** — $33.07 in iteration 020, against $9.21 when the + adjudicator ran on the cheap model. My prior, stated in advance: **merged costs parity to ~20% more** + in total. CG arms have historically sent 32-35% fewer tokens and billed 12-14% more, because a removal + inside a cached prefix invalidates everything after it. The case for this design is reward, not cost. +* **Deferral.** `summarize.acted / requests`. And it must not be read as quality: a badly-chosen removal + defers `summarize` exactly as well as a well-chosen one. +* **Yield.** `saved_tokens_unique`, never `savings_pct` — the latter is inflated 3-8× by frozen replays + re-crediting the same removal (iteration 012). +* **Mechanism health**, or the arm is uninterpretable regardless of its reward: `merged_offered` vs + answered (65% in iteration 020), `merged_unparseable`, `merged_reply_truncated`, + `merged_quote_not_verbatim`, `merged_drop_contradicts_obligation`, `prefix_ask_cache_read_ZERO`, + `adjudicate_stray`, `expand_unresolved_missing`. + +## Checkpoints + +* **~200 requests, arm B:** offered-vs-answered ≥ 50%, `prefix_ask_cache_read_ZERO` ≈ 0, + `merged_reply_truncated` = 0. If the mechanism is not running, **abort** — a completed arm that never + ran the design is what iterations 014, 016 and 018 each produced. +* **Both arms:** one proxy only, bound port verified. The stale-proxy bug invalidated two arms in + iteration 013 by running a previous arm's pipeline on a reused port. +* **Reminder from iteration 020:** LOCA prints `Overall Success`, which counts runs that did not error. + It is **not** the solve count. The comparable metric is accuracy-weighted, and the two differed by 3×. + +## Cost and scope + +Two arms, n=75 each, ~$240 per arm, **~$480 total**. If arm B wins, the decomposition arms (`coref` +alone; per-output `extract_llm` alone) run on the **same binary and seeds** for another ~$480 without +re-running the baseline. Note in advance that a per-output arm cannot use prefix asks, so it would +conflate "merged vs per-output" with "prefix-ask vs none". diff --git a/docs/experiments/loca/iter021/results.md b/docs/experiments/loca/iter021/results.md new file mode 100644 index 00000000..59b69c48 --- /dev/null +++ b/docs/experiments/loca/iter021/results.md @@ -0,0 +1,106 @@ +# Iteration 021 — results + +**Status: RUNNING.** This file was created before launch to record the frozen inputs, which is the +control iterations 014, 016 and 018 lacked and the reason they cannot be compared to each other. + +## Frozen inputs + +| | | +|---|---| +| binary | `cg-proxy-v19`, 37,080,187 bytes | +| SHA-256 (first 32) | `ecc02f28417fe8d5edbcfaf3cc13505b` | +| code commit | `fcf78cd` — everything since is documentation only | +| task config | `task-configs/final_128k_set_config.json` — 15 tasks × 5 seeds (`state0..state4`) | +| arm A config | `deploy/harbor/cfg-iter021-A-baseline.yaml` | +| arm B config | `deploy/harbor/cfg-iter021-B-merged.yaml` | +| environment | `INJECT_EXPAND=always`, `CACHE_MODE=on`, 128k declared window, LOCA clearing at 128k, `--max-workers 8` | +| adjudication model | the request's own model (`aws/claude-sonnet-5`) | +| summarize model | cheap client (`aws/claude-haiku-4-5`) | + +**One binary serves both arms.** Any code change before both arms finish invalidates both, not one. + +## Pipelines — one component apart + +``` +A [format, toon, dedup, failed_run, cmdfilter, extract, cachesplit, summarize] +B [format, toon, dedup, failed_run, cmdfilter, extract_llm, extract, cachesplit, summarize] +``` + +## Results + +**A null on the primary endpoint.** Merged adds **+1.00 solve on 75 pairs, p = 1.0000, at +2% total +cost.** The pre-registered reading for this outcome, written before the run: *"clustered null, both +directions flat → no detectable marginal value at this n and this cost → close merged; keep the +deterministic pipeline; report the ceiling honestly."* + +| | arm A (baseline) | arm B (+ merged) | +|---|---|---| +| **ITT solves (all 75 pairs, errored = 0)** | **14.00 / 75** | **15.00 / 75** | +| **clustered p, 15 clusters (GOVERNING)** | — | **1.0000** — 2 better, 2 worse, 11 tied | +| per-pair p, 75 pairs (sensitivity) | — | 1.0000 — 7 gained, 6 harmed, 62 unchanged | +| harm, 95% Clopper-Pearson upper bound | — | 15.2% — **does not block** | +| errors | **17** | **10** | +| LOCA cost | $217.38 | $187.22 | +| CG model spend | $6.68 | **$41.44** | +| **total cost** | **$224.06** | **$228.66 (+2%)** | +| `summarize` fired on | 71% of requests | **56%** | +| requests | 3,446 | 2,466 | +| unique tokens removed by `extract_llm` | — | 6,545,956 | +| expand restored / unresolved | 361 / 1 | 717 / 0 | + +### Per-task, mean accuracy over 5 seeds + +| task | A | B | B−A | +|---|---|---|---| +| AcademicWarningS2LEnv | 1.00 | 0.40 | **−0.60** | +| NhlB2bAnalysisS2LEnv | 0.20 | 0.00 | **−0.20** | +| PayableInvoiceCheckerS2LEnv | 0.60 | 1.00 | **+0.40** | +| MachineOperatingS2LEnv | 0.00 | 0.60 | **+0.60** | +| ExcelMarketResearchS2LEnv | 0.60 | 0.60 | 0 | +| SetConfCrDdlS2LEnv | 0.40 | 0.40 | 0 | +| *the other nine tasks* | 0.00 | 0.00 | 0 | + +**Eleven of fifteen tasks score zero in BOTH arms.** That is the dominant fact about this benchmark's +power: the comparison rests on four tasks, two moving each way. No configuration change could have +produced a detectable difference here without a large effect. + +## What merged did deliver, all secondary + +* **7 fewer errored runs** (17 → 10). Both arms hit `"prompt is too long"` on oversized tool outputs; + arm B hit it less. **But this is NOT clean evidence that model compaction prevents runaways** — see + the `collapse` limitation below. +* **15 points less summarization** (71% → 56%). Real, and much smaller than the 4× that comparing + against iteration 020 had suggested — which is why iteration 020 was never a valid comparator. +* **28% fewer requests** (3,446 → 2,466), consistent with fewer runaway sessions burning turns. +* **6.5M unique tokens removed**, with recovery working: 717 restores, **0 unresolved**. + +**Cost prediction, stated in advance, was right:** "parity to ~20% worse" → **+2%**. LOCA spend fell +$30 and CG spend rose $35. + +## Limits — none of which rescue the result + +1. **Coverage 61%.** Arm B answered verdicts for 2,114 of 3,461 offered candidates, and ran below its + own pre-registered 50% checkpoint for the first third of the run (amendment 2). Its numbers are a + **floor**: the honest null statement is *"merged at 61% coverage adds nothing detectable"*, not + *"merged adds nothing"*. +2. **Neither arm had `collapse` or `mask`**, so neither is any shipped preset. And `collapse` would + probably not have helped anyway: it skips outputs of ≤40 lines (`headLines+tailLines`), and a database + or API result serialised as JSON is often ONE line. A single-line multi-megabyte payload falls through + `collapse` as well as through everything else — which is a product gap with no owner, not a + configuration mistake. +3. **`summarize` ran alongside in-place offloaders**, which `config.go` explicitly advises against + ("run it alone so no other component's in-place edits race apply's rebuild"). Its own preset is + `{summarize}` alone. +4. **`merged_unparseable` 146** — replies that were neither a tool call nor parseable JSON, so those + calls changed nothing. + +## Conclusion + +On the question asked — *what does merged add on top of the shipped deterministic pipeline?* — the +answer at this n, this coverage and this cost is **nothing detectable in reward**, with modest +operational gains (fewer errors, less summarization, fewer requests) at cost parity. + +Two things would have to change before that verdict could be called final rather than +underpowered: **coverage** (61% → near 100%, and the untested lead is a SMALLER batch, since a probe +answered 6 of 6 on six candidates against ~61% on twelve), and **a benchmark with room to move** — 11 +of 15 tasks scoring zero in both arms leaves almost nothing to detect. diff --git a/docs/how-to/choose-a-preset.md b/docs/how-to/choose-a-preset.md index 49a871e9..6b01ed54 100644 --- a/docs/how-to/choose-a-preset.md +++ b/docs/how-to/choose-a-preset.md @@ -118,10 +118,15 @@ Tuned for long agentic sessions (e.g. Claude Code on SWE-bench) where the domina transcript of old tool outputs re-sent every turn. - **Fits:** long-running agents with a growing transcript. -- **Caveat:** **`mask` is the biggest lever here** — age-based GC of tool outputs older than - `keep_recent`. In the SWE-bench sweep it delivered ~27% mean content-token savings (up to 93.5% - on a long session) with no reward loss ([Benchmarks](../RESULTS.md)). Order matters: lossless - first, then offload old-then-large, cache last. +- **Caveat:** **`extract_llm` is the biggest lever here**, not `mask`. The ~27% mean content-token + saving from the SWE-bench sweep (up to 93.5% on a long session, no reward loss — + [Benchmarks](../RESULTS.md)) came from `extract_llm`'s LLM trimming of large file reads, which + some of the team call the "programming masker" — which is how the number got attached to `mask` + in these docs. The arm that produced it (`codesmart`) contains **no `mask`**. + On a caching backend note two things: `mask` is + [structurally inert](../components/mask.md#when-its-inert), and `extract_llm` is **disabled by + default** (`allow_on_caching_backend` unset = false) and restricted to the uncached **tail** when + enabled. Order matters: lossless first, then offload old-then-large, cache last. ### `general` — `[format, toon, dedup, failed_run, cmdfilter, mask, extract, extract_llm, collapse, cachesplit]` The recommended all-round pipeline: the reward-neutral levers of `agent` plus the situational diff --git a/docs/how-to/measure-savings.md b/docs/how-to/measure-savings.md index efade32c..581862d9 100644 --- a/docs/how-to/measure-savings.md +++ b/docs/how-to/measure-savings.md @@ -225,9 +225,11 @@ The dashboard does not replace any of these — it captures out of band from ## Benchmarks -For the full per-component SWE-bench evaluation — where `mask` delivers ~27% -content-token savings with no reward loss, and how the `/stats` within-run metric is -derived — see [Benchmarks](../RESULTS.md). To view a harness run in the dashboard, point +For the full per-component SWE-bench evaluation — where **`extract_llm`** delivers ~27% +content-token savings with no reward loss (this was previously miscredited to `mask`; see +[comparison.md](../results/comparison.md#per-component--per-compressor), which attributes the +savings to `extract_llm` + `extract` + `cmdfilter`/`dedup` and never mentions `mask`), and how the +`/stats` within-run metric is derived — see [Benchmarks](../RESULTS.md). To view a harness run in the dashboard, point `--dashboard-bench-dirs` at its jobs root: each run's `summary.json` + `rows-.json` is ingested, with cost-vs-reward per arm and per-task drill-down. diff --git a/docs/proposals/coref-compaction.md b/docs/proposals/coref-compaction.md new file mode 100644 index 00000000..5c49b8aa --- /dev/null +++ b/docs/proposals/coref-compaction.md @@ -0,0 +1,814 @@ +# Co-reference-aware compaction (`coref`) + +**Status:** mechanism implemented and tested; the §7 measurement pass has **run on three corpora** +(Claude Code, UltraHorizon, LOCA-bench) — [results](../results/coref-density.md) — but not yet on the +eval-box captures the acceptance criteria are written against. The `coref` component ships **opt-in, in no preset**, +with the calibrated (`closed`) cut **off by default**. See [implementation status](coref-implementation.md). +**Headline numbers:** unreferenced tool-output mass is **21% on interactive traffic and ~70% on +benchmark traffic** (a 3.3x workload difference, not a constant), a reference consumes a median +18.7% of what its output introduced, and — the result that most affects the design — **recency is +nearly inert while reference count does all the discrimination**. +A later [held-out experiment](../results/coref-selection-experiment.md) added the decision-quality +numbers the density pass could not: the deterministic index keeps **95%** of what the agent went on +to need while removing 11.8% of mass, no model-in-the-verdict arm beat it, `cut_unreferenced` +carries an irreducible **11% false-drop**, and selective removal **cannot replace a summarizer** at +any aggression setting — see [§8b](#8b-what-the-held-out-experiment-settled). +**Related:** [the component reference](../components/coref.md) · [cheat sheet](../reference/coref-glossary.md) · +[dynamic, model-aware triggers](../components.md) · +[improvement plan §0, §C](../results/improvement-plan.md) · [agent compaction](../how-to/agent-compaction.md) + +The idea, as originally posed: once a session passes a context threshold, compact — but pick +*what* to drop by looking at **back-references**. If a later turn references an earlier tool +result, that reference means one of two things: + +- **A.** The model already took what it needed from that output (it lifted one value out of a + large response and does not need the rest). +- **B.** The model has effectively marked the output as important, and may need it again. + +The original framing separates A from B by **distance from the current turn**: B for recent +references, A for early ones. And it notes the asymmetry that matters — A licenses a *large* +cut, a large cut rewrites the cache, so A must only fire when confidence is high. + +This doc does three things: says what a reference actually is in our traffic, argues that +distance is the wrong discriminator and proposes a better one, and then prices the whole idea +against numbers already measured in this repo. The pricing is the part that changes the design. + +!!! info "Scope: this proposal is now explicitly about the **caching** regime only" + Earlier drafts priced both regimes. That was wasted breath: on a non-caching backend every + turn re-sends the whole transcript at full input price, so the bill grows quadratically in + session length and the workload is uneconomic before any component's decisions matter. + **The non-caching regime is out of scope by decision, not by oversight.** + + This narrows the proposal in a useful way. `coref` no longer has to justify itself as a + general token reducer; it has exactly one job — **be worth a cache-write** — and every + section below is a test of that. It also removes the only argument for a `coref` variant that + scans all messages instead of the tail: the tail restriction was never the point, the + cache-write was. + + Two conventions are being **changed** as a consequence, and both are recorded where they + live: `TailOnly` is no longer treated as inviolable for backward-looking offloaders (§5.3), + and `allow_on_caching_backend` is no longer a blanket veto on model-calling offloaders — the + gate becomes "does this pay for the rewrite", which is a measurement rather than a policy. + +--- + +## 1. Why today's relevance signal cannot do this + +Every LLM component gets its relevance signal from `conversationGoal` +([`components/offload/common.go`](https://github.com/rossoctl/context-guru/blob/main/components/offload/common.go)): the first user turn +(the task), plus the most recent assistant and user turns (current intent). Tool outputs are +deliberately excluded — they are the mass being reduced, not the goal. + +That signal is **forward-looking and position-free**, and both halves matter: + +- *Forward-looking* — it describes the destination ("fix the failing auth test"), not the history. +- *Position-free* — it is one blob of text. Nothing in it says which message an output was, or + which earlier output a later turn leaned on. + +Concretely. Suppose turn 4 read `src/auth.py`, turn 5 said "the bug is `TOKEN_GRACE_SECONDS`", +and thirty turns later the agent is editing tests. Asked "is the turn-4 output still needed?", +`conversationGoal` can only answer "well, the task is still about auth" — which is true of every +output in the session and therefore decides nothing. What actually settles it is that the one +value the agent ever took out of that output (`TOKEN_GRACE_SECONDS`) is sitting in turn 5, and +turn 5 is not going anywhere. That is a *positional, backward-looking* fact, and today's signal +cannot represent it at all. + +Co-reference is therefore not a tuning change to an existing input; it is a new input, and it is +the only input that can justify dropping a *large*, *early* span rather than projecting a recent one. + +The good news is that the repo already contains most of the machinery, pointed the other way. +Two existing pieces, and what each contributes: + +- **`internal/extract/deterministic.go`** keeps a list of "important keys" (`id`, `status`, + `state`, `name`, `error`, `reason`, `date`, `time`) used to decide which leaves of a JSON blob + are worth keeping when shrinking it. That list is already an answer to *"which parts of an + output is a model likely to carry forward?"* — the same question a reference detector asks. +- **`internal/extract/contain.go`** checks that a shrunken output is a *subset of* its original — + a guard that the shrinker did not invent text. + +The second is the reusable idea, run backwards. Today it asks *"is this compacted text contained +in the original output?"*. Invert the direction and ask *"is this span of the original output +contained in a **later message**?"* and the same containment test becomes a reference detector. +Same primitive, opposite direction: today it validates a rewrite, inverted it measures reuse. + +That is the beginning of a reference index — beginning, because §2 shows raw containment is +badly wrong on its own. + +## 2. What a reference is, in three tiers + +Ordered by how deterministically it can be detected: + +| Tier | Signal | Example | Detectable | +|---|---|---|---| +| **1** | `tool_use_id` ↔ `tool_result` pairing; and **literal carry-over** — a span introduced by tool result *i* reappearing verbatim in a later `tool_use` argument or assistant text (paths, symbols, line numbers, IDs, hashes, error strings) | output: `TOKEN_GRACE_SECONDS = 0` → later turn: `Edit(old="TOKEN_GRACE_SECONDS = 0")`. The string is *identical*, so a substring test finds it. | exact, zero LLM | +| **2** | **transformed** carry-over — the agent summed the rows, converted units, reworded the finding. No substring match exists | output: `[{"ms": 1200}, {"ms": 1800}]` → later turn: *"total latency is 3 seconds"*. `3` appears nowhere in the output; it was computed. Same for `1200ms` → `1.2s`, or `ETIMEDOUT` → *"the request timed out"*. | no; this is the deterministic ceiling | +| **3** | **semantic** — "as I noted earlier", "per the schema", a plan step that depends on an observation without naming it | output: a directory listing → later turn: *"as I saw earlier, the tests live beside the source"*. The reference is unmistakable to a reader and carries **no shared token at all**. | LLM only | + +Tier 2 is the objection raised on the thread — values drift through paraphrase and unit +conversion, so exact matching will miss real references — and the "maybe it covers 90% of +cases" instinct is the right *shape* of estimate. It is also a measurable quantity rather than +an arguable one, which is what §7 is for. + +### The confound that decides whether Tier 1 means anything + +A naive implementation will report near-total reference density and be wrong. If the agent +calls `Read(src/auth.py)`, the path appears in the **`tool_use` argument**, is echoed in the +`tool_result`, and appears again in a later `Edit(src/auth.py)`. Exact matching sees a +reference from the output to a later turn. But the path was never *extracted from the output* — +it was already in context before the output existed. + +The only sound signal is **novel** tokens: identifiers the tool output *introduced*, which do +not appear anywhere at or before the `tool_use` that produced it, and which are then reused +later. Everything else is echo. The measurement script implements this as a prior-vocabulary +exclusion, and it is not an optional refinement — without it the numbers are meaningless. + +## 3. Distance is a proxy; open-vs-closed is the discriminator + +Distance conflates two different things. A span referenced three times, forty turns ago, is not +case A — it is a hot span that happens to be old. What actually distinguishes A from B is +whether the reference is **closed**: + +- **Closed (A).** The span yielded a value that now has a **surviving copy** — written to a + file, carried into a plan or todo, or restated in a later message that will itself survive + the cut. The original is redundant *with something still in context*. Cut hard. +- **Open (B).** Referenced **repeatedly**, or **most recently**, or referenced without any + specific value having been lifted out (the agent is still surveying, still searching). Keep. + +The practical payoff is that "certain enough" stops being a confidence score on a distance +curve and becomes a **verifiable predicate**: take the large cut only when you can point at the +surviving copy. Score on `(reference count, how long ago the last reference was, witness +present)` — with recency measured **from the head of the transcript**, not from the output's own +position, since "recent messages vs early messages" is a statement about now. + +And the witness looks free. `coref` only ever cuts **tool outputs**; references live in +**assistant** turns, which are never cut. So a reference is always a *surviving copy of +something* — which is what makes the closed case cheap to establish rather than a second search. + +**But "a surviving copy of something" is not "a surviving copy of what is needed", and the gap +is a real hole in the argument above.** Raised in review, with this counter-example: + +```jsonc +// tool output +[{"name": "david", "id": 123, "address": "foobarbaz"}, + {"name": "osher", "id": 235, "address": "banana"}] + +// the model's next turn +"I need to remember david 123 address." +``` + +The reference is real. `david` and `123` genuinely came from the output and genuinely reappear. +But the value the model actually needs — `foobarbaz` — was **never copied into a model turn**. It +referenced an **anchor** precisely *in order to* point at a payload it did not restate. Cut the +output and the address is gone. + +So the witness argument holds only when the reference *carries* the value, and an exact matcher +cannot tell an anchor reference from a payload reference. Three consequences: + +1. **`closed` cannot rest on "referenced once, long ago" alone.** That is exactly the anchor + case's signature, so it is the *reason* `cut_closed` ships off rather than merely an + abundance of caution. Distinguishing anchors needs a signal this index does not have — most + plausibly requiring the reference to have consumed a large share of what the output + introduced, which is `used_frac`, and which cuts against the reading in §7 that a *low* + `used_frac` is evidence for case A. A low `used_frac` is in fact **ambiguous**: "took the + value, rest is chaff" and "took an anchor, still needs the payload" look identical. +2. **It is not a contradiction of case B, it is a demonstration that the A/B labels are not + observable from a reference alone** — which is the same conclusion §3 reaches about distance, + arrived at from the other side. +3. **The record shape it uses is exactly the shape this index is blind to.** `david`, `123`, + `foobarbaz` are short lowercase words and a 3-digit number — none survive the precision rules + in §2, so the index sees *no* novel tokens at all. That produced a separate and worse defect, + now fixed: see `opaque` below. + +### `opaque`: absence of evidence is not evidence of deadness + +The counter-example above exposed a defect in the first implementation. An output that +introduced **no trackable identifier** was scored `unreferenced` — because both states satisfy +`refs == 0` — and `unreferenced` is what the DEFAULT configuration cuts. But the two mean +opposite things: + +| state | meaning | +|---|---| +| introduced 200 identifiers, no later turn touched one | evidence of deadness → the safe cut | +| introduced nothing the index can see | **no evidence at all** → no opinion | + +Measured, this is not a corner case: **8% of tool-output mass on interactive traffic, 20% on +UltraHorizon and 40% on LOCA-bench** introduces nothing the index can track. On LOCA that is 11 +outputs averaging 22k tokens — bulk record and spreadsheet dumps of human-readable values, the +exact shape of the counter-example. The default config was about to delete them on no evidence. + +`opaque` is therefore its own class and is **never cut**. The asymmetry is deliberate: an opaque +output costs tokens, while a wrongly cut one costs an `expand` round-trip plus a cache-write, and +can cost the task. + +### The opportunity floor + +The same review raised the mirror-image error: an output near the *tail* has had no chance to be +referenced yet, so scoring it as unused would make a batched pass preferentially cut the most +**recent** context — the worst possible choice. `mask` avoids this with `keep_recent`; `coref` +now expresses it in turns (`min_later_turns`, default 8): an output with fewer model turns after +it than that is treated as `open` regardless of everything else. §7's measurement had *bounded* +this bias but nothing had *guarded* against it. + +Framed this way, `coref` is `dedup` generalized: from "this tool output is byte-identical to +another" to "this tool output's useful content survives elsewhere in the request". + +## 4. The economics, and why they reshape the design + +This is where the proposal has to survive contact with what the repo already measured +([improvement plan §0 and §C](../results/improvement-plan.md)). + +**Three measured facts:** + +1. **The agent appends.** 1124/1124 consecutive turn pairs carry the entire previous turn as a + byte-identical prefix; 232/232 re-sent large outputs live at exactly one stable message + index for the whole session. Nothing is "re-sent as new bytes" — a re-read is a cache-read + of a stable prefix. +2. **A cache-write costs 11.5 cache-reads** — `($2.50 − $0.20)/$0.20`. Mutating already-cached + content rewrites the entire suffix. +3. **Steps dominate the bill.** `corr(Δsteps, Δcost) = +0.95` on every arm and both benchmarks. + Unique token removal is a rounding error: 0.024% of billed input on SWE-bench, 0.127% on + Terminal-Bench. The SWE-bench win was a −13.7% step reduction that multiplied 165k removed + tokens into −18.3M cache-read tokens — **110× leverage**. + +Now price a cut. Cut `S` tokens at message index `i`, with `W` tokens of transcript after `i`, +and `T` turns remaining in the session. The prefix hash breaks at `i`, so the suffix is +cache-**written** once, then read cheaply thereafter: + +``` +cost = W × (2.50 − 0.20) = 11.5 × W (in cache-read-equivalents) +benefit = S × T × 0.20 = S × T +break-even: S × T > 11.5 × W +``` + +Put a real shape to it. A 5k-token output sitting at 20% depth of a 150k-token transcript +gives `W ≈ 120k`, so it needs `5k × T > 1.38M` → **T > 276 turns**. That does not happen. + +> **A single early cut can never pay for itself on token savings alone.** This is not an +> argument against the idea — it is the quantified version of the original concern about cache +> rewrites, and it dictates the design. + +!!! note "`W` is not "everything after `i`" — it is set by the breakpoints, and that is favourable" + The formula above prices `W` as the whole suffix, which is the conservative reading and the + one the measurement used. The provider is more forgiving than that. Anthropic caching has + **at most 4 `cache_control` breakpoints** and reads from the deepest one whose prefix still + matches, with a 20-block lookback. So a cut at index `i` invalidates only from **the nearest + live breakpoint at or before `i`** — the blocks before it are still served from cache. The + honest cost term is: + + ``` + cost = 11.5 × (nearest live breakpoint at or before i → end) + ``` + + Two consequences worth acting on. **Cutting several outputs that share one breakpoint span is + free relative to cutting one of them** — which strengthens the batching argument in a way §4 + understates. And **where the breakpoints sit is a lever `coref` could pull**, via + [`cachesplit`](../components/cachesplit.md): placing a breakpoint just below the intended cut + depth bounds the damage in advance. + + Unmeasured, and it needs to be before this is leaned on: whether adding a breakpoint over + bytes the provider has *already* cached itself incurs a write charge. If it does, the lever + costs what it saves. Cheap to answer with a two-request probe against real usage figures. + +Three things *can* pay, and they are the design: + +- **Batching.** One rewrite serves every cut taken at that boundary, so `S` is the **sum** of + all cuts in the pass. Cutting 60k of a 150k transcript at 20% depth needs `T > 23` turns — + plausible in a long session. So: a **rare, batched, threshold-triggered pass**, never a + per-output per-turn decision. The original instinct — A fires only when we are certain, and + it is a big cut — is exactly right, and this is why. +- **Step reduction.** At `corr = 0.95`, removing rot that costs the agent even a few turns + dominates all token arithmetic. The objective function is **steps and reward, not bytes**. +- **Deferring the agent's own compaction.** Claude Code compacts at + `min(window, configured) − min(maxOutput, 20000) − 13000` (167k on a 200k model), counted from + the **API's** reported usage — so our reduction genuinely pushes that back + ([agent compaction](../how-to/agent-compaction.md)). Staying under that threshold avoids a + full-transcript summarization that is both a large cache event and a quality loss. Plausibly + the largest prize here, and the reason a threshold-triggered regime is the right frame. + +One more asymmetry, stated carefully because the obvious version of it is wrong. It is +tempting to say a wrong cut is not a wrong answer, only a `context_guru_expand` round-trip +plus a cache-write. **That holds only when the model notices.** Expansion is +model-initiated — the tool is advertised and the host loop merely answers a call — and +nothing in the system detects a bad cut. So a wrong cut has three outcomes: + +| | Outcome | Cost | +|---|---|---| +| 1 | The model notices and expands the right marker | one round-trip + a cache-write | +| 2 | It notices something is missing but cannot tell which marker holds it | several expands, or it proceeds without | +| 3 | **It never notices** | it answers from less than it had — silently | + +Only (1) is the cheap case. **Reversibility is a capability, not a guarantee:** the stash +guarantees the bytes are recoverable, never that they get recovered. + +Row 3 is the one that matters, and Tier 3 is where it lives. A missing Tier-1 reference is a +token the model goes looking for and cannot find. A missing *semantic* reference is the model +reasoning from something it no longer has — there is nothing to look up, so nothing prompts +the expand call. The result is a plausible answer built on less evidence. + +Two consequences the rest of this document depends on: + +- **`expand` rate is a precision metric for noticed errors only.** It is still the right + inner loop — cheap, available on any traffic, no scoring — but it is *blind to row 3* by + construction. Anything that improves it by making the model expand less could be an + improvement or could be row 3 getting worse, and the metric cannot tell you which. +- **Reward is therefore the only instrument that sees the worst failure**, which is why §7 + and §8 treat it as a gate rather than as one number among several. + +What the design *can* influence is the gap between rows 1 and 2: whether the residue left in +place lets the model tell that this marker is where the thing it wants lives. That is why the +marker describes the SHAPE of structured content (`200 records, fields: address, id, name`) +rather than only peeking at its first line, and why it never asserts that the cut was safe — +an earlier version wrote "no later turn referred back to it", which is precisely the claim +that is false in the Tier-2/3 case, and which reads as reassurance not to bother expanding. + +## 5. Hard constraints the codebase imposes + +These are not preferences; each one is a property of existing machinery. + +1. **Decisions must be latched, not re-derived.** `repairLostFreeze` + (`components/offload/state.go`) is documented as safe *only* for offloaders whose + replacement is a pure function of `(content, config)` — `mask` and `failed_run` qualify + because their output is position-independent. A co-reference decision is **history-dependent + by construction**: whether to cut span *i* depends on messages after *i*, so re-deriving it + next turn may yield different bytes, which is precisely the flip that rewrites the suffix. + `coref` must therefore **store its decision per session** and replay the latched bytes. + `extract_llm` is already excluded from freeze-repair for the analogous reason (sampled + output); `coref` inherits that exclusion. +2. **One-way: keep → cut only.** New evidence can never resurrect a span, because un-cutting is + a second prefix rewrite. Monotonicity is a cache-cost requirement, not tidiness. +3. **`TailOnly` is being deliberately violated, so it must be budgeted.** Every other + age-based offloader refuses to touch the cached prefix (`Ctx.TailOnly`, + `components/component.go`). `coref`'s entire purpose is to mutate the prefix. That makes it + the first component that must **spend** cache-writes on purpose, under an explicit + per-session budget (a bounded number of rewrite events), with the spend reported next to the + benefit — the dashboard already has the slots (cache-frozen tokens, restorations, reverts). +4. **`freeze` / `reapplyFrozen` are mandatory.** Improvement plan §C3 records 101 + compacted→full→compacted flip turns and 15 non-byte-stable replays because 5 of 7 offloaders + never wired freeze. A component that mutates the prefix cannot ship without it. +5. **Reversibility and the expand-loop guard.** Standard `<>` + Store stash, and + `MarkKeptVerbatim` on anything the agent expands, so a restored span is never re-cut. +6. **Fail open, never worse.** Unchanged: any error reverts this component only; a pass that + would not shrink the request is reverted. +7. **`skipReduce` makes `coref` and `extract_llm` mutually exclusive per output, first-come.** + Every offloader consults `skipReduce` (`components/offload/state.go:292`), which refuses any + content already carrying an offload marker. So whichever of the two reaches an output first + owns it: once `coref` has replaced it with `<>`, `extract_llm` sees a placeholder and + declines, and vice versa. They do not compose or stack on the same output — pipeline order is + the whole policy. That is a **design constraint that has never been stated**, and it decides + the shape of the answer: a "coref-aware `extract_llm`" and a separate `coref` component are + not additive alternatives, they are competing owners of the same candidate set. If the two + ideas are to combine, they must combine **inside one component's decision**, not as two + components in a pipeline. +8. **The kept-verbatim guard is cross-session and best-effort, and both halves are surprises.** + `MarkKeptVerbatim` keys purely by content hash (`keptKey(ck) = "cg:keep:" + ck`, + `state.go:275`) with **no session in the key** — the comment says "session-independent" and + means it. So one expand of a given byte-identical output permanently exempts *that content in + every future session*. For genuinely per-session content that is harmless; for content that + recurs across sessions — a config file, a repeated banner, a standard schema dump, exactly the + high-value repeated mass — **one agent asking for it back once opts it out of compaction + globally, for every session thereafter.** Nothing reports this, so the effect is a slow, + invisible erosion of yield that looks like the component getting worse over time. + + And it is best-effort in the other direction: the flag goes through `Store.Put`, so it carries + the store's default TTL *and* competes for capacity in the same LRU as multi-kilobyte stash + payloads. A one-byte guard flag can be evicted by payload pressure, after which the content is + re-cut and the expand loop the guard exists to prevent can recur. + + Neither behaviour is `coref`-specific and neither is a `coref` bug — but `coref` is the first + component whose cuts are **latched and never revisited**, so it is the first for which a lost + guard flag is unrecoverable rather than self-healing next turn. + + **Both halves are now fixed.** `keptKey` is scoped by session — the minimal scope that still + prevents every loop the guard was built for, since the loop is intra-session by construction — + and `store.KeptPrefix` joined `DefaultPinPrefixes`, so a one-byte guard no longer competes for + LRU capacity against the multi-kilobyte stashes it guards. The scoped session travels from + `apply.Trace.Session` through to the proxy's expand loop rather than being recomputed there, + so the mark is always written under the id the pipeline compacted under. An empty session is a + no-op rather than a global mark. Two tests cover the half that used to be wrong: the exemption + does not leak to another session, and it still holds for the session that earned it. + +## 6. Trigger integration + +`coref` is an expensive, prefix-mutating pass, so it gates on the existing `Trigger` +(`components/trigger.go`) rather than inventing thresholds — `MinRequestFrac` against the +dynamically resolved `Ctx.CtxWindow` is the natural dial, which keeps it model-general. + +Two additions the current `Trigger` cannot express, both implied by §4: + +- **A remaining-turn estimate `T`.** The break-even inequality needs it. Cheap proxies: + elapsed turns against the threshold distance, or the observed step rate. Firing at 90% of the + window is firing when `T` is nearly zero — i.e. paying a rewrite for nothing. The + counter-intuitive consequence: **the profitable moment to compact is earlier than the + moment of maximum pressure.** +- **A rewrite budget** (constraint 3), which is a policy field rather than a shape threshold. + +### The deferral gate: designed, unquantified + +`min_batch_frac` is a correct implementation of the **token** argument in §4 and a poor proxy for +the **deferral** argument — which is the one §7's measurement says actually pays. This subsection +records the gap, the corrected arithmetic, and the order in which it should be closed. None of it +is built. + +**The prize is a step function, and clearing the threshold is not enough.** Cutting to exactly the +threshold buys one turn: the next turn grows the transcript, crosses again, and now you either eat +the compaction anyway or pay a **second** cache-write — at the point where `W` is largest and each +write is most expensive. So the requirement is not `deficit`, it is: + +``` +required cut ≥ (usage − threshold) + growthPerTurn × headroomTurns +``` + +Measured on the 19 real sessions in [the corpus](../results/coref-density.md) that passed Claude +Code's 167k threshold, expressed as a share of the request: + +| headroom bought | required cut | achievable with `unreferenced` + `closed` | +|---|---|---| +| H = 0 (bare clear) | 7.3% | 10/19 | +| H = 20 | 12.6% | 5/19 | +| H = 40 | 18.0% | **0/19** | +| H = 60 | 23.5% | **0/19** | + +So a bar high enough to avoid paying twice (≈20–25% of the request, which is what 40–60 turns of +headroom costs) is a bar **Tier-1 matching cannot clear**. Mean available cut is 4.4% of the +request for `unreferenced` and 9.6% including `closed`. That is also how the old +`min_batch_frac: 0.15` default was found to admit **1 of 19** sessions, and 0 at the shipped cut +set — a gate no traffic can clear is an off switch that looks like a threshold. + +!!! warning "One figure here is partly an artifact" + Peak request ≈180k and deficit ≈13k are shaped by `cc_capture.py` segmenting transcripts at + 180k tokens, so peaks cluster there by construction. The durable number is the one that does + not depend on it: **available cuttable mass is 4–10% of the request.** + +**What the gate should ask instead.** `coref` never runs alone, and it is the only component that +pays a prefix rewrite — `mask`, `extract` and `cmdfilter` all work in the uncached tail and are +cache-safe (`extract_llm`'s tail pass measured 12.5% on SWE-bench, 27.5% on Terminal-Bench — +figures long miscredited to `mask`, which is inert behind the tail gate). So the deferral +prize is mostly earned by the components that pay nothing for it, and `coref` is a marginal +contributor paying the most. Its gate should therefore not ask "is my cut large?" but "**does my +cut change the outcome?**": + +| Case | Condition, evaluated at `coref`'s entry | Action | Why | +|---|---|---|---| +| **Already safe** | under the threshold with headroom before `coref` runs | **do not cut** | The prize is already won; a rewrite buys nothing | +| **Decisive** | over the threshold before, under it after | **cut** | `coref` is what tips it — the only case that justifies a rewrite | +| **Unreachable** | still over the threshold even after `coref` | **do not cut** | The agent compacts regardless, so we would pay the rewrite *and* eat the compaction | + +The counter-intuitive case is the first: `coref` should be **less** aggressive when the rest of the +pipeline is doing well. `min_batch_frac` cannot express any of this — a 6% cut is "too small" +whether it is the decisive 6% or an irrelevant one. + +**It all reduces to one scalar: how many tokens until the agent compacts.** Which is the hard part, +because the threshold is compared against the **provider's own reported usage**, not against +anything we compute. Claude Code sums all four tiers of the most recent response's `usage` +(`input_tokens + cache_creation + cache_read + output_tokens`) plus a local estimate for the +trailing user turn ([agent compaction](../how-to/agent-compaction.md)). That figure includes the +`system` array, the tool definitions and last turn's *output* — none of which a component can see, +which is why `Ctx.ExistingBreakpoints` exists at all. `schema.MessagesTokens` is therefore a +systematic undercount by an unknown amount. + +**Three routes, in increasing cost — and the order matters, because the first may make the others +unnecessary:** + +1. **Measure whether the prize is even in play.** `modes.Tracker` already detects the agent's + compaction resets, so on real traffic we can ask how often the agent compacts and whether a + `coref` pass moves that at all. This needs *nothing new*, and it is ground truth rather than an + estimate. If the answer is "rarely, or not measurably", the whole gate is solving a problem we + do not have and `min_batch_frac` is adequate. +2. **Let the host supply the distance.** The proxy holds the raw body, including `system` and + `tools`, so it can count the full request and pass one number down against a configured + threshold. Self-contained, and it avoids the response path entirely; last turn's `output_tokens` + is a correction (capped ~20k against 167k), not the substance. +3. **Calibrate and learn.** Only if (2) proves too coarse: record per-session the offset between our + count and the previous turn's reported usage, and the observed marginal growth per turn, in the + Store — session-scoped like `sumCheckpoint`, with a cross-session prior like `markSeenContent` so + turn one is not cold. The offset is not constant within a session (tool definitions change, + output varies), so it wants a recent estimate rather than a lifetime average. + +**Start conservative, because the asymmetry is sharp.** Over-estimating growth means demanding more +headroom and cutting less often — safe. Under-estimating means cutting, crossing again, and paying a +second rewrite at maximum `W` — the disaster case. So a cold start should bias growth *high* and +headroom *large*, and relax only as evidence accumulates. + +**And none of this touches reward.** Every route above can tell us whether a compaction was +deferred. None can tell us whether the content we removed was needed — that is the silent failure in +§4, and it remains visible only in reward. + +**Status: the prize is argued, not measured.** This document has claimed throughout that deferring +the agent's own compaction is plausibly the largest win, without ever measuring how often it is +reachable. Route (1) is the resolution, and nothing else here should be built before it. + +## 7. What we do not know — the measurement pass + +Nothing above should be built before the substrate is measured, and it can be measured for +**zero API dollars**: the proxy already captures pristine inbound bodies as JSONL +(`CONTEXT_GURU_CAPTURE`, `proxy/proxy.go`), the same corpus that refuted `xdedup` (1,325 +requests / 51 sessions across `capture-tb`, `capture-swe`, `capture-swebench`). + +A second input path needs no proxy run at all: +[`deploy/harbor/cc_capture.py`](https://github.com/rossoctl/context-guru/blob/main/deploy/harbor/cc_capture.py) converts a Claude Code +session transcript — the agent's own append-only log of what it sent — into the same capture shape. That +is what produced [the measured results](../results/coref-density.md) when the eval box was out of reach, and +it is the cheapest way to re-measure on any workstation. + +[`deploy/harbor/coref.py`](https://github.com/rossoctl/context-guru/blob/main/deploy/harbor/coref.py) computes, per capture file and +per session, over Tier-1 novel-token references only: + +- **Referenced mass** — what fraction of tool-output tokens are ever referenced later at all. +- **Reference recency** — how many messages **ago** the last reference was, measured from the + head of the transcript. This is the A/B axis, and getting it right matters: the tempting + quantity is the output's own depth, or the gap from the output to its reference, and neither + is what "recent messages vs early messages" means. Reported alongside it, separately, is + **consume lag** (output → last use), which says how long an output stayed live. +- **Reference-count distribution** — 1× vs 2× vs 3+, the other half of the open/closed axis. +- **Reuse fraction** — of the novel tokens an output introduced, how many the model actually + carried forward. If this is low, "took a value, doesn't need the rest" is confirmed as the + dominant pattern rather than assumed. +- **Unreferenced mass** — outputs no later model turn touches. The safe, free, deterministic + cut, and the honest ceiling for a zero-LLM implementation. +- **Break-even table** — for each candidate cut set, the rewritten suffix `W`, the `T` it would + need, and the `T` the session actually had, so §4's inequality is answered with this traffic + rather than with an illustrative example. + +Note the pleasant simplification the open/closed predicate gets for free: because `coref` only +ever cuts **tool outputs**, and references live in **assistant** turns which are never cut, any +reference at all *is* a surviving copy. "Referenced by a model turn" ≡ "the value that was +taken still exists in the request". The witness needs no separate search. + +### Validated against a known-answer fixture + +[`deploy/harbor/coref_fixture.py`](https://github.com/rossoctl/context-guru/blob/main/deploy/harbor/coref_fixture.py) builds four tool +outputs whose correct classification is fixed by construction — one closed, two unreferenced, +one open — including the echo confound from §2 (a `Read(src/config.py)` whose only later +overlap is the path that arrived as the `tool_use` argument). All four classify correctly. + +The **negative control** is the part worth keeping. *Echo-exclusion guard* is the mechanism from +§2 above, named: only tokens the output **introduced** are eligible, so any token already present +at or before the producing tool call is discarded as an echo of context that predates the output. +Disabling it means counting every token match, echoes included. + +With the guard disabled, the +`src/config.py` output flips from `unreferenced` to `open`, and measured cuttable mass drops +**49% → 23%**. The guard is not a refinement — it is the difference between a usable +measurement and one that reports everything as load-bearing. + +```sh +python3 deploy/harbor/coref_fixture.py /tmp/cap.jsonl +python3 deploy/harbor/coref.py /tmp/cap.jsonl sweep=1 +# then, on real traffic: +CONTEXT_GURU_CAPTURE=/tmp/capture-swe.jsonl ./bin/context-guru-proxy --preset off +python3 deploy/harbor/coref.py /tmp/capture-swe.jsonl window=200000 fire_frac=0.6 sweep=1 +``` + +`sweep=1` prints closed-mass share across a `closed_dist` × `open_reps` grid, so the thresholds +come from the data instead of from this document. + +It reports Tier-2/3 mass as **unknown**, never as unreferenced. An exact matcher cannot +distinguish "never used" from "used after transformation", and conflating them is how a +compactor cuts something load-bearing. + +**Decision rules this pass feeds:** + +- If unreferenced mass is large and concentrated, the first version is deterministic and needs + no model call. +- If referenced mass is dominated by low reference counts with distant last-references, the + closed-case predicate is worth building. +- If the break-even table says no realistic `T` clears `11.5 × W` even when batched, the + component is a step-reduction and agent-compaction-deferral play only, and must be evaluated + that way — or not built. + +**Every one of those rules is about cost, and cost alone can never authorize shipping this.** A +compactor that saves tokens and loses a single task is a regression, because the value of the +context it removed was never denominated in tokens. So **reward is a gate, not a metric**: no arm +of any experiment here is interpretable without it, and a cost win alongside an unmeasured reward +is not a result. §8's acceptance criteria are deliberately ordered with reward parity first, and +the measurement in this section is explicitly *not* an experiment — it reads what a cut would +remove from traffic that already happened, and can say nothing about reward by construction. That +is a limitation of the measurement, not a reason to defer the question. + +## 8. Consequences for benchmark selection + +A benchmark only tests this if its traffic contains co-reference **at the tier the detector +targets**. That criterion cuts against the intuitive ordering: + +| Vehicle | Role | +|---|---| +| **SWE-bench Verified** (already wired, `deploy/harbor/swebench.py`) | **Tier-1-rich.** `Read → Edit → Bash` flows reference earlier outputs by exact path, symbol, line, error string. The right substrate for the deterministic detector, and the incumbent cost/reward regression floor (cache-read-dominated: 64% of the bill) | +| **Terminal-Bench 2.0** (already wired) | A **different cost regime** — output tokens are 47% of the bill, larger than cache-read. Must be tuned separately, and it is where a step-reduction claim is won or lost | +| **LOCA-bench** (MIT, native `anthropic` SDK + `LOCA_ANTHROPIC_BASE_URL` → direct attach) | The **controlled instrument**: context length is a dial (8K→256K) with fixed task semantics, deterministic binary scoring, and built-in `memory_tool` / `ptc` / context-editing arms — the naive-compaction baselines to beat. But its BigQuery/Sheets/Snowflake domains *aggregate and compute over* tool results, so references arrive transformed: it is a **Tier-2/3 stress test**, not a showcase for exact matching. Note its native trimmer orphans `tool_use`/`tool_result` pairs at 64K and provokes provider 400s. **Port the existing fix rather than rediscovering it:** `repair_tool_pairing()` in forever's `forever/benchmarks/_anthropic_auth_hop.py` already solves this rig-side in two phases — drop `tool_result` blocks whose `tool_use` was trimmed away (and any message left empty), then synthesize placeholder results for `tool_use` blocks left unanswered — and it counts the repairs so the rate is visible instead of silent. Worth noting separately that `coref` cannot *cause* this bug: it rewrites a tool message's text in place and never removes a message, so pairing is structurally preserved (unlike `summarize`, which restructures the list) | +| **UltraHorizon** | The most extreme regime (200k+ tokens, 400+ tool calls, hard in-context wipe), but LLM-judged, capability-gated, expensive, no license. Not a driver | +| **SlopCodeBench** | Resets per checkpoint, so sessions never approach the threshold. Structurally cannot test this | + +**Acceptance criteria** (in priority order, from §4): reward parity or better; **steps** down; +**cache-write** within a stated budget rather than "unchanged"; `expand` rate below a +pre-registered ceiling; billed cost down. And two methodology guards the corpus has already +paid for: do not stop at first significance (`p = 0.036` at `n ≈ 22` regressed to `p = 0.22` at +`n = 30`), and prevent-and-measure rather than filter-after-the-fact (dropping anomalous runs +introduced survivorship bias when the failure rate was arm-imbalanced). + +!!! info "Implementation status lives in its own document" + What is built, what is deliberately inert, and the ordered next steps are in + **[`coref` implementation status](coref-implementation.md)**. It is kept separate because it + goes stale on every commit while the argument above does not — and because a proposal that + doubles as a changelog stops being reviewable as a proposal. + +## 8b. What the held-out experiment settled + +§7's measurement pass reads what a cut *would* remove. A separate, later experiment asked the +harder question — **do the decisions come out right** — by holding out the future of 885 real tool +outputs and scoring ten arms against it. Full method, arms and limitations: +[the selection experiment](../results/coref-selection-experiment.md). Four of its results change +this proposal. + +**1. The deterministic index is the strongest discriminator measured, not a fallback.** It keeps +95% of what the agent went on to need while removing 11.8% of mass. Every arm that put a model in +the verdict path — including a merged prompt seeing both the content and the reference evidence — +scored *worse on both axes*, and no combination of index and model beat the index alone. This +**refutes** the intermediate design the discussion around this proposal had converged on +(demote the index to an evidence supplier, move the verdict into `extract_llm`'s prompt). + +**2. `cut_unreferenced` is not the free safe cut §3 calls it.** Measured false-drop is **11%**, +it is not a boundary artifact (57% of the errors land 51+ turns past the firing point), and it is +irreducible with the features the index has. §3's language has been corrected in the +[cheat sheet](../reference/coref-glossary.md); it should be read as a *cheap* cut with a bounded +error rate, never a free one. + +**3. Prompt framing is worth ~26 points of accuracy — and reassurance is the failure mode.** A +prompt telling the model its cuts "stay recoverable on request" produced 91% removal at 6% +live-kept. Replacing that with the real cost — *the agent usually does not notice the gap and +answers from worse information instead of asking for it back* — moved live-kept to 58%. This is +the same finding as the marker rule in [`corefstub.go`](../components/coref.md), arrived at from +the other end: **any surface that tells a decider a cut is cheap makes the decider careless**, +whether that decider is the compacting model or the agent reading the marker. + +**4. Selection cannot replace a summarizer, and this is arithmetic rather than a measurement.** +Let `g` be the mass arriving per turn and `f` the fraction that ever becomes removable. Sustained +removal is `f × g`, and `f < 1` always — dead content is a subset of arriving content — so removal +is strictly less than growth. Selection cannot hold the line; it multiplies time-to-threshold by +`1/(1−f)`: + +| removable share of request | session extension | +|---|---| +| 4.4% (`unreferenced`, measured) | 1.05× | +| 9.6% (`+closed`, measured) | 1.11× | +| 18% (most aggressive model arm) | 1.22× | +| 24% (most aggressive arm at 34% false-drop) | 1.32× | + +A summarizer reaches ~96% reduction because it compresses **live** content; `coref` can only +remove **dead** content. So the ambition of replacing the agent's own compaction — running a +lighter pass at 60% and a heavier one at the threshold — is **not available to a selective +component at any aggression setting.** `coref` is a deferral play, permanently. The honest +framing is that it buys 5–30% more turns before the summarizer runs, and its case rests on +whether those turns are worth a cache-write. + +### Two design notes from review, neither implemented + +**1. The MERGED call — what "fold" actually means, and what was built instead.** + +Review's proposal is that the co-reference *reasoning* belongs **inside `extract_llm`'s prompt**, +not beside it. The argument is economic and it is strong: Tier-2 (transformed) and Tier-3 (semantic) +references are invisible to exact matching by construction, Tier-1 carries a measured **11% +false-drop**, and *a model call is already being made* to decide what to trim. The marginal cost of +adding "…and consider whether later turns referred back to this, including in paraphrase" to a +prompt that is already being sent is approximately **zero**. One call, both jobs. + +What is implemented (`allow_cached_prefix`) is **not that**. It uses the index as an *eligibility +gate* — the index decides which prefix outputs are candidates, then the model decides how much of +each to keep. Two sequential mechanisms; the deadness judgement stays deterministic and stays +Tier-1-blind. + +The distinction matters because it is easy to think the merged design has already been refuted. It +has not. [The selection experiment](../results/coref-selection-experiment.md) refuted +**model-as-decider** — eight arms where the model judged deadness from content plus evidence, best +of them 58% live-kept against the index's 95%, and no intersection or union beating the index alone. +That is a different question. It never tested *adding a criterion to a call already being paid for*, +and the economic objection that ran through it — that calls cost money the saving cannot repay — +does not apply when the call happens regardless. The nearest data point is the intersection arm: +**96% live-kept at 8% removed**, marginally safer than the index alone at lower yield. Suggestive, +not damning. + +So the merged design is **untested**, and testing it is a prompt change plus passing the reference +evidence alongside the content — not new machinery. + +**Naming, fixed here to stop the two being conflated.** "Fold" was used loosely for both and is +retired: + +| name | what it is | status | +|---|---|---| +| **merged** | co-reference reasoning **inside** `extract_llm`'s prompt; one call, both jobs | **not implemented, untested** | +| **prefix reach with an index gate** | `allow_cached_prefix`: index selects candidates, model trims them | implemented, measured | + +The measured arms in [iterations 004–005](../experiments/README.md) exercised the *second*, and +several of them additionally ran `coref` as a standalone component in the same pipeline — so those +numbers cannot separate the gate from standalone dropping. [Iteration 006](../experiments/loca/iter006/results.md) +isolates `coref` against `format`-only for that reason. + +**2. A cache rewrite is sometimes already free, and §4 never accounts for it.** + +`S × T > 11.5 × W` prices a cache-write that the mutation *causes*. But when the provider's cache has +already expired, the prefix is re-written on the next turn **regardless** — so at that moment a +prefix mutation costs nothing incremental, and no break-even test is needed at all. A component that +can never justify a rewrite could act for free whenever the gap between turns exceeds the cache TTL: +a slow tool, a queue, a human thinking between turns. + +Measured on LOCA (`deploy/harbor/cache_opportunity.py`, direct from per-step +`cache_read_input_tokens`): **zero such moments**. That is a property of the benchmark rather than an +answer — LOCA drives local mock MCP servers, so turns land seconds apart and a 5-minute TTL never +lapses. The workloads where it would appear are the slow-tool ones: SWE-bench container builds and +`pytest` runs, Terminal-Bench, anything with a human in the loop. Note the irony that SWE-bench was +[ruled out](../results/component-gating.md) for having tool outputs too small for these components, +yet its slow tools may make it the right vehicle for *this* question. + +**Deferred deliberately, to be examined on a different benchmark.** Acting on it needs TTL state at +*decision* time, and the provider reports cache usage in the **response** — after the decision — so +it would have to be inferred from the previous turn's usage plus elapsed wall-clock. That is a design +question with a real failure mode (guess wrong and you pay 11.5× believing it was free), not a patch. + +### The hypothesis this proposal should be tested against + +Everything above narrows the claim to one testable sentence, which is what §8's acceptance +criteria should be pointed at: + +> **On a caching backend, a batched, latched, one-way `coref` pass firing once per session at +> 55–70% of the context window removes 10–25% of the request at ≤11% false-drop, defers the +> agent's own compaction by 20+ turns, and does so at reward parity — with the cache-write it +> spends repaid by the deferred summarization rather than by the tokens it removed.** + +It is falsifiable in four independent places, and three of the four are cheap: + +| Clause | How it fails | Cost to test | +|---|---|---| +| "removes 10–25%" | Measured 4.4–9.6% on interactive traffic at the shipped cut set | **done** — it currently fails | +| "defers by 20+ turns" | 0/19 sessions could reach 40 turns of headroom **when fired late**; fired at the crossing the deficit term vanishes and 20–60 turns is affordable, but only in the 17–29% of sessions that compact at all | **partly answered** — see [reachability](../results/coref-reachability.md) | +| "cache-write repaid by deferred summarization" | Needs the deferral prize to be reachable at all | cheap — `modes.Tracker` reset detection, no new machinery | +| "at reward parity" | Any task lost to a false drop | expensive — the eval box, and the only real gate | + +One clause **fails on measured traffic** and one is **partly rescued** by firing earlier, which is +why the component ships opt-in and in no preset. Recording that plainly is more useful than restating the ambition: the +remaining case for `coref` is that the corpus it failed on is the wrong one (interactive research +traffic, mostly one author, `opaque`-heavy), and the corpus the acceptance criteria are written +against has never been measured. + +## 9. Open questions + +- ~~**How often is the deferral prize actually reachable?**~~ **Measured** — see + [reachability](../results/coref-reachability.md). The prize exists in **17% of sessions (6/35)** + and **29% of sessions past 200 model turns**, so every expected-value argument here must be + multiplied by ~0.17–0.29 — a factor no version of this document carried. The same pass produced + the first clause of the hypothesis that does *not* fail: fired **at** the threshold crossing the + deficit term vanishes (it was an artifact of measuring after a late fire), leaving only + `growth × headroom`, which the available cut can supply for 20–60 turns. That vindicates §6's + counter-intuitive claim about the profitable moment from a second direction. It is sensitive to a + growth estimator two measurements disagree about by 2×, and it still inherits the 11% false-drop. +- **Is `xdedup` back on the table?** §C left one caveat explicitly open: compaction is the one + regime that could make cross-turn dedup viable, because it removes the first copy while later + re-reads land in the mutable tail. `coref` *creates* that regime. C1 should be re-measured + after, not assumed still refuted. +- **Where does the reference index live?** Recomputing it per turn on a 150k transcript is + latency the sync path may not absorb (budget: ~117 ms added today, ~450 ms with the LLM + trimmer). An incremental per-session index in `store`/`session` is the likely answer, but it + is state the components layer does not currently keep. +- **Does `observe` mode suffice for the first read?** It measures what a pipeline would have + done with zero request modification, which fits — but the cache-write cost of a prefix + mutation is exactly the thing observe cannot observe, since nothing is forwarded. +- **Tier-2 escalation shape.** If the deterministic ceiling is low, the escalation is an + `extract_llm`-style cheap-model pass restricted to far-field large spans. That pass is + *sampled* output, so per constraint 1 its decision must be latched on first computation and + never recomputed. + + +--- + +## FUTURE CONSIDERATION: reversibility is only coherent if the substitution is total + +Raised in review and recorded here rather than acted on, because it does not affect current behaviour. + +Expand recovery now works by rewriting the client's own `tool_result` on the request path +(`expand.RestoreResults`), because the response loop structurally cannot satisfy an expand call that +arrives alongside a real tool call — and live traffic showed that is 102 of 107 cases. + +That rewrite touches a message the model has **already seen**. It is coherent only if applied +**deterministically on every turn**: + +- **Intermittent substitution is incoherent.** The model saw `Tool 'context_guru_expand' not found` on + turn N and may have acted on it — re-run the tool, written the content off. If turn N+5 shows the + content where the failure was, the record contradicts the model's own reasoning that is still sitting + next to it. +- **Intermittent substitution is also a cache miss**, from the position of the changed message, every + time it flips. The client resends its originals each turn, so the rewrite must be re-derived每 turn or + the prefix flaps — the same discipline the freeze/reapply machinery exists to enforce. + +Being keyed only by marker id and store contents, the rewrite is deterministic **for as long as the +stash lives**. Two things protect that, and both are worth revisiting: + +1. `MarkKeptVerbatim` (applied on both the response and request paths) stops expanded content being + re-compacted, so no later turn needs a marker for it. +2. Stash durability. The in-memory store loses everything on restart, and + `expand_unresolved_missing` in `/stats` counts exactly the cases where a marker was issued and its + original is gone — i.e. where a cut advertised as reversible was not. + +**The open question is whether reversibility is worth this much machinery.** The measured facts do not +obviously favour it: across 225 runs the agent reached for recovery constantly and was refused, and its +fallback was to re-run the tool rather than to give up — which is more expensive than the "one +round-trip" the design assumes. If recovery cannot be made reliably total, `marker_mode: summary` (drop +without pretending reversibility) or simply not removing the content are the honest alternatives, and +they should be compared on reward rather than assumed inferior. diff --git a/docs/proposals/coref-implementation.md b/docs/proposals/coref-implementation.md new file mode 100644 index 00000000..46ee1f15 --- /dev/null +++ b/docs/proposals/coref-implementation.md @@ -0,0 +1,196 @@ +# `coref` implementation status + +What exists in the tree, what is deliberately inert, and what has to happen next. Split out of +[the proposal](coref-compaction.md), which is the design argument and should stay readable as one — +this is the part that goes stale with every commit. + +**Related:** [the proposal](coref-compaction.md) · [component reference](../components/coref.md) · +[cheat sheet](../reference/coref-glossary.md) · [measured density](../results/coref-density.md) + + +Two pieces exist, and the split between them is the point: the **mechanism** is a matter of getting the +definition of a reference right, which a known-answer fixture can settle; the **thresholds** are a +matter of what real traffic looks like, which only §7's pass can settle. So the first is built and the +second is not, and the component is configured to be inert on anything that depends on the second. + +**Built.** + +- [`internal/coref`](https://github.com/rossoctl/context-guru/blob/main/internal/coref/coref.go) — the Tier-1 index: identifier + tokenizer, novel-token (echo) exclusion, boilerplate exclusion, sibling exclusion, reference count, + recency from the head, consume lag, used-fraction, and the open/closed/unreferenced predicate. No + bifrost, no components, no tokenizer dependency, so it is a pure function of a flattened message list + — deliberately, because it must stay interchangeable with `coref.py`'s definition. The Go fixture is + the twin of `coref_fixture.py` down to the four known answers **and the negative control**: with the + echo guard disabled the `src/config.py` read flips out of `unreferenced` and measured cuttable mass + falls, and the test fails if it does *not* flip — the control is asserted, not just run once. +- [`components/offload/coref.go`](https://github.com/rossoctl/context-guru/blob/main/components/offload/coref.go) — the Offload + component, with each of §5's constraints as a tested behaviour rather than a comment: latched + decisions replayed byte-for-byte even when fresh evidence would reclassify the span (constraint 1, + and `repairLostFreeze` deliberately not consulted), keep→cut only (2), prefix mutation on purpose + under a per-session `rewrite_budget` (3), `freeze`/`reapplyFrozen` wired from the start (4), + `<>` + stash + kept-verbatim (5), and side-effect-free planning so a batch that fails a gate + leaves the request byte-identical (6). +- §4's arithmetic as an actual gate, not a note: `min_batch_frac` for batching, and `break_even` + applying `S × T > 11.5 × W` with `T` estimated from the transcript's observed growth rate and `W` + bounded to the *cached* span (content past the cache boundary would be written this turn regardless). + The counter-intuitive consequence from §6 is what the test pins: at the window edge `T ≈ 0` and the + pass correctly declines. + +**Deliberately not built, and why the "measure first" rule in §7 is not being broken.** §7 says nothing +should be built before the substrate is measured. What that rule protects against is *calibrating* a +component against numbers nobody has — so the implementation is scoped to the part that has no +calibration in it. `cut_unreferenced` needs no threshold: "no later turn used anything this output +introduced" is a fact about the transcript. `cut_closed` needs two (`closed_dist`, `open_reps`), which +are precisely what §7 produces, so it defaults to **off** and the shipped values are placeholders +carried over from `coref.py`'s defaults for comparability, not recommendations. `coref` is in **no +preset** for the same reason. + +**Also not built, and deliberately: the deferral gate.** `min_batch_frac` implements the token +argument and is a poor proxy for the deferral argument — it cannot ask whether this cut is the +*decisive* one. The design, the measured numbers (a bar high enough to avoid paying two cache-writes +is 20–25% of the request; Tier-1 finds 4–10%) and a three-step order of attack are in +[the deferral gate](coref-compaction.md#the-deferral-gate-designed-unquantified). Step one is to +measure whether the prize is reachable at all, using `modes.Tracker`'s existing reset detection — +which needs nothing new and may make the rest unnecessary. Nothing else there should be built first. + +Also not built: the Tier-2 LLM escalation ([open questions](coref-compaction.md#9-open-questions)), and the incremental per-session reference index. The +index is currently recomputed per firing turn — acceptable because the trigger makes firings rare, but +it is the latency question in the proposal's [open questions](coref-compaction.md#9-open-questions) and it is unmeasured. + +**Measured, on three corpora.** The pass has run — on **Claude Code transcripts, UltraHorizon runs and +LOCA-bench trajectories**, none of which is the eval-box capture set (unreachable). Full write-up and +caveats: [co-reference density](../results/coref-density.md). + +The single most useful result is that the three corpora **disagree by a factor of three**, so +`unreferenced` mass is a property of the workload rather than a constant: + +| | Claude Code (interactive) | UltraHorizon | LOCA-bench | +|---|---|---|---| +| `unreferenced` | 23% | 78% | 95% | +| `closed` | 15% | 8% | **0%** | +| `open` | 60% | 13% | 4% | +| …restricted to outputs with ≥20 later turns | 21% | 70% | 70% | + +Interactive work on a coherent codebase keeps returning to the same files and errors; benchmark tasks +survey, extract, and move on. The last row bounds the obvious bias (an output near the end has no later +turns that *could* reference it) and the ordering survives it: **benchmark traffic carries ~3.3× the +unreferenced mass of interactive traffic.** LOCA's 0% `closed` is also §8's own prediction landing — it +argued LOCA would be a Tier-2/3 stress test where references arrive transformed past what a substring +match can see, and an exact matcher finds not one output in 166 that was referenced once or twice and +then left alone. + +Four things it settles, and one it overturns: + +- **`cut_unreferenced` is justified as the default** — 21% of mass on interactive traffic and ~70% on + benchmark traffic, with no calibrated threshold and no model call. Decision rule one from §7 is + answered yes on every corpus. +- **A reference consumes a median 18.7% of what its output introduced** (11.5% on UltraHorizon). + Hypothesis A — "took one value, does not need the rest" — is confirmed rather than assumed. +- **Tier-2 leakage measures ~2% of model turns.** Unpacking that, because it is a proxy and not a + direct measurement: Tier 2 is a reference that arrived *transformed* (the model summed the rows or + converted the units), so by definition no substring match can find it. What can be counted instead is + a **symptom** — a model turn that states a numeric value appearing nowhere in any earlier message. If + the model says "3 seconds" and `3` is nowhere upstream, it computed that number from something, and + that something was almost certainly a tool output. On the interactive corpus 2% of turns look like + that, which says the deterministic ceiling is not badly compromised and a zero-LLM first version is + viable. + + Three caveats. It is a *lower* bound even for Tier 2: only numeric transformations leave this trace, so + a reworded finding is invisible to it. Tightening the identifier rules (see the results write-up) also + blinded it further — bare numbers now need 5+ digits, and most computed values are small — so its + **0% on LOCA means "none among tokens the tokenizer still accepts", not "none"**. + + And the third is a scope point worth keeping straight: **this proxy is Tier-2 only, and Tier 3 has no + measurement at all** — not a degraded one, none. A semantic reference ("as I noted earlier", "per the + schema") carries no shared token *and* no novel numeric, so nothing here can see it, by design rather + than by regression. So on a corpus with 0% `closed` and 40% `opaque`, the honest reading is that + **Tier-2 and Tier-3 references there are both common and unmeasured**, for different reasons: Tier 2 + has a detector that is nearly blind, Tier 3 has none. Tier 2 needs its own detector; Tier 3 needs a + model call, which is why it sits in open questions rather than in a measurement. +- **Break-even is workload-dependent, and better on benchmarks than on long interactive sessions**: + median required `T` is 95 turns for Claude Code (15/30 sessions clear it) against 17 for UltraHorizon + (7/10) and 14 for LOCA (4/9) — the cut is a far larger share of a smaller transcript. §4's arithmetic + holds everywhere; batching moves break-even from unreachable to *comfortable on benchmarks* and + *marginal on long interactive sessions*, so decision rule three still applies and steps plus deferred + agent-compaction remain the load-bearing justification. One trap: a break-even figure measured against + a window the traffic never used is a construction, not a result — UltraHorizon reads 0/10 at a 200k + window purely because its peak request is 30k and the trigger never fires. +- **Overturned: distance is not merely a lossy proxy, it is nearly inert.** Sweeping `closed_dist` over a + 10× range moves closed mass by 2–3 points; sweeping `open_reps` from 2 to 6 moves it by 18. And 44% of + all mass was last referenced 40+ messages ago while 60% is `open` — most referenced mass is old *and + still hot*. A distance-based A/B split would confidently cut repeatedly-referenced content. §3's + reframe is load-bearing, `open_reps` is the only dial worth tuning, and `closed_dist` should be left + alone. + +One methodological result deserves promoting out of the write-up, because it nearly invalidated the +measurement: **the identifier/prose rule decided the answer.** An earlier tokenizer accepted any 10+ +character token, so `description`, `transparency`, `efficiency` and `conditions` scored as references and +referenced mass came out at 71% instead of 60%. A manufactured reference makes an output look +load-bearing, so that class of bug fails by **silently declining to compact** — invisible to any metric +that counts only what the component did. Every false positive is now a regression case in +`internal/coref/coref_test.go`, and the residual (lowercase hyphenated compounds, indistinguishable from +real names like `context-guru`) is bounded at ~6 points of *under*-reporting rather than argued away. + +**What has to happen next**, in order: + +1. ~~Re-run `coref.py` over `capture-tb` / `capture-swe` / `capture-swebench` on the eval box.~~ + **Done** — [eval-box measurement](../results/coref-evalbox.md). `unreferenced` is **28%** on + `capture-swebench` (50 sessions, 433 outputs), the best measured and double the interactive + figure, confirming §8's claim that SWE-bench is the Tier-1-rich substrate. Two findings came with + it: a session-key collision in `coref.py` was silently discarding 98% of that capture (fixed — + `metadata.user_id` carries the real session id), and the corpus is **too shallow to test + deferral at all** (peak request 12.6k against a 167k threshold). `capture-tb` and `capture-swe` + turn out to be smoke captures (6 and 2 outputs), so only one of the three supports a claim. +2. Then, and only on that corpus, flip `cut_closed` on. `open_reps: 3` is the conservative setting; + `closed_dist` is inert and should stay at its default. +3. `observe` mode on real traffic to read `expand` rate — the precision inner loop from §4 — before any + scored benchmark run. +4. Only then §8's benchmarks, with the multi-seed and don't-stop-at-first-significance guards. + +~~Separately and in parallel, because it needs no API budget and no eval box: measure how often the +agent's own compaction is reachable at all.~~ **Done** — [reachability](../results/coref-reachability.md). +17% of sessions, 29% past 200 model turns; and firing at the crossing rather than late removes the +deficit term, which makes 20–60 turns of headroom affordable where the density pass had 0/19. The +deferral gate is therefore worth building, but it should gate on *fire early*, not just on batch size. + +`cut_closed` nonetheless **stays off**: it is 20% of mass on `capture-swebench` and 0% on LOCA, so +the workload spread that made it undefendable is unchanged by having the right corpus. + +## What the held-out experiment removed from this list + +A separate [selection experiment](../results/coref-selection-experiment.md) ran ten arms against +held-out ground truth. It **closed three questions** that were queued here, and it did so mostly by +ruling things out — which is the cheapest kind of progress this list can make: + +- **Do not build a model-in-the-verdict variant.** Eight model arms across two models and four + prompt shapes; none beat the deterministic index on both axes, and no intersection or union of + index and model beat the index alone. The intermediate design (`coref` supplies evidence, + `extract_llm`'s prompt supplies the verdict) is refuted, not merely unproven. +- **Do not specify trims as free text.** Models returned text that was not verbatim in the original + in the majority of trim verdicts. Any trim path must go through `extract_llm`'s sandboxed-filter + mechanism, which enforces containment structurally + (`internal/extract/contain.go`), rather than through a "return the part worth keeping" prompt. +- **Do not pursue replacing the agent's summarizer.** Sustained selective removal is `f × g` with + `f < 1`, so it multiplies time-to-threshold by `1/(1−f)` — 1.05–1.32× on measured traffic — and + cannot hold a context flat at any aggression setting. `coref` is a deferral play permanently. + +And it **added one item**, ahead of everything above because it is a correctness issue in shipped +code rather than a calibration question: + +0. ~~**Fix the kept-verbatim guard before `coref` goes in any preset.**~~ **Done.** + `MarkKeptVerbatim` keyed by content hash with no session component, so one expand exempted that + byte-identical content in *every future session* — permanently eroding yield on exactly the + recurring content worth cutting — and the one-byte flag shared the payload LRU with the + multi-kilobyte stashes it guards, so it could be evicted and the guard silently lost. + + `keptKey` is now session-scoped (the loop it prevents is intra-session by construction, so that + is the minimal correct scope), `store.KeptPrefix` is pinned against LRU eviction, and the proxy + threads `apply.Trace.Session` into the expand loop so the mark lands under the same id the + pipeline compacted under. Empty session is a no-op rather than a global mark. Covered by + `TestKeptVerbatimDoesNotLeakAcrossSessions` and + `TestMarkKeptVerbatimIgnoresAnEmptySession`. Rationale in + [the proposal, §5.8](coref-compaction.md#5-hard-constraints-the-codebase-imposes). + +Two other things it did **not** settle, and neither is cheap: nothing here touches **reward**, and +the experiment ran on captured traffic with a single firing point per transcript. The ordering above +is unchanged — reward is still the gate. diff --git a/docs/reference/coref-glossary.md b/docs/reference/coref-glossary.md new file mode 100644 index 00000000..18b75743 --- /dev/null +++ b/docs/reference/coref-glossary.md @@ -0,0 +1,271 @@ +# `coref` cheat sheet + +Every term the co-reference work uses, in one page, in the order you meet them. Full argument in +[the proposal](../proposals/coref-compaction.md); measured numbers in +[the density pass](../results/coref-density.md) and [the selection +experiment](../results/coref-selection-experiment.md); config in [the component reference](../components/coref.md). + +## The one-sentence version + +`coref` cuts tool outputs that **nothing later in the conversation ever used** — and because cutting deep +in a transcript forces the provider to re-write its cache, it does all its cutting **at once, rarely, and +never changes its mind**. + +--- + +## 1. The core idea + +| Term | Means | +|---|---| +| **co-reference** | A later turn pointing back at an earlier tool output. The whole component is built on detecting these. | +| **reference** | Concretely: an identifier that a tool output *introduced* shows up again in a later **model** turn (its prose, or a tool-call argument). | +| **novel token** | An identifier the output **introduced** — one that appears nowhere at or before the tool call that produced it. Only these can constitute a reference. | +| **echo** | The opposite, and the trap. `Read(src/auth.py)` puts the path in the tool-call *argument*; the result echoes it; a later `Edit(src/auth.py)` uses it again. Exact matching sees a reference, but nothing was ever taken *from the output*. | +| **echo guard** (prior-vocabulary exclusion) | Excluding tokens already in context before the output existed. Not optional: without it the measurement reports nearly everything as load-bearing, and the component then declines to cut anything. | + +## 2. The two cases from the original idea + +The idea started with: if a later turn references an earlier output, that means one of two things. + +| Term | Means | +|---|---| +| **case A** | The model already **took what it needed** — lifted one value out of a big response and doesn't need the rest. Licenses a *large* cut. | +| **case B** | The model has effectively **marked it important** and may need it again. Keep. | + +The original framing separated A from B by **distance** (B = recent, A = early). The measurement says +distance barely works — see `closed_dist` below. + +## 3. The four verdicts (what the classifier outputs) + +For each tool output, exactly one of: + +| Verdict | Means | Cut it? | +|---|---|---| +| **`opaque`** | The output introduced **nothing the index can track** — so there is no evidence either way. | **Never.** Absence of evidence is not evidence of deadness (see the box below). | +| **`unreferenced`** | It **did** introduce trackable identifiers, and no later turn used any of them. | **Yes — the shipped default** (`cut_unreferenced`). No threshold, no model call. **Not free:** measured **11% false-drop** against held-out ground truth — see the box below. | +| **`closed`** | Referenced **once or twice, and not for a long time**. Whatever the model took survives in the turn that took it, so the original is redundant *with content still in the request*. This is **case A** made checkable. | Optional (`cut_closed`, **off by default**). | +| **`open`** | Referenced **recently, or repeatedly**. Still load-bearing. This is **case B**. | **No.** | + +!!! warning "`unreferenced` never means 'unused' — and it is wrong 11% of the time" + It means "no later **exact** use". A value the model summed, converted or reworded leaves no substring + to match, so it lands here too. Always an **upper bound** on what is safe to cut. + + Now measured, not argued. Held out the future of 885 real tool outputs and asked how often + "unreferenced at the firing point" was contradicted later: **11%**. It is **not a boundary + artifact** — 0% of those errors are one turn past the firing point and **57% are 51+ turns + past it** — and it is **irreducible with the features the index has**: demanding more + introduced identifiers makes it *worse* (11% → 21%), demanding longer dormancy barely moves + it while costing 2.5× the mass. An output that lies dormant for a hundred turns and is then + used carries no signal, at the moment of decision, that distinguishes it from one dormant + forever. + + Since ground truth is Tier-1 matching only, **11% is a lower bound.** Full method and the + ten arms it was measured against: [the selection + experiment](../results/coref-selection-experiment.md). + +!!! danger "`opaque` vs `unreferenced` — the distinction that took a review to find" + Both have zero references, and they are opposites. "Introduced 200 identifiers, nobody touched one" is + *evidence of deadness*. "Introduced nothing I can see" is *absence of evidence*. + + It matters because it is common, not exotic. A tool returning + `[{"name":"david","id":123,"address":"foobarbaz"}]` yields **no** trackable tokens — short lowercase + words and 3-digit numbers are exactly what the precision rules exclude. Measured: 8% of tool-output + mass on interactive traffic, 20% on UltraHorizon, **40% on LOCA-bench**. Folded into `unreferenced`, + all of it was a silent vote to delete under the default config. + +**Why "closed" looks cheap to establish:** `coref` only ever cuts *tool outputs*, and references live in +*model turns*, which it never cuts. So a reference is always a surviving copy of *something* — no separate +search for the **witness** is needed. + +!!! danger "…but a surviving copy of *something* is not a copy of what's *needed*" + Given the records above and a model that says *"I need to remember david 123 address"*, the reference + (`david`, `123`) is real — and the value actually needed (`foobarbaz`) was **never copied into a model + turn**. The model referenced an **anchor** in order to point at a payload it did not restate. + + An exact matcher cannot tell an anchor reference from a payload reference. That is the real reason + `cut_closed` ships **off**, and it is why a low `used_frac` is *ambiguous* rather than evidence for + case A: "took the value, rest is chaff" and "took an anchor, still needs the payload" look identical. + +## 4. The two thresholds that decide `closed` vs `open` + +| Knob | Default | Means | Verdict from the data | +|---|---|---|---| +| **`closed_dist`** | 12 | How many messages **ago** the last reference must be before the output counts as `closed`. Newer than this ⇒ `open`. | **It is load-bearing but flat.** Set it to 0 and the `closed` class stops existing, so it *matters*; but anywhere in 4–40 gives the same answer within 2–3 points, so there is no gain from tuning it. Leave it at the default and spend the effort on `open_reps`. | +| **`open_reps`** | 3 | Referenced at least this many times ⇒ `open` **regardless of age**, because a span referenced repeatedly is a hot span that happens to be old. | **This is the dial.** 2→6 moves the answer 18 points. 3 is the conservative setting. | +| **`min_later_turns`** | 8 | The **opportunity floor**: an output with fewer model turns after it is `open` regardless of everything else. | **Justified structurally, not by safety.** Near the tail, "no references yet" and "recent" are the same thing, so without it a batched pass preferentially cuts the **most recent** context — the worst possible choice, and `mask`'s `keep_recent` idea expressed in turns. But it does **not** buy accuracy: swept against held-out ground truth, `min_later=0` yields *more* mass (363k vs 207k) at a *lower* false-drop rate (8% vs 11%). Keep it for the structural reason; do not claim it as a safety margin. | + +## 5. The three measurements per output (and the one that's easy to get wrong) + +| Term | Means | +|---|---| +| **ref count** | How many later turns used something this output introduced. Feeds `open_reps`. | +| **ref age / recency** | How many messages ago the **last** reference was, counted **from the head of the transcript** (i.e. from *now*). Feeds `closed_dist`. | +| **consume lag** | How many messages **after the output** its last reference was — i.e. how long it stayed live. A *different* axis, reported separately. | +| **used fraction** | Of the identifiers the output introduced, the share the model actually carried forward. Measured median ~19% — but see the anchor box in §3: a *low* value is **ambiguous**, not evidence for case A. | +| **later turns** | How many model turns follow the output — its **opportunity** to be referenced. Feeds `min_later_turns`. | + +!!! note "recency ≠ consume lag, and conflating them is the bug" + "Recent messages vs early messages" is a statement about **now**, so recency must be measured from the + head. The tempting alternatives — the output's own depth, or the gap from output to reference — are + different quantities. An early output referenced immediately has a *small* consume lag and a *large* + ref age; swapping them makes every ancient output look freshly used. (This mis-modelling was caught by + the fixture, not by inspection.) + +## 6. The three tiers of reference (what's detectable) + +| Tier | Signal | Detectable? | +|---|---|---| +| **Tier 1** | Literal carry-over — a path, symbol, id, hash or error string reappearing verbatim | **Yes, exactly, no LLM.** This is all that ships. | +| **Tier 2** | **Transformed** carry-over — the model summed the rows, converted the units, reworded the finding. No substring match exists | No. This is the deterministic ceiling. | +| **Tier 3** | **Semantic** — "as I noted earlier", "per the schema" | LLM only. | + +Tier 2 is the objection raised on the original thread (values drift through paraphrase and unit +conversion). Measured at ~2% of model turns on interactive traffic — but only via a numeric proxy, and +**Tier 3 is not measured at all**, so treat that 2% as a floor on invisible references rather than a +total. Real, small on this traffic, and the reason +`unreferenced` is an upper bound. + +## 7. The cache economics (why the component has this shape) + +| Term | Means | +|---|---| +| **cache-read / cache-write** | Re-sending an unchanged prefix is a cheap cache-**read**. Mutating it breaks the prefix hash and forces a cache-**write** of everything after the mutation. | +| **11.5×** | One cache-write costs 11.5 cache-reads: `($2.50 − $0.20) / $0.20` per MTok. | +| **`S`** | Tokens the cut removes (the saving, collected on every later turn). | +| **`W`** | The **rewritten suffix** — tokens from the shallowest cut to the end of what the provider already cached. The cost. | +| **`T`** | Turns remaining in the session, i.e. how many times the saving gets collected. Nobody has this, so it's estimated from how fast the transcript has been growing. | +| **break-even** | `S × T > 11.5 × W`. | + +**Why this reshapes everything:** a single early cut can never clear it — 5k cut from 20% depth of a 150k +transcript needs `T` > 276 turns. Three consequences, and they *are* the design: + +| Term | Means | +|---|---| +| **batching** | One rewrite must serve **every** cut in the pass, so `S` is the sum of all of them. That's what makes break-even reachable (60k of a 150k transcript needs `T` > 23). Hence a rare, threshold-triggered pass — never a per-output, per-turn decision. | +| **`min_batch_frac`** | The operational form of that: the pass must cut at least this fraction of the request (default 0.05) or it declines and leaves the request byte-identical. A correct implementation of the *token* argument, and a **poor proxy for the deferral argument** — it cannot express "is my cut the decisive one?". See the box below. | +| **`rewrite_budget`** | Prefix-rewrite passes allowed per session (default 3). `coref` is the only component that spends cache-writes **on purpose**, so the spend is capped and reported. | +| **step reduction** | The real prize. `corr(Δsteps, Δcost) = +0.95`; unique token removal is ~0.02% of the bill. The objective is **steps and reward, not bytes**. | +| **deferring agent compaction** | Claude Code compacts itself at ~167k on a 200k model. Staying under that avoids a full-transcript summarization — a large cache event *and* a quality loss. Plausibly the biggest win. | + +**The counter-intuitive consequence:** firing at 90% of the context window means `T` ≈ 0 — paying a +rewrite for a saving collected once. **The profitable moment to compact is earlier than the moment of +maximum pressure.** + +### What all this means at 200k vs 1M + +Raised in review, and the answer is less obvious than "bigger window, more to cut". + +**The break-even inequality is scale-invariant.** Rearranged, `S × T > 11.5 × W` says `T > 11.5 × (W/S)` +— the turns you need depend only on the **ratio** of rewritten suffix to cut mass, never on absolute size. +On the measured interactive corpus that ratio is ~15 (a 10.5k cut against a 157k suffix), hence `T > 138`. +A 1M-token transcript with the same *density* of cuttable mass has the same ratio and the same required +`T`. So a larger window neither rescues nor damns the token economics; it only moves **when** the trigger +fires. What actually improves the ratio is cutting a larger share of what lies *after* the shallowest cut — +which is an argument for cutting deep and rarely, not for cutting more. + +**Three things genuinely do change:** + +| At a 1M window | Effect | +|---|---| +| Cache-read is the whole bill | Re-reading ~1M cached tokens every turn dominates cost long before the window is a constraint. That makes `coref` a **cost** play at 1M rather than a *fit* play, and it is the strongest argument for it there. | +| The agent's own compaction recedes | Claude Code compacts at ~967k instead of ~167k, so the deferral prize becomes **rarer but much larger** — avoiding one summarization of a 1M transcript. As you note, it is also the one prize that is cheap to *measure* deterministically: compare the API-reported usage against the documented threshold and count the turns of headroom the cut bought. No benchmark scoring, no seeds. That belongs in the metrics, and it is not there yet. | +| The index gets 5× more expensive | Recomputing the reference index over 1M tokens per firing turn is the open latency question, and it scales linearly with the window. An incremental per-session index stops being an optimization and becomes a requirement. | + +**And the prize is a step function, not a slope.** You either drop below the agent's compaction +threshold or you don't; cutting 90% of what was needed is worth nothing. Worse, clearing it *exactly* +buys one turn — the next turn grows past it again, and then you pay a **second** cache-write at the +point where `W` is largest. So the real requirement is +`(usage − threshold) + growthPerTurn × headroomTurns`, which on measured traffic works out at 20–25% +of the request for 40–60 turns of headroom — and Tier-1 matching finds only 4–10%. + +!!! danger "The gate is measuring the wrong thing, and the right thing is unmeasured" + `min_batch_frac` asks "is my cut large relative to the request?". The question that matters is + "**does my cut change the outcome?**" — because `coref` is the only component paying a prefix + rewrite, while `extract_llm` does its measured ~27% from the cache-safe tail for free (a figure + long miscredited to `mask`, which is structurally inert there). So `coref` should + cut **only when it is decisive**: not when the pipeline is already under the threshold (the prize + is won, a rewrite buys nothing), and not when even `coref` cannot get it under (the agent + compacts anyway, so we pay the write *and* eat the compaction). + + That reduces to one scalar — **tokens until the agent compacts** — which is hard because the + threshold is compared against the provider's *reported usage*, including `system`, tool + definitions and last turn's output, none of which a component can see. + + Fully worked through, with the measured numbers and a three-step order of attack, in the + proposal's [deferral gate](../proposals/coref-compaction.md#the-deferral-gate-designed-unquantified). + Unbuilt on purpose: **how often the prize is even reachable has never been measured**, and that + measurement needs nothing new. + +## 8. Mechanism terms (how it stays cache-safe) + +| Term | Means | +|---|---| +| **latching** | The decision is stored per session and **replayed byte-for-byte** thereafter, never re-derived. A co-reference decision depends on *history*, so re-deriving it against a longer transcript could emit different bytes — which is exactly the prefix flip that costs a second cache-write. | +| **one-way / monotonic** | Keep → cut only. New evidence can never un-cut, because un-cutting is another rewrite. Monotonicity is a cost requirement, not tidiness. | +| **`freeze` / `reapplyFrozen`** | The mechanism that does it: record the replacement text against the original's content hash, and replay it on every later turn at any depth. | +| **`TailOnly`** | A helper on `Ctx` that answers "may I safely modify the message at index *i*?" It returns false for anything the provider has already cached (index ≤ `MaxCachedIdx`), because editing cached content breaks the prefix hash and forces a cache-write. Every *other* age-based offloader (`mask`, `failed_run`, `collapse`) consults it and simply declines. `coref` deliberately ignores it — reaching into the cached prefix **is** its purpose, since by the time a session crosses the threshold all the mass is back there — which is exactly why its spend has to be budgeted (`rewrite_budget`) instead of forbidden. | +| **`repairLostFreeze`** | Background: an offloader `freeze`s its replacement text against the original's content hash and replays it every turn, so the bytes stay stable. If the store *drops* that record (TTL, eviction), the offloader would normally decline to act at depth — but then the message reverts to full text, which is *itself* a prefix change. So `mask` and `failed_run` are allowed to re-derive the decision even deep in the prefix: their replacement is a pure function of `(content, config)`, so re-deriving reproduces byte-for-byte what the provider already cached. **`coref` must never use this.** Its decision depends on the whole transcript, so re-deriving against a longer one can yield a different class and different bytes — the precise byte-flip the repair exists to prevent. A lost `coref` freeze therefore declines and the output stays verbatim. | +| **marker / `<>`** | What's left in place of cut content, resolvable back to the stashed original via `context_guru_expand`. | +| **head peek** | A one-line snippet of the cut output left inside the marker, so the model knows *what* went missing without a blind `expand` round-trip. | +| **kept-verbatim** | Once the agent expands something, it's marked never-re-cut — otherwise it expands again every turn (an **expand loop**). | +| **`expand` rate** | The precision inner loop: how often the model asks for cut content back. Cheap, needs no scoring, no seeds, no n=30. **But it counts *noticed* errors only** — see the box below — so a falling expand rate is ambiguous rather than good news. | +| **fail open** | Any error reverts this component only; the original request is always forwardable. | + +!!! danger "Reversible does not mean recovered" + Expansion is **model-initiated**. The stash guarantees the bytes *can* come back; only the model + decides to ask, and nothing detects a bad cut. Three outcomes, not one: + + 1. it notices and expands the right marker → one round-trip + a cache-write; + 2. it notices but cannot tell which marker holds it → several expands, or it proceeds without; + 3. **it never notices** → it answers from less than it had, silently. + + Tier 3 is where (3) lives: a missing semantic reference leaves nothing to look up, so nothing + prompts recovery. (3) is invisible to every counter this component keeps, which is exactly why + reward is a **gate** and not one metric among several. + + What the design can influence is the 1-vs-2 gap, which is what the marker's structural descriptor + (`200 records, fields: address, id, name`) is for — and why the marker never asserts that the cut + was safe. + +## 9. Where things live + +| Thing | Where | +|---|---| +| The reference index (pure, shared definition) | `internal/coref/coref.go` | +| The offload component | `components/offload/coref.go` | +| Offline measurement | `deploy/harbor/coref.py` | +| Known-answer fixture + negative control | `deploy/harbor/coref_fixture.py`, `internal/coref/coref_test.go` | +| Claude Code transcript → capture | `deploy/harbor/cc_capture.py` | +| Benchmark run log → capture | `deploy/harbor/runlog_capture.py` | + +The Go index and the Python script must agree on what a reference is. If they drift, the thresholds the +measurement produces are calibrated for a different algorithm than the one that ships — silently. That's +why the same fixture and the same false-positive regression cases exist on both sides. + +## 10. Status in one line + +Mechanism implemented and tested; `cut_unreferenced` on by default, justified on yield (21% of mass on +interactive traffic, ~70% on benchmark traffic) and now bounded on accuracy (**11% false-drop**); +`cut_closed` **off** until measured on the eval-box corpus; component **opt-in, in no preset**; no +reward measurement exists, which remains the gate. + +## 11. Experiment terms (the held-out measurement) + +Introduced by [the selection experiment](../results/coref-selection-experiment.md), and worth having +here because they are how any future claim about this component should be scored. + +| Term | Means | +|---|---| +| **firing point `F`** | The message index a compaction pass is imagined to fire at. Everything before it is a candidate; everything after it is ground truth the decider never sees. | +| **evidence window** | `(output, F]` — the only thing any arm may look at. | +| **held-out future** | `(F, end]` — what the agent actually went on to do. Scoring only. | +| **false-drop** | Of what an arm removed, the share that *was* referenced later. A definite error. Bounded above by the base rate, which is why it is misleading alone. | +| **live-kept** | Of what *was* referenced later, the share an arm correctly kept. **The discrimination metric**, and base-rate independent. | +| **null baseline** | Drop everything. Scores exactly the base rate at 0% live-kept. A high removal rate paired with a base-rate false-drop rate means an arm is doing nothing but deleting. | +| **base rate** | The share of candidates referenced after `F` at all. 4% on LOCA, 11% on UltraHorizon, 46% on interactive traffic — so **the same arm scores wildly different false-drop rates on different corpora**, and comparing arms across corpora without this control produces artifacts. It produced two in this work. | + +!!! danger "Where this metric stops being valid" + It scores **verbatim survival**, so it can compare arms that keep-or-drop text and **cannot** + score one that *paraphrases*. A summary reading "found the grace-period bug in the auth module" + has preserved the information while containing none of the identifiers. Any comparison between + selective compaction and summarization needs **downstream task outcome**, not this. diff --git a/docs/reference/presets.md b/docs/reference/presets.md index 9fd5dfd8..7ff19ac8 100644 --- a/docs/reference/presets.md +++ b/docs/reference/presets.md @@ -15,7 +15,7 @@ taken exactly from the `presets` map in `config/config.go`. | `aggressive` | `format` → `dedup` → `failed_run` → `cmdfilter` → `smartcrush` → `extract` → `extract_llm` → `cachesplit` | `balanced` plus `smartcrush` (crush long homogeneous arrays), deterministic `extract` (noise collapse), and `extract_llm` (cheap-model relevance trim) for deeper savings. | | `coding` | `format` → `skeleton` → `cmdfilter` → `cachesplit` | Coding agents: `skeleton` reduces big source-file reads to their structure via tree-sitter. **Needs a `cg_skeleton` build** — see below. | | `mcp` | `format` → `smartcrush` → `cachesplit` | Tool/MCP servers returning long homogeneous JSON arrays (list endpoints, search hits). | -| `agent` | `format` → `dedup` → `failed_run` → `mask` → `extract` → `extract_llm` → `cachesplit` | Long agentic sessions (e.g. Claude Code on SWE-bench) where re-sent tool outputs dominate cost. `mask` is the biggest lever — ~27% content-token savings with no task-reward loss (see [Benchmarks](../RESULTS.md)). | +| `agent` | `format` → `dedup` → `failed_run` → `mask` → `extract` → `extract_llm` → `cachesplit` | Long agentic sessions (e.g. Claude Code on SWE-bench) where re-sent tool outputs dominate cost. **`extract_llm` is the biggest lever** — ~27% content-token savings with no task-reward loss (see [Benchmarks](../RESULTS.md)); that figure was previously miscredited to `mask`, which is not even present in the arm that produced it and is [structurally inert on caching traffic](../components/mask.md#when-its-inert). | | `general` | `format` → `toon` → `dedup` → `failed_run` → `cmdfilter` → `mask` → `extract` → `extract_llm` → `collapse` → `cachesplit` | The recommended all-round pipeline: the reward-neutral levers of `agent` plus the situational shrinkers (`toon` / `cmdfilter` / `collapse`) that cost nothing when they don't fire. | | `summarize` | `summarize` | Long trajectories where the transcript itself is the cost. **Runs alone** — it restructures the whole transcript (changes the message count), so no other component's in-place edits race the rebuild. | diff --git a/docs/results/REPRODUCE.md b/docs/results/REPRODUCE.md index ea71a072..9374f8f2 100644 --- a/docs/results/REPRODUCE.md +++ b/docs/results/REPRODUCE.md @@ -11,7 +11,8 @@ cache-aware token/cost metrics. ## 1. Prerequisites (one-time) -- **OS**: Linux (RHEL/Fedora family here), passwordless `sudo`. +- **OS**: Linux (RHEL/Fedora family here), passwordless `sudo`. **On x86_64 — this is a hard + requirement, not a preference. See the box below before trying an Apple-silicon Mac.** - **Go 1.26** + `CGO_ENABLED=1` (context-guru's `cg_skeleton` build tag needs cgo/tree-sitter). - **Python 3.13** + [`uv`](https://docs.astral.sh/uv/) (Harbor needs ≥3.12). - **Docker** (each task builds a container). Use it via `sg docker -c '...'` — do **not** @@ -23,11 +24,49 @@ cache-aware token/cost metrics. `aws/claude-sonnet-5` (agent) and `aws/claude-haiku-4-5` (context-guru's cheap compaction model). +!!! danger "Not reproducible on Apple silicon (arm64), and the failure looks like a bad result" + Tried and characterised on an M-series Mac with Docker 29.1.3. It does not work, and the way it + fails is worse than not working. + + **Every task image is linux/amd64 only**, for both benchmarks. SWE-bench's are + `swebench/sweb.eval.x86_64.*` by name. Terminal-Bench 2.0 *looks* portable — its task + Dockerfiles use multi-arch bases (`python:3.13-slim`, `ubuntu:24.04`) — but all 89 `task.toml` + files pin a prebuilt `alexgshaw/:20251031` image that overrides the Dockerfile, and those + are single-arch amd64. A container started from one reports `x86_64`. So both benchmarks run + under QEMU emulation. + + Emulation itself works. **Claude Code does not run under it.** The agent is a bun-compiled + single-file native executable, and it dies immediately: + + ``` + panic(main thread): Segmentation fault at address 0x490BF8348C5B9 + qemu: uncaught target signal 11 (Segmentation fault) - core dumped + ``` + + Installing via `npm install -g @anthropic-ai/claude-code` instead of the native bootstrap + installer does **not** help — the npm package ships the same executable, so the install + succeeds and then `claude --version` segfaults. There is no configuration that avoids this. + + **Why this matters even if you have a Linux box:** Harbor reports the segfault as + `NonZeroAgentExitCodeError`, which is **indistinguishable from an agent failure** unless you + read the container log. A whole run comes back with `reward=0` on every task and reads as "the + preset is catastrophic" rather than "the agent never started". Same failure mode as the + `CG_LAN` and port-clash gotchas already documented below — check the container log before + believing a zero. + + If you only have arm64 hardware, the options are an x86_64 VM or remote host; substituting a + pure-Python agent (`mini-swe-agent`, `swe-agent`) runs fine emulated but is **not** comparable + to the published `claude-code` numbers. + ### Docker Hub authentication (required — avoids the 429 pull-quota wall) SWE-bench task images live on Docker Hub (`docker.io/swebench/…`). Harbor pulls one per -task; ~100 pulls exhausts the **anonymous** quota (HTTP 429) and environments fail to -build. Authenticate once (an authenticated account has a separate 200-pulls/6h quota): +task; the **anonymous** quota (HTTP 429) is per-IP and environments then fail to build. +Authenticate once. Measured against the live registry on 2026-08-20, the quotas are +`RateLimit-Limit: 100;w=3600` anonymous and `200;w=3600` authenticated — a 2× improvement +rather than the order of magnitude this section used to imply, but still the difference +between finishing a 50-task run and dying partway, and the anonymous limit is shared with +anything else pulling from your IP: ``` sg docker -c 'docker login -u ' # paste a Read-only Personal Access Token diff --git a/docs/results/component-gating.md b/docs/results/component-gating.md new file mode 100644 index 00000000..bfadc14e --- /dev/null +++ b/docs/results/component-gating.md @@ -0,0 +1,122 @@ +# What actually fires, and why + +Before spending anything on a scored benchmark, a cheap question: **on real captured traffic, which +components actually act, and which are silently declining?** The answer reframed a planned +experiment badly enough to cancel it, so it is worth reading before designing any arm. + +Everything here is replay against captured request bodies via the proxy's `/compact` endpoint — +no agent, no task, no reward. Total spend **~$3**. + +Reproduce: `deploy/harbor/replay2.py --configs ...`, with `CACHE_MODE` set per run. + +## 1. The tail gate costs more than the docs imply — **measured** + +`replay2.py` inherits `CACHE_MODE`, so the same configs can be replayed with cache-awareness on and +off. The delta *is* the tail gate. 200 requests of `capture-swebench`: + +| component | non-tail (`CACHE_MODE=off`) | tail-gated (`on`) | effect of the gate | +|---|---|---|---| +| **`mask`** | **50.67%** (685,055 tok) | **3.33%** (45,020 tok) | **loses 93% of its effect** | +| **`failed_run`** | 1.29% | **0%** | **fully inert** | +| `extract` | 0.84% | 0.84% | unaffected — not tail-gated | +| `cmdfilter` | 0.23% | 0.23% | unaffected | +| `dedup` | 0% | 0% | never fires on this corpus either way | + +Only `mask`, `failed_run` and `extract_llm` consult `TailOnly`. The others are pure functions of +content, so replaying them each turn is byte-stable and needs no gate. + +This is the quantified form of the geometry in [`mask`](../components/mask.md#when-its-inert): +`TailOnly(i)` permits only `i > MaxCachedIdx = prevLen − 1`, while `mask`'s candidates are by +definition outputs that were present last turn. `failed_run` lands in the same place and reaches +exactly zero. Both are in shipped presets — `failed_run` in `codesmart`, `agent`, `general`, +`codesafe` and `balanced`. + +## 2. `extract_llm` cannot fire on SWE-bench at all — **measured** + +0% saved in **every** configuration tried: economic gate on and off, cache-aware and not. The +recorded gate reason is `below_output_floor`. + +The cause is a quantity nobody had measured: **tool-output size**. + +| corpus | p50 | max | ≥3,000 (its floor) | ≥30,500 (its break-even, as published) | +|---|---|---|---|---| +| SWE-bench, a fresh run made for this pass | 71 | **2,760** | **0** | **0** | +| `capture-swebench` | 106 | 5,674 | a handful | **0** | +| `capture-tb` | ~0 | 1,906 | **0** | **0** | +| **LOCA-bench** | 185 | **59,857** | **54** | **7** | + +**`extract_llm`'s output floor is larger than the largest tool output SWE-bench produces.** Its +break-even is an order of magnitude larger. It is structurally incapable of acting there, and the +same is true of Terminal-Bench. + +### So `codesmart` saves ~1% on this workload + +`codesmart` is `[format, toon, dedup, failed_run, cmdfilter, extract_llm, extract, cachesplit]`. On +caching traffic `extract_llm` is off by default and `failed_run` is inert, leaving `extract` (0.84%) ++ `cmdfilter` (0.23%) ≈ **1%**. Any arm labelled "the SWE-bench winning config" should be described +that way rather than by reputation. + +### And the ~27% figure is not reproducible here + +No configuration tried reproduces it on this corpus with current code. That is consistent with +`config.go`'s own note that the published numbers "describe an ancestor" of the preset. Whatever +produced it was a different workload with much larger outputs, or pre-gate code with a far lower +floor. See also the [reattribution](../components/mask.md) — the figure was long credited to `mask`, +which was never in the arm that produced it. + +## 3. A mispricing in the economic gate — **found, fixed** + +Raised in review: *tail-only `extract_llm` has no break-even constraint since it doesn't invalidate +the cache.* Correct in mechanism, and it exposed a real bug. + +`savedTokenValue` priced **every** saved token at the cache-read rate whenever the request was +cache-aware, reasoning that re-sent content is already in the cached prefix. True of a **replay** +turn; false of the turn the cut is made — and when cache-aware `extract_llm` acts *only* on the +tail, which by definition has never been cached. On that turn the content is billed as a +cache-**write** (`$3.75/MTok`, dearer than fresh input) or as plain fresh input. + +Confirmed from live usage rather than argued: the fresh SWE-bench trial reported **52,561 +`cache_creation` tokens** against 746,047 cache-read over 18 turns. + +| case | as published | corrected | gain | +|---|---|---|---| +| caching, recurring | 30,397 | **11,550** | 2.63× | +| caching, first sight | 42,556 | **12,900** | 3.30× | +| non-caching | unchanged | unchanged | — (one rate, so `first + r·rate ≡ (1+r)·rate`) | + +**The verdict survives the correction; the number did not.** SWE-bench's largest output (2,760) is +still ~4× short, so keeping the component off by default there remains right. What changes is +large-output workloads: on LOCA the eligible set goes from **7 to 31** of 1,639 outputs. + +Two side effects worth knowing before tuning: the cached/non-caching break-even ratio falls from +~20× to **~6.4×**, and **recurrence becomes a much weaker lever** — ×1.12 rather than ×1.40 — +because the applied turn now dominates the sum. Three tests that had encoded the old arithmetic +were updated rather than deleted, including the drift guard that ties the code to +[`extract_llm`'s doc](../components/extract_llm.md). + +## 4. Unexplained, and recorded as such + +`extract_llm` burned **12.8 s across 20 requests** (~640 ms each) while reporting `acted: 0`, +`discarded_changes: 0`, and nothing above INFO in the proxy log. Something in that path spends real +wall-clock without producing *or* discarding a change. Not diagnosed; it should be a bug rather than +a guess. + +## What this changed + +A planned reward-safety arm on SWE-bench Verified was **cancelled**. With ~1% of content available +to remove, a reward-neutral result would have been vacuous — it would have shown that removing +almost nothing breaks almost nothing, at ~$100. + +The binding constraint turns out not to be context length, which is where the argument had been +focused, but **tool-output size**. SWE-bench and Terminal-Bench both have outputs an order of +magnitude below the thresholds these components need. **LOCA-bench is the only benchmark in the set +where anything can fire** — and independently the only one whose context length is a dial. That is +now the vehicle. + +Sobering even there: only 31 of 1,639 LOCA outputs clear the corrected `extract_llm` break-even. +`coref` has a better case on the same corpus — its floor is 300 tokens, giving it 580 candidate +blocks (35%) — which is the one place the two components' economics genuinely diverge. + +See also: [`mask`](../components/mask.md) · [`extract_llm`](../components/extract_llm.md) · +[density](coref-density.md) · [the eval-box measurement](coref-evalbox.md) · +[reachability](coref-reachability.md) diff --git a/docs/results/coref-density.md b/docs/results/coref-density.md new file mode 100644 index 00000000..c7bdc650 --- /dev/null +++ b/docs/results/coref-density.md @@ -0,0 +1,313 @@ +# Co-reference density on real agent traffic + +The measurement pass [`coref-compaction.md` §7](../proposals/coref-compaction.md) says must run before +the `coref` component is calibrated. It has now run, on **three corpora** — and the headline is that they +disagree by roughly 4x, which is itself the most useful result. These numbers are the SECOND +version: review of PR #80 found a defect that inflated the first set badly, and +[what review changed](#what-review-changed) records both the defect and the delta. + +All of it cost **zero API dollars**: the runs already happened and left their logs on disk. + +## The three corpora + +| Corpus | What it is | Sessions | Outputs ≥300 tok | Tool-output mass | Median peak request | +|---|---|---|---|---|---| +| **Claude Code** | Interactive research/documentation work on one project, via [`cc_capture.py`](https://github.com/rossoctl/context-guru/blob/main/deploy/harbor/cc_capture.py) | 31 | 1,344 | 1,329,009 tok | 179,660 tok | +| **UltraHorizon** | `loopb_uh` benchmark runs (sequence-exploration game, `python_execute`), via [`runlog_capture.py`](https://github.com/rossoctl/context-guru/blob/main/deploy/harbor/runlog_capture.py) | 10 | 134 | 170,477 tok | 30,220 tok | +| **LOCA-bench** | `all_trajectories.json` from 35 LOCA task runs (MCP filesystem/sheets/email envs) | 9 | 166 | 591,932 tok | 52,258 tok | + +```sh +# interactive traffic — Claude Code writes a transcript of every session it runs +python3 deploy/harbor/cc_capture.py /tmp/cc.jsonl ~/.claude/projects//.jsonl + +# benchmark traffic — the harnesses already logged their request bodies +python3 deploy/harbor/runlog_capture.py /tmp/uh.jsonl --label ultrahorizon .../llm_calls.jsonl +python3 deploy/harbor/runlog_capture.py /tmp/loca.jsonl --label loca .../all_trajectories.json + +python3 deploy/harbor/coref.py /tmp/uh.jsonl window=32000 fire_frac=0.6 sweep=1 +``` + +!!! warning "None of this is the eval-box corpus — which has since been measured, and is *smaller*" + §8's acceptance criteria are written against `capture-swe` / `capture-tb` (SWE-bench Verified and + Terminal-Bench, cache-read-dominated, the incumbent regression floor). Those captures live on the eval + box and were not reachable. What is measured here is real agent traffic from three *other* workloads. + It answers §7's questions with data instead of argument, and it turns out to answer them + **differently per workload** — which is a stronger reason to re-run on the eval box, not a weaker one. + + **That re-run has since happened** ([eval-box measurement](coref-evalbox.md)), and it reverses the + apology in this box. `capture-swebench` is 50 **shallow** sessions (433 outputs, peak request + 12,607 tokens), and `capture-tb` / `capture-swe` turn out to be smoke captures of 6 and 2 outputs. + The three corpora on this page are **larger and deeper** than the corpus they defer to. Read + `capture-swebench` as the most *relevant* corpus and these as the most *substantial* — neither is + sufficient alone. + +## The headline: reference density is a property of the workload + +Measured with the `opaque` class and the opportunity floor both in force (see +[what review changed](#what-review-changed) — the first version of these numbers was materially wrong): + +| | Claude Code | UltraHorizon | LOCA-bench | +|---|---|---|---| +| `opaque` — introduced nothing trackable, **no evidence** | 8% | 20% | **40%** | +| `unreferenced` — introduced identifiers, nothing used them | **13%** | **51%** | **22%** | +| `closed` — value taken, survives above | 15% | 6% | 0% | +| `open` — recent, repeated, or too new to judge | 62% | 21% | 36% | +| **cut at the shipped default** (`unreferenced` only) | **13%** | **51%** | **22%** | + +Interactive work on a coherent codebase keeps returning to the same files, symbols and errors, so 62% of +its tool-output mass is still load-bearing. UltraHorizon's game-exploration traffic surveys, verifies, and +moves on, so half its mass is provably dead. `coref`'s value is not a single number; it is +workload-dependent, and it differs by ~4× across three corpora. + +LOCA is the interesting case and the reason the `opaque` class exists: its raw `unreferenced` share looked +like **95%**, and 40 points of that was mass the index cannot see into at all (11 outputs averaging 22k +tokens — bulk record and spreadsheet dumps of human-readable values), with most of the remainder outputs +too near the tail to have been referenced yet. The honest free cut there is 22%, not 95%. + +### And LOCA behaves exactly as §8 predicted + +§8 argued LOCA would be a **Tier-2/3 stress test** rather than a showcase for exact matching, because its +BigQuery/Sheets/Excel envs *aggregate and compute over* tool results, so references arrive transformed past +the point where a substring match can see them. The signature is unmistakable: **0% closed — not one +output in 166 was referenced once or twice and then left alone** — alongside the largest `opaque` share of +any corpus at 40%. References there are either immediate-and-repeated (the `open` 36%) or invisible to an +exact matcher. That is the predicted result, and it is the strongest reason never to read `unreferenced` +as "unused" on that corpus. + +## What §7's other questions came back with + +**A reference consumes a small fraction of what its output introduced** — median 18.7% (Claude Code), +11.5% (UltraHorizon), 50% (LOCA). Hypothesis A — "took one value out of a large response, does not need +the rest" — is confirmed on the two long-horizon corpora and much weaker on LOCA, whose outputs are +smaller and more thoroughly consumed. + +**Distance is not the discriminator. Repetition is.** On the Claude Code corpus, sweeping `closed_dist` +over a 10× range (4→40 messages) moves closed mass by 2–3 points; sweeping `open_reps` from 2 to 6 moves +it by 18: + +| `open_reps` ≥ | `closed_dist` 4 | 8 | 12 | 24 | 40 | +|---|---|---|---|---|---| +| 2 | 10% | 10% | 10% | 9% | 9% | +| **3** | 16% | 15% | **15%** | 14% | 13% | +| 4 | 21% | 20% | 20% | 19% | 18% | +| 6 | 28% | 27% | 26% | 25% | 23% | + +The corroborating figure: **44% of all Claude Code tool-output mass was last referenced 40+ messages +ago**, and yet 62% of mass is `open`. Most referenced mass is *old and still hot*. A policy separating +case A from case B by distance — the original framing — would confidently cut repeatedly-referenced +content believing it was taking the safe early cut. §3's reframe from distance to open-vs-closed is not a +refinement; it is the difference between the policy working and not. **`closed_dist` is not worth tuning; +`open_reps` is the dial.** + +**Break-even is workload-dependent too, and it is worse than the first pass claimed** — necessarily, since +`opaque` and tail-protected outputs left the cut set and `S` shrank: + +| | Claude Code | UltraHorizon | LOCA-bench | +|---|---|---|---| +| median cut `S` | 10,539 tok | 14,164 tok | 21,234 tok | +| median rewritten suffix `W` | 157,189 tok | 26,044 tok | 49,611 tok | +| turns `T` needed | **138** | **23** | **34** | +| sessions whose observed `T` cleared it | **9/30** | **4/8** | **2/6** | + +§4's arithmetic holds everywhere, and the margin is thin everywhere: **roughly a third of sessions repay +the cache-write on tokens**, and on long interactive transcripts the median session would need 138 more +turns. Batching moves break-even from impossible (T > 276 for a single early cut) to *merely unlikely* on +tokens alone. This is the third decision rule in §7 firing: **`coref` must be justified on step reduction +and on deferring the agent's own compaction, and evaluated that way.** `corr(Δsteps, Δcost) = +0.95` says +tokens were never the interesting axis; these numbers say it is not even a supporting one. + +Window choice matters here and is easy to get wrong: measured against a 200k window, UltraHorizon shows +**0** sessions clearing break-even — but its peak request is 30k, so `fire_frac × window` is never reached +and `T` collapses to zero by construction. At a 32k window (matching what those runs actually held) it is +4/8. A break-even figure is meaningless without a window the traffic actually used. + +## Can it defer the agent's own compaction? + +The proposal's largest claimed win is pushing back Claude Code's self-compaction (167k on a 200k +model). Of the 31 Claude Code sessions, **19 passed that threshold**, so the question is answerable +on this corpus. The requirement is not merely to clear the threshold but to stay clear — cutting to +exactly the line buys one turn, and then you either eat the compaction or pay a *second* cache-write +at maximum `W`: + +``` +required cut ≥ (usage − threshold) + growthPerTurn × headroomTurns +``` + +| headroom bought | required cut, as a share of the request | achievable with `unreferenced` + `closed` | +|---|---|---| +| H = 0 (bare clear) | 7.3% | 10/19 | +| H = 20 | 12.6% | 5/19 | +| H = 40 | 18.0% | **0/19** | +| H = 60 | 23.5% | **0/19** | + +Mean available cut is **4.4%** of the request (`unreferenced`) and **9.6%** (`+closed`), against a +mean deficit of 12.9k on a ~180k request and mean growth of ~514 tokens/turn. So a bar high enough +to avoid paying twice — 20–25%, which is what 40–60 turns of headroom costs — is a bar Tier-1 +matching **cannot clear on this corpus**. + +The same figures condemned the original `min_batch_frac: 0.15`, which admitted **1 of 19** sessions +with `cut_closed` on and **0 of 19** at the shipped cut set. It is now 0.05 (16/19), recorded as a +starting point rather than a claim. + +!!! danger "The deficit column is an artifact of firing LATE, and the peak column is not usable at all" + Peak request ≈180k and deficit ≈13k are shaped by `cc_capture.py` segmenting at 180k tokens, so + peaks cluster there by construction. The durable finding is the one independent of it: + **available cuttable mass is 4–10% of the request.** + + Two later corrections, both from [the reachability pass](coref-reachability.md): + + 1. **The deficit term should be ~0, not 7.3%.** Segmenting at 180k places the measurement point + *past* the threshold; at the moment the agent actually compacts, usage *is* the threshold by + definition. Fired at the crossing, the requirement collapses to `growth × headroom` — and the + H=40 row below goes from 0/19 to affordable. The table measures a late fire, not the design. + 2. **Absolute request sizes are not recoverable from this corpus at all.** Claude Code + transcripts are trees (25–51 forks, 338–632 leaves per compacted transcript) and the + `parentUuid` graph is too fragmented to reconstruct the active branch. A linear read spans + multiple context windows. Duplicate tool-output mass is only 3% pooled / 2% median, so the + **share**-based results on this page are unaffected — but read every absolute token figure as + indicative only. + +The design consequence — a gate that asks whether `coref`'s cut is the *decisive* one rather than +whether it is large, and what it would take to know the distance to the threshold — is worked +through in the proposal's +[deferral gate](../proposals/coref-compaction.md#the-deferral-gate-designed-unquantified). It is +unbuilt, and deliberately so: how often the prize is reachable at all has never been measured, and +`modes.Tracker`'s reset detection answers that with no new machinery. + +## What review changed + +Review of PR #80 raised a counter-example that turned out to invalidate the first version of every number +above. It is worth recording in full, because the defect was invisible in the arithmetic. + +**The counter-example.** A tool output returns records; the model's next turn references one: + +```jsonc +[{"name": "david", "id": 123, "address": "foobarbaz"}, + {"name": "osher", "id": 235, "address": "banana"}] +// model: "I need to remember david 123 address." +``` + +The reference is real, but the value needed — `foobarbaz` — was never copied into a model turn. The model +referenced an **anchor** in order to point at a payload it did not restate. So §3's "any reference is a +surviving copy" is too strong, and `closed` cannot rest on "referenced once, long ago" alone. + +**The worse defect it exposed.** Run through the actual index, that output yields **zero** trackable +tokens: `david`, `123`, `foobarbaz` are short lowercase words and a 3-digit number, precisely what the +precision rules below exclude. With no novel tokens there are no references, so it scored `unreferenced` — +**the class the default configuration cuts.** Two states satisfy `refs == 0` and they are opposites: + +| state | meaning | +|---|---| +| introduced 200 identifiers, nobody touched one | evidence of deadness → safe cut | +| introduced nothing the index can see | **no evidence** → no opinion | + +`opaque` is now its own class and is never cut, and an **opportunity floor** (`min_later_turns`) stops an +output too near the tail from being scored as unused. The delta on the same corpora: + +| | Claude Code | UltraHorizon | LOCA-bench | +|---|---|---|---| +| `unreferenced` as first reported | 23% | 78% | 95% | +| `unreferenced` after the fix | **13%** | **51%** | **22%** | +| of which reclassified `opaque` | 8% | 20% | **40%** | +| sessions clearing break-even | 15/30 → **9/30** | 7/10 → **4/8** | 4/9 → **2/6** | + +The first version would have deleted 40% of LOCA's tool-output mass on no evidence at all, under the +default config. Both the class and the floor are now tested on both sides of the implementation. + +## Two further methodological results + +### 1. The identifier/prose rule decided the answer + +An earlier run of this measurement reported **71%** of Claude Code mass as referenced. Nothing about the +corpus changed — only the rule deciding whether a token is an identifier or an English word. The original rule accepted any token of 10+ characters, any +token containing punctuation anywhere, and any bare number of 3+ digits, and its top +reference-producing "identifiers" were: + +``` +395 description 140 transparency 116 final_score 104 conditions +142 forever_fetch 135 2026 112 integration 94 persistent +``` + +`description`, `transparency`, `integration`, `efficiency`, `conditions`, `orientation`, +`effectiveness` — all ≥10 characters, all ordinary English. Plus `e.g.` / `try:` / `None:` / `memory.` +(prose plus a sentence mark) and `2026` (a year). Each collision manufactures a reference, and a +manufactured reference makes an output look load-bearing — so this class of bug fails by **silently +declining to compact**, which is invisible to any metric that only counts what the component did. + +The rules were tightened to require *interior* structure after trimming edge punctuation, a digit, or +camelCase — with no bare length rule and no stopword list (a stopword list does not survive a change of +domain, or of language). After the fix the top drivers are `forever_fetch`, `final_score`, `OpenManus`, +`db_state`, `ANTHROPIC_BASE_URL`, `session_id`, `v1/messages`, `GraphStore`, `json.dumps`, +`os.path.join` — identifiers. Every false positive above is now a regression case in +`internal/coref/coref_test.go`, because the Go component and this script must agree or the thresholds +measured here are calibrated for a different algorithm than the one that ships. + +**Residual, bounded rather than argued.** Lowercase hyphenated compounds (`cross-session`, `end-to-end`, +`dead-end`) still pass, and nothing structural separates them from real names like `context-guru` or +`terminal-bench`. Rejecting the whole class as a sensitivity arm moves cuttable mass from 39% to 45%. The +shipped tokenizer is therefore **conservative by ~6 points** — it under-reports what can be cut, which is +the safe direction. + +### 2. That fix also blinded the Tier-2 detector, and the report reflects it + +`derived_evidence` — the Tier-2 proxy, "numeric values the model states that appear nowhere earlier" — +runs over the same token set. Requiring bare numbers to carry 5+ digits or a separator means most +*computed* values (sums, counts, converted units — small numbers) no longer register at all. Tier-2 +evidence now reads 2% on Claude Code and **0% on LOCA**, and that 0% must not be read as "LOCA has no +transformed references": the 0% closed share says the opposite. It means *no transformed references among +the tokens this tokenizer still accepts*. Precision for the primary signal was bought at the cost of +recall for the caveat signal. Measuring Tier 2 properly needs its own detector, not this one. + +Keep the scope straight, because it is easy to over-read a 0%: this proxy covers **Tier 2 only**. Tier 3 +— a semantic reference like "as I noted earlier" — carries no shared token *and* no novel numeric, so it +has **no measurement here at all**, by design rather than by regression. The two therefore fail +differently: Tier 2 has a detector that is nearly blind, Tier 3 has none. On LOCA, with 0% `closed` and +40% `opaque`, the defensible statement is that Tier-2 *and* Tier-3 references are both common there and +both unmeasured — not that transformed references are absent. + +## What this settles, and what it does not + +**Settled enough to act on:** + +- `cut_unreferenced` (the shipped default) is justified **on yield** everywhere, and its size is + workload-dependent: 21% of mass on interactive traffic, ~70% on benchmark traffic, with no + calibrated threshold and no model call. Its **accuracy** is a separate question this pass could + not ask, and [the held-out experiment](coref-selection-experiment.md) since answered it: + **11% false-drop**, irreducible, and a lower bound. Read the yield figures here alongside that. +- `closed_dist` is nearly inert; `open_reps` is the dial. Leave the recency threshold alone. +- Break-even needs a window the traffic actually used, or it reports a construction rather than a result. + +**Not settled — and why `cut_closed` stays off by default:** + +`closed` mass is 15% on interactive traffic, 8% on UltraHorizon and **0% on LOCA**. A knob whose yield +ranges from 0% to 15% by workload has no defensible default, and none of these corpora is the one §8's +acceptance criteria are written against. Enabling `cut_closed` per config for a measured arm is +reasonable; shipping it on is not. + +Still outstanding, unchanged: the eval-box re-run on `capture-swe`/`capture-tb`, `observe`-mode `expand` +rate as the precision inner loop, and only then the scored benchmarks. + +## Caveats + +- **Not the eval-box corpus** (see the warning above) — the single largest caveat. +- **Small n, and one author's traffic.** 31 + 10 + 9 sessions; the Claude Code corpus is mostly one + project. No seeds, no variance estimates. Every figure is a point estimate of unknown spread. +- **Tail bias is now guarded, not merely bounded** — `min_later_turns` (default 8) treats an output with + too few later model turns as `open`. Before that guard existed, LOCA's headline was the most affected. + Note the guard is justified **structurally** (it stops a batched pass preferring the newest + context) and **not** by accuracy: swept against held-out ground truth, `min_later=0` gives more + mass at a *lower* false-drop rate. See + [the experiment](coref-selection-experiment.md#5-min_later_turns-does-not-earn-its-keep-on-this-metric-measured). +- **Session boundaries are reconstructed.** Claude Code transcripts are cut at 180k tokens to approximate + compaction boundaries the transcript does not record; UltraHorizon runs are cut where the harness's own + context wipe drops the message count. Measuring across a boundary the model cannot see across would + invent cuttable mass out of the harness's reset. +- **`thinking` blocks count as reference-bearing** — model-authored, and the real wire body carries them. +- **Token counts are a ~4-chars/token proxy**, consistent with `analyze_content.py`. Fine for shares, not + for billing arithmetic. +- **Two additive fields** (`turn_tokens`, `conv`) were added to what `coref.py` accepts, for converted + logs only. A real capture sets neither and its behaviour is unchanged. + +See also: [the proposal](../proposals/coref-compaction.md) · [the held-out selection +experiment](coref-selection-experiment.md) · [the component](../components/coref.md) · +[glossary / cheat sheet](../reference/coref-glossary.md) · [improvement plan](improvement-plan.md) diff --git a/docs/results/coref-evalbox.md b/docs/results/coref-evalbox.md new file mode 100644 index 00000000..62d31753 --- /dev/null +++ b/docs/results/coref-evalbox.md @@ -0,0 +1,126 @@ +# The eval-box measurement, finally run + +Every co-reference document in this repo carries the same caveat: *"none of this is the eval-box +corpus."* [Implementation status](../proposals/coref-implementation.md) makes re-running on +`capture-swe` / `capture-tb` / `capture-swebench` **step 1**, ahead of everything else, because +those are the captures §8's acceptance criteria are written against. + +It has now run, on the box itself. It cost **$0** — the captures already existed. + +It also found a bug in `coref.py` that was **silently discarding 98% of the corpus**, and the +headline result inverts a framing that runs through all the earlier documents. + +## The bug: a session key that collapses benchmark traffic + +`coref.py` grouped requests into sessions by hashing `json.dumps(first_user_message)[:200]`. That is +sound for interactive traffic, where every session opens on a different human sentence. It is +**catastrophic on benchmark traffic**, where every task's instruction opens with the same long +standard preamble. + +Measured on `capture-swebench.jsonl`: the 200-character prefix has only **19 distinct values, and the +most common one covers 1,771 of 1,795 requests.** Because only the largest member of each group is +analysed, 18 of the 19 groups contributed nothing but stray single-message calls, and the +measurement reported **one session**. + +The capture already carried the right key all along — the Anthropic clients pack +`{device_id, account_uuid, session_id}` into `metadata.user_id`. Preferring it recovers **50 +sessions** from the same file: + +| | before the fix | after | +|---|---|---| +| sessions | 1 usable (of 19 groups) | **50** | +| tool outputs ≥300 tok | 25 | **433** | +| tool-output mass | 20,655 tok | **355,771 tok** | + +A **17× larger corpus**, from the same bytes. This is the same class of defect as the `conv` +collision already fixed for `cc_capture.py` — fixed there, never fixed for real captures — and it +fails the same silent way: it does not error, it just measures less and reports confidently. + +!!! warning "It is worth stating what this means for reading the earlier docs" + Any figure previously produced from a real capture with no explicit `conv` field was computed + over whatever survived the collision. The three corpora in + [the density pass](coref-density.md) were all converted with explicit `conv` keys by + `cc_capture.py` / `runlog_capture.py`, so **they are unaffected**. But it is the reason the + eval-box captures had looked empty on earlier glances. + +## The result on `capture-swebench` (SWE-bench Verified, 50 sessions) + +| bucket | outputs | mass | share | | +|---|---|---|---|---| +| `opaque` | 42 | 24,781 | 6% | no evidence — never cut | +| **`unreferenced`** | 151 | 99,845 | **28%** | the shipped default cut | +| `closed` | 85 | 74,380 | 20% | opt-in | +| `open` | 155 | 156,765 | 44% | keep | + +Alongside: reference consumes a median **14.3%** of what its output introduced (the +"took a value, dropped the rest" pattern, strongest of any corpus measured); Tier-2 numeric evidence +2%; `open_reps` is again the dial and `closed_dist` again nearly inert. + +### This is the best yield yet, on the corpus that counts + +`cut_unreferenced` in context: + +| corpus | `unreferenced` | `+closed` | +|---|---|---| +| Claude Code (interactive) | 13% | 28% | +| LOCA-bench | 22% | 22% | +| **SWE-bench Verified (eval box)** | **28%** | **48%** | +| UltraHorizon | 51% | 57% | +| Terminal-Bench (eval box, n=3 — see caveat) | 71% | 71% | + +§8 predicted SWE-bench would be the Tier-1-rich substrate, and it is: **more than double the +interactive yield**, with a healthy `closed` share (20%) where LOCA had 0%. The proposal's +benchmark-selection argument is confirmed on the corpus it was written about. + +## But the same corpus removes the deferral case entirely + +**Peak request, median: 12,607 tokens.** Against an agent-compaction threshold of 167,000. + +These sessions are not remotely long enough to compact. So on the authoritative corpus the largest +claimed win in the proposal — deferring the agent's own compaction — **cannot occur at all**, not +because the cut is too small but because the pressure never arrives. That is consistent with +[reachability](coref-reachability.md), which found the prize present in only 17% of interactive +sessions; here it is 0%. + +And the token economics stay thin. At a window this traffic actually uses (20k), **6 of 48 sessions +clear break-even** (`S × T > 11.5 × W`; median cut 2,550 tok against a 9,028-token rewritten suffix, +needing T ≈ 37). Measured against a 200k window it is 0/48 — the window artifact the density pass +warned about, and a reminder that a break-even figure without a window the traffic used is a +construction, not a result. + +## The caveat that turned out to matter most + +**Two of the three eval-box captures are too small to conclude from.** + +| capture | sessions | outputs ≥300 tok | mass | peak request | +|---|---|---|---|---| +| `capture-swebench.jsonl` | 50 | 433 | 355,771 tok | 12,607 | +| `capture-tb.jsonl` | 3 | 6 | 4,707 tok | 6,391 | +| `capture-swe.jsonl` | 1 | 2 | 897 tok | 4,201 | + +`capture-tb`'s striking 71% `unreferenced` rests on **four outputs**. It is a smoke capture, not a +corpus, and the same is true of `capture-swe`. Only `capture-swebench` supports a claim. + +**This inverts the caveat every earlier document carried.** Those documents apologised for measuring +31 interactive sessions / 1,344 outputs instead of "the real corpus" — and the real corpus is one +usable capture of 50 shallow sessions plus two smoke files. The interactive corpus was **larger and +deeper** than the thing it was deferring to. The honest ranking is that `capture-swebench` is the +most *relevant* corpus and the interactive one is the most *substantial*, and neither is sufficient +alone. + +## What this settles + +| question | answer | +|---|---| +| Yield on the corpus the criteria are written against | **28% (`unreferenced`), 48% (`+closed`)** — the best measured, and double interactive | +| Was §8 right that SWE-bench is the Tier-1-rich substrate? | **Yes**, confirmed | +| Can `coref` defer agent compaction here? | **No** — peak request 12.6k vs a 167k threshold. Structurally impossible on this corpus | +| Do the token economics work? | **6/48 sessions**, and only at a realistic window | +| Is `cut_closed` now safe to enable? | **Still no.** 20% here vs 0% on LOCA — the workload spread that keeps it off is unchanged | +| Reward | **Still unmeasured.** Still the gate. The box can now run it | + +Next: this box has Harbor, Docker with authenticated Hub pulls, 16 CPUs and x86_64, so the scored +benchmark runs the acceptance criteria actually require are runnable here for the first time. + +See also: [density](coref-density.md) · [selection experiment](coref-selection-experiment.md) · +[reachability](coref-reachability.md) · [the proposal](../proposals/coref-compaction.md) diff --git a/docs/results/coref-loca.md b/docs/results/coref-loca.md new file mode 100644 index 00000000..d566694b --- /dev/null +++ b/docs/results/coref-loca.md @@ -0,0 +1,126 @@ +# Deferring summarization on LOCA-bench + +Does paying a cache-write to compact the full request body **defer the summarization** an agent +otherwise has to run when its context fills? That is the largest claimed win in +[the proposal](../proposals/coref-compaction.md), and this is the first end-to-end measurement of it. + +**Result: yes, and by a lot — 72% fewer summarizations.** But not for the reason the design +predicted, and the measurement found a design error worth more than the number. + +Total spend: **~$0.16** across five arms. + +## Setup + +LOCA-bench, because [component gating](component-gating.md) established it is the only benchmark in +the set whose tool outputs clear these components' thresholds *and* whose requests reach real context +pressure (median 52,027 tokens, max 233,288 — SWE-bench never exceeds 46,405). + +**Sequential replay.** LOCA traffic is append-only, so turn *k* of a conversation is reconstructed as +`messages[0:k]` of its final request, cut at assistant boundaries: **197 turns across 9 +conversations**, replayed in order under a stable per-conversation session id so `MaxCachedIdx` +advances turn by turn as it does in production. + +!!! warning "This supersedes an earlier version of this page, which was wrong" + The first run sent **one request per conversation** (the deepest). Every request was therefore a + cold first turn with `MaxCachedIdx = -1`, so the tail gate never engaged. That inflated `mask` to + 52.3% — essentially its non-tail figure — and produced a "`mask` removes 8.3× more than `coref`" + comparison that was an artifact of the setup. It also reported **zero `closed`** outputs on LOCA, + which the sequential replay shows is false (25 of them): `closed` requires a reference that has + since gone stale, which cannot exist when every request is turn 1. + +**`summarize` is wired at a 60,000-token context max and runs last**, mimicking an agent compacting +when full. It therefore fires only when compaction failed to keep the turn under the max, which makes +**firings the deferral measurement**. + +## Result + +| arm | pipeline | **summarizer firings** | vs baseline | request shrink | +|---|---|---|---|---| +| **S1** | `summarize` alone | **71** | — | 64.6% | +| **S2** | `codesmart` − `extract_llm` | **46** | **−35%** | 58.2% | +| **S3** | + **tail** `extract_llm` | **46** | **+0** | 58.2% | +| **S4** | + `coref` | **20** | **−72%** | 45.6% | +| **S4b** | + `extract_llm` **prefix reach** | **20** | −72% | 45.6% | + +Shrink *falls* as firings fall, and that is the correct direction: `summarize` is the largest single +shrinker, so deferring it removes its contribution. Shrink is not the objective here — firings are. + +Three findings, in order of how much they matter. + +## 1. Deferral works, and it is `coref` that does it — **measured** + +**72% fewer summarizations.** `coref` acted 28 times, removing 1,008,646 tokens, and that alone took +firings from 46 to 20. This is the first clearly positive end-to-end result for this line of work. + +The deterministic pipeline gets a third of the way there (71 → 46) for free. **The tail +`extract_llm` lever adds exactly nothing** (S2 and S3 are byte-identical), consistent with every +other measurement of it on this corpus. + +## 2. The fold engages correctly and contributes nothing — **measured** + +`allow_cached_prefix` demonstrably worked. The gate counters are the proof: + +| gate | S4 (prefix reach off) | S4b (on) | +|---|---|---| +| `cached_prefix` | 6,597 | **gone** | +| `cached_prefix_above_floor` | 279 | **gone** | +| `prefix_still_referenced` | — | **6,519** | +| `economic_gate` | 25 | **103** | + +The blanket refusal is replaced by the index pre-filter, which rejected **6,519 prefix candidates for +free**; the economic gate then declined what survived. And the outcome is **byte-identical** to S4 — +same 20 firings, same 45.6%, `extract_llm acted=0`. + +Two reasons. 98.8% of prefix candidates are still referenced (matching the 96% `open` share). And the +handful that pass cannot clear break-even. + +## 3. The pre-filter selects the wrong class — **a design error, and the useful result** + +The deeper problem is not yield. **For the `unreferenced` class, dropping strictly dominates +trimming.** The pre-filter hands `extract_llm` only content the index has shown nothing ever used — +and for that content a model call can at best preserve *part* of what is already spent, while paying +a call and a cache-write. `coref` removes it outright for free. There is no work for the model to do +in the class it is allowed to see. + +The class where trimming belongs is **`closed`** — referenced once, long ago; the value was taken and +the remainder is chaff. That content is still partly live, so deciding what to keep genuinely needs +judgement: + +| class | who should act | why | +|---|---|---| +| `unreferenced` | **`coref` — drop** | provably spent; a call could only preserve less | +| **`closed`** | **`extract_llm` — trim** | partly live; needs judgement | +| `open` / `opaque` | neither | still in use, or no evidence | + +The sequential replay makes this testable for the first time: it surfaced **`class_closed: 25`**, +where the earlier one-request setup structurally could not produce any. Repointing the pre-filter +from `unreferenced` to `closed` is a one-line change and is the obvious next experiment. + +## Also: the biggest lever is lossless, and it is none of these components + +| component | firings | tokens saved | +|---|---|---| +| **`format`** (lossless JSON repack) | 119 | **1,266,088** | +| `coref` | 28 | 1,008,646 | +| `dedup` | 54 | 15,546 | +| `extract_llm` | **0** | **0** | + +`format` recovers more than `coref` does, losslessly and for free. Every lossy component here is +competing for what is left after it. + +## Caveats + +- **No reward.** Nothing here says the 20-vs-71 tradeoff preserved task success. That remains the + gate, and it is not answerable from replay. +- **9 conversations, 197 turns.** Direction and mechanism, not calibration. +- **Turns are reconstructed prefixes**, valid because LOCA is append-only, but each `/compact` call is + independent — a real session would carry the *compacted* transcript forward, so effects that + compound across turns are not captured. +- **The 60,000-token max is a choice**, made because it fires often enough to measure (66 of 197 + turns exceed it). Claude Code's real threshold is ~167,000, which only 2 of 197 turns reach. +- **The summarizer is cheap here because of freeze/replay** — 71 firings cost 17 model calls. An agent + whose transcript actually changes after each summary would pay more. + +See also: [component gating](component-gating.md) · [density](coref-density.md) · +[eval box](coref-evalbox.md) · [reachability](coref-reachability.md) · +[the proposal](../proposals/coref-compaction.md) diff --git a/docs/results/coref-reachability.md b/docs/results/coref-reachability.md new file mode 100644 index 00000000..07b14964 --- /dev/null +++ b/docs/results/coref-reachability.md @@ -0,0 +1,121 @@ +# Is the deferral prize reachable at all? + +[The proposal](../proposals/coref-compaction.md) calls deferring the agent's own compaction its +largest claimed win. [Implementation status](../proposals/coref-implementation.md) calls how often +that prize is reachable **"the largest unexamined claim in the proposal"**, and notes it needs no +API budget and no eval box. This is that measurement. It cost **$0**. + +It also found a defect in how every earlier measurement in this repo read its corpus, and one +result that is the **first positive finding** for the proposal. + +Reproduce: `python3 deploy/harbor/coref_reachability.py` + +## 1. The prize does not exist in most sessions — **measured** + +Counted over real `isCompactSummary` events, i.e. the points at which Claude Code actually +compacted itself. Not reconstructed, not inferred. + +| population | sessions that ever compacted | events | +|---|---|---| +| all main-conversation transcripts | **6/35 (17%)** | 9 | +| ≥ 50 model turns | 5/30 (16%) | 7 | +| ≥ 100 model turns | 5/22 (22%) | 7 | +| ≥ 200 model turns | **5/17 (29%)** | 7 | + +So the prize is a **long-session feature that is still absent from two thirds of long sessions**. +Any expected-value argument for `coref` has to be multiplied by ~0.17–0.29, and that multiplier +has been missing from every version of the proposal. + +Subagent transcripts (197 of them, under `/subagents/`) are excluded: they are separate +conversations with their own context, so counting them would inflate the denominator with sessions +that structurally cannot reach a threshold. + +## 2. The corpus cannot support absolute request sizes — **methodological defect, quantified** + +The first attempt at this measurement produced a "pre-compaction request" of **777,339 tokens** on a +200k model. That is impossible, and chasing it found a real problem. + +A Claude Code transcript is **not a linear conversation**. It is a tree: `--resume` and message +edits fork it, and the compacted transcripts here carry **25–51 forks and 338–632 leaf entries** +each. Reading the file in order concatenates every abandoned branch. Worse, the `parentUuid` graph +is too fragmented to fix by walking it — the longest reconstructible root→leaf chain collapses to +**5–78 entries out of 1,486–5,217**. + +**So absolute request size is not recoverable from this corpus**, and any figure derived from a +linear read spans multiple context windows rather than describing one request. + +!!! success "But share-based results are safe, and that was worth checking rather than assuming" + If branches duplicated content heavily, every class share and reference statistic in + [the density pass](coref-density.md) and [the selection experiment](coref-selection-experiment.md) + would be contaminated. Measured directly: exact-duplicate tool outputs are **16% by count but + only 3% of mass pooled, 2% median per transcript, 8% worst case**. The duplicates are small + repeated reads, not the large outputs the measurements turn on. + + Those documents' **share**-based findings therefore stand. What does not stand is any absolute + size claim — and the density doc's peak-request and deficit columns were already labelled + artifacts of 180k segmentation, so the two caveats compound rather than conflict. + +## 3. Firing at the crossing removes the deficit term — **the first positive finding** + +The requirement is `required cut ≥ (usage − threshold) + growth × headroom`. The density pass +measured the first term at **7.3% of the request** and concluded H=40 was unreachable (0/19). + +But that term is an artifact of measuring **after** the crossing — `cc_capture.py` segments at +180k, which places the measurement point past the threshold by construction. **At the moment the +agent compacts, usage *is* the threshold, by definition — that is what triggered it.** So a pass +that fires at the crossing faces only the growth term: + +``` +required cut ≈ growth_per_turn × headroom_turns +``` + +which needs no absolute size measurement at all — only growth per turn, which is a ratio of two +quantities the branch problem inflates alike, and therefore robust. + +Measured growth in the 6 sessions that actually compacted: **min 201, median 239, max 9,765 +tok/turn** (the max is one short session and not representative). + +| headroom bought | required cut | as share of a 167k request | `unreferenced` (4.4%) | `+closed` (9.6%) | +|---|---|---|---|---| +| H = 20 | 4,778 | 2.9% | **yes** | yes | +| H = 40 | 9,555 | 5.7% | no | **yes** | +| H = 60 | 14,333 | 8.6% | no | **yes** | +| H = 80 | 19,110 | 11.4% | no | no | + +**This is the first clause of [the hypothesis](../proposals/coref-compaction.md#the-hypothesis-this-proposal-should-be-tested-against) that does not fail.** Fired at +the crossing rather than at maximum pressure, the available cut is enough for 20–60 turns of +headroom. It vindicates the proposal's own counter-intuitive claim that **the profitable moment to +compact is earlier than the moment of maximum pressure** — which had been argued from cache +economics and is now also true of the deferral prize. + +### And it is sensitive to the growth estimator, which is not pinned down + +The density doc reports **~514 tok/turn** from a different estimator (request-size deltas inside a +180k segment); this script measures **239** (total content ÷ model turns). The verdict flips: + +| growth estimate | H = 20 | H = 40 | H = 60 | +|---|---|---|---| +| 239 tok/turn (this script) | `unreferenced` suffices | needs `cut_closed` | needs `cut_closed` | +| 514 tok/turn (density doc) | needs `cut_closed` | **neither** | **neither** | + +`cut_closed` **ships off by default**, so even the optimistic column requires enabling a knob whose +yield ranges 0–15% by workload. Pinning the estimator down needs request-level data this corpus +cannot provide. + +## What this settles + +| claim | status | +|---|---| +| How often the deferral prize is reachable | **Answered: 17% of sessions, 29% of long ones.** Was unmeasured. | +| Can `coref` supply the required cut? | **Yes for 20–60 turns if it fires at the crossing** — reversing the density pass's 0/19, which measured a late fire. | +| Is that robust? | **No.** It flips on a growth estimator two measurements disagree about by 2×. | +| Do absolute request sizes from Claude Code transcripts mean anything? | **No** — tree-structured transcripts, unreconstructable branches. Share-based results unaffected (3% duplicate mass). | +| Reward | **Still unmeasured**, still the gate. Nothing here touches it. | + +Every "yes" above inherits the **11% false-drop** measured in +[the selection experiment](coref-selection-experiment.md), and applies only inside the 17–29% of +sessions where the prize exists at all. + +See also: [the proposal](../proposals/coref-compaction.md) · +[density](coref-density.md) · [selection experiment](coref-selection-experiment.md) · +[implementation status](../proposals/coref-implementation.md) diff --git a/docs/results/coref-selection-experiment.md b/docs/results/coref-selection-experiment.md new file mode 100644 index 00000000..ef6a49a7 --- /dev/null +++ b/docs/results/coref-selection-experiment.md @@ -0,0 +1,308 @@ +# Does an LLM decide better than the reference index? A held-out experiment + +The [co-reference proposal](../proposals/coref-compaction.md) and the design discussion around it +produced one claim that decides the whole component, and it had never been measured: **if a single +model call may either *drop* a tool output entirely or *trim* it, does that produce better decisions +than the deterministic reference index — and does it pay for itself?** + +This is that experiment. **$43.88** of model spend, 8,105 recorded decisions, ten arms. Every number +below is reproducible from `~/cg-coref-experiment-20260819/` (code, all decisions, logs). + +Two of its results **retract earlier claims in this repo's own docs**, and they are marked as such. +Confidence is labelled per finding: **measured**, **measured, with caveat**, **structural argument** +(arithmetic, not observation), or **withdrawn**. + +!!! warning "What could not be run, and why it matters" + SWE-bench Verified and Terminal-Bench 2.0 — the two benchmarks this repo reports — **were not + runnable**. [`REPRODUCE.md`](REPRODUCE.md) requires Linux, a Harbor checkout at a fixed path, and + Docker Hub authentication; the work was done on macOS with no Harbor. So this experiment measures + **decision quality on captured traffic**, never task outcome. It cannot speak to reward, and + reward is the axis the proposal itself says is decisive. + +## Method: decide at a firing point, score against the future + +The design is what makes the numbers trustworthy, so it is worth stating precisely. + +For each captured transcript, pick a **firing point `F`** at 60% of the way through its model turns — +the point where a compaction pass would plausibly fire. Every tool output at a message index below +`F` and at least 300 tokens becomes a candidate. Then split the transcript: + +| window | contents | who sees it | +|---|---|---| +| **evidence `(i, F]`** | which identifiers the output introduced, which later turns reused them, how many times, how long ago, how many turns elapsed | every arm | +| **held-out future `(F, end]`** | what the agent actually went on to do | nothing — ground truth only | + +Every arm decides `drop` / `trim` / `keep` from the evidence window alone. Scoring: + +- **false-drop** — of the outputs an arm removed, the share that *were* referenced by exact match in + `(F, end]`. A definite error: we removed something the agent went on to use. +- **live-kept** — of the outputs that *were* referenced in `(F, end]`, the share an arm correctly + kept. **This is the discrimination metric**, and it is the one that separates the arms. +- **null baseline** — drop everything. Scores exactly the base rate, and reveals that a high + removal rate paired with a base-rate false-drop rate means *no discrimination at all*. + +`false-drop` alone is misleading because it is bounded by the base rate; `live-kept` is not. + +### Corpora + +| corpus | candidates | mass | referenced after `F` | +|---|---|---|---| +| LOCA-bench | 140 | 254,244 tok | 4% | +| UltraHorizon | 94 | 116,764 tok | 11% | +| **Claude Code** | **885** | **943,747 tok** | **46%** | + +Claude Code carries the conclusions: at a 4% base rate there is almost nothing to discriminate, so +LOCA and UltraHorizon cannot separate a good arm from a reckless one. + +### Arms + +Two deterministic (the `coref` classifier, with and without `cut_closed`) and eight model arms across +two models × four prompt shapes: + +- **digest** — goal + output content + reference evidence, per output. +- **nodigest** — same, evidence withheld (ablation: does the index earn its place?). +- **evidence** — reference evidence only, *no output content* (tests whether cost can collapse ~10×). +- **strict** — digest, but the prompt states the real cost of a wrong removal instead of reassuring + the model that cuts are recoverable, and insists on verbatim copying. +- **bulk** — one call adjudicates ~15 outputs together, so the model judges comparatively. + +## Results (Claude Code, n=885, null = 46%) + +| arm | removed | false-drop | **live-kept** | $/call | break-even | +|---|---|---|---|---|---| +| *null: drop everything* | *100%* | *46%* | *0%* | — | — | +| **deterministic, unref only** | 11.8% | **11%** | **95%** | $0 | — | +| **deterministic, +closed** | 26.5% | 21% | 81% | $0 | — | +| sonnet bulk | 33.6% | 29% | 57% | $0.0054 | 10,675 | +| haiku bulk | 40.2% | 36% | 58% | **$0.0010** | 1,606 | +| **sonnet strict** | 49.0% | 30% | 58% | $0.0118 | **15,987** | +| haiku strict | 64.0% | 34% | 32% | $0.0028 | 2,952 | +| haiku nodigest | 88.2% | 45% | 11% | $0.0026 | 1,966 | +| haiku digest | 90.9% | 40% | 6% | $0.0027 | 1,984 | +| haiku evidence-only | 90.9% | 43% | 9% | $0.0012 | 844 | +| sonnet digest | 87.3% | 42% | 14% | $0.0107 | 8,205 | + +## Findings + +### 1. The per-output merged design is refuted — **measured** + +This is the design the discussion converged on: one call, seeing the output and the reference +evidence, deciding drop-or-trim. It fails. Haiku keeps **6%** of what the agent actually needed; +dropping everything blindly keeps 0%. Sonnet is *worse* (14% live-kept, 42% false-drop). Both sit +inside the null model's error bar. + +Giving a model the content **and** the evidence, per output, does not produce discrimination. It +produces a machine that drops nearly everything. + +### 2. Two interventions genuinely work — **measured** + +**Bulk adjudication** lifts live-kept from 6% → **58%**, at the *lowest* cost per output (overhead +amortises across ~15 outputs). Comparative judgment beats absolute judgment: shown fifteen outputs +side by side with their evidence, the model ranks them; shown one, it just drops it. + +**Prompt framing is worth ~26 points of live-kept.** The first prompt reassured the model that cuts +"stay recoverable on request". That single clause produced 91% removal at 6% live-kept. Replacing it +with the real cost — *"the agent usually does not notice the gap and answers from worse information +instead of asking for it back"* — moved haiku to 64%/32% and sonnet to 49%/**58%**. Telling a model +its mistakes are cheap makes it careless. + +### 3. The deterministic index is the best discriminator measured — **measured** + +95% live-kept at 11% false-drop, at zero marginal cost. **No combination of index and model beat +it**: intersecting the index with the best model arm gives 8% removed at 10% false-drop — strictly +worse than the index alone. Every model-involving operating point is either lower yield or higher +error than a deterministic one, up to 26.5% removal. + +This reverses the design conclusion the discussion had reached, which was to demote the index to an +evidence supplier and move the verdict into the model's prompt. + +### 4. `cut_unreferenced` is not the "free safe cut" it ships as — **measured** + +It has an **11% false-drop rate** under Tier-1 exact ground truth: outputs dormant at `F` that the +agent used later. "Unreferenced" is a claim about the past, and the future contradicts it one time in +nine. + +**Revised upward to 21–24%** by [iteration 009](../experiments/loca/iter009/results.md), which widened +ground truth with deterministic normalization (numeric reformatting, case, substring; the 24% figure +additionally allows path-basename matching and is an upper bound). So the "free safe cut" is roughly +**twice as lossy as first published**, and since Tier-3 semantic reuse is still invisible, even 21% is +a lower bound. + +It is **not a boundary artifact**, which was the first thing to check: + +| first future reference | share of false drops | +|---|---| +| 1 turn after `F` | **0%** | +| 2–5 turns | 5% | +| 6–10 turns | 21% | +| 11–25 turns | 10% | +| 26–50 turns | 5% | +| **51+ turns after `F`** | **57%** | + +Discarding every reference within 50 turns of the boundary still leaves 6%. The errors are genuine +long-range dormancy — an output goes a hundred turns untouched, then the agent reaches for it. + +**And it is irreducible with the features available.** Requiring more introduced identifiers makes it +*worse* (11% → 21%); requiring longer observed dormancy barely helps while cutting mass 2.5×. An +output dormant for 100+ turns and then used carries no signal at `F` distinguishing it from one +dormant forever. + +### 5. `min_later_turns` does not earn its keep on this metric — **measured** + +Sweeping the opportunity floor added earlier: + +``` +min_later= 0: drops=352 false= 8% mass=362,915 +min_later= 8: drops=210 false=11% mass=207,331 <- the shipped default +min_later= 20: drops=162 false= 9% mass=146,209 +min_later= 80: drops= 95 false=14% mass= 57,042 +``` + +Zero gives *more* mass and *lower* false-drop than the default. The floor was added for a different +reason — stopping a batched pass from preferentially cutting the newest context — and that rationale +stands, but it is not buying safety and the component docs should not imply it does. + +### 6. The trim contract as specified is unusable — **measured** + +Asking a model to return the content worth keeping produced text that was **not verbatim in the +original** in 94–172 of every ~140 trim verdicts. Models paraphrase, reformat, and reconstruct. This +is exactly what `internal/extract/contain.go` exists to catch, and it means the "return the kept +text" contract must be replaced by `extract_llm`'s sandboxed-filter mechanism, which enforces +containment structurally. + +**The anchor guard also failed.** The prompt instruction — *"if a later turn referred to something +from this output in order to point at a value it did not restate, that value must be kept"* — did not +work: over half of trims on future-referenced outputs dropped the very identifier the agent needed. + +### 7. The cost/capability bind — **measured** + +| | $/call | break-even | live-kept | +|---|---|---|---| +| haiku strict | $0.0028 | 2,952 tok | 32% | +| sonnet strict | $0.0118 | 15,987 tok | **58%** | + +**The model cheap enough to pay for cannot discriminate; the model that discriminates cannot be paid +for.** No prompt moves this — it is cost against capability. + +And almost no real output clears the higher bar: + +| corpus | > 2,952 tok | > 15,987 tok | max output | +|---|---|---|---| +| LOCA | 11 (7%) | **5 (3%)** | 39,713 | +| UltraHorizon | 0 | **0** | 2,500 | +| Claude Code | 62 (7%) | **0** | 11,399 | + +### 8. The break-even mechanism is real; its magnitude was overstated — **measured, correcting this repo** + +`B = callCost / (ratio × (1 + reuses) × perToken)`, so break-even is inversely proportional to the +effective ratio. `extract_llm` measures ratio 0.10–0.12 because *trim* is its only legal outcome. +Adding a *drop* outcome raises the population ratio, and `B` falls proportionally. **The mechanism is +confirmed.** + +But the eye-catching figures (844–2,952) belong to arms removing 64–91% of mass at 34–43% false-drop +— correct arithmetic on an unusable policy. At the best defensible operating point (sonnet strict, +ratio 0.49) break-even is **15,987 tokens**: 2× better than `extract_llm`'s measured 30,500, not the +10–15× first claimed. **Holding call cost fixed, the ratio improvement buys ~4.5×.** + +And break-even here is denominated in *cache-read savings*, which this repo already measured as 0.024% +of billed input. Clearing it means only that the call cost stopped being the binding constraint — not +that the pass is worth making. + +### 9. Selection cannot replace a summarizer — **structural argument** + +Sustained selective removal is `f × g`, where `f` is the fraction of arriving mass that ever becomes +removable and `g` is the growth rate. Since `f < 1`, removal is always **less** than growth: selection +cannot hold the line, only slow the approach, by `1/(1−f)`. + +| removable fraction of request | session extension | +|---|---| +| 4.4% (deterministic unref) | 1.05× | +| 9.6% (deterministic +closed) | 1.11× | +| 18% (sonnet strict) | 1.22× | +| 24% (haiku strict) | 1.32× | + +Even the aggressive arms buy 22–32% more turns. That is **deferral, not replacement**. A summarizer +achieves ~96% because it compresses **live** content; selection can only remove **dead** content. To +replace a summarizer you must paraphrase, which means being one. + +### 10. The incumbent comparison is **withdrawn** — the metric cannot make it + +An attempt to score the agent's own auto-compaction the same way (7 real `isCompactSummary` events, +640k tokens of tool output → 27.7k of summary) first appeared to show the summarizer dominating every +selective arm at 96% removal and 20% false-drop. **That was an artifact of a missing base-rate +control** — the same error this document warns about in its own method section. With the null added, +the population's base rate is 23% against the summarizer's 20% false-drop: the summary rescued **3 +percentage points**, i.e. 9% live-kept. + +**Neither figure is trustworthy, and the comparison should not be made with this instrument.** +Identifier matching scores *verbatim survival*. A prose summary that reads "found the grace-period +bug in the auth module" has preserved the information while containing none of the identifiers, so +the metric punishes paraphrase by construction and cannot bound how much the summary really carried. +It is valid *within* the selective arms — they all keep-or-drop verbatim — and invalid across the +verbatim/paraphrase boundary. + +**One number does survive**, because it does not require scoring the summary's content: +**11% of post-compaction model turns reference identifiers the summary did not carry and that were +not re-read** — the agent visibly reaching for something it no longer has. That is the incumbent's +operational damage rate. + +## Corrections to earlier claims in this repo and its discussion + +| claim | status | +|---|---| +| "`coref` amortises one cache-write across a batch; `extract_llm` pays per output" | **Wrong.** `extract_llm` applies all projections in one request, so its write cost is also one rewrite. `coref`'s real advantage is zero model calls. | +| "Break-even collapses 10–15×" | **Overstated ~3×.** ~4.5× at a defensible operating point. | +| "Sonnet is dramatically safer (4% false-drop)" | **Artifact.** It had only processed LOCA (4% base rate). On Claude Code it is 42%. | +| "The summarizer dominates every selective arm" | **Withdrawn** (finding 10). | +| "`cut_unreferenced` is the free safe cut, no calibrated threshold needed" | **False.** 11% false-drop, irreducible, lower bound. | +| "Fold the verdict into the model's prompt; demote the index to evidence supplier" | **Refuted** (findings 1, 3). | + +## Limitations + +- **No reward, no benchmark.** Decision quality on captured traffic only. The proposal's own + acceptance criteria put reward first, and nothing here touches it. +- **Ground truth is Tier-1 exact matching**, so it cannot see transformed or semantic reuse. Every + false-drop figure is a **lower bound**, for every arm. **Partly quantified** + ([iteration 009](../experiments/loca/iter009/results.md)): widening it with deterministic + normalization (numeric reformatting, case, substring) moves referenced candidates 408 → **473** + (+16%) and raises every arm's false-drop — `cut_unreferenced`'s goes **11% → 21%**, see finding 4. + The gap between the index and the best model arm does not narrow; it slightly widens. The + deterministic slice of Tier-2 turns out to be nearly empty, so closing this bias properly needs a + judge, which reintroduces the noise that ruled out UltraHorizon + ([measurement-limits §6](measurement-limits.md)). The bound is tighter, not removed. +- **One firing point** (`F` = 60% of model turns). A real pass fires at a threshold crossing, at + varying depth with varying future remaining. +- **An asymmetry that flatters the deterministic arms:** `min_later_turns` is a hard structural guard + present only in them. Model arms received `later_turns` as information with no enforced floor. + **Now done** ([iteration 009](../experiments/loca/iter009/results.md), $0, no new model calls): it + changes almost nothing. The floor overrides only 6–23 of 885 decisions per arm and moves live-kept + by 0–2 points; removing it from the deterministic arms costs them one point. The asymmetry was a + reasonable suspicion and is not the explanation for the gap. +- **`n = 7` compaction events** for finding 10, from 4 sessions, one contributing 3. +- **The Claude Code corpus is read linearly out of a tree-structured transcript**, so abandoned + `--resume`/edit branches are present. Measured contamination is small — exact-duplicate tool + outputs are 3% of mass pooled, 2% median ([reachability §2](coref-reachability.md)) — so class + shares and the arm comparison hold. But it introduces a bias *opposing* the Tier-1 one: an + abandoned branch can supply a "later reference" that the live conversation never made, which + inflates false-drop. So the 11% is bracketed by two biases of unknown relative size rather than + being a clean lower bound, and the earlier claim that it is purely a lower bound was too strong. +- **"Referenced later" ≠ "the agent was harmed."** Removed content is recoverable via `expand`, so + the true cost is a round-trip *if the model notices*. Treating a later reference as an error is an + assumption inside the metric. + +## What this settles, and what it does not + +**Settled:** the per-output merged design does not work; bulk adjudication and cost-honest prompting +are the only model shapes worth pursuing; the deterministic index is the strongest discriminator +measured; `cut_unreferenced` carries an irreducible ~11% error floor; the trim contract needs a +sandboxed filter, not free-text; selection cannot replace a summarizer. + +**Not settled — and each needs a different instrument:** whether any of this improves *reward*; +whether recovery via `expand` actually fires often enough to make a 30% recoverable false-drop +preferable to a 20% permanent one; how often the agent's own compaction is reachable at all +(`modes.Tracker` reset detection, free, still not run); ~~and whether a floor-symmetric comparison +narrows the gap between the index and the bulk arm~~ — **answered: it does not** +([iteration 009](../experiments/loca/iter009/results.md)). + +See also: [co-reference density](coref-density.md) · [the proposal](../proposals/coref-compaction.md) +· [implementation status](../proposals/coref-implementation.md) diff --git a/docs/results/measurement-limits.md b/docs/results/measurement-limits.md new file mode 100644 index 00000000..110429c2 --- /dev/null +++ b/docs/results/measurement-limits.md @@ -0,0 +1,361 @@ +# What can and cannot be measured here + +This page exists because the hardest part of evaluating compaction turned out not to be building +it, but establishing what any given experiment is *capable* of showing. Several results in this +repo were invalid for reasons that had nothing to do with the components — sample sizes with no +power, a harness blind to a whole class of defect, and yields that measured a component's +*economic throttle* rather than its capability. + +Read this before designing an arm or believing a number. + +## 1. Statistical power — reward at small n shows nothing + +Reward on LOCA is **binary per task**, so a paired comparison uses McNemar's exact test on +discordant pairs. That test is brutal at small n. + +### Headroom: n is not the only thing that limits detection + +Before any n calculation, ask how many tasks are *able* to show the effect at all. Direction matters, +and the two pools are disjoint: + +- **Harm can only appear on tasks that currently pass.** A task that already fails cannot be broken + visibly — it fails either way. +- **Improvement can only appear on tasks that currently fail.** A task that already passes cannot + pass more. + +Measured pools, from the matched 15-task comparison in +[iteration 008](../experiments/loca/iter008/results.md): + +| band | pass → **can show harm** | fail → **can show gain** | +|---|---|---| +| 64k | **3** (of 12 clean) | 9 | +| 32k | **8** (of 15) | 7 | + +At 64k the whole harm signal had to come from **three tasks**. At a 10% harm rate that is 0.3 +expected visible events, which is why [iteration 007](../experiments/loca/iter007/results.md) could +bound harm only at ≤26% — there was almost nothing for harm to act on. Nor were the 9 failing tasks +useful: they failed under `format` alone, which is **lossless**, so they fail for reasons no +compaction component can influence. They cost full price and return a tie. + +**Why ~50% is the optimum rather than merely "better".** For a binary outcome the available variance +is `p(1-p)`, maximal at `p = 0.5`: + +| base rate | p(1-p) | +|---|--:| +| 0.25 | 0.19 | +| **0.53** | **0.25** | +| 1.00 | 0 | + +For the *same* underlying effect, the observable signal is largest near 50%. The whole band table in +section 6 is this one fact: + +| band | rate | failure mode | +|---|---|---| +| 8k | 100% | **ceiling** — only harm can appear, improvement is invisible | +| **32k** | **53%** | both pools populated | +| 64k | 25% | near-**floor** — harm pool nearly empty | +| 128k | ~0% | **floor** — nothing passes, so nothing can degrade | + +This bites hardest on the two-sided claim below. If deferring a wipe can *raise* accuracy, that needs +a pool of currently-failing-but-recoverable tasks — and at a 25% base rate the large failing pool was +mostly tasks failing for unrelated reasons. + +### The headroom limit was ours, not the benchmark's + +Worth recording as a reasoning error, not just a result. The conclusion drawn at 64k was "LOCA lacks +the headroom for this" — a claim about the *benchmark*. The 32k run used the **identical 15 tasks**, +same classes and seeds, differing only in data volume (600 games / 6 teams instead of 1200 / 8), and +scored 53%. + +Same benchmark, same tasks, one config knob, **25% → 53%**. The limitation was a property of the +chosen configuration that had been generalised into a property of the benchmark. That is the same +shape as the three rig artifacts in section 7 (chunked bodies, EAGAIN, the 128k "collapse"): each +looked like a fact about the world and was a fact about the setup. + +**Standing rule: treat "the benchmark cannot do X" as a hypothesis about your own configuration until +a matched run says otherwise.** A matched run is cheap — this one cost $85 and saved several hundred. + +(Caveat kept visible: the 64k figure is 3/12 because three tasks errored on the broken shim, so its +true rate carries some uncertainty. The 25%-vs-53% gap is far too wide for that to account for.) + +[iteration 004b](../experiments/loca/iter004b/results.md), 12 tasks: + +| comparison | gained | lost | discordant | p | +|---|--:|--:|--:|--:| +| deterministic vs baseline | 0 | 0 | **0** | 1.000 | +| + prefix reach vs baseline | 2 | 1 | 3 | **1.000** | + +**What n=12 can detect at all:** + +| discordant pairs, all one direction | p | +|---|--:| +| 4 | 0.125 | +| 5 | 0.0625 | +| **6** | **0.031 ← first significant** | + +So compaction must flip **≥6 of 12 tasks (50 percentage points)** to register. Nothing smaller is +detectable **in either direction**, which means a null result at this size is not evidence of +safety. + +**What a realistic effect needs:** + +| discordant rate | split | n (80% power, α=.05) | +|---|---|--:| +| 25% | 70/30 | **194** | +| 25% | 65/35 | **347** | +| 40% | 65/35 | 216 | + +**~200–350 tasks per arm.** LOCA has 75, so even the full set detects only ~20pp effects. + +### Two consequences that keep being forgotten + +**Zero discordance is stronger evidence than a p-value.** The deterministic arm's per-task outcome +string was *byte-identical* to the baseline — `010000101010`, same tasks solved, same tasks failed. +That is not "no significant difference", it is "no difference", and it is the most reassuring number +in this work. Still n=12, but qualitatively unlike a 4-vs-5 split. + +**"Reward-neutral" is a non-inferiority claim.** It needs *more* power than showing a difference, +because failing to find an effect is not finding its absence. Every reward statement here should be +read as **unmeasured**, not confirmed. + +### The effect is two-sided: compaction can *raise* accuracy, not only risk it + +The non-inferiority framing below is necessary but incomplete, and taking it as the whole story +understates what compaction can do. + +"coref can only harm accuracy" is true against a baseline that keeps the **full** context. That is +not the baseline in any run here. LOCA's native trimmer **drops whole messages** once a request +outgrows its limit — that is precisely why `loca_repair_shim.py` has to repair `tool_use`/`tool_result` +pairing at all. So at any band where the trimmer fires, **the baseline is itself lossy**, and +selective removal that keeps what is still referenced can preserve *more* usable context than a blunt +drop. That is a mechanism for coref to be **better**, not merely not-worse. + +**The same argument applies to summarization, and that is the core of the deferral claim.** Replacing +a blunt context wipe with `summarize` does not remove the phenomenon — a summary is lossy too, and +lossy in a way that is arguably *worse* for co-reference: a trim removes messages wholesale, so what +is gone is at least knowable, whereas a summary paraphrases exact identifiers into prose, silently +corrupting the literal tokens Tier-1 matching depends on. Summarizing early therefore destroys +exactly what selective removal would have kept, which is why deferring it is expected to help rather +than merely cost less. + +Two consequences for design: + +1. **Use a two-sided test, not a one-sided harm bound.** A two-sided test has more power at the same + n, and a one-sided non-inferiority frame cannot register a gain even when one occurs. +2. **The baseline arm must be named precisely.** "vs baseline" is ambiguous between *full context* + (where only harm is possible) and *lossy default trim or summary* (where gain is possible). These + are different experiments with different expected signs, and conflating them makes any result + uninterpretable. + +### The affordable version of the claim, priced + +Superiority on a binary reward is the expensive claim; **bounding harm is the cheap one, and is +usually what is actually being asked.** Priced below at $7.59 per task per arm, which was **wrong by +5×** — the true figure is **$1.52 per run** ([iteration 010](../experiments/loca/iter010/PREREGISTRATION.md) +amendment 1). Divide every dollar figure in this table by ~5; the *ratios* between rows, which are the +point, are unaffected: + +| pairs | cost (2 arms) | upper 95% bound on harm if 0 tasks harmed | +|---|---|---| +| 10 | $152 | ≤ 26% — worthless | +| 20 | $304 | ≤ 14% | +| 40 | $607 | ≤ 7% | +| 60 | $911 | ≤ 5% | +| 100 | $1,518 | ≤ 3% | + +Compare the superiority cost at the observed ~10% discordance rate: 119 pairs at a realistic 80/20 +split, i.e. **$1,800**, to detect an effect nobody claimed. **The margin must be declared before the +run and the budget chosen to buy it** — otherwise the run silently purchases the top row. + +### Contrast: token measurements are near-exact + +Component savings (`format` at 92–99% of total, `extract_llm` 38 firings/494k tokens, `coref` 61/638k) +are counts over a fixed request set from deterministic components, not sampled estimates. They need +no significance testing. **The asymmetry is the point: we know precisely what is removed and almost +nothing about what it costs.** + +## 2. Replay is not validation + +The `/compact` endpoint runs a pipeline and returns the rewritten body **without forwarding it +upstream**. No provider ever validates it. So replay can tell you *what a component removed* and is +**structurally incapable** of telling you *whether the result is a sendable request*. + +This blind spot hid three provider-rejecting defects in `summarize` +([iteration 005](../experiments/loca/iter005/results.md)), each masked by the previous one, found +only by sending live traffic one at a time. It silently covered every `/compact`-based result here: +[density](coref-density.md), [the eval-box pass](coref-evalbox.md), +[component gating](component-gating.md), and +[iteration 002](../experiments/loca/iter002/results.md) — whose deferral figure came from a pipeline +that 400s in production. + +**Mitigated, not solved.** `schema.ValidateShape` + the all-presets test now catch this class +statically in 0.07s (verified against the real defects). Live runs remain the other half, but no +longer carry the whole burden. + +## 3. Eligible ≠ acted — yield measures the throttle, not the capability + +`coref` acts only after clearing **four** economic gates, all on by default: + +| gate | default | refuses when | +|---|---|---| +| `trigger` | request shape | the request is too small to bother | +| `min_batch_frac` | **0.05** | the batch cuts <5% of the request | +| `break_even` | **true** | `S × T ≤ 11.5 × W` | +| `rewrite_budget` | **3** | 3 prefix rewrites already spent this session | + +So a reported yield is **"what `coref` could remove *and* justify"**, never "what `coref` could +remove". Three numbers must be reported separately or the result is uninterpretable: + +- **eligible mass** — from `class_*` gates: what the index judged dead +- **acted mass** — what survived all four gates +- **refused-for-economics** — `batch_too_small` + `break_even` + `rewrite_budget` + +**Why it changes the conclusion.** If acted-yield is negligible *but* refused-for-economics is +large, the finding is **not** "`coref` does not work" — it is "`coref` is economically throttled". +Those demand opposite responses, and without the gate breakdown they are indistinguishable. + +## 4. Break-even is sometimes vacuous + +`S × T > 11.5 × W` prices a cache-write the mutation *causes*. When the provider's cache has already +expired, the prefix is rewritten **regardless** — the mutation costs nothing incremental and needs +no break-even test at all. Those moments arise whenever the inter-turn gap exceeds the cache TTL: a +slow tool, a queue, a human thinking. + +Combined with §3: **`coref`'s ceiling is set by break-even, and break-even is sometimes wrong.** Its +real capability is higher than any measured yield, by exactly the mass refused for economics at +moments when the rewrite was already free. + +Measured on LOCA (`deploy/harbor/cache_opportunity.py`, from per-step `cache_read_input_tokens`): +**zero such moments** — LOCA drives local mock MCP servers, so turns land seconds apart and a +5-minute TTL never lapses. A property of the benchmark, not an answer. See +[the proposal's design notes](../proposals/coref-compaction.md#two-design-notes-from-review-neither-implemented). + +## 5. Cost model, measured + +Per-task cost at the 64k band, from real runs — **not** estimates: + +| arm shape | $/task | 75 tasks | model calls | +|---|--:|--:|--:| +| passthrough baseline | 1.78 | **134** | 0 | +| `format` only | ~1.15 | **~86** | 0 | +| deterministic (`+dedup +extract`) | ~1.22 | ~92 | 0 | +| **`+coref`** | ~1.25 | ~94 | **0** | +| `+extract_llm` (tail) | 1.89 | ~142 | haiku | +| `+extract_llm` (prefix) | 1.89 | ~142 | haiku | + +**Deterministic arms cost *less* than the baseline** — 31% less in one measured case — because +compaction reduces the agent's own token bill and makes no model calls. `coref` is deterministic, so +only the two `extract_llm` arms carry LLM spend. + +Six arms ≈ **$690** at k=1. Earlier I quoted "$315/arm", which was actually the *three-arm total* and +ignored that deterministic arms undercut the control. + +**But removing tokens is not saving money.** The prefix-reach arm removed 20.3% and cost *more* than +baseline ($22.64 vs $21.34): model calls plus pipeline overhead outweighed the saving. + +## 6. Benchmark suitability + +| benchmark | long context | tool outputs big enough | reward signal | slow tools (TTL) | verdict | +|---|---|---|---|---|---| +| SWE-bench Verified | **no** (max 46k) | **no** (max 2,760 tok) | binary, n=500 | **yes** | ruled out for compaction; **right for the TTL question** | +| Terminal-Bench 2.0 | no (~6k) | no (max 1,906) | binary, n=89 | yes | ruled out | +| **LOCA-bench** | **yes** (dial 8k→256k) | **yes** (max 59,857) | deterministic but **binary**; n=75 runs, but only **15 independent tasks** (5 seeds each) | **no** | viable for savings; reward limited by 15 clusters, not by n | +| UltraHorizon | yes (200k+) | yes | **LLM-judged** | ? | noise we cannot afford; no licence | +| Claude Code transcripts | yes | yes | **none** | yes | no reward → cannot gate | + +**The unclosable gap: nothing is both long-context and Tier-1-rich.** LOCA is long but Tier-2/3-heavy +(adverse to an exact-match detector); SWE-bench is Tier-1-rich but short. So a null result on LOCA +**cannot be generalised**, and that limit should be stated in advance rather than discovered. + +LOCA band behaviour — **three of these five rows were never actually measured:** + +| band | runs | baseline accuracy | usable | +|---|---|---|---| +| 8K (`debug`) | yes | **1.0** saturated | regression control only; only `format` fires | +| 32k | **never measured** | — | probe died in the rig, not the band | +| **64k** | **yes** | **20% (2/10 clean tasks)** | pressure yes, headroom **thin** | +| 96k | **never measured** | — | same rig failure as 32k | +| 128k | attempted | reported **0.0** / errors | **suspect** — collected during the broken-shim window | + +All three bad rows were my rig, not the benchmark. The 32k and 96k probes died with +`[Errno 11] write could not complete without blocking` — the EAGAIN pipe trap in the table below — +with `messages: 0`, i.e. before the agent ever ran. The 128k row was collected while +`repair_shim.py` was silently dropping chunked bodies, so its zero floor is not established. + +**Consequence for design:** "64k is the only band with pressure *and* headroom" was never tested +against its neighbours. Solve rate falls from 1.0 at 8k to 0.20 at 64k, so an intermediate band +plausibly has both, and the headroom problem in section 1 may be an artifact of having picked 64k. +Measuring 32k on the fixed rig is the cheapest available move, and should precede any decision to +buy more pairs at 64k or to move the agent to a larger model. + +### Two properties of LOCA that cap what any reward arm here can conclude + +**~~`group_by_seed` silently divides your n by five.~~ FALSE — and the real trap is the opposite one.** +It defaults to `True` and is genuinely not exposed as a CLI flag, but it groups for *reporting* only: +**all 75 configs execute either way.** The actual trap is on the reading side — `state0`…`state4` are +the 5 seeds, so globbing `tasks/*/state0/eval.json` silently reads 15 of 75 completed runs and +**overstates per-run cost by 5×**. That is what iterations 007 and 008 did +([iteration 010](../experiments/loca/iter010/PREREGISTRATION.md) amendment 1). + +**What actually limits reward power here is 15 independent tasks, not n.** 75 runs are 5 seeds of 15 +tasks, so they are clustered: extra seeds buy precision *within* a task and do not add independent +observations. Any harm bound must be quoted on the task-clustered end (~≤18% at zero events), and +tightening it needs more distinct tasks, which LOCA's 32k set does not have. + +**The 64k base solve rate is ~33%, and accuracy is binary.** (Measured over all 75 runs; the "20%" +first reported came from reading `state0` only.) Every +accuracy value is exactly 0.0 or 1.0, so there is no partial-credit signal to recover power from. A 20% +ceiling means most tasks fail for reasons no context-management component can affect — they cannot +register improvement *or* degradation, so they consume budget while contributing nothing but a tie. +Trajectories were 9–53 tool calls, well short of the ~106 ceiling, and the agent terminated on its own: +these are genuine task failures, not truncation. + +Together these are why [iteration 007](../experiments/loca/iter007/results.md) was stopped rather than +completed. + +## 7. Rig traps that produced valid-looking wrong numbers + +Each of these yielded a numerically plausible result with a broken cause. Check them before +believing any benchmark output. + +| trap | symptom | real cause | +|---|---|---| +| Claude Code under QEMU | `NonZeroAgentExitCodeError`, `reward=0` on every task | bun binary segfaults; amd64-only images on arm64 | +| `\| tail` on LOCA's stdout | `[Errno 11] write could not complete without blocking` on every band >8K | **my pipe**, not LOCA's MCP transport | +| `.venv/bin/loca` invoked directly | tools silently absent; agent reports an empty workspace | `.venv/bin` not on `PATH`, so spawned MCP servers got system python 3.9 | +| stale proxy binary | a feature arm behaves exactly like the arm without it | binary predated the feature; **every arm must record its build** | +| `summarize` in a shared pipeline | HTTP 400s | it must run alone — and separately had 3 shape defects | +| back-to-back arms | later arms look cheaper | they inherit the earlier arm's **prompt cache**; run the baseline last and never read cost as a clean saving | +| `pkill -f ` | command dies mid-script, silently | the pattern matches **its own** command line; use `arms6[4].sh` | +| `tool_success_counter: 0` | looks like broken tools | appears on **any** failure; not diagnostic | +| `--max-tool-uses` | looked like a cap | agent stopped at ~106 calls regardless of a 4× cap change | + +**The meta-lesson:** a broken *environment* produces a plausible transcript and a real-looking score. +Before believing a benchmark number, confirm the tools worked — per-task `eval.json`, gate counters, +and a transparency assertion on the `off` arm (0 saved, no component acted). + +See also: [the proposal](../proposals/coref-compaction.md) · +[experiment log](../experiments/README.md) · +[selection experiment](coref-selection-experiment.md) + +### Untracked scratch tooling is the least-reviewed code in the measurement path + +`repair_shim.py` — an ~80-line HTTP hop between LOCA and the gateway, living only in `/tmp` on the +eval box, never committed, never reviewed, never tested — silently dropped the body of every request +the client happened to send with `Transfer-Encoding: chunked`, and forwarded `chunked` together with +`Content-Length: 0`. Result: intermittent `400 Bad request` HTML that was attributed first to a +component under test and then to the benchmark, across two iterations. + +Two structural lessons, not one: + +1. **Error counts that move opposite to the amount of machinery indict the harness.** The arm running + `format`+`coref` produced *one* such error; the arm running only `format` produced *three*, and + succeeded on none of the three tasks the other arm passed. No component-caused defect has that + shape. That signal was in the data before the root cause was found. +2. **A permissive component will hide a protocol violation until it reaches a strict one.** A local + Python echo server accepted the contradictory framing with a 200. Only the real gateway rejected + it. Testing the shim against a lenient stand-in would have "passed." + +The shim is scratch tooling by necessity (it repairs bodies for replay), but everything in the +measurement path deserves the same suspicion as the thing being measured — and it is the code *not* +in the repo that gets none. diff --git a/docs/results/min-tokens-vs-economic-gate.md b/docs/results/min-tokens-vs-economic-gate.md new file mode 100644 index 00000000..5a957b04 --- /dev/null +++ b/docs/results/min-tokens-vs-economic-gate.md @@ -0,0 +1,109 @@ +# The floor and the gate: `min_tokens` vs `economic_gate` + +`min_tokens` and `economic_gate` pull against each other. **That interaction predates PR #80 by +weeks** — but it only became a *correctness* bug when PR #80 added a selection mode that makes one +model call per request. + +| | | +|---|---| +| **Pre-existing** (since 2026-08-10) | The two filters overlap, and `min_tokens` silently stops mattering below the gate's implied economic floor. Redundant and hard to observe — **not wrong**. | +| **New in PR #80** | `selection_mode: merged` breaks the assumption the gate is built on, so the gate suppresses candidates that cost almost nothing to include. | + +Provenance: `min_tokens` on `extract_llm` arrived in `7f0379a` (2026-07-25); `economic_gate` in +`2adb476` (2026-08-10, #28/#51); `selection_mode: merged` is PR #80 and not yet in `main`. + +## What each one does + +**The floor** — `min_tokens`, resolved through `Trigger.OutputFloor(window, min_tokens)` — is a size +test on each tool output: is it big enough to be worth considering? Smaller outputs are skipped and +counted as `below_output_floor`. + +**The gate** — `economic_gate` — is a cost/benefit test on each candidate that clears the floor. It +compares an expected saving (size × the compression ratio this workload has actually been achieving, +learned rather than assumed) against an expected cost (a model call's tokens), and charges for a cache +write when the removal falls inside an already-cached prefix. + +## Why they pull against each other + +Lowering the floor admits *more* candidates, but each one is *smaller*. Smaller candidates have +smaller expected savings, so the gate rejects more of them. **Widening the floor to increase coverage +feeds the gate exactly the candidates it is designed to refuse.** + +Measured on this workload after dropping `min_tokens` from 3000 to 800 — expecting more work, getting +less: + +``` +suppressed: cache-aware, saving below call cost 1497 +allow: recurring content, amortised 613 +allow: expected saving exceeds cost 29 + +candidates that reached the model 224 +``` + +## Why this was not a bug before PR #80 + +In the per-output design — the only one that existed until now — the pipeline makes **one model call +per candidate**. So a candidate *is* a call, and pricing a candidate against the cost of a call is +exactly right. When the gate refuses a small output it is making a true statement: that output cannot +repay its own call. + +On that design the floor and the gate are two filters **agreeing**. The gate is simply the stricter and +better-informed of the two, since it knows the workload's real compression ratio and the cache cost +while the floor knows only a byte count. + +Two things do deserve attention independently of PR #80: + +* `min_tokens` is effectively **advisory** below the gate's economic floor. An operator who lowers it + to widen coverage may see no behavioural change at all. +* The override is **invisible unless you know which counter to read**. Nothing reports "your floor was + honoured and then overruled on economic grounds" — you have to notice `economic_gate` in the gate + map and compare it against what reached the model. + +## What PR #80 changed + +`selection_mode: merged` makes **one model call per request**, adjudicating the whole batch together, +because the comparative judgement it relies on needs peers to compare against. The marginal cost of +adding the seventh candidate is therefore its share of one prompt — not a call. + +With prefix asks (also PR #80) the outputs are read from the provider's cached transcript rather than +shipped, so a candidate's marginal cost is a single inventory line, on the order of thirty tokens. + +The gate still prices each candidate against a whole call. In this configuration that estimate is wrong +by two to three orders of magnitude, and it rejects candidates that are nearly free to include — which +starves the batch and removes the very comparison the design depends on. + +## Evidence that the floor is a real lever + +Same three tasks, same everything, only `min_tokens` varying, with the gate disabled so the floor could +be seen in isolation: + +| `min_tokens` | candidates / call | batch hit the 12-cap | +|---|---|---| +| 800 | 1.29 | no | +| 300 | 2.22 | 4× | +| **120** | **5.97** | 11× | + +**Only the candidates-per-call column is trustworthy here.** The three runs diverged in trajectory — +total traffic spread over 7×, and `summarize` fired on 57%, 66% and 2% of requests respectively — so +drop counts and token savings are not comparable across rows. + +## Suggested changes + +1. **Make the gate batch-aware.** Compare the batch's total expected saving against *one* call's cost, + and treat a candidate's marginal cost as prompt tokens. The only change that fixes the cause rather + than working around it. +2. **Log when the gate overrules the operator's floor.** A configured `min_tokens` that has no effect + should say so, in either design. +3. **Report candidates per call.** Verdicts divided by calls is one line of arithmetic, and it would + have caught this at the first arm rather than the third. + +The third point is the one with history. Three separate experiments were run and interpreted as "the +model declines to act" when the model was being shown roughly one candidate at a time — first by a +co-reference pre-filter, then by a per-request call cap, then by this gate. All three are the same +family of defect: **cost machinery written for per-output calls, applied to a one-call design.** + +## Where the numbers come from + +Gate counters read from the proxy's `/stats` during a 128k-band LOCA arm; the floor sweep was run +offline on a 3-task configuration. Full detail in `docs/experiments/loca/iter019/results.md` and +`docs/experiments/loca/iter020/`. diff --git a/expand/availability_test.go b/expand/availability_test.go new file mode 100644 index 00000000..6d94676c --- /dev/null +++ b/expand/availability_test.go @@ -0,0 +1,36 @@ +package expand + +import ( + "strings" + "testing" +) + +// WHEN IS THE EXPAND TOOL ACTUALLY AVAILABLE TO THE MODEL? +// +// Live runs showed the model calling context_guru_expand and the CLIENT answering +// "Tool context_guru_expand not found" -- 17 of 38 attempts in one arm, 48 of 108 in another. This +// pins the advertise rule that decides availability, instead of paraphrasing the comments around it. +func TestWhenIsExpandAdvertised(t *testing.T) { + withMarker := `{"model":"claude-x","tools":[{"name":"Read"}],"messages":[ + {"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","content":"200 records <>"}]}]}` + noMarker := `{"model":"claude-x","tools":[{"name":"Read"}],"messages":[ + {"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","content":"200 records, nothing removed"}]}]}` + noTools := `{"model":"claude-x","messages":[ + {"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","content":"x <>"}]}]}` + + for _, tc := range []struct { + name, body string + persists bool + mode string + }{ + {"auto + tools + marker", withMarker, true, InjectAuto}, + {"auto + tools + NO marker", noMarker, true, InjectAuto}, + {"auto + marker but NO tools", noTools, true, InjectAuto}, + {"auto + marker + store does NOT persist", withMarker, false, InjectAuto}, + {"always + NO marker", noMarker, true, InjectAlways}, + } { + out, injected := Inject("anthropic", tc.mode, []byte(tc.body), tc.persists) + t.Logf("%-42s injected=%-5v tool_in_body=%v", tc.name, injected, + strings.Contains(string(out), ToolName)) + } +} diff --git a/expand/expand.go b/expand/expand.go index 983ec315..def18b89 100644 --- a/expand/expand.go +++ b/expand/expand.go @@ -108,6 +108,10 @@ func ParseMarkers(s string) []string { func Resolve(s store.Store, key string) (string, bool) { b, ok := s.Get(key) if !ok { + // Classify the miss. A well-formed id with nothing behind it means a marker was issued and + // its original is gone -- a cut advertised as reversible that is not. That is the number + // worth alerting on, and it was previously invisible; see unresolved.go. + noteUnresolved(WellFormedID(key)) return "", false } return string(b), true diff --git a/expand/restore.go b/expand/restore.go new file mode 100644 index 00000000..afb8da85 --- /dev/null +++ b/expand/restore.go @@ -0,0 +1,138 @@ +package expand + +import ( + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + + "github.com/rossoctl/context-guru/store" +) + +// RESTORING EXPAND RESULTS ON THE REQUEST PATH. +// +// Why this exists rather than a response-side interception. The response loop can only auto-continue +// when the expand call arrives ALONE: if the model also called a real tool, the client -- not the proxy +// -- must execute it, so the response has to be relayed and the loop bails (`otherTools` in +// ResponseCalls). Measured on live traffic, that is the DOMINANT case: of 107 turns whose expand call +// was refused, 102 carried two or more tool_use blocks and only 5 carried one. So a response-side fix +// addresses 5% of the problem. +// +// The client then executes the real tools, finds no `context_guru_expand` among its own tools -- the +// proxy injected it, so the client has never heard of it -- and answers with something like +// `Tool 'context_guru_expand' not found`. The model loses its recovery path and re-runs the original +// tool instead, paying a full tool execution plus fresh output and enlarging the transcript that +// provoked the cut. +// +// This works WITH the client's loop instead of against it. On the next request, the client's own +// tool_result for the expand call is replaced with the stashed original before the request goes +// upstream. No response splitting, no need for the client to implement anything, and the mixed-call +// case is covered because the real tools were executed normally by whoever owns them. +// +// DETERMINISM IS REQUIRED, NOT OPTIONAL. The client resends its whole history every turn, so this +// substitution must be applied identically on every turn or the prefix flaps and the model sees +// content appear and disappear. Both consequences are real: a flapping prefix is a cache miss from the +// point of the change, and a model that saw a failure and reasoned about it should not later find the +// record contradicting itself. Being keyed only by marker id and store contents, the rewrite is +// deterministic for as long as the stash lives -- which is what MarkKeptVerbatim protects, by ensuring +// expanded content is not re-compacted and so needs no marker on later turns. +// +// See docs/proposals/coref-compaction.md, "reversibility": the durability of the stash is what makes +// this coherent, and `expand_unresolved_missing` in /stats counts the cases where it was not. + +// RestoreResults replaces the client's tool_result for every expand call with the stashed original. +// Returns the rewritten body and the ids restored, so the caller can protect them from re-compaction. +// Unresolvable ids are left exactly as the client wrote them: the model is already reading a failure, +// and inventing a different failure string would only add a second story. +func RestoreResults(provider string, body []byte, s store.Store) ([]byte, []string) { + if s == nil { + return body, nil + } + msgs := gjson.GetBytes(body, "messages") + if !msgs.IsArray() { + return body, nil + } + // tool_use id -> marker hash, for expand calls only. + want := map[string]string{} + msgs.ForEach(func(_, m gjson.Result) bool { + m.Get("content").ForEach(func(_, blk gjson.Result) bool { + if blk.Get("type").String() == "tool_use" && blk.Get("name").String() == ToolName { + if id := blk.Get("id").String(); id != "" { + want[id] = blk.Get("input.id").String() + } + } + return true + }) + // OpenAI dialect: tool_calls on the assistant message. + m.Get("tool_calls").ForEach(func(_, tc gjson.Result) bool { + if tc.Get("function.name").String() == ToolName { + if id := tc.Get("id").String(); id != "" { + want[id] = gjson.Get(tc.Get("function.arguments").String(), "id").String() + } + } + return true + }) + return true + }) + if len(want) == 0 { + return body, nil + } + + out := body + var restored []string + arr := gjson.GetBytes(out, "messages").Array() + for i := range arr { + base := "messages." + itoa(i) + // Anthropic: tool_result blocks inside a user message. + blocks := gjson.GetBytes(out, base+".content") + if blocks.IsArray() { + for b, blk := range blocks.Array() { + if blk.Get("type").String() != "tool_result" { + continue + } + hash, ok := want[blk.Get("tool_use_id").String()] + if !ok { + continue + } + orig, found := Resolve(s, hash) + if !found { + continue // leave the client's own failure text in place + } + if nb, err := sjson.SetBytes(out, base+".content."+itoa(b)+".content", orig); err == nil { + out = nb + restored = append(restored, hash) + } + } + continue + } + // OpenAI: a role=tool message answering one call. + if gjson.GetBytes(out, base+".role").String() == "tool" { + hash, ok := want[gjson.GetBytes(out, base+".tool_call_id").String()] + if !ok { + continue + } + if orig, found := Resolve(s, hash); found { + if nb, err := sjson.SetBytes(out, base+".content", orig); err == nil { + out = nb + restored = append(restored, hash) + } + } + } + } + if len(restored) > 0 { + restoredCount.Add(int64(len(restored))) + } + return out, restored +} + +func itoa(i int) string { + if i == 0 { + return "0" + } + var b [20]byte + p := len(b) + for i > 0 { + p-- + b[p] = byte('0' + i%10) + i /= 10 + } + return string(b[p:]) +} diff --git a/expand/unresolved.go b/expand/unresolved.go new file mode 100644 index 00000000..3c9481d3 --- /dev/null +++ b/expand/unresolved.go @@ -0,0 +1,60 @@ +package expand + +import ( + "regexp" + "sync/atomic" +) + +// UNRESOLVED-EXPAND ACCOUNTING. +// +// An expand call the proxy cannot satisfy is the failure mode that makes lossy compaction unsafe, and +// until now it was INVISIBLE: nothing in /stats recorded it, so three experiment iterations ran while +// the model was calling the tool and being refused. The refusal was only found by grepping the +// benchmark client's own transcripts for the string "not found". +// +// Two causes, kept apart because they call for opposite responses: +// +// MALFORMED — the id does not look like a marker this proxy ever issues. The model invented or +// garbled it. Nothing to fix here; it is the model's error, and a placeholder is the right answer. +// +// MISSING — the id is well formed but nothing is stashed under it. THIS is a context-guru defect: +// a marker was issued and its original is gone, so a cut advertised as reversible is not. Every +// count here is a case where reversibility silently failed. +// +// Package-level like modes.compactionResets and offload's frozen counters, for the same reason: the +// host merges them into the snapshot at serve time, since metrics cannot import this package. +var ( + unresolvedMalformed atomic.Int64 + unresolvedMissing atomic.Int64 + // restoredCount counts tool_results rewritten back to their stashed original on the REQUEST path. + // That is the path that covers a mixed turn -- an expand call alongside a real tool -- which the + // response loop structurally cannot handle, and which live traffic showed to be 102 of 107 cases. + restoredCount atomic.Int64 +) + +// noteUnresolved records one expand id the proxy could not satisfy. wellFormed distinguishes a +// garbled id from a lost stash. +func noteUnresolved(wellFormed bool) { + if wellFormed { + unresolvedMissing.Add(1) + return + } + unresolvedMalformed.Add(1) +} + +// Restored returns how many expand tool_results were rewritten back to their originals. +func Restored() int64 { return restoredCount.Load() } + +// Unresolved returns (malformed, missing) since process start. missing > 0 means reversibility +// failed for that many cuts. +func Unresolved() (malformed, missing int64) { + return unresolvedMalformed.Load(), unresolvedMissing.Load() +} + +// WellFormedID reports whether id has the shape of a marker this proxy issues. Used to classify an +// unresolvable expand call rather than to validate input. +func WellFormedID(id string) bool { return idShapeRe.MatchString(id) } + +// idShapeRe is the marker payload shape from markerRe in expand.go, anchored so a whole id +// can be tested. Kept beside the classifier so the two cannot drift apart silently. +var idShapeRe = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) diff --git a/internal/adjudicate/tool.go b/internal/adjudicate/tool.go new file mode 100644 index 00000000..19e79c67 --- /dev/null +++ b/internal/adjudicate/tool.go @@ -0,0 +1,216 @@ +// Package adjudicate declares the context-maintenance tool the compaction pipeline uses to ask the +// request's own model which tool outputs are spent, and the wire helpers that keep it byte-stable in +// the prompt-cache prefix. +// +// WHY A TOOL AT ALL, rather than asking for JSON in the reply text. Measured against the live route: +// +// tool_choice reply shape cache verdict coverage +// {"type":"none"} prose / thinking only read (free) 0 of 6 -- no answer at all +// {"type":"tool",...} tool_use MISS + rewrite 6 of 6 +// (omitted) tool_use read (free) 6 of 6, 4 trials out of 4 +// +// Three things follow, and each was the opposite of an earlier assumption in this repo: +// +// - Setting tool_choice:none to stop the model answering with a tool_use is what forced it into +// PROSE. That reply was then scored as an unparseable failure, which is a large part of what read +// as "the model declines to act" across three iterations. +// - FORCING the tool is not free: it produced a separate cache entry (8378 tokens written against +// the 8268 already cached), so tool_choice does participate in the key when it names a tool, even +// though `none` does not. +// - Merely DECLARING the tool, with a description that says who it is for, gets a schema-shaped +// answer covering the whole batch, at cache-read price. +// +// The tool is therefore injected on EVERY request rather than only when the pipeline is about to ask. +// `tools` hashes before system and messages, so a tool that appears and disappears invalidates the +// prefix from position zero — the same flap that expand's `always` mode exists to prevent. +package adjudicate + +import ( + "sync/atomic" + + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +// strayAnswered counts tool_results rewritten because the AGENT called this tool. Non-zero is +// expected rather than alarming -- models do call advertised tools they were told to leave alone -- +// but the rate is the signal for whether the description is doing its job. +var strayAnswered atomic.Int64 + +// StrayAnswered returns how many stray calls have been answered. Surfaced in /stats. +func StrayAnswered() int64 { return strayAnswered.Load() } + +// ToolName is the wire name. Prefixed like the expand tool so an operator reading a transcript can +// tell at a glance which tools the proxy injected and which the client owns. +const ToolName = "context_guru_adjudicate" + +// strayAnswer is what a stray call from the AGENT gets. The model will call an advertised tool it was +// told not to — directly observed with the expand tool, which it called at step 2 of a run — and the +// client cannot execute a tool the proxy injected, so it answers "not found" and the agent loses a +// turn to a dead end. This gives it a definite, uninteresting answer instead. +const strayAnswer = "Context maintenance runs automatically in the background. No action is required " + + "from you, and you do not need to call this tool. Continue with the task." + +// anthropicDef and openAIDef are the tool definitions, kept as raw JSON so injection is a byte splice +// and the cached prefix stays stable to the byte. +const anthropicDef = `{"name":"` + ToolName + `","description":"` + toolDesc + `","input_schema":` + schemaJSON + `}` + +const openAIDef = `{"type":"function","function":{"name":"` + ToolName + `","description":"` + toolDesc + + `","parameters":` + schemaJSON + `}}` + +// toolDesc tells the model who the tool is for. "Do not call this yourself" does not reliably stop it +// (see strayAnswer), but it costs nothing and reduces the rate. +const toolDesc = "Internal to the context manager. Reports which earlier tool outputs are spent and " + + "safe to remove from the transcript. This is invoked by the context manager, not by you - do not " + + "call it yourself." + +// schemaJSON constrains the answer. The label is a small INTEGER, never the tool_use id: asked for +// opaque ids the model regularised them (answering toolu_01..07 for toolu_probe_00..07), because +// reproducing a random identifier from thousands of tokens back is a copying task rather than a +// judgement. With integer labels it was 0 bad labels across 40+ trials. +const schemaJSON = `{"type":"object","properties":{"verdicts":{"type":"array","description":` + + `"One entry per label you were shown. Answer for EVERY label.","items":{"type":"object","properties":{` + + `"i":{"type":"integer","description":"The label you were shown."},` + + `"needed_by":{"type":"string","enum":["a","b","c","none"],"description":` + + `"Which outstanding obligation still needs this output, or none if it is spent."},` + + `"quote":{"type":"string","description":"Verbatim transcript text creating that obligation; empty when needed_by is none."},` + + `"verdict":{"type":"string","enum":["keep","drop"],"description":"drop requires needed_by to be none."}},` + + `"required":["i","needed_by","verdict"]}}},"required":["verdicts"]}` + +// ToolDefRaw returns the provider-shaped tool definition. +func ToolDefRaw(provider string) []byte { + if provider == "anthropic" { + return []byte(anthropicDef) + } + return []byte(openAIDef) +} + +// HasTool reports whether body already declares the tool. This is the ADVERTISE test, and a host must +// answer stray calls exactly when it is true. +func HasTool(provider string, body []byte) bool { + field := "function.name" + if provider == "anthropic" { + field = "name" + } + found := false + gjson.GetBytes(body, "tools").ForEach(func(_, t gjson.Result) bool { + if t.Get(field).String() == ToolName { + found = true + return false + } + return true + }) + return found +} + +// Inject appends the tool to body's tools array, byte-stably and idempotently. +// +// Appended LAST so the client's own tools keep their order, and skipped when a forcing tool_choice is +// present so tool selection is never perturbed. Also skipped when the request declares no tools at +// all: adding the first tool to a tool-free request changes what the model believes it can do. +// Fail-open — any trouble returns the original body. +func Inject(provider string, body []byte) (out []byte, injected bool) { + if tc := gjson.GetBytes(body, "tool_choice"); tc.Exists() && !toolChoiceIsAuto(tc) { + return body, false + } + tools := gjson.GetBytes(body, "tools") + if !tools.Exists() || !tools.IsArray() || len(tools.Array()) == 0 { + return body, false + } + if HasTool(provider, body) { + return body, false + } + nb, err := sjson.SetRawBytes(body, "tools.-1", ToolDefRaw(provider)) + if err != nil { + return body, false + } + return nb, true +} + +func toolChoiceIsAuto(tc gjson.Result) bool { + if tc.Type == gjson.String { + s := tc.String() + return s == "auto" || s == "" + } + t := tc.Get("type").String() + return t == "" || t == "auto" +} + +// AnswerStrayCalls replaces the client's tool_result for any call the AGENT made to this tool with a +// definite answer, and reports how many it replaced. +// +// Same request-path shape as expand's RestoreResults, and for the same reason: the client executes the +// real tools itself, finds no tool by this name because the proxy injected it, and answers something +// like "Tool 'context_guru_adjudicate' not found". Left alone, the model reads a failure it cannot act +// on and may retry. Rewriting it on the next request works WITH the client's loop instead of against +// it, needs nothing implemented client-side, and is deterministic — the same substitution every turn, +// so the prefix does not flap. +func AnswerStrayCalls(provider string, body []byte) (out []byte, answered int) { + msgs := gjson.GetBytes(body, "messages") + if !msgs.IsArray() { + return body, 0 + } + ours := map[string]bool{} // tool_use id -> made by this tool + msgs.ForEach(func(_, m gjson.Result) bool { + m.Get("content").ForEach(func(_, blk gjson.Result) bool { + if blk.Get("type").String() == "tool_use" && blk.Get("name").String() == ToolName { + if id := blk.Get("id").String(); id != "" { + ours[id] = true + } + } + return true + }) + m.Get("tool_calls").ForEach(func(_, tc gjson.Result) bool { + if tc.Get("function.name").String() == ToolName { + if id := tc.Get("id").String(); id != "" { + ours[id] = true + } + } + return true + }) + return true + }) + if len(ours) == 0 { + return body, 0 + } + out = body + for i := range gjson.GetBytes(out, "messages").Array() { + base := "messages." + itoa(i) + blocks := gjson.GetBytes(out, base+".content") + if blocks.IsArray() { + for b, blk := range blocks.Array() { + if blk.Get("type").String() != "tool_result" || !ours[blk.Get("tool_use_id").String()] { + continue + } + if nb, err := sjson.SetBytes(out, base+".content."+itoa(b)+".content", strayAnswer); err == nil { + out, answered = nb, answered+1 + } + } + continue + } + if gjson.GetBytes(out, base+".role").String() == "tool" && + ours[gjson.GetBytes(out, base+".tool_call_id").String()] { + if nb, err := sjson.SetBytes(out, base+".content", strayAnswer); err == nil { + out, answered = nb, answered+1 + } + } + } + if answered > 0 { + strayAnswered.Add(int64(answered)) + } + return out, answered +} + +func itoa(i int) string { + if i == 0 { + return "0" + } + var b [20]byte + p := len(b) + for i > 0 { + p-- + b[p] = byte('0' + i%10) + i /= 10 + } + return string(b[p:]) +} diff --git a/internal/adjudicate/tool_test.go b/internal/adjudicate/tool_test.go new file mode 100644 index 00000000..b6edb07b --- /dev/null +++ b/internal/adjudicate/tool_test.go @@ -0,0 +1,113 @@ +package adjudicate + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/tidwall/gjson" +) + +// The tool must land LAST and be idempotent, because `tools` hashes before system and messages: a tool +// inserted anywhere else, or twice, invalidates the prompt-cache prefix from position zero. +func TestInjectIsByteStableAndIdempotent(t *testing.T) { + body := []byte(`{"model":"m","tools":[{"name":"Read","description":"d","input_schema":{"type":"object"}}],` + + `"messages":[{"role":"user","content":"go"}]}`) + out, ok := Inject("anthropic", body) + if !ok { + t.Fatal("did not inject into a request that declares tools") + } + tools := gjson.GetBytes(out, "tools").Array() + if len(tools) != 2 { + t.Fatalf("expected 2 tools, got %d", len(tools)) + } + if tools[0].Get("name").String() != "Read" { + t.Error("the client's own tool moved; its order must be preserved exactly") + } + if tools[1].Get("name").String() != ToolName { + t.Errorf("our tool is not last: %s", tools[1].Get("name").String()) + } + // The schema must actually parse, or the provider rejects every request carrying it. + var schema map[string]any + if err := json.Unmarshal([]byte(tools[1].Get("input_schema").Raw), &schema); err != nil { + t.Fatalf("input_schema is not valid JSON: %v", err) + } + again, ok2 := Inject("anthropic", out) + if ok2 { + t.Error("injected twice; a duplicated tool changes the prefix on every turn") + } + if string(again) != string(out) { + t.Error("a second injection altered the body") + } +} + +// Two cases where injecting would change what the model believes it can do, or which tool it is +// compelled to call. Both must be refused. +func TestInjectRefusesWhenItWouldPerturbSelection(t *testing.T) { + noTools := []byte(`{"model":"m","messages":[{"role":"user","content":"go"}]}`) + if _, ok := Inject("anthropic", noTools); ok { + t.Error("injected into a request with NO tools; that hands the model its first tool and " + + "changes what it believes it can do") + } + forced := []byte(`{"model":"m","tool_choice":{"type":"tool","name":"Read"},` + + `"tools":[{"name":"Read","description":"d","input_schema":{"type":"object"}}],` + + `"messages":[{"role":"user","content":"go"}]}`) + if _, ok := Inject("anthropic", forced); ok { + t.Error("injected under a forcing tool_choice; tool selection must never be perturbed") + } +} + +// A stray call from the AGENT must get a definite answer, and the real tools' results must survive +// untouched. The model does call advertised tools it was told to leave alone -- directly observed with +// the expand tool, which it called at step 2 of a run -- so this path is load-bearing, not defensive. +func TestAnswerStrayCallsLeavesRealResultsAlone(t *testing.T) { + body := []byte(`{"model":"m","messages":[ + {"role":"assistant","content":[ + {"type":"tool_use","id":"u1","name":"` + ToolName + `","input":{"verdicts":[]}}, + {"type":"tool_use","id":"u2","name":"Read","input":{"path":"a.py"}}]}, + {"role":"user","content":[ + {"type":"tool_result","tool_use_id":"u1","content":"Tool '` + ToolName + `' not found"}, + {"type":"tool_result","tool_use_id":"u2","content":"real file contents"}]} + ]}`) + out, n := AnswerStrayCalls("anthropic", body) + if n != 1 { + t.Fatalf("answered %d stray calls, want 1", n) + } + s := string(out) + if strings.Contains(s, "not found") { + t.Error("the client's dead-end refusal reached the model; the point is to replace it") + } + if !strings.Contains(s, "runs automatically") { + t.Error("no substitute answer was written") + } + if !strings.Contains(s, "real file contents") { + t.Error("a REAL tool's result was overwritten; only our own calls may be touched") + } + if StrayAnswered() < 1 { + t.Error("the stray call was not counted; the rate is the signal for whether the tool's " + + "description is working") + } + // OpenAI dialect: a role=tool message answering one call. + oa := []byte(`{"model":"m","messages":[ + {"role":"assistant","tool_calls":[{"id":"c1","function":{"name":"` + ToolName + `","arguments":"{}"}}]}, + {"role":"tool","tool_call_id":"c1","content":"Tool not found"}]}`) + out2, n2 := AnswerStrayCalls("openai", oa) + if n2 != 1 || !strings.Contains(string(out2), "runs automatically") { + t.Errorf("OpenAI dialect not handled: answered=%d body=%.180s", n2, out2) + } +} + +// A body with no calls to our tool must come back byte-identical: this runs on every request, and a +// gratuitous rewrite would change the prefix and cost a cache write for nothing. +func TestAnswerStrayCallsIsANoOpWhenUninvolved(t *testing.T) { + body := []byte(`{"model":"m","messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"u9","name":"Read","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u9","content":"data"}]}]}`) + out, n := AnswerStrayCalls("anthropic", body) + if n != 0 { + t.Errorf("answered %d calls on a body that never called our tool", n) + } + if string(out) != string(body) { + t.Error("body was rewritten with nothing to do; that costs a cache write for nothing") + } +} diff --git a/internal/cheapmodel/anthropic.go b/internal/cheapmodel/anthropic.go index d1e119a1..b2db7f7f 100644 --- a/internal/cheapmodel/anthropic.go +++ b/internal/cheapmodel/anthropic.go @@ -10,7 +10,11 @@ import ( "encoding/json" "fmt" "net/http" + "strconv" "strings" + + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" ) // Anthropic calls a small Anthropic model with a single user prompt and returns the @@ -118,3 +122,146 @@ func (a Anthropic) CompleteSystem(ctx context.Context, system, prompt string) (s } return "", nil } + +// PrefixUsage is what one CompletePrefixed call cost, straight from the provider's usage block. +// Returned rather than only recorded because the CALLER has to gate on it: a prefix ask whose whole +// justification is the cache read must be able to see that the read happened. +type PrefixUsage struct { + CacheRead int + CacheWrite int + Fresh int + Output int +} + +// CompletePrefixed sends `ask` as a trailing user message appended to prefixBody — an ENTIRE +// previously-sent Anthropic request — so the provider reads the prompt cache that body populated +// instead of being charged fresh for the transcript again. +// +// Measured against the live gateway (docs/experiments/loca/iter019/results.md §2): +// +// - appending a trailing user message to a byte-identical prefix reads the whole prefix from +// cache and writes nothing: 19,595 read / 0 created. +// - `tool_choice` is NOT part of the cache key, so forcing it to "none" is free — and necessary, +// because the prefix carries the agent's tools and the model will otherwise answer with a +// tool_use instead of the verdicts. +// - `tools` ARE part of the key. They are therefore left exactly as the prefix had them; dropping +// them read a different, smaller entry (19,129) i.e. a separate cache line and a fresh write. +// - this route REJECTS assistant prefill ("the conversation must end with a user message"), which +// the appended user message satisfies by construction — but it means prefixBody must not be +// extended any other way. +// +// Everything else about the body is preserved untouched, because every byte before the appended +// message is prefix and any edit to it costs the cache read this method exists for. `stream` is the +// one exception: the caller wants a single JSON answer, and a streamed response is not that. +func (a Anthropic) CompletePrefixed(ctx context.Context, prefixBody []byte, ask string) (string, PrefixUsage, error) { + var u PrefixUsage + if !gjson.GetBytes(prefixBody, "messages").IsArray() { + return "", u, fmt.Errorf("cheapmodel: prefix body has no messages array") + } + n := len(gjson.GetBytes(prefixBody, "messages").Array()) + body, err := sjson.SetBytes(prefixBody, "messages."+strconv.Itoa(n), + map[string]any{"role": "user", "content": ask}) + if err != nil { + return "", u, err + } + // NO tool_choice. This is deliberate and was measured, twice over: + // + // - Setting {"type":"none"} -- which an earlier version of this method did, to stop the model + // answering with a tool_use -- drove it into PROSE and produced no answer at all: 0 of 6 labels + // across trials, while the caller scored the prose as an unparseable failure. + // - Setting {"type":"tool","name":...} to force the answer is NOT free: it wrote a second cache + // entry (8,378 tokens against the 8,268 already cached), so naming a tool participates in the + // cache key even though "none" does not. + // - Omitting it entirely reads the same cache entry as "none" and the model calls the advertised + // tool of its own accord: 6 of 6 labels on 4 of 4 trials. + // + // So the answer arrives schema-shaped, in full, at cache-read price -- provided the tool is present + // in the prefix, which is why it is injected on every request rather than only here. + // 16k, not the 2048 default, and this is a correctness matter rather than generosity. + // + // MEASURED: with 2048 this path produced UNPARSEABLE replies on ~70% of calls (24 of 34). Two + // things stack up. The request model runs adaptive thinking, which consumes the output budget + // before any text is emitted -- at max_tokens 900 a probe returned thinking blocks and no text at + // all. And a verdict array over a 12-item batch, each entry carrying an obligation label and a + // VERBATIM quote, is simply long. The array was being cut mid-flight, leaving no closing bracket, + // so the parse failed and the caller changed nothing -- indistinguishable in the counters from a + // model that declined to act, which is exactly how it was misread for three iterations. + // + // Output tokens bill as GENERATED, not as budgeted, so a ceiling this high costs nothing until it + // is used. An operator who wants a tighter bound sets MaxTokens explicitly. + maxTok := a.MaxTokens + if maxTok == 0 { + maxTok = 16000 + } + if body, err = sjson.SetBytes(body, "max_tokens", maxTok); err != nil { + return "", u, err + } + body, _ = sjson.DeleteBytes(body, "stream") + + base := a.BaseURL + if base == "" { + base = "https://api.anthropic.com" + } + client := a.Client + if client == nil { + client = http.DefaultClient + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + strings.TrimRight(base, "/")+"/v1/messages", bytes.NewReader(body)) + if err != nil { + return "", u, err + } + req.Header.Set("content-type", "application/json") + req.Header.Set("anthropic-version", "2023-06-01") + if a.APIKey != "" { + req.Header.Set("x-api-key", a.APIKey) + req.Header.Set("authorization", "Bearer "+a.APIKey) + } + resp, err := client.Do(req) + if err != nil { + return "", u, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", u, fmt.Errorf("cheapmodel: prefixed status %d", resp.StatusCode) + } + var out struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + Name string `json:"name"` + Input json.RawMessage `json:"input"` + } `json:"content"` + Usage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheCreationTok int `json:"cache_creation_input_tokens"` + CacheReadTok int `json:"cache_read_input_tokens"` + } `json:"usage"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", u, err + } + u = PrefixUsage{CacheRead: out.Usage.CacheReadTok, CacheWrite: out.Usage.CacheCreationTok, + Fresh: out.Usage.InputTokens, Output: out.Usage.OutputTokens} + recordUsageCache(out.Usage.InputTokens, out.Usage.OutputTokens, + out.Usage.CacheCreationTok, out.Usage.CacheReadTok) + // A TOOL_USE INPUT BEATS TEXT. When the prefix advertises a structured-answer tool the model uses + // it, and its input arrives already schema-shaped -- which removes three failure modes the text path + // had: prose instead of JSON, verdicts for only part of the batch, and a JSON array cut off by the + // output budget mid-flight. The raw input is returned as-is so the caller's existing parser reads it + // unchanged. + for _, c := range out.Content { + if c.Type == "tool_use" && len(c.Input) > 0 { + return string(c.Input), u, nil + } + } + // No tool call: fall through to text, which covers a model that answered in prose anyway and any + // provider that does not return tool_use here. The caller counts what it cannot parse. + for _, c := range out.Content { + if c.Text != "" { + return c.Text, u, nil + } + } + return "", u, nil +} diff --git a/internal/cheapmodel/prefixed_test.go b/internal/cheapmodel/prefixed_test.go new file mode 100644 index 00000000..7d3bae59 --- /dev/null +++ b/internal/cheapmodel/prefixed_test.go @@ -0,0 +1,100 @@ +package cheapmodel + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/tidwall/gjson" +) + +// The wire construction is the whole mechanism: every byte before the appended message is prefix, and +// any edit to it costs the cache read this exists for. Measured facts being enforced here (see +// docs/experiments/loca/iter019/results.md §2): tools ARE part of the cache key, tool_choice is NOT, +// and this route rejects assistant prefill so the ask must be a trailing USER message. +func TestCompletePrefixedPreservesThePrefix(t *testing.T) { + var got []byte + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got, _ = io.ReadAll(r.Body) + w.Header().Set("content-type", "application/json") + io.WriteString(w, `{"content":[{"type":"thinking","text":""},{"type":"text","text":"[{\"i\":2}]"}], + "usage":{"input_tokens":12,"output_tokens":7,"cache_read_input_tokens":19595,"cache_creation_input_tokens":0}}`) + })) + defer srv.Close() + + prefix := []byte(`{"model":"claude-x","stream":true,"system":"be terse", + "tools":[{"name":"Read","description":"d","input_schema":{"type":"object"}}], + "messages":[{"role":"user","content":"go"}, + {"role":"assistant","content":[{"type":"text","text":"working"}]}]}`) + + a := Anthropic{BaseURL: srv.URL, Model: "claude-x", APIKey: "k", Client: srv.Client(), MaxTokens: 4000} + reply, u, err := a.CompletePrefixed(context.Background(), prefix, "ADJUDICATE THIS") + if err != nil { + t.Fatalf("CompletePrefixed: %v", err) + } + if reply != `[{"i":2}]` { + t.Errorf("reply = %q; a leading thinking block must be skipped, not returned", reply) + } + if u.CacheRead != 19595 || u.CacheWrite != 0 { + t.Errorf("usage not surfaced: %+v -- the caller must be able to see whether the cache read "+ + "actually happened, since a miss costs ~10x and looks identical otherwise", u) + } + + msgs := gjson.GetBytes(got, "messages").Array() + if n := len(msgs); n != 3 { + t.Fatalf("expected the ask appended as a 3rd message, got %d", n) + } + if r := msgs[2].Get("role").String(); r != "user" { + t.Errorf("appended message role = %q, want user: this route rejects assistant prefill", r) + } + if c := msgs[2].Get("content").String(); c != "ADJUDICATE THIS" { + t.Errorf("appended content = %q", c) + } + // The prefix must be untouched, byte-for-byte, or there is no cache hit. + if msgs[0].Raw != gjson.GetBytes(prefix, "messages.0").Raw || + msgs[1].Raw != gjson.GetBytes(prefix, "messages.1").Raw { + t.Error("an existing message was rewritten; every byte before the appended ask is prefix") + } + if gjson.GetBytes(got, "system").String() != "be terse" { + t.Error("system was altered; it hashes before messages, so this misses from position zero") + } + if gjson.GetBytes(got, "tools").Raw != gjson.GetBytes(prefix, "tools").Raw { + t.Error("tools were altered; tools ARE part of the cache key -- dropping them read a " + + "different, smaller cache entry when measured") + } + // tool_choice must be ABSENT. This assertion used to demand {"type":"none"}, on the reasoning that + // the prefix carries the agent's tools so a tool_use reply had to be suppressed. Measurement + // reversed it: with "none" the model answered in PROSE and produced no verdicts at all (0 of 6 + // labels), while with tool_choice omitted it called the advertised structured-answer tool and + // covered the whole batch (6 of 6, on 4 of 4 trials) at the same cache-read price. Forcing a + // named tool is a third option and NOT free -- it wrote a separate cache entry. + if tc := gjson.GetBytes(got, "tool_choice"); tc.Exists() { + t.Errorf("tool_choice was set to %q; it must be omitted, or the model answers in prose "+ + "instead of calling the structured-answer tool", tc.Raw) + } + if gjson.GetBytes(got, "stream").Exists() { + t.Error("stream survived; the caller wants one JSON answer, not a stream") + } + if mt := gjson.GetBytes(got, "max_tokens").Int(); mt != 4000 { + t.Errorf("max_tokens = %d, want 4000", mt) + } +} + +// A prefix that is not a conversation must be refused rather than silently sent, because the caller's +// fallback (a plain completion) is correct and a malformed send is not. +func TestCompletePrefixedRejectsNonConversation(t *testing.T) { + a := Anthropic{BaseURL: "http://127.0.0.1:1", Model: "m", APIKey: "k"} + if _, _, err := a.CompletePrefixed(context.Background(), []byte(`{"model":"m"}`), "ask"); err == nil { + t.Error("a body with no messages array was accepted") + } else if !strings.Contains(err.Error(), "messages") { + t.Errorf("unhelpful error: %v", err) + } + var js map[string]any + if json.Unmarshal([]byte(`{"model":"m"}`), &js) != nil { + t.Fatal("fixture is not JSON") + } +} diff --git a/internal/coref/coref.go b/internal/coref/coref.go new file mode 100644 index 00000000..aebfbdf4 --- /dev/null +++ b/internal/coref/coref.go @@ -0,0 +1,392 @@ +// Package coref builds the Tier-1 co-reference index that co-reference-aware +// compaction decides from: for each tool output in a request, which identifiers that +// output INTRODUCED, and whether any later turn carried them forward. +// +// It is deliberately free of the bifrost schema, of the components package and of the +// tokenizer. The index is a pure function of a flattened message list, which is what +// lets the Go component and the offline measurement pass +// (deploy/harbor/coref.py) share ONE definition of "reference" and be checked against +// the same known-answer fixture. A component whose notion of a reference had drifted +// from the script's would be calibrated against thresholds measured for a different +// algorithm — the thresholds are the whole output of the measurement, so that drift +// would be silent and total. +// +// See docs/proposals/coref-compaction.md for what the index is FOR: §2 (the three +// tiers and the echo confound), §3 (open vs closed, and why recency is measured from +// the head of the transcript rather than from the output). +package coref + +import ( + "regexp" + "strings" +) + +// identRe matches an identifier-ish token: the things a model actually carries forward +// out of a tool output — paths, symbols, ids, hashes, error codes, line numbers. Prose +// is filtered out by distinctive below rather than by a stopword list, which does not +// survive a change of domain (or of natural language). +var identRe = regexp.MustCompile(`[A-Za-z_][A-Za-z0-9_./:\-]{2,63}|\b\d{3,}\b|\b[0-9a-f]{7,40}\b`) + +// numericRe matches a token that is purely a number (with thousands/decimal separators). +var numericRe = regexp.MustCompile(`^\d[\d,._]*$`) + +// punctEdge is the punctuation trimmed from a token's ends before it is judged. Interior +// punctuation is structure; trailing punctuation is usually a sentence mark. +const punctEdge = "._:-/" + +// distinctive keeps tokens that look like an identifier rather than an English word: after +// trimming surrounding punctuation they carry INTERIOR structure (_ . / : -), a digit, or +// CamelCase. +// +// The point is precision, not recall. A token that also occurs in ordinary prose produces a +// spurious reference, and a spurious reference makes an output look load-bearing when it is +// not — which suppresses compaction silently. Missing a real reference is the safe +// direction: it can only make the index report LESS cuttable mass than exists. +// +// All three rules below were measured rather than guessed. An earlier version also accepted +// any token of 10+ characters, any token containing punctuation anywhere, and any number of +// 3+ digits; run over real agent traffic, the top "references" it produced were +// `description`, `transparency`, `efficiency`, `conditions`, `e.g.`, `try:`, `None:` and +// `2026`, which inflated referenced mass from 51% to 71% of all tool-output tokens: +// +// - No bare length rule. Long English words are still English words, and a real +// identifier almost always carries structure, a digit, or camelCase (`src/auth.py`, +// `session_id`, `GraphStore`). One carrying none of those cannot be told from prose. +// - Punctuation must be INTERIOR. `e.g.` / `try:` / `memory.` are prose plus a sentence +// mark; trimmed they become `e.g` / `try` / `memory` and fail on their own merits, while +// `src/auth.py` and `v1/messages` are untouched. +// - A bare number needs 5+ digits or a separator. `2026` is a year and `447` is a line +// number or a count; both recur everywhere. Hashes, ids and versions survive. +func distinctive(t string) bool { + t = strings.Trim(t, punctEdge) + if len(t) < 4 { + return false + } + if numericRe.MatchString(t) { + digits := 0 + for i := 0; i < len(t); i++ { + if t[i] >= '0' && t[i] <= '9' { + digits++ + } + } + return digits >= 5 || strings.ContainsAny(t, ",._") + } + if strings.ContainsAny(t, "_./:-") { + return true + } + for i := 0; i < len(t); i++ { + if t[i] >= '0' && t[i] <= '9' { + return true + } + } + return hasCamel(t) +} + +// hasCamel reports whether t contains a lower→upper ASCII transition (camelCase). +// identRe only ever yields ASCII, so a byte scan is equivalent to the `[a-z][A-Z]` +// pattern the measurement script uses. +func hasCamel(t string) bool { + for i := 1; i < len(t); i++ { + if t[i-1] >= 'a' && t[i-1] <= 'z' && t[i] >= 'A' && t[i] <= 'Z' { + return true + } + } + return false +} + +// Idents returns the distinctive identifier tokens in s, as a set. Tokens are trimmed of +// surrounding punctuation so `memory.` and `memory` are ONE token rather than two that can +// never match each other. +func Idents(s string) map[string]struct{} { + out := map[string]struct{}{} + for _, t := range identRe.FindAllString(s, -1) { + if distinctive(t) { + out[strings.Trim(t, punctEdge)] = struct{}{} + } + } + return out +} + +// Result is one tool output under evaluation, identified by the tool-call id that +// produced it (a single turn can carry several). +type Result struct { + ID string + Text string +} + +// Message is one flattened transcript entry. Texts is the reference-BEARING surface +// (prose, plus tool-call names and arguments); Results is the mass under evaluation. +// +// Both surfaces feed the echo-exclusion set, and only Texts can constitute a +// reference — see Index. Callers build these 1:1 with their own message indices so a +// Record's Idx points back at the message it came from. +type Message struct { + Texts []string + Results []Result +} + +// Class is the verdict for one tool output. +type Class string + +// The four verdicts. Cutting Unreferenced needs no reference model beyond "nobody +// ever used this"; cutting Closed is the large, early cut that needs the thresholds. +const ( + // Opaque — this output introduced NO identifier the index can track, so there is no + // evidence either way and the only honest verdict is no opinion. Never cut. + // + // This is not a corner case, and collapsing it into Unreferenced was a real defect: + // the two look identical in the arithmetic (refs == 0) and mean opposite things. + // "Introduced 200 identifiers, nobody touched one" is evidence of deadness; + // "introduced nothing I can see" is absence of evidence. A tool returning records of + // human-readable values — `[{"name":"david","id":123,"address":"foobarbaz"}]` — yields + // no distinctive tokens at all, because short lowercase words and 3-digit numbers are + // exactly what the precision rules in distinctive exclude. Classified as Unreferenced, + // that output is cut by the DEFAULT config while the agent still needs the address. + // + // So the blind spot is now a class rather than a silent vote for deletion, and the + // asymmetry is deliberate: an opaque output costs tokens, a wrongly cut one costs an + // expand round-trip plus a cache-write and can cost the task. + Opaque Class = "opaque" + // Unreferenced — the output introduced identifiers and no later turn reused any of + // them. The safest cut on Tier 1, and blind to Tier 2 (a value that was transformed + // before being restated leaves no exact match). Never read this as "unused". + Unreferenced Class = "unreferenced" + // Closed — referenced a small number of times, and not for a long time. Whatever + // the model took survives in the turn that took it, so the original is redundant + // with content still in the request. The case-A large-cut candidate. + Closed Class = "closed" + // Open — referenced recently, or repeatedly. Still load-bearing; keep. + Open Class = "open" +) + +// Record is the per-output measurement the classifier decides from. +type Record struct { + // Idx is the caller's message index; ID the tool-call id within it. + Idx int + ID string + // SizeTokens is the output's own size — the mass a cut would recover. + SizeTokens int + // Novel counts the identifiers this output INTRODUCED (see Index). + Novel int + // Refs counts the later turns that reused at least one novel identifier. + Refs int + // RefAge is how many messages AGO the last reference was, counted from the head of + // the transcript; -1 when there was none. This is the A/B axis. The tempting + // quantities — the output's own depth, or the gap from the output to its reference — + // are both something else: "recent messages vs early messages" is a statement about + // now, so it has to be measured from now. + RefAge int + // ConsumeLag is how many messages after the output its LAST reference was; -1 when + // there was none. Reported separately from RefAge because it answers a different + // question — how long the output stayed live — and conflating the two is what makes + // a hot old span look like a cold one. + ConsumeLag int + // UsedFrac is the share of the novel identifiers the model actually carried + // forward. A low value on a referenced output is the "took one value, does not need + // the rest" pattern, measured rather than assumed. + // + // It is NOT sufficient on its own to justify a cut, and the reason is worth stating + // where it will be read. A low UsedFrac is ambiguous: it can mean the model took the + // value it needed and the remainder is chaff, or it can mean the model took an ANCHOR + // (a name, an id) precisely in order to point at a payload it never copied. Given + // `[{"name":"david","id":123,"address":"foobarbaz"}, ...]` and a model that says "I + // need to remember david 123 address", the reference is real, the payload is not in + // the model's turn, and cutting the output loses the address. + UsedFrac float64 + // LaterTurns is how many model turns follow this output — its OPPORTUNITY to be + // referenced. Near the tail this approaches zero, and an output that has not had a + // chance to be used must not be scored as unused. See Classify's minLater. + LaterTurns int +} + +// Classify applies the open/closed predicate. closedDist is the recency floor (a last +// reference NEWER than this many messages ago keeps the output open); openReps is the +// repetition ceiling (referenced at least this many times keeps it open regardless of +// age, because a span referenced repeatedly is a hot span that happens to be old). +// +// minLater is the opportunity floor: an output with fewer than this many model turns +// after it is reported Open regardless of everything else, because it has not yet HAD a +// chance to be referenced. Without it the newest outputs classify as Unreferenced purely +// for being new, and a batched pass would preferentially cut the most recent context — +// the worst possible choice, and the reason mask carries keep_recent. 0 disables. +func Classify(r Record, closedDist, openReps, minLater int) Class { + if r.Novel == 0 { + return Opaque // no evidence either way; see Opaque + } + if minLater > 0 && r.LaterTurns < minLater { + return Open // too new to have been referenced yet — absence of opportunity + } + if r.Refs == 0 { + return Unreferenced + } + if (openReps > 0 && r.Refs >= openReps) || r.RefAge < closedDist { + return Open + } + return Closed +} + +// Index computes one Record per tool output at least minOutputTokens in size, using +// tok to measure size (nil falls back to the ~4-chars/token proxy the measurement +// script uses). +// +// Two exclusions do the real work, and neither is optional: +// +// ECHO. Only identifiers the output INTRODUCED are eligible. If the agent calls +// Read(src/auth.py), the path arrives in the tool-call ARGUMENT, is echoed by the +// result, and appears again in a later Edit(src/auth.py) — an exact matcher sees a +// reference from the output to a later turn, but nothing was ever taken FROM the +// output. So a token is novel only if it appears nowhere at or before this message +// (in any surface), nor in a sibling result of the same turn. On the fixture, dropping +// this guard flips a plainly-unreferenced file read to open and halves the measured +// cuttable mass: it is the difference between a usable measurement and one that +// reports everything as load-bearing. +// +// BOILERPLATE. A token echoed by many outputs (more than max(5, outputs/4)) is +// session furniture — a banner, a prompt, a repeated header — not a carried value. +// +// Outputs below minOutputTokens get no Record but still contribute to both exclusion +// sets, because they are part of the context the model saw. +func Index(msgs []Message, minOutputTokens int, tok func(string) int) []Record { + return index(msgs, minOutputTokens, tok, true) +} + +// index is Index with the echo guard made switchable, so the test can run the negative +// control that proves the guard is what produces the result. The knob is unexported on +// purpose: priorGuard=false is a KNOWN-WRONG index (it counts the tool-call argument +// echoed by its own result as a reference), and no caller should be able to select it. +func index(msgs []Message, minOutputTokens int, tok func(string) int, priorGuard bool) []Record { + if tok == nil { + tok = approxTokens + } + n := len(msgs) + + // Per-surface token sets. refTokens is the reference-bearing surface of each + // message; resTokens the outputs, keyed by tool-call id. + refTokens := make([]map[string]struct{}, n) + resTokens := make([]map[string]map[string]struct{}, n) + nOut := 0 + for i := range msgs { + refTokens[i] = Idents(strings.Join(msgs[i].Texts, " ")) + resTokens[i] = make(map[string]map[string]struct{}, len(msgs[i].Results)) + for _, r := range msgs[i].Results { + resTokens[i][r.ID] = Idents(r.Text) + nOut++ + } + } + + // firstSeen[t] is the lowest message index at which t occurs in ANY surface, so + // "t was already in context before message i" is firstSeen[t] < i. This replaces a + // per-message snapshot of the running union — same answer, but O(distinct tokens) + // memory instead of O(messages × tokens), which matters at the transcript sizes + // this component fires on. + firstSeen := make(map[string]int) + note := func(i int, set map[string]struct{}) { + for t := range set { + if _, ok := firstSeen[t]; !ok { + firstSeen[t] = i + } + } + } + for i := range msgs { + note(i, refTokens[i]) + for _, toks := range resTokens[i] { + note(i, toks) + } + } + + // spread[t] is how many distinct outputs carry t; past a threshold it is furniture. + spread := make(map[string]int) + for i := range msgs { + for _, toks := range resTokens[i] { + for t := range toks { + spread[t]++ + } + } + } + commonAt := nOut / 4 + if commonAt < 5 { + commonAt = 5 + } + + var recs []Record + for i := range msgs { + for _, r := range msgs[i].Results { + size := tok(r.Text) + if size < minOutputTokens { + continue + } + // Sibling results of this same turn: the producing tool call normally lands in + // the previous message (and so in firstSeen), but a batched turn carries several + // results at once and they must not credit each other. + siblings := map[string]struct{}{} + for id, toks := range resTokens[i] { + if id == r.ID { + continue + } + for t := range toks { + siblings[t] = struct{}{} + } + } + novel := map[string]struct{}{} + for t := range resTokens[i][r.ID] { + if fs, ok := firstSeen[t]; priorGuard && ok && fs < i { + continue // already in context before this output existed + } + if _, ok := siblings[t]; ok { + continue + } + if spread[t] > commonAt { + continue // session furniture + } + if _, ok := refTokens[i][t]; ok { + continue // the same turn's own prose/arguments + } + novel[t] = struct{}{} + } + + rec := Record{Idx: i, ID: r.ID, SizeTokens: size, Novel: len(novel), RefAge: -1, ConsumeLag: -1} + used := map[string]struct{}{} + last := -1 + for j := i + 1; j < n; j++ { + // A later MODEL turn, judged by whether it has a model-authored surface at all + // — not by whether that surface happens to contain trackable identifiers. The + // distinction matters twice: it is the definition coref.py uses (so the two + // must agree), and a turn with no identifiers is still an opportunity that was + // declined rather than an opportunity that never existed. + if len(msgs[j].Texts) == 0 { + continue + } + rec.LaterTurns++ + hit := false + for t := range novel { + if _, ok := refTokens[j][t]; ok { + used[t] = struct{}{} + hit = true + } + } + if hit { + rec.Refs++ + last = j + } + } + if last >= 0 { + rec.RefAge = n - last + rec.ConsumeLag = last - i + } + if len(novel) > 0 { + rec.UsedFrac = float64(len(used)) / float64(len(novel)) + } + recs = append(recs, rec) + } + } + return recs +} + +// approxTokens is the ~4-chars/token proxy the offline pass uses, so an Index built +// without a real tokenizer sizes outputs the same way the measurement did. +func approxTokens(s string) int { + if n := len(s) / 4; n > 1 { + return n + } + return 1 +} diff --git a/internal/coref/coref_test.go b/internal/coref/coref_test.go new file mode 100644 index 00000000..be839cd5 --- /dev/null +++ b/internal/coref/coref_test.go @@ -0,0 +1,403 @@ +package coref + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +// The fixture below is the Go twin of deploy/harbor/coref_fixture.py: four tool outputs +// whose correct classification is fixed by construction. It is duplicated rather than +// loaded so this package's test has no Python dependency, and so a change to the index +// that silently disagrees with the offline measurement fails HERE — the two must share +// one definition of a reference or the thresholds the measurement produces are +// calibrated for a different algorithm. +// +// #1 src/auth.py read -> closed (novel TOKEN_GRACE_SECONDS lifted out once, early) +// #2 src/config.py read -> unreferenced (only overlap is the echoed path == the argument) +// #3 ls -R listing -> unreferenced (nothing ever comes back to it) +// #4 pytest failure -> open (novel error id reused 3x, most recently 1 turn ago) + +// filler builds per-output distinct lines. Shared filler would be dropped as session +// boilerplate, which would mask whether the novel-token logic works at all. +func filler(tag string, n int) string { + var b strings.Builder + for i := 0; i < n; i++ { + fmt.Fprintf(&b, "%4d\t%s_line_%d = compute_%s_%d(arg_%d)\n", i, tag, i, tag, i, i) + } + return b.String() +} + +// fixture returns the flattened transcript, ending on the last MODEL turn. Ending there +// (rather than on the trailing tool result) is what the offline fixture captures, and it +// is the state in which the last turn's references are visible — recency is measured from +// the head, so the two must agree on where the head is. +func fixture() []Message { + toolUse := func(name string, args map[string]any) string { + b, _ := json.Marshal(args) + return name + " " + string(b) + } + text := func(t string) Message { return Message{Texts: []string{t}} } + asst := func(t, name string, args map[string]any) Message { + return Message{Texts: []string{t, toolUse(name, args)}} + } + res := func(id, t string) Message { return Message{Results: []Result{{ID: id, Text: t}}} } + + msgs := []Message{ + text("Fix the failing test test_auth_expiry in src/auth.py"), + + // #1 closed + asst("Reading the auth module.", "Read", map[string]any{"path": "src/auth.py"}), + res("t1", "src/auth.py\n"+filler("auth", 240)+"\nTOKEN_GRACE_SECONDS = 0 # novel\n"), + asst("The bug is TOKEN_GRACE_SECONDS is 0; it must be 300.", "Edit", + map[string]any{"path": "src/auth.py", "old": "TOKEN_GRACE_SECONDS = 0", "new": "TOKEN_GRACE_SECONDS = 300"}), + res("t2", "ok"), + + // #2 unreferenced — the echo confound + asst("Checking config.", "Read", map[string]any{"path": "src/config.py"}), + res("t3", "src/config.py\n"+filler("config", 240)), + asst("Config is fine, adjusting anyway.", "Edit", map[string]any{"path": "src/config.py", "old": "a", "new": "b"}), + res("t4", "ok"), + + // #3 unreferenced + asst("Surveying the tree.", "Bash", map[string]any{"cmd": "ls -R"}), + res("t5", filler("tree", 240)), + + // #4 open + asst("Running the suite.", "Bash", map[string]any{"cmd": "pytest -q"}), + res("t6", "1 failed, 42 passed\n"+filler("pytest", 240)+"\nE AssertionError: XPIRE_DRIFT_7f3a\n"), + } + for k := 0; k < 3; k++ { + msgs = append(msgs, + asst(fmt.Sprintf("XPIRE_DRIFT_7f3a again; attempt %d.", k), "Bash", map[string]any{"cmd": "pytest -q"}), + ) + if k < 2 { // the transcript ends on the model turn, so the last result is not sent + msgs = append(msgs, res(fmt.Sprintf("r%d", k), "1 failed\nE AssertionError: XPIRE_DRIFT_7f3a\n")) + } + } + return msgs +} + +// The offline pass's defaults, so the Go index is checked at the same operating point. +const ( + testClosedDist = 12 + testOpenReps = 3 + testMinOutput = 300 + // The fixture's outputs sit near the tail of a short transcript, so the opportunity + // floor is disabled for the ground-truth cases; it has its own test below. + testMinLater = 0 +) + +func classifyFixture(t *testing.T, guard bool) map[string]Class { + t.Helper() + recs := index(fixture(), testMinOutput, nil, guard) + got := map[string]Class{} + for _, r := range recs { + got[r.ID] = Classify(r, testClosedDist, testOpenReps, testMinLater) + } + return got +} + +func TestFixtureClassification(t *testing.T) { + got := classifyFixture(t, true) + want := map[string]Class{"t1": Closed, "t3": Unreferenced, "t5": Unreferenced, "t6": Open} + if len(got) != len(want) { + t.Fatalf("recorded outputs = %v, want exactly the four above min_output", got) + } + for id, w := range want { + if got[id] != w { + t.Errorf("output %s classified %q, want %q", id, got[id], w) + } + } +} + +// TestEchoGuardIsLoadBearing is the negative control. Without the prior-vocabulary +// exclusion, the src/config.py read is scored as REFERENCED — its only later overlap is +// the path that arrived as the tool-call argument, so the output introduced nothing that +// was carried forward. An index that gets this wrong reports nearly all mass as +// load-bearing, which reads as "there is nothing to cut" rather than as a bug. +func TestEchoGuardIsLoadBearing(t *testing.T) { + if got := classifyFixture(t, true)["t3"]; got != Unreferenced { + t.Fatalf("with the guard, t3 = %q, want %q", got, Unreferenced) + } + if got := classifyFixture(t, false)["t3"]; got == Unreferenced { + t.Fatal("without the guard, t3 stayed unreferenced: the control proves nothing, " + + "so the guard is no longer what produces the result") + } +} + +// Cuttable mass (unreferenced + closed) must be materially larger with the guard on. +// This is the measurement-level statement of the control: the guard's effect is not a +// reclassified edge case, it is most of the answer. +func TestEchoGuardChangesCuttableMass(t *testing.T) { + mass := func(guard bool) (cuttable, total int) { + for _, r := range index(fixture(), testMinOutput, nil, guard) { + total += r.SizeTokens + if c := Classify(r, testClosedDist, testOpenReps, testMinLater); c != Open { + cuttable += r.SizeTokens + } + } + return cuttable, total + } + onCut, onTotal := mass(true) + offCut, offTotal := mass(false) + if onTotal != offTotal || onTotal == 0 { + t.Fatalf("total mass differs between arms (%d vs %d): the arms are not comparable", onTotal, offTotal) + } + if onCut <= offCut { + t.Errorf("cuttable mass with guard = %d/%d, without = %d/%d; the guard must INCREASE it", + onCut, onTotal, offCut, offTotal) + } +} + +func TestRecencyIsMeasuredFromTheHead(t *testing.T) { + recs := index(fixture(), testMinOutput, nil, true) + n := len(fixture()) + byID := map[string]Record{} + for _, r := range recs { + byID[r.ID] = r + } + // #1 is referenced once, by the turn immediately after it. Its RefAge must be the + // distance from the HEAD (large — the reference is ancient), while its ConsumeLag is + // small (the value was taken immediately). Swapping the two is the modelling error + // this assertion exists to catch: it makes every early output look freshly used. + r := byID["t1"] + if r.Refs != 1 { + t.Fatalf("t1 refs = %d, want 1", r.Refs) + } + if r.RefAge != n-3 { + t.Errorf("t1 RefAge = %d, want %d (messages ago, from the head)", r.RefAge, n-3) + } + if r.ConsumeLag != 1 { + t.Errorf("t1 ConsumeLag = %d, want 1 (the very next turn took the value)", r.ConsumeLag) + } + if r.RefAge <= r.ConsumeLag { + t.Error("t1 RefAge must exceed ConsumeLag here; the two axes have been conflated") + } + // An unreferenced output reports both as absent rather than as zero — zero would read + // as "referenced by the current turn", the opposite of the truth. + if u := byID["t5"]; u.RefAge != -1 || u.ConsumeLag != -1 { + t.Errorf("t5 (unreferenced) RefAge/ConsumeLag = %d/%d, want -1/-1", u.RefAge, u.ConsumeLag) + } +} + +func TestUsedFracShowsPartialConsumption(t *testing.T) { + // #1 introduced ~240 lines' worth of identifiers and the model carried exactly one + // value forward. "Took a value, does not need the rest" should therefore be visible + // as a small UsedFrac rather than assumed. + for _, r := range index(fixture(), testMinOutput, nil, true) { + if r.ID != "t1" { + continue + } + if r.Novel < 100 { + t.Fatalf("t1 novel tokens = %d, want the filler identifiers to count", r.Novel) + } + if r.UsedFrac <= 0 || r.UsedFrac > 0.05 { + t.Errorf("t1 UsedFrac = %.4f, want a small positive fraction", r.UsedFrac) + } + } +} + +func TestClassifyBoundaries(t *testing.T) { + for _, tc := range []struct { + name string + rec Record + want Class + }{ + {"no trackable identifiers is opaque, not unreferenced", Record{Novel: 0, Refs: 0, RefAge: -1}, Opaque}, + {"never referenced", Record{Novel: 20, Refs: 0, RefAge: -1, LaterTurns: 99}, Unreferenced}, + {"referenced exactly at the recency floor is closed", Record{Novel: 20, Refs: 1, RefAge: 12, LaterTurns: 99}, Closed}, + {"one message newer than the floor is open", Record{Novel: 20, Refs: 1, RefAge: 11, LaterTurns: 99}, Open}, + {"repetition keeps it open however old", Record{Novel: 20, Refs: 3, RefAge: 9999, LaterTurns: 99}, Open}, + {"just under the repetition ceiling, and old", Record{Novel: 20, Refs: 2, RefAge: 9999, LaterTurns: 99}, Closed}, + } { + if got := Classify(tc.rec, testClosedDist, testOpenReps, testMinLater); got != tc.want { + t.Errorf("%s: got %q, want %q", tc.name, got, tc.want) + } + } +} + +func TestDistinctiveRejectsProse(t *testing.T) { + // Precision matters more than recall: a prose word scored as an identifier mints a + // spurious reference, and a spurious reference silently suppresses a cut. + // + // Every entry in the second group below is a REGRESSION CASE — each was among the top + // reference-producing "identifiers" on real agent traffic before the rules in + // distinctive were tightened, and together they inflated referenced mass from 51% to 71%. + for _, w := range []string{"the", "have", "should", "config", "failing", "module"} { + if distinctive(w) { + t.Errorf("distinctive(%q) = true, want false (prose)", w) + } + } + for _, w := range []string{ + "description", "transparency", "integration", "efficiency", "conditions", + "persistent", "orientation", "effectiveness", "environment", "conversation", + "e.g.", "try:", "None:", "memory.", "2026", "447", + } { + if distinctive(w) { + t.Errorf("distinctive(%q) = true, want false (measured false positive)", w) + } + } + for _, w := range []string{ + "src/auth.py", "TOKEN_GRACE_SECONDS", "XPIRE_DRIFT_7f3a", "camelCaseName", "12345", + "v1/messages", "session_id", "GraphStore", "config.py", "claude-sonnet-5", "1.2.3", + } { + if !distinctive(w) { + t.Errorf("distinctive(%q) = false, want true (identifier)", w) + } + } +} + +// A token's surrounding punctuation must not split it in two: `memory.` at the end of a +// sentence and `memory` in a tool argument have to match, or a real reference is missed. +func TestIdentsTrimEdgePunctuation(t *testing.T) { + got := Idents("wrote src/auth.py. then read src/auth.py") + if _, ok := got["src/auth.py"]; !ok { + t.Errorf("Idents lost the trimmed form: %v", got) + } + if _, ok := got["src/auth.py."]; ok { + t.Errorf("Idents kept an untrimmed duplicate: %v", got) + } +} + +func TestSiblingResultsDoNotCreditEachOther(t *testing.T) { + // A batched turn carries several results at once. If one sibling's identifiers count + // as a reference to another's, a parallel tool call makes both look load-bearing. + body := filler("batch", 240) + msgs := []Message{ + {Texts: []string{"go"}}, + {Results: []Result{{ID: "a", Text: body}, {ID: "b", Text: body}}}, + {Texts: []string{"done"}}, + } + for _, r := range index(msgs, testMinOutput, nil, true) { + if r.Refs != 0 { + t.Errorf("output %s refs = %d, want 0 (its only overlap is its sibling)", r.ID, r.Refs) + } + } +} + +func TestBoilerplateIsNotAReference(t *testing.T) { + // A banner repeated by many outputs is furniture. Counting it makes the FIRST output + // that emitted it look referenced by every later turn that echoes it. + const banner = "=== build_harness_v2.1 /opt/ci/run.sh ===" + var msgs []Message + msgs = append(msgs, Message{Texts: []string{"start"}}) + for i := 0; i < 12; i++ { + msgs = append(msgs, + Message{Results: []Result{{ID: fmt.Sprintf("o%d", i), Text: banner + "\n" + filler(fmt.Sprintf("run%d", i), 240)}}}, + Message{Texts: []string{banner + " again"}}, + ) + } + for _, r := range index(msgs, testMinOutput, nil, true) { + if r.Refs != 0 { + t.Errorf("output %s refs = %d, want 0 (its only later overlap is the banner)", r.ID, r.Refs) + } + } +} + +func TestBelowFloorOutputsStillExclude(t *testing.T) { + // A small output gets no Record, but the identifiers it introduced must still be in + // the exclusion set: otherwise a large output re-emitting them looks like it + // introduced them, and a later mention of them looks like a reference to it. + const novel = "GRACE_WINDOW_88fa" + msgs := []Message{ + {Texts: []string{"start"}}, + {Results: []Result{{ID: "small", Text: novel}}}, + {Results: []Result{{ID: "big", Text: novel + "\n" + filler("big", 240)}}}, + {Texts: []string{"using " + novel}}, + } + for _, r := range index(msgs, testMinOutput, nil, true) { + if r.ID != "big" { + t.Fatalf("unexpected record for %q: the small output is below the floor", r.ID) + } + if r.Refs != 0 { + t.Errorf("big refs = %d, want 0: %s was introduced by the earlier small output", r.Refs, novel) + } + } +} + +func TestToolResultsAreNotReferenceBearing(t *testing.T) { + // The environment repeating a token is not the model using it. Otherwise a flaky + // command that prints the same error every turn keeps its own first output alive. + msgs := []Message{ + {Texts: []string{"start"}}, + {Results: []Result{{ID: "first", Text: "E AssertionError: DRIFT_9c2b\n" + filler("first", 240)}}}, + {Results: []Result{{ID: "second", Text: "E AssertionError: DRIFT_9c2b\n" + filler("second", 240)}}}, + } + for _, r := range index(msgs, testMinOutput, nil, true) { + if r.ID == "first" && r.Refs != 0 { + t.Errorf("first refs = %d, want 0 (only a later tool_result echoes it)", r.Refs) + } + } +} + +func TestIndexHandlesEmptyAndNil(t *testing.T) { + if recs := Index(nil, testMinOutput, nil); len(recs) != 0 { + t.Errorf("Index(nil) = %v, want none", recs) + } + if recs := Index([]Message{{}, {Results: []Result{{ID: "x", Text: ""}}}}, 0, nil); len(recs) != 1 { + t.Errorf("an empty output should still record (at the zero floor), got %v", recs) + } +} + +// An output whose values the tokenizer cannot see must come back `opaque`, never +// `unreferenced`. Both have refs == 0 and they mean opposite things: "introduced 200 +// identifiers, nobody touched one" is evidence of deadness, "introduced nothing I can see" +// is absence of evidence. Collapsing them made the DEFAULT config cut a record set the +// agent had explicitly said it still needed. +// +// Raised in review on PR #80 with exactly this shape: the agent references an ANCHOR +// (`david`, `123`) in order to point at a payload (`foobarbaz`) it never copied. +func TestRecordsOfPlainValuesAreOpaqueNotUnreferenced(t *testing.T) { + people := strings.Repeat( + `[{"name":"david","id":123,"address":"foobarbaz"},{"name":"osher","id":235,"address":"banana"}]`, 60) + msgs := []Message{ + {Texts: []string{"look up the people directory"}}, + {Texts: []string{"Querying.", "query_people {}"}}, + {Results: []Result{{ID: "t1", Text: people}}}, + {Texts: []string{"I need to remember david 123 address."}}, + } + for i := 0; i < 20; i++ { + msgs = append(msgs, Message{Texts: []string{fmt.Sprintf("unrelated step %d", i)}}) + } + recs := index(msgs, testMinOutput, nil, true) + if len(recs) != 1 { + t.Fatalf("expected one record, got %d", len(recs)) + } + if recs[0].Novel != 0 { + t.Fatalf("fixture assumption broken: Novel = %d, expected the tokenizer to see nothing "+ + "in short lowercase words and 3-digit numbers", recs[0].Novel) + } + if got := Classify(recs[0], testClosedDist, testOpenReps, testMinLater); got != Opaque { + t.Errorf("classified %q, want %q — an output the index cannot see into must not be "+ + "a silent vote to delete", got, Opaque) + } +} + +// An output near the tail has had no chance to be referenced, so scoring it as unused would +// make a batched pass preferentially cut the most RECENT context. This is why mask carries +// keep_recent, and coref needs the same idea expressed in turns. +func TestOpportunityFloorProtectsRecentOutputs(t *testing.T) { + body := filler("recent", 240) + msgs := []Message{ + {Texts: []string{"start"}}, + {Texts: []string{"Reading.", "Read {\"path\":\"x\"}"}}, + {Results: []Result{{ID: "fresh", Text: body}}}, + {Texts: []string{"ok"}}, // exactly one later model turn + } + recs := index(msgs, testMinOutput, nil, true) + if len(recs) != 1 || recs[0].Novel == 0 { + t.Fatalf("fixture assumption broken: %+v", recs) + } + if recs[0].LaterTurns != 1 { + t.Errorf("LaterTurns = %d, want 1", recs[0].LaterTurns) + } + if got := Classify(recs[0], testClosedDist, testOpenReps, 0); got != Unreferenced { + t.Errorf("with the floor disabled: got %q, want %q", got, Unreferenced) + } + if got := Classify(recs[0], testClosedDist, testOpenReps, 8); got != Open { + t.Errorf("with the floor at 8: got %q, want %q — one later turn is not an "+ + "opportunity to be referenced", got, Open) + } +} diff --git a/internal/extract/bulk.go b/internal/extract/bulk.go new file mode 100644 index 00000000..6a3d04c7 --- /dev/null +++ b/internal/extract/bulk.go @@ -0,0 +1,236 @@ +package extract + +import ( + "encoding/json" + "strconv" + "strings" +) + +// BULK ADJUDICATION — the only model shape the held-out selection experiment found worth pursuing. +// +// docs/results/coref-selection-experiment.md measured ten arms over 8,105 recorded decisions and +// settled two things that this file exists to implement, and one it exists to avoid: +// +// REFUTED — the per-output merged design. One call, shown ONE output plus its reference evidence, +// deciding drop-or-trim, scored 6% live-kept on haiku and 14% on sonnet, both inside the +// drop-everything null model's error bar. Shown a single output, a model simply drops it. +// +// WORKS — bulk adjudication. One call shown ~15 outputs TOGETHER lifted live-kept from 6% to 58% +// at the LOWEST cost per output, because the overhead amortises and, more importantly, because +// comparative judgement beats absolute judgement: ranking fifteen candidates against each other +// is a question a model can answer, "is this one output expendable" is not. +// +// WORKS — cost-honest framing, worth ~26 points of live-kept on its own. The first prompt +// reassured the model that removals "stay recoverable on request"; that single clause produced +// 91% removal at 6% live-kept. Replacing it with the real consequence moved haiku to 64%/32% and +// sonnet to 49%/58%. Telling a model its mistakes are cheap makes it careless, so this prompt +// states the true cost and never mentions recoverability. +// +// The deterministic index still beats every model arm measured (95% live-kept at 11% false-drop, at +// zero marginal cost), and a floor-symmetric re-score plus a widened Tier-2 ground truth did not +// close that gap (docs/experiments/loca/iter009/results.md). So this is not shipped as a default — +// it exists to answer the one question decision-quality experiments structurally cannot: whether it +// moves REWARD. + +// BulkItem is one candidate output offered for adjudication, with the co-reference evidence the +// deterministic index already computed for it. +type BulkItem struct { + Index int // caller's message index, echoed back in the verdict + ID string // tool-call id, for the operator's logs + SizeTokens int + Evidence string // rendered reference evidence (see RenderEvidence) + Sample string // the output itself, truncated +} + +// BulkVerdict is one decision. +// +// NeededBy and Quote are the FORCED EVIDENCE, and they are the only thing measured to protect the +// marginal case (docs/experiments/loca/iter019/results.md §6). Arms carrying an identical criterion +// differed ONLY in whether the model had to emit which obligation still needs the output, and the +// arm that had to emit it halved the false-drop rate: instructions the model can skim past are +// inert, a required output field is not. +// +// `Kept` is GONE along with the trim verdict. Offered across 21 probe opportunities trim was chosen +// zero times, keep/drop scored identically to keep/drop/trim on every metric, and in production it +// was accepted once against eight rejected as invented -- it was the only verdict requiring the +// model to transport text, which is what it is worst at. +type BulkVerdict struct { + Index int `json:"i"` + Verdict string `json:"verdict"` // keep | drop + NeededBy string `json:"needed_by"` // a | b | c | none (see bulkContract's CRITERION) + Quote string `json:"quote,omitempty"` +} + +// bulkContract is deliberately blunt about consequences. See the cost-honest framing note above: +// every softening of this text measured WORSE. +const bulkContract = `You are shown several tool outputs from one agent's transcript, each with evidence +about whether the agent has referred back to it since. Decide, for EACH output, whether the agent +still needs it. + +CRITERION. An output is SPENT only if it is needed for NONE of the following: + (a) the step the agent is on right now; + (b) any instruction the user has given that is NOT YET COMPLETE; + (c) any step the agent has EXPLICITLY STATED it will take and has not yet taken. +Only obligations WRITTEN IN THE TRANSCRIPT count -- do not invent hypothetical future needs. An +output whose information has already been captured elsewhere (a filed total, a recorded conclusion) +AND which no outstanding obligation needs in raw form is spent. + +WHAT A WRONG REMOVAL ACTUALLY COSTS. If you remove something the agent still needs, it usually does +NOT notice the gap and does not ask for the content back. It answers from worse information and gets +the task wrong. There is no safety net you should count on. A wrong removal is a silent, permanent +loss of task quality; a wrong retention costs only tokens. + +JUDGE THEM AGAINST EACH OTHER. You are given several outputs precisely so you can compare. Rank them: +the ones whose information has clearly been consumed and superseded are the candidates. If they all +look load-bearing, keep them all -- "keep everything" is a valid and often correct answer. + +READING THE EVIDENCE. novel = identifiers this output introduced. refs = how many later turns reused +one. ref_age = how many messages ago the last reuse was. used_frac = what share of its identifiers +were carried forward. later_turns = how many turns the output has HAD to be referenced in. + + - refs=0 with many later_turns is the strongest signal of deadness -- but it is exact-match + evidence only, so an output whose values were TRANSFORMED (summed, reformatted, reworded) before + being restated leaves refs=0 while still being load-bearing. Your job on those is to VETO the + index, not to rubber-stamp it. + - a LOW used_frac on a referenced output is ambiguous and must not be read as "the rest is chaff". + The agent may have taken an ANCHOR -- a name, an id -- precisely in order to point at a payload it + never copied. Keep the payload. + - novel=0 means the index could see nothing trackable. That is absence of evidence, not evidence of + absence. Default to keep. + - few later_turns means the output has not yet had a chance to be used. Keep it. + +FOR EACH OUTPUT, ANSWER THE CRITERION FIRST, THEN DECIDE: + "needed_by" -- which of (a)/(b)/(c) still needs this output, or "none" if it is spent. + "quote" -- when needed_by is a/b/c, the transcript text that creates that obligation, copied + VERBATIM. Leave empty only when needed_by is "none". + "verdict" -- keep (still needed, or you are unsure -- this is the default) or drop (its + information is spent; a short descriptor of its shape will remain in its place). + A verdict of "drop" REQUIRES needed_by "none": if any obligation still needs the + output, the verdict must be keep. + +Reply with ONLY a JSON array, one object per output, no prose: +[{"i": , "needed_by": "a|b|c|none", "quote": "", "verdict": "keep|drop"}]` + +// BuildBulkPrompt renders the adjudication request. goal is what the agent is currently doing, so +// relevance is judged toward the live task rather than in the abstract. +func BuildBulkPrompt(goal string, items []BulkItem) string { + var b strings.Builder + b.WriteString(bulkContract) + b.WriteString("\n\nWHAT THE AGENT IS DOING NOW (judge relevance toward this):\n") + g := strings.TrimSpace(goal) + if g == "" { + g = "(no explicit goal stated)" + } + if len(g) > 4000 { + g = g[:4000] + } + b.WriteString(g) + b.WriteString("\n\n") + for _, it := range items { + b.WriteString("=== OUTPUT ") + b.WriteString(strconv.Itoa(it.Index)) + b.WriteString(" (") + b.WriteString(strconv.Itoa(it.SizeTokens)) + b.WriteString(" tokens)\nevidence: ") + b.WriteString(it.Evidence) + b.WriteString("\ncontent:\n") + b.WriteString(it.Sample) + b.WriteString("\n\n") + } + return b.String() +} + +// ParseBulkVerdicts reads the model's reply, returning the verdicts and whether the reply PARSED. +// +// The two must be distinguished. An empty array is a legitimate answer -- "keep everything", which +// the contract explicitly invites -- while unparseable output is a prompt or model failure. Folding +// both into "no verdicts" made a deliberate keep-all indistinguishable from junk in the gate +// counters, and those counters are the only way to tell "the model declined to act" from "the model +// was never successfully asked", which is exactly the distinction one live arm turned on. +func ParseBulkVerdicts(reply string) ([]BulkVerdict, bool) { + s := stripFences(strings.TrimSpace(reply)) + i, j := strings.Index(s, "["), strings.LastIndex(s, "]") + if i < 0 || j <= i { + return nil, false + } + var out []BulkVerdict + if err := json.Unmarshal([]byte(s[i:j+1]), &out); err != nil { + return nil, false + } + keep := out[:0] + for _, v := range out { + switch v.Verdict { + case "keep", "drop": + keep = append(keep, v) + case "trim": + // Trim is no longer offered. A model that answers with it anyway is asking for partial + // retention we cannot perform, so degrade to the safe direction rather than discarding + // the verdict -- dropping it entirely would leave the output unjudged and looks + // identical to "the model said nothing about it". + v.Verdict = "keep" + keep = append(keep, v) + } + } + return keep, true +} + +// BuildPrefixAsk renders the adjudication question for a PREFIX ASK — a call whose prefix is the +// transcript the agent already sent, read from the provider's prompt cache. +// +// The difference from BuildBulkPrompt is what is NOT here: the outputs themselves. BuildBulkPrompt +// must ship a truncated sample of every candidate because a bare completion has no other way to show +// them, which caps each at mergedSampleChars and forces the model to judge an excerpt. A prefix ask +// has the whole transcript above it, so shipping samples would pay fresh tokens for content already +// being read from cache — and would show a TRUNCATED copy of something the model can read in full. +// +// What remains is an inventory: a small integer label per candidate, its tool-call id, its size, and +// a short head so the model can locate it unambiguously in the transcript above. +// +// The labels are integers for a measured reason. Asked to answer with opaque tool_use ids, the model +// regularised them -- `toolu_01..07` for `toolu_probe_00..07` -- because reproducing a random +// identifier from thousands of tokens back is a copying task, not a judgement. With integer labels it +// was 0 bad labels in 40+ trials. The rule generalises: give the model short things it cannot get +// wrong, and keep every mapping on our side. +func BuildPrefixAsk(goal string, items []BulkItem) string { + var b strings.Builder + b.WriteString(bulkContract) + b.WriteString("\n\nThe transcript above is the agent's own. Read the tool outputs from it directly.\n") + b.WriteString("\nWHAT THE AGENT IS DOING NOW (judge relevance toward this):\n") + g := strings.TrimSpace(goal) + if g == "" { + g = "(no explicit goal stated)" + } + if len(g) > 4000 { + g = g[:4000] + } + b.WriteString(g) + b.WriteString("\n\nTOOL OUTPUTS UNDER CONSIDERATION. Refer to them by these labels only:\n") + for _, it := range items { + b.WriteString(" [") + b.WriteString(strconv.Itoa(it.Index)) + b.WriteString("] ") + b.WriteString(strconv.Itoa(it.SizeTokens)) + b.WriteString(" tokens, tool_use id ") + b.WriteString(it.ID) + b.WriteString(", evidence: ") + b.WriteString(it.Evidence) + b.WriteString("\n begins: ") + b.WriteString(headLine(it.Sample, 90)) + b.WriteString("\n") + } + return b.String() +} + +// headLine returns a single-line, bounded opening of s, for locating an output in the transcript +// above. Newlines are collapsed so one candidate stays one line and the inventory stays readable to +// a model counting labels. +func headLine(s string, max int) string { + s = strings.TrimSpace(s) + if i := strings.IndexAny(s, "\r\n"); i >= 0 { + s = s[:i] + } + if len(s) > max { + s = s[:max] + "…" + } + return s +} diff --git a/metrics/metrics.go b/metrics/metrics.go index b694182f..341201b9 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -614,6 +614,28 @@ type Snapshot struct { // off a cliff mid-session is the regression it exists to expose. Filled by the host at // serve time (the counter lives in `modes`, which metrics cannot import). CompactionResets int64 `json:"compaction_resets"` + // Expand calls the proxy could not satisfy, split by cause. ExpandUnresolvedMissing is the + // alerting number: a well-formed marker with nothing stashed behind it is a cut that was + // advertised as reversible and is not, so every count is a case where reversibility silently + // failed. Malformed ids are the model's own invention and need no action. Neither existed + // before, which is how three experiment iterations ran while the model was calling the expand + // tool and being refused — the refusal was found by grepping the benchmark client's transcripts, + // not from any counter here. Filled by the host at serve time (the counters live in `expand`, + // which metrics cannot import). + // AdjudicateStray counts tool_results rewritten because the AGENT called the context-maintenance + // tool, which is injected by the proxy and therefore unknown to the client. Expected to be + // non-zero -- a model calls advertised tools it was told to leave alone, directly observed with + // the expand tool -- and the rate says whether the tool's description is working. + AdjudicateStray int64 `json:"adjudicate_stray"` + // ExpandRestored counts client tool_results rewritten back to their stashed original on the + // REQUEST path. This is the counter that says the recovery loop is working: the client cannot + // execute a proxy-injected tool, so it always refuses, and its refusal is only harmless because + // this substitution happens before the request goes upstream. Without it there is no way to tell + // a working restore from a silent no-op -- which is exactly the gap that let the refusals run + // unnoticed for three iterations. + ExpandRestored int64 `json:"expand_restored"` + ExpandUnresolvedMalformed int64 `json:"expand_unresolved_malformed"` + ExpandUnresolvedMissing int64 `json:"expand_unresolved_missing"` // cmdfilter attribution: which command FAMILIES pay off (builds/tests/iac/pkg/net), // which individual filters fire, and which output shapes matched no filter (the // backlog of filters worth writing). Additive fields — nothing above is renamed. diff --git a/mkdocs.yml b/mkdocs.yml index eb17e28c..20deceed 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -152,6 +152,8 @@ nav: - extract_llm: components/extract_llm.md - smartcrush: components/smartcrush.md - mask: components/mask.md + - coref: components/coref.md + - "coref cheat sheet": reference/coref-glossary.md - summarize: components/summarize.md - The DSL filter engine: - Filter language: components/dsl.md @@ -172,6 +174,13 @@ nav: - Component internals & examples: results/components.md - Reproduce the results: results/REPRODUCE.md - Improvement plan: results/improvement-plan.md + - "Co-reference density (measurement)": results/coref-density.md + - "Co-reference selection (experiment)": results/coref-selection-experiment.md + - "Co-reference reachability (measurement)": results/coref-reachability.md + - "Co-reference on the eval box": results/coref-evalbox.md + - "What can and cannot be measured": results/measurement-limits.md + - "What actually fires (component gating)": results/component-gating.md + - "coref on LOCA-bench": results/coref-loca.md - "SWE-bench Verified: per-arm detail": - baseline: results/baseline.md - context-guru: results/context-guru.md @@ -184,6 +193,19 @@ nav: - context-guru: results/terminal-bench-context-guru.md - headroom: results/terminal-bench-headroom.md - rtk: results/terminal-bench-rtk.md + - Experiment log: + - Overview: experiments/README.md + - "captures — iter001": experiments/captures/iter001/results.md + - "LOCA — iter001 (retracted)": experiments/loca/iter001/results.md + - "LOCA — iter002": experiments/loca/iter002/results.md + - "LOCA — iter003 (reward)": experiments/loca/iter003/results.md + - "LOCA — iter004 (reward @64k)": experiments/loca/iter004/results.md + - "LOCA — iter004b (reward, clean)": experiments/loca/iter004b/results.md + - "LOCA — iter005 (deferral; summarize bug)": experiments/loca/iter005/results.md + - "LOCA — iter006 (stage 1 ablation)": experiments/loca/iter006/results.md + - Proposals: + - "Co-reference-aware compaction": proposals/coref-compaction.md + - "…implementation status": proposals/coref-implementation.md - Integrations: - Host adapters: integrations.md - bifrost plugin: how-to/bifrost-plugin.md diff --git a/proxy/expandmixed_test.go b/proxy/expandmixed_test.go new file mode 100644 index 00000000..16639789 --- /dev/null +++ b/proxy/expandmixed_test.go @@ -0,0 +1,72 @@ +package proxy_test + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/rossoctl/context-guru/expand" +) + +// THE DOMINANT REAL CASE: an expand call arriving ALONGSIDE a real tool call. +// +// The response loop cannot satisfy this -- only the client can execute the real tool, so the response +// must be relayed and ResponseCalls sets otherTools, which bails. Live traffic: 102 of 107 refused +// turns carried two or more tool_use blocks, 5 carried one. So the client answers the expand call +// itself with "Tool 'context_guru_expand' not found", the model loses its recovery path, and it +// re-runs the original tool -- a full tool execution plus fresh output, enlarging the transcript that +// provoked the cut. +// +// The fix is on the REQUEST path: when the client sends its results back, its own failed tool_result +// for the expand call is replaced with the stashed original before the request goes upstream. This +// asserts the model actually receives the content. +func TestExpandRestoredOnMixedToolTurn(t *testing.T) { + var lastUp []byte + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + lastUp, _ = io.ReadAll(r.Body) + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"thanks"}}]}`)) + })) + defer upstream.Close() + + h, st := buildHandler(t, "pipeline: []\n", upstream.URL) + st.Put("HASH", []byte("THE ORIGINAL CONTENT")) + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + + // What the client sends back after a MIXED turn: it executed Read fine, and could not execute + // the proxy-injected expand tool, so it wrote its own failure for that one. + body := `{"model":"gpt-x","messages":[ + {"role":"user","content":"go"}, + {"role":"assistant","tool_calls":[ + {"id":"c_read","type":"function","function":{"name":"Read","arguments":"{}"}}, + {"id":"c_exp","type":"function","function":{"name":"context_guru_expand","arguments":"{\"id\":\"HASH\"}"}}]}, + {"role":"tool","tool_call_id":"c_read","content":"file contents here"}, + {"role":"tool","tool_call_id":"c_exp","content":"Tool 'context_guru_expand' not found"} + ],"tools":[{"type":"function","function":{"name":"Read"}}]}` + + before := expand.Restored() + resp, err := http.Post(srv.URL+"/openai/v1/chat/completions", "application/json", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + io.ReadAll(resp.Body) + resp.Body.Close() + + up := string(lastUp) + if !strings.Contains(up, "THE ORIGINAL CONTENT") { + t.Errorf("the model never received the content it asked for; the client's failure was forwarded "+ + "verbatim. upstream body: %.500s", up) + } + if strings.Contains(up, "not found") { + t.Errorf("the client's 'not found' text reached the model: %.300s", up) + } + if !strings.Contains(up, "file contents here") { + t.Error("the real tool's result must be preserved untouched") + } + if got := expand.Restored() - before; got != 1 { + t.Errorf("expected 1 restored tool_result, counted %d", got) + } +} diff --git a/proxy/expandoffered.go b/proxy/expandoffered.go new file mode 100644 index 00000000..df9ee561 --- /dev/null +++ b/proxy/expandoffered.go @@ -0,0 +1,47 @@ +package proxy + +import "sync" + +// Which sessions have been OFFERED the expand tool. +// +// The model may call a tool it saw on an earlier turn even when the current request does not list it, +// so interception cannot be gated on the current request alone -- see the comment at `advertised` in +// serve. This is the minimum state needed to keep the fast path: a session that has never been +// offered the tool is never inspected, and no response of its is buffered. +// +// Bounded deliberately. An unbounded map keyed by session id grows with traffic on a long-lived +// proxy, so past the cap the set stops admitting new sessions rather than growing without limit. The +// consequence of falling off is only that a session reverts to advertise-gated interception, which is +// the previous behaviour, so the failure mode is a return to the old bug for the coldest sessions +// rather than unbounded memory. +const maxExpandOfferedSessions = 50_000 + +type expandOfferedSet struct { + mu sync.RWMutex + seen map[string]struct{} +} + +func (h *Handler) noteExpandOffered(sess string) { + if h.expandSeen == nil { + return + } + h.expandSeen.mu.Lock() + defer h.expandSeen.mu.Unlock() + if h.expandSeen.seen == nil { + h.expandSeen.seen = make(map[string]struct{}, 1024) + } + if len(h.expandSeen.seen) >= maxExpandOfferedSessions { + return + } + h.expandSeen.seen[sess] = struct{}{} +} + +func (h *Handler) expandOffered(sess string) bool { + if h.expandSeen == nil { + return false + } + h.expandSeen.mu.RLock() + defer h.expandSeen.mu.RUnlock() + _, ok := h.expandSeen.seen[sess] + return ok +} diff --git a/proxy/expandunresolved_test.go b/proxy/expandunresolved_test.go new file mode 100644 index 00000000..7e011533 --- /dev/null +++ b/proxy/expandunresolved_test.go @@ -0,0 +1,122 @@ +package proxy_test + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/rossoctl/context-guru/expand" +) + +// NOTHING RESOLVED must be answered BY THE PROXY, never relayed to the client. +// +// The old behaviour replayed the model own tool_use when no id resolved. A client that does not +// implement this tool then answers "Tool context_guru_expand not found", the model loses its recovery +// path, and it re-runs the original tool instead -- a full tool execution plus fresh output, enlarging +// the transcript that provoked the cut. Measured on LOCA: 48 of 108 attempts refused in one arm, with +// exact-repeat tool calls at 9.9-25.3% against 0.4% in a lossless baseline. +func TestUnresolvedExpandIsAnsweredNotRelayed(t *testing.T) { + var calls int + var secondBody []byte + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + calls++ + w.Header().Set("Content-Type", "application/json") + if calls == 1 { + // The model asks for an id that is NOT in the store. + w.Write([]byte(`{"choices":[{"message":{"role":"assistant","tool_calls":[` + + `{"id":"call_1","type":"function","function":{"name":"context_guru_expand","arguments":"{\"id\":\"GONE\"}"}}` + + `]},"finish_reason":"tool_calls"}]}`)) + return + } + secondBody = b + w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"understood, it is gone"}}]}`)) + })) + defer upstream.Close() + + h, st := buildHandler(t, "pipeline: []\n", upstream.URL) + st.Put("HASH", []byte("SOMETHING ELSE")) // store has content, but not under GONE + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + + body := expandableBody("HASH") + resp, err := http.Post(srv.URL+"/openai/v1/chat/completions", "application/json", strings.NewReader(string(body))) + if err != nil { + t.Fatal(err) + } + final, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + if calls < 2 { + t.Fatalf("the proxy relayed the unresolved call instead of answering it: only %d upstream call(s). "+ + "A client without this tool answers \"Tool not found\" and the model re-runs the original tool.", calls) + } + if !strings.Contains(string(secondBody), "no longer available") { + t.Errorf("the continuation should tell the model the content is gone; got %.400s", secondBody) + } + if strings.Contains(string(final), "context_guru_expand") { + t.Errorf("the client was handed a tool_use for a tool it may not implement: %.300s", final) + } + // And the miss must be COUNTED, split by cause -- it was invisible in /stats before. + mal, miss := expand.Unresolved() + if mal+miss == 0 { + t.Error("an unresolved expand call was not counted; reversibility can fail silently again") + } + t.Logf("unresolved counters: malformed=%d missing=%d", mal, miss) +} + +// INTERCEPTION MUST OUTLIVE ADVERTISEMENT. +// +// Under InjectAuto the tool is advertised only on requests carrying a marker, and interception used to +// be gated on the same test. But a model that saw the tool on one turn calls it on a later turn, and +// that later turn often has no marker -- so nothing intercepted and the tool_use was relayed to a +// client with no such tool. Interception now also covers any session that has EVER been offered it. +func TestExpandInterceptedAfterAdvertisementStops(t *testing.T) { + var calls int + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.ReadAll(r.Body) + calls++ + w.Header().Set("Content-Type", "application/json") + // On the SECOND client request (which carries no marker) the model calls expand anyway. + if calls == 2 { + w.Write([]byte(`{"choices":[{"message":{"role":"assistant","tool_calls":[` + + `{"id":"call_1","type":"function","function":{"name":"context_guru_expand","arguments":"{\"id\":\"HASH\"}"}}` + + `]},"finish_reason":"tool_calls"}]}`)) + return + } + w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"ok"}}]}`)) + })) + defer upstream.Close() + + h, st := buildHandler(t, "pipeline: []\n", upstream.URL) + st.Put("HASH", []byte("THE ORIGINAL CONTENT")) + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + + // Turn 1 CARRIES a marker, so the tool is advertised and the session is remembered. + turn1 := strings.Replace(string(expandableBody("HASH")), `{"model":`, + `{"metadata":{"user_id":"sess-decouple"},"model":`, 1) + r1, err := http.Post(srv.URL+"/openai/v1/chat/completions", "application/json", + strings.NewReader(turn1)) + if err != nil { + t.Fatal(err) + } + io.ReadAll(r1.Body) + r1.Body.Close() + + // Turn 2 carries NO marker -- same session id, since buildHandler's bodies share one. + plain := `{"metadata":{"user_id":"sess-decouple"},"model":"gpt-x","messages":[{"role":"user","content":"carry on"}],"tools":[{"type":"function","function":{"name":"Read"}}]}` + r2, err := http.Post(srv.URL+"/openai/v1/chat/completions", "application/json", strings.NewReader(plain)) + if err != nil { + t.Fatal(err) + } + final, _ := io.ReadAll(r2.Body) + r2.Body.Close() + + if strings.Contains(string(final), "context_guru_expand") { + t.Errorf("a marker-free turn relayed the expand tool_use to the client: %.300s", final) + } + t.Logf("upstream calls=%d final=%.120s", calls, final) +} diff --git a/proxy/modes.go b/proxy/modes.go index 0266179f..6de1daba 100644 --- a/proxy/modes.go +++ b/proxy/modes.go @@ -45,6 +45,7 @@ func (h *Handler) applyMode(r *reqInfo) ([]byte, time.Duration, apply.Trace) { Provider: r.provider, Body: r.body, Session: r.session, Tenant: r.tn.ID, Bypass: r.bypassed, Models: r.models, Window: r.window, CacheMode: h.opts.CacheMode, Mode: mode, Tracker: h.tracker, + PrefixAsk: h.prefixAskerFor(r.provider, r.models), }) added := time.Since(start) if res.Body == nil { diff --git a/proxy/prefixask.go b/proxy/prefixask.go new file mode 100644 index 00000000..20ee00de --- /dev/null +++ b/proxy/prefixask.go @@ -0,0 +1,139 @@ +package proxy + +import ( + "context" + "os" + "sync" + + bschemas "github.com/maximhq/bifrost/core/schemas" + + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/internal/cheapmodel" +) + +// PREFIX ASKS — putting a question to the request's own model with the previous turn's SENT body as +// the prefix, so the provider reads its prompt cache instead of being re-sent the transcript. +// +// Why the SENT body. The cache upstream was populated by what context-guru emitted, which is the +// compacted form. The incoming body is uncompacted, so it diverges from the cached bytes at the +// first thing any component removed and everything past that point is a fresh charge. Appending to +// the bytes actually sent is the only construction that reliably reads the cache — measured at +// 19,595 read / 0 written in docs/experiments/loca/iter019/results.md §2. +// +// Why it is worth the machinery. A component that must judge whether a tool output is still needed +// cannot answer that from the output alone: need is relevance MINUS whatever has already been +// captured elsewhere in the transcript, and that second term lives in the later turns. Sending those +// turns fresh costs ~10x a cache read; on the cheap model the required verbatim quoting also +// degraded to 20.8% at the batch sizes the bulk mechanism needs, against 0 of 59 on the request +// model. So the judgement wants the agent's own model AND the whole transcript, and only a cache +// read makes that affordable. +// +// OFF BY DEFAULT. It holds request bodies in memory, and a feature whose benefit is a cache hit must +// not be enabled by a host that cannot verify the hit is happening. + +// prefixAskEnabled gates the whole mechanism. Opt-in, and read once per call so a run can be +// configured without a rebuild. +func prefixAskEnabled() bool { + v := os.Getenv("CONTEXT_GURU_PREFIX_ASK") + return v == "1" || v == "true" || v == "on" +} + +// Bounds on the stash. These are deliberately small: the stash exists to serve the NEXT turn of an +// ACTIVE session, so retention beyond that is pure memory cost. A body larger than the per-body cap +// is not stashed at all rather than evicting others to hold it. +const ( + maxSentSessions = 64 + maxSentBody = 1_500_000 + maxSentBytes = 96_000_000 +) + +// sentStash holds the last body forwarded upstream per session. +type sentStash struct { + mu sync.Mutex + m map[string][]byte + bytes int +} + +func newSentStash() *sentStash { return &sentStash{m: map[string][]byte{}} } + +// put records this session's forwarded body, replacing any previous one. +// +// Eviction is crude on purpose: when either bound is hit the whole stash is dropped. A prefix ask +// that finds nothing simply falls back to a plain completion, so the cost of over-eviction is a +// cache miss on one call — whereas an LRU here would be state to get wrong for no measurable gain. +func (s *sentStash) put(session string, body []byte) { + if s == nil || session == "" || len(body) == 0 || len(body) > maxSentBody { + return + } + cp := make([]byte, len(body)) + copy(cp, body) + s.mu.Lock() + defer s.mu.Unlock() + if old, ok := s.m[session]; ok { + s.bytes -= len(old) + } + if len(s.m) >= maxSentSessions || s.bytes+len(cp) > maxSentBytes { + s.m = map[string][]byte{} + s.bytes = 0 + } + s.m[session] = cp + s.bytes += len(cp) +} + +func (s *sentStash) get(session string) []byte { + if s == nil || session == "" { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + return s.m[session] +} + +// prefixAsker is the components.PrefixAsker the pipeline receives. It is created per request and +// holds no lock; the stash owns its own. +type prefixAsker struct { + stash *sentStash + cli cheapmodel.Anthropic +} + +// Ask appends the question to this session's last forwarded body and returns the model's text plus +// what it actually cost. A missing stash is an error rather than a silent empty answer, so the +// caller falls back to a plain completion instead of treating "no prefix" as "no verdicts". +func (p prefixAsker) Ask(ctx context.Context, session, ask string) (string, components.PrefixUsage, error) { + body := p.stash.get(session) + if len(body) == 0 { + return "", components.PrefixUsage{}, errNoPrefix + } + reply, u, err := p.cli.CompletePrefixed(ctx, body, ask) + return reply, components.PrefixUsage(u), err +} + +// errNoPrefix is the first-turn case: nothing has been forwarded for this session yet. +var errNoPrefix = errNoPrefixType{} + +type errNoPrefixType struct{} + +func (errNoPrefixType) Error() string { return "no stashed prefix for this session" } + +// prefixAskerFor builds the asker for one request, or nil when any precondition is missing. Nil is +// the normal case on a first turn and whenever the feature is off, and callers must degrade. +// +// Anthropic only: the appended-message construction and the tool_choice/tools cache-key facts were +// measured on that dialect, and guessing at another provider's cache semantics is how a claimed +// cache read becomes a silent 10x bill. +func (h *Handler) prefixAskerFor(provider bschemas.ModelProvider, models components.ModelSpec) components.PrefixAsker { + if !prefixAskEnabled() || provider != bschemas.Anthropic { + return nil + } + cli, ok := models.Incoming.(cheapmodel.Anthropic) + if !ok { + // No incoming client means ModelSpec.For would hand the component the STATIC cheap model, + // which lives in a different cache namespace and could not read this prefix anyway. + return nil + } + // NO pre-flight stash check. The first turn of a session has nothing stashed, and that case must + // surface as an error from Ask -- counted as prefix_ask_failed and falling back to a plain + // completion -- rather than as a nil asker. A nil asker is indistinguishable from "the feature is + // off", which is exactly how the session-key mismatch above stayed invisible. + return prefixAsker{stash: h.sent, cli: cli} +} diff --git a/proxy/prefixask_test.go b/proxy/prefixask_test.go new file mode 100644 index 00000000..74cbf939 --- /dev/null +++ b/proxy/prefixask_test.go @@ -0,0 +1,94 @@ +package proxy + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/internal/cheapmodel" +) + +// THE KEY MUST MATCH, and this test exists because it did not. +// +// Live symptom: prefix_ask_used = 0 with prefix_ask_failed = 0 -- the asker was never built, so the +// mechanism silently never ran and looked exactly like a feature that was switched off. Cause: the +// stash is written under the pipeline's RESOLVED, tenant-scoped session id, while the asker was being +// built from the caller's `x-context-guru-session` header, which this workload never sends. Two +// different keys, no error anywhere. +// +// The component-level tests all injected a fake asker, so none of them touched this wiring. That is +// the gap: a fake that satisfies the interface proves nothing about who supplies the key. +func TestPrefixAskerLooksUpByTheSessionItIsGiven(t *testing.T) { + t.Setenv("CONTEXT_GURU_PREFIX_ASK", "1") + var got int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got++ + w.Header().Set("content-type", "application/json") + io.WriteString(w, `{"content":[{"type":"text","text":"[]"}], + "usage":{"input_tokens":5,"output_tokens":2,"cache_read_input_tokens":14651,"cache_creation_input_tokens":0}}`) + })) + defer srv.Close() + + h := &Handler{sent: newSentStash(), client: srv.Client()} + models := components.ModelSpec{ + Incoming: cheapmodel.Anthropic{BaseURL: srv.URL, Model: "m", APIKey: "k", Client: srv.Client()}, + } + a := h.prefixAskerFor(bschemas.Anthropic, models) + if a == nil { + t.Fatal("no asker built even though the feature is on and an incoming client exists") + } + + // FIRST TURN: nothing stashed. This must be an ERROR the caller can count and fall back on, not a + // nil asker and not an empty answer -- "no prefix yet" and "the feature is off" must not look alike. + if _, _, err := a.Ask(context.Background(), "resolved-session-1", "ask"); err == nil { + t.Error("Ask succeeded with an empty stash; the first turn of a session must report an error") + } + if got != 0 { + t.Error("a request was sent upstream with no prefix to append to") + } + + // The host stashes what it forwarded, under the RESOLVED id. + h.sent.put("resolved-session-1", []byte(`{"model":"m","messages":[{"role":"user","content":"hi"}]}`)) + + _, u, err := a.Ask(context.Background(), "resolved-session-1", "ask") + if err != nil { + t.Fatalf("Ask failed for the session that was stashed: %v", err) + } + if u.CacheRead != 14651 { + t.Errorf("cache read not surfaced: %+v", u) + } + + // A DIFFERENT session must not read another session's transcript. This is the cross-session + // leak the key mismatch could have become had the empty-string key ever matched a stash entry. + if _, _, err := a.Ask(context.Background(), "resolved-session-2", "ask"); err == nil { + t.Error("Ask returned another session's prefix; the stash must be keyed strictly by session") + } + if _, _, err := a.Ask(context.Background(), "", "ask"); err == nil { + t.Error("an empty session id resolved to some stash entry; it must never match") + } +} + +// Off by default, and only where the cache semantics were measured. +func TestPrefixAskerPreconditions(t *testing.T) { + h := &Handler{sent: newSentStash()} + models := components.ModelSpec{Incoming: cheapmodel.Anthropic{BaseURL: "http://x", Model: "m", APIKey: "k"}} + + if a := h.prefixAskerFor(bschemas.Anthropic, models); a != nil { + t.Error("built an asker with CONTEXT_GURU_PREFIX_ASK unset; it must be opt-in") + } + t.Setenv("CONTEXT_GURU_PREFIX_ASK", "1") + if a := h.prefixAskerFor(bschemas.OpenAI, models); a != nil { + t.Error("built an asker for OpenAI; the appended-message and tool_choice cache facts were " + + "measured on Anthropic only, and guessing another provider's cache semantics is how a " + + "claimed cache read becomes a silent 10x bill") + } + if a := h.prefixAskerFor(bschemas.Anthropic, components.ModelSpec{}); a != nil { + t.Error("built an asker with no incoming client; ModelSpec.For would hand the component the " + + "static cheap model, which is in a different cache namespace entirely") + } +} diff --git a/proxy/proxy.go b/proxy/proxy.go index fb553987..46991ffc 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -28,6 +28,7 @@ import ( "github.com/rossoctl/context-guru/components/offload" "github.com/rossoctl/context-guru/dash" "github.com/rossoctl/context-guru/expand" + "github.com/rossoctl/context-guru/internal/adjudicate" "github.com/rossoctl/context-guru/internal/cheapmodel" "github.com/rossoctl/context-guru/internal/modelinfo" "github.com/rossoctl/context-guru/metrics" @@ -169,6 +170,14 @@ type Handler struct { agg *metrics.Aggregator opts Options client *http.Client + // expandSeen records which sessions have been offered the expand tool, so a call arriving on a + // later turn that does not advertise it is still intercepted rather than relayed to a client + // that has no such tool. See expandoffered.go. + expandSeen *expandOfferedSet + // sent holds the last body forwarded upstream per session, so a component can put a question to + // the request's own model with the bytes the provider's cache was populated from. See + // prefixask.go; bounded, and populated only when CONTEXT_GURU_PREFIX_ASK is on. + sent *sentStash // tracker owns the per-session cached-prefix boundary. Always present: every mode // benefits from reading and recording it in one locked step (the previous // read-then-deferred-write raced between concurrent turns of a session). @@ -218,7 +227,8 @@ func New(pipe *components.Pipeline, st store.Store, agg *metrics.Aggregator, opt c = &http.Client{Timeout: 5 * time.Minute} } h := &Handler{pipe: pipe, store: st, agg: agg, opts: opts, client: c, - tracker: modes.NewTracker(0), rec: opts.Dashboard} + tracker: modes.NewTracker(0), rec: opts.Dashboard, expandSeen: &expandOfferedSet{}, + sent: newSentStash()} if h.mode() == components.ModeObserve { h.pool = modes.NewPool(opts.Observe.MaxQueue, opts.Observe.Workers) h.shadow = store.NewMemory(store.Options{}) @@ -647,6 +657,12 @@ func (h *Handler) chat(provider bschemas.ModelProvider, static upstream, pick fu // panic anywhere here must forward the PRISTINE inbound body, never 500 the client. // apply.BodyFull has its own recover; this backstops expand.Inject and anything else. orig := body + // The pipeline's scoped session id, needed after the forward: the expand loop marks + // recovered content kept-verbatim under it, and that mark must use the SAME id the + // pipeline compacted under or the guard is written where nothing reads it. Only + // apply computes it (tenant + explicit header + session-head hash), so it comes back + // out through the trace rather than being recomputed here and risking divergence. + var sess string func() { defer func() { if rec := recover(); rec != nil { @@ -666,6 +682,38 @@ func (h *Handler) chat(provider bschemas.ModelProvider, static upstream, pick fu window: window, tn: tn, }) + sess = tr.Session + // RESTORE EXPAND RESULTS, after the pipeline and before the forward. + // + // The response loop can only satisfy an expand call that arrives ALONE: when the model + // also called a real tool the client must execute it, so the response is relayed and the + // loop bails on `otherTools`. Live traffic showed that is the dominant shape -- 102 of 107 + // refused turns carried two or more tool_use blocks, 5 carried one -- so the client + // answers the expand call itself with "Tool 'context_guru_expand' not found", the model + // loses recovery, and it re-runs the original tool instead. + // + // Rewriting the client's own tool_result on its way back upstream covers that case with + // no response splitting and nothing required of the client. AFTER the pipeline + // deliberately: restored content must not be handed straight back to the components that + // just cut it, which would compact it into another marker and another expand call. The + // kept-verbatim mark uses the pipeline's own session id for the same reason the expand + // loop does -- written under any other id, the guard sits where nothing reads it. + if !bypassed && tn.Store != nil { + if nb, restored := expand.RestoreResults(string(provider), body, tn.Store); len(restored) > 0 { + // A stray call the AGENT made to the maintenance tool gets a definite answer instead of the + // client's "tool not found". Same request-path shape, and necessary for the same reason: the + // model calls advertised tools it was told not to -- observed with expand at step 2 of a run. + if nb, answered := adjudicate.AnswerStrayCalls(string(provider), body); answered > 0 { + body = nb + } + body = nb + for _, hh := range restored { + if orig2, ok := expand.Resolve(tn.Store, hh); ok { + offload.MarkKeptVerbatim(tn.Store, sess, orig2) + } + } + } + } addedMs := float64(added.Microseconds()) / 1000.0 cp.noteCG(addedMs) cp.noteTrace(tr) @@ -690,9 +738,15 @@ func (h *Handler) chat(provider bschemas.ModelProvider, static upstream, pick fu im = expand.InjectAuto } body, _ = expand.Inject(string(provider), im, body, tn.Store.Persists()) + // The context-maintenance tool goes on EVERY request, not only when the pipeline is + // about to ask. `tools` hashes before system and messages, so a tool that comes and + // goes invalidates the prefix from position zero -- the flap expand's `always` mode + // exists to avoid. Declaring it is also what makes the model answer in a schema + // instead of prose; see internal/adjudicate. + body, _ = adjudicate.Inject(string(provider), body) } }() - h.serve(w, r, provider, up, body, bypassed, cp, tn) + h.serve(w, r, provider, up, body, bypassed, cp, tn, sess) } } @@ -732,7 +786,10 @@ var errNoUpstream = errors.New("no upstream configured") // → /stats sse_streamed / sse_buffered). It previously matched the expand tool // description this proxy injects itself, so it was unconditionally true and the // zero-added-latency promise above never held for any request (issue #26). -func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschemas.ModelProvider, up upstream, body []byte, bypassed bool, cp *capture, tn *Tenancy) { +// sess is the pipeline's scoped session id (empty when the pipeline never ran, e.g. observe +// mode or a request with no messages). The expand loop needs it to scope the kept-verbatim +// guard; nothing else on this path depends on it. +func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschemas.ModelProvider, up upstream, body []byte, bypassed bool, cp *capture, tn *Tenancy, sess string) { // ONE condition governs both halves of the loop: the tool is intercepted exactly when // it is advertised on the outgoing request. Those used to be different conditions — // advertised when the request had tools, intercepted (for SSE) when it had markers — @@ -742,7 +799,25 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschema // // Since injection now requires markers (expand.Inject, InjectAuto), this keeps the // documented fast path: no offload yet → no tool → no buffering, zero added latency. + // ADVERTISED vs INTERCEPTED, and why these are no longer the same condition. + // + // They were deliberately unified: never declare a tool whose use you will not handle. But the + // unification assumed the model only calls tools listed on the CURRENT request, and it does not. + // A model that saw the tool on turn N calls it on turn N+1, and under InjectAuto turn N+1 often + // carries no marker, so the tool is not advertised, nothing intercepts, and the raw tool_use is + // relayed to a client that has no such tool. Measured on LOCA: `Tool 'context_guru_expand' not + // found` on 17 of 38 attempts in one arm and 48 of 108 in another, after which the model gave up + // on recovery and re-ran the original tool instead. + // + // So interception now also covers any session that has EVER been offered the tool. The fast path + // survives for sessions that never were: no offload, no advertisement, no inspection, no + // buffering. Once a session has seen it, inspecting that session's responses is exactly the cost + // of being able to answer a call it was invited to make. advertised := expand.HasTool(string(provider), body) + if advertised && sess != "" { + h.noteExpandOffered(sess) + } + mayCall := advertised || (sess != "" && h.expandOffered(sess)) // SSE accounting is PER CLIENT REQUEST, not per upstream round: one client request // that drives several expand rounds waited for all of them, so timing a single // round would report a healthy TTFB for a client that waited three round-trips. @@ -766,9 +841,18 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschema } cp.finish(usage, usageOK, h.captureContentFor(tn), h.contentCap(), h.contentMax()) }() + zeroResolved := 0 // rounds where no id resolved; capped so placeholders cannot loop for round := 0; ; round++ { upStart := time.Now() resp, err := h.doUpstream(r, up, body) + if err == nil && resp.StatusCode == http.StatusOK { + // The bytes the provider's prompt cache is now populated from. Stashed here rather than + // anywhere earlier because only what was ACTUALLY forwarded is a valid prefix -- an + // aborted or rejected send caches nothing. See prefixask.go. + if prefixAskEnabled() { + h.sent.put(sess, body) + } + } if err != nil { // LOG it, and record it on the captured row. An upstream failure used to be // invisible in both places: the caller got a 502 and the operator got nothing @@ -803,7 +887,20 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschema // hit the round cap. Nothing else can produce a call to it, so this is both the // necessary and the sufficient condition — and for SSE it is what decides whether we // pay the buffering cost. - checkExpand := advertised && round < maxExpandRounds + // Whether to inspect this response for an expand call. + // + // mayCall covers "this request advertised the tool, or an earlier turn of this session did". + // But a model may call the tool on a turn where neither holds -- it remembers the tool from + // its own history -- and relaying that call to a client that does not implement it is the + // worst outcome available (measured: 48 of 108 attempts refused, after which the model + // abandons recovery and re-runs the original tool). + // + // So for a NON-STREAMING response, always inspect. The cost is nil: a JSON body is read in + // full either way, so there is nothing to buffer and no latency to add. The advertise/session + // gate is kept only for SSE, where inspecting means buffering the whole stream and the client + // genuinely loses its incremental output -- which is the cost expand/inject.go weighed, and it + // applies to streaming alone. + checkExpand := (mayCall || !isSSE) && round < maxExpandRounds if !checkExpand { // sseBuffered is sticky: if an earlier round was buffered the client already // lost its stream, so this request counts as buffered however it ends. @@ -858,7 +955,7 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschema got++ // The agent needed this content back — don't re-compact it on later turns // (that would loop it straight back into another expand). Keep it verbatim. - offload.MarkKeptVerbatim(tn.Store, orig) + offload.MarkKeptVerbatim(tn.Store, sess, orig) back := schema.TextTokens(orig) if h.agg != nil { h.agg.RecordExpand(back) // bounce: offload had to come back @@ -869,11 +966,33 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschema } } next, ok := expand.Continuation(string(provider), body, msg, resolved) - if got == 0 || !ok { - writeRaw(w, resp, respBody) // nothing recovered; return the model's own call + if !ok { + // The continuation could not be BUILT — an internal failure, not a miss. Replaying is + // the honest fail-open: we cannot construct a valid next request. + writeRaw(w, resp, respBody) return } - body = next // loop: re-invoke with the originals in hand + if got == 0 { + // NOTHING RESOLVED. This used to replay the model's own tool_use to the client, which is + // the worst available option: a client that does not implement this tool answers "Tool + // 'context_guru_expand' not found", the model loses its recovery path, and it re-runs the + // original tool instead — paying a full tool execution plus fresh output, and enlarging + // the very transcript that provoked the cut. + // + // Answer the model ourselves instead. `resolved` already carries an explicit placeholder + // per call id (see above), so the continuation is well formed and tells the model the + // content is gone — information it can act on, unlike a missing tool. + // + // Capped at ONE such round. A model that asks again gets the same placeholders, so + // continuing indefinitely would burn a round-trip per attempt for no new information; + // after one, fall through and replay. + if zeroResolved > 0 { + writeRaw(w, resp, respBody) + return + } + zeroResolved++ + } + body = next // loop: re-invoke with the originals (or the placeholders) in hand } } @@ -1025,6 +1144,14 @@ func (h *Handler) stats(w http.ResponseWriter, r *http.Request) { // indistinguishable from a run that had little to compact — the arm reads as fast // because it silently stopped working. llm_call_timeout_ms travels with the counts // because a timeout total is meaningless without the budget it was measured against. + // Unresolved expand calls, split by cause. `expand_unresolved_missing` is the one that matters: + // a well-formed marker with nothing stashed behind it is a cut that was advertised as reversible + // and is not. It was invisible before, which is how three iterations ran while the model was + // being refused recovery — the failure was only found by grepping the benchmark client's own + // transcripts. See expand/unresolved.go. + snap.ExpandRestored = expand.Restored() + snap.AdjudicateStray = adjudicate.StrayAnswered() + snap.ExpandUnresolvedMalformed, snap.ExpandUnresolvedMissing = expand.Unresolved() snap.LLMTimeouts = offload.LLMTimeouts() snap.LLMErrors = offload.LLMErrors() snap.LLMCallTimeoutMs = offload.LLMCallTimeout().Milliseconds() diff --git a/proxy/stats_golden_test.go b/proxy/stats_golden_test.go index 8676e0a2..919c164b 100644 --- a/proxy/stats_golden_test.go +++ b/proxy/stats_golden_test.go @@ -26,6 +26,20 @@ var statsGoldenTopLevel = []string{ "cache_write_tokens", "cg_added_ms_avg", "compaction_resets", + // Unresolved expand calls, split by cause. expand_unresolved_missing is a well-formed marker with + // nothing stashed behind it, i.e. a cut advertised as reversible that is not; malformed is the + // model inventing an id. Added because the refusal of expand calls was previously invisible here + // and had to be found by grepping a benchmark client's transcripts. + // expand_restored is the counter that distinguishes a working recovery loop from a silent + // no-op; the client always refuses a proxy-injected tool, so only this says the substitution + // happened before the request went upstream. + "expand_restored", + // adjudicate_stray counts tool_results rewritten because the AGENT called the injected + // context-maintenance tool. A rising rate means the tool's "do not call this yourself" description + // is not working and the agent is losing turns to it. + "adjudicate_stray", + "expand_unresolved_malformed", + "expand_unresolved_missing", "components", "extract", "fresh_input_tokens", diff --git a/schema/validate.go b/schema/validate.go new file mode 100644 index 00000000..bacb22f7 --- /dev/null +++ b/schema/validate.go @@ -0,0 +1,131 @@ +package schema + +import ( + "fmt" + + schemas "github.com/maximhq/bifrost/core/schemas" +) + +// Static validation of a message list against the provider's message-SHAPE rules — the +// rules that make a request well-formed regardless of its content. +// +// WHY THIS EXISTS. Three separate shape violations shipped in `summarize` and were caught +// only by sending live traffic to a real API, one after another, each masked by the one +// before it: +// +// 400 messages.1: role 'system' must precede an 'assistant' message or end the array +// 400 messages.N.content.M: unexpected `tool_use_id` found in `tool_result` blocks +// 400 messages.N: `tool_use` ids were found without `tool_result` blocks immediately after +// +// None was findable by the project's existing methods. No test asserted them, and every +// offline measurement replayed through the `/compact` endpoint, which runs the pipeline and +// returns the rewritten body WITHOUT forwarding it upstream — so no provider ever validated +// it. Replay can tell you what a component removed; it is structurally incapable of telling +// you whether the result is a sendable request. +// +// All three are checkable statically, with no provider and no model. That is what this is +// for: a component that mutates a message list can be asserted well-formed in a unit test, +// closing the blind spot without paying for live traffic on every change. +// +// DELIBERATELY NOT A REQUEST VALIDATOR. It checks shape invariants that hold across +// providers with an Anthropic-style tool protocol; it does not check token limits, model +// names, sampling parameters, or anything content-dependent. + +// ShapeViolation is one broken invariant, phrased to point at the message that broke it. +type ShapeViolation struct { + Index int // message index, or -1 when the violation is about the list as a whole + Rule string // short, stable identifier for the invariant + Msg string // human-readable, mirroring the provider's own wording where possible +} + +func (v ShapeViolation) String() string { + if v.Index < 0 { + return fmt.Sprintf("[%s] %s", v.Rule, v.Msg) + } + return fmt.Sprintf("messages.%d [%s] %s", v.Index, v.Rule, v.Msg) +} + +// ValidateShape reports every message-shape invariant the list breaks. An empty result +// means the list is well-formed in the ways a provider enforces structurally. +// +// The invariants, and why each one exists: +// +// 1. system-position — a system-role message may appear only at index 0. Providers expect +// system content in a dedicated top-level field; one spliced mid-array is rejected. +// This is the defect that made `summarize` unusable on every single call. +// 2. answered-tool-use — every tool call must be answered by a result in the NEXT message. +// Removing a span can delete the answer while keeping the call. +// 3. paired-tool-result — every result must answer a call that appeared earlier. Removing a +// span can delete the call while keeping the answer. The mirror of (2), and the reason +// both are checked: fixing one alone leaves the other live, which is exactly what +// happened here. +// +// (2) requires the answer in the immediately following message, matching the provider's +// wording. (3) permits a call at any earlier position, because a legitimately inserted +// summary may sit between a call and its result. +func ValidateShape(msgs []schemas.ChatMessage) []ShapeViolation { + var out []ShapeViolation + seenCall := map[string]bool{} + + for i := range msgs { + m := msgs[i] + + if m.Role == schemas.ChatMessageRoleSystem && i != 0 { + out = append(out, ShapeViolation{Index: i, Rule: "system-position", + Msg: "role 'system' must precede an 'assistant' message or end the array"}) + } + + // A result must answer a call seen earlier (possibly at a distance). + if m.Role == schemas.ChatMessageRoleTool && m.ChatToolMessage != nil && + m.ChatToolMessage.ToolCallID != nil { + if id := *m.ChatToolMessage.ToolCallID; !seenCall[id] { + out = append(out, ShapeViolation{Index: i, Rule: "paired-tool-result", + Msg: fmt.Sprintf("unexpected `tool_use_id` found in `tool_result` blocks: %s", id)}) + } + } + + if m.ChatAssistantMessage == nil || len(m.ChatAssistantMessage.ToolCalls) == 0 { + continue + } + var ids []string + for _, tc := range m.ChatAssistantMessage.ToolCalls { + if tc.ID != nil { + seenCall[*tc.ID] = true + ids = append(ids, *tc.ID) + } + } + // Every call must be answered before any non-tool message intervenes. + // + // PARALLEL CALLS. One assistant message may carry several tool_use blocks, and Anthropic + // requires every result in the single user message that follows. bifrost's schema, though, + // represents each result as its OWN role=tool message, so the wire's one-user-message maps + // to a RUN of consecutive tool messages here. Inspecting only msgs[i+1] therefore reported + // a violation on every ordinary parallel exchange -- the second call always looked + // unanswered: + // + // messages.1 [answered-tool-use] ... without `tool_result` blocks immediately after: call_b + // + // which is indistinguishable from the real defect being hunted, and would have made this + // validator useless precisely where it was needed. Scanning the whole run of consecutive + // tool messages is the representation-correct reading; the invariant is unchanged, because + // the run must still be contiguous -- any other role in between ends it. + answered := map[string]bool{} + for j := i + 1; j < len(msgs); j++ { + n := msgs[j] + if n.Role != schemas.ChatMessageRoleTool { + break + } + if n.ChatToolMessage != nil && n.ChatToolMessage.ToolCallID != nil { + answered[*n.ChatToolMessage.ToolCallID] = true + } + } + for _, id := range ids { + if !answered[id] { + out = append(out, ShapeViolation{Index: i, Rule: "answered-tool-use", + Msg: fmt.Sprintf("`tool_use` ids were found without `tool_result` blocks "+ + "immediately after: %s", id)}) + } + } + } + return out +} diff --git a/store/store.go b/store/store.go index 08c55be2..2dd1614d 100644 --- a/store/store.go +++ b/store/store.go @@ -62,15 +62,24 @@ type FrozenLoser interface { // The prefixes are declared by their OWNERS (components/offload, apply) and passed in via // Options.PinPrefixes — the store must not know what a component names its keys. const ( - FrozenPrefix = "cg:frz:" // mask / failed_run freeze decisions - ResultPrefix = "cg:res:" // extract_llm's replayed result (projection + summary, one key) - LenPrefix = "cg:len:" // apply's prev-turn message count (the MaxCachedIdx boundary) + FrozenPrefix = "cg:frz:" // mask / failed_run freeze decisions + ResultPrefix = "cg:res:" // extract_llm's replayed result (projection + summary, one key) + LenPrefix = "cg:len:" // apply's prev-turn message count (the MaxCachedIdx boundary) + KeptPrefix = "cg:keep:" // content the agent expanded: never re-compact it ) // DefaultPinPrefixes is the shipped set of key namespaces whose loss is cache-destructive. // Callers that build their own Store may pass a different set; the zero value means "none", // so a host that opts out simply gets plain TTL+LRU. -var DefaultPinPrefixes = []string{FrozenPrefix, ResultPrefix, LenPrefix} +// +// KeptPrefix belongs here by the same criterion as the rest, which is easy to miss because +// its payload is one byte. Losing it does not lose data — it loses the FACT that the agent +// already asked for this content back, so the next turn re-compacts it, the agent expands it +// again, and every turn thereafter pays a round-trip plus a fresh cache-write. That is the +// expand loop the flag exists to prevent, and it is cache-destructive in exactly the sense +// this namespace is for. Before it was pinned, a one-byte guard competed for LRU capacity +// against the multi-kilobyte rewind stashes it guards, and lost. +var DefaultPinPrefixes = []string{FrozenPrefix, ResultPrefix, LenPrefix, KeptPrefix} // pinned reports whether key belongs to one of the configured pin namespaces. func (m *Memory) isPinPrefix(key string) bool {