From 1270e86842818bb3f7b926911a0aab057e3cf3c3 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Thu, 17 Sep 2026 08:50:44 +0200 Subject: [PATCH 1/4] feat(combo): add agentic-stall response classifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a standalone, side-effect-free classifier that detects when a model answers a tool-result turn with prose instead of the tool call the loop is waiting for — the "agentic stall" that leaves an agent hanging until timeout. This commit is the classifier and its tests ONLY. It is not wired into any dispatch path here: the wiring in services/combo.ts depends on refactors that do not exist on this branch, so porting it would mean rewriting unrelated code. Landing the pure logic first keeps this reviewable and lets wiring follow in a separate PR. Design notes: - Pure functions, zero imports; never mutates or consumes the caller's Response (clone-only contract, covered by a test). - Returns null on anything it cannot confidently parse, so an unknown content-type can never break the request path. - Kill switch: OMNIROUTE_AGENTIC_STALL_FAILOVER=0 disables classification. Two exemptions prevent false positives observed in production: - Intentional silence: an exact "[SILENT]" (or blocked-state) reply is a valid answer, not a stall. - Instructed terminal echo: when the request itself instructs the model to reply with a specific short literal, echoing it verbatim is compliance. Matching is verbatim and length-capped, so a drifted paraphrase still classifies as a stall. Tests: 64 cases over both providers x JSON/SSE, both exemptions, the kill switch, the null-safety path and the no-consume contract. --- open-sse/services/combo/agenticStall.ts | 442 +++++++++++++++++ tests/unit/agentic-stall.test.ts | 605 ++++++++++++++++++++++++ 2 files changed, 1047 insertions(+) create mode 100644 open-sse/services/combo/agenticStall.ts create mode 100644 tests/unit/agentic-stall.test.ts diff --git a/open-sse/services/combo/agenticStall.ts b/open-sse/services/combo/agenticStall.ts new file mode 100644 index 000000000000..62af8db09b5c --- /dev/null +++ b/open-sse/services/combo/agenticStall.ts @@ -0,0 +1,442 @@ +/** + * Agentic-stall failover guard. + * + * Live defect: some upstream providers (reproduced across more than one + * model and provider family) + * answer a MID-AGENTIC-TURN request — tools present, conversation tail a tool + * result — with `finish: stop` + a summary/narration as content and ZERO tool + * calls. Agent harnesses read stop+no-tool-call as turn-over; + * the user re-prompts "continue"; the model summarizes again — an infinite + * loop with zero progress. Every quality gate passes the response because it + * is a syntactically valid, non-empty completion. + * + * Classification (ALL required, conservative — a false positive costs one + * retry, a false negative costs the loop): + * 1. The request had `tools` AND the conversation tail is a tool result + * (OpenAI: last message role "tool"; Anthropic: last user message carries + * a tool_result block) — i.e. a tool call was expected next. + * 2. The response finished with stop ("stop" / "end_turn") and contains NO + * tool calls. + * 3. The text content matches the stall shape: a summary/narration block + * (`…`, `# Summary`, `Summary: …`) or short structureless prose + * that reports progress or a no-op instead of answering ("Silent retry + * succeeded"). A literal the request itself requested is exempt — see below. + * + * Three terminal-protocol shapes are EXEMPT because their response is turn-over + * by design, not the summarization defect this guard exists to catch: + * - a genuine blocked-state report ("Blocked: #12345 still OPEN"), + * - an intentional-silence token ("[SILENT]" / "NO_REPLY") — see + * `isIntentionalSilenceNarration`, and + * - a literal the REQUEST ITSELF quoted as the required terminal reply + * ("just say 'Nothing to save.' and stop") — see + * `isInstructedTerminalEcho`. + * + * The guard is provider-agnostic: it keys on request/response SHAPE, never on + * model id (same lesson as the kimiToolCallNarration recovery — a model-id + * gate silently skips every future imitator). + * + * The classifier reads the response via clone() ONLY — never disturbs the + * body the client pipeline still needs (same contract as + * applyComboStepResponseGuards). It must NEVER throw or break the request + * path: every failure mode returns null (not a stall). + * + * Kill switch: OMNIROUTE_AGENTIC_STALL_FAILOVER=0 disables classification + * (default: enabled). + */ + +export const AGENTIC_STALL_SIGNATURE = "agentic_stall_no_toolcall"; + +/** Cap on bytes read from a cloned body while classifying — a pathological + * multi-MB stream must not stall the combo loop on inspection. */ +const MAX_STALL_INSPECT_BYTES = 1024 * 1024; + +export function isAgenticStallFailoverEnabled(): boolean { + return process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER !== "0"; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** + * Condition 1: tools were offered AND the conversation tail is a tool result, + * so the next model action was expected to be a tool call (or a substantive + * answer informed by one) — not a bare summary. + */ +export function requestExpectsToolCall(body: unknown): boolean { + if (!isRecord(body)) return false; + const tools = body.tools; + if (!Array.isArray(tools) || tools.length === 0) return false; + const messages = Array.isArray(body.messages) ? body.messages : null; + if (!messages || messages.length === 0) return false; + const last = messages[messages.length - 1]; + if (!isRecord(last)) return false; + // OpenAI chat shape: tool results arrive as role:"tool" messages. + if (last.role === "tool") return true; + // Anthropic shape: tool results are tool_result blocks inside a user message. + if (last.role === "user" && Array.isArray(last.content)) { + return last.content.some((blk) => isRecord(blk) && blk.type === "tool_result"); + } + return false; +} + +/** Summary/narration openers observed in the wild ("…" context- + * compaction blocks, markdown "Summary" headings, "Summary: …" prose). */ +const SUMMARY_PREFIX = + /^\s*(?:]|#{1,6}\s+(?:conversation\s+|context\s+)?summary\b|summary\s*[:—–-])/i; + +/** Structure that marks a substantive answer (code, lists, tables, headings). + * Its presence keeps the short-prose heuristic from firing. */ +const STRUCTURE_MARKERS = /```|^\s*(?:[-*•+]|\d+[.)])\s|^\s*#{1,6}\s|\|\s*-{2,}/m; + +/** Max length for the "short structureless narration" branch. Kept small: + * longer prose is a legitimate final answer unless it opens as a summary. + * Lane-instructed terminal literals ("Nothing to save.") are handled earlier by + * `isInstructedTerminalEcho`, so this branch is now genuinely about drift. */ +const SHORT_NARRATION_MAX_CHARS = 240; + +/** Terminal blocked-state openers: the model did its tool work and is reporting + * a genuine blocker as its final answer ("Blocked: #12345 still OPEN...", + * "Cannot proceed — CI red"). This is turn-over BY DESIGN, not a stall; + * failing over re-runs the same blocked task on the next member and each + * member re-verifies and re-reports. Exempt before the short-prose branch. */ +const BLOCKED_STATE_OPENER = + /^\s*(?:still\s+|attempt\s+\d+[^\w\s]*(?:\s+blocked)?[.:]?\s+|blocked\s+again[.:]?\s+|blocked[.:,\s]|cannot\s+proceed[\s:—–-]|unable\s+to\s+(?:proceed|continue|complete)[\s:—–-]|no\s+(?:path|way)\s+forward[\s:—–-])/i; + +/** Intentional-silence control tokens emitted by autonomous agent harnesses + * (cron/webhook lanes, bot mode) to suppress delivery when there is nothing + * to report. Mirrors the common harness marker set so the gateway and the + * harness cannot drift on what counts as silence. Lower-cased; matching is + * case-folded. */ +const SILENCE_MARKERS = new Set([ + "[silent]", + "silent", + "no_reply", + "no reply", + "[静默]", + "静默", + "[沉默]", + "沉默", +]); + +/** Bracketed marker OPENING the response — the autonomous lane's own prefix + * rule, where "[SILENT] No changes detected this tick." still suppresses + * delivery. Bracketed only: bare "Silent retry succeeded" must stay + * classifiable as narration. */ +const SILENCE_MARKER_OPENER = /^\s*\[(?:silent|静默|沉默)\]/i; + +/** Drop stray edge punctuation (".NO_REPLY", "*SILENT*") while keeping brackets + * structural, so a malformed "[SILENT" cannot become a marker. */ +function stripEdgeSilencePunctuation(text: string): string { + const keep = /[\p{L}[\]]/u; + let start = 0; + let end = text.length; + while (start < end && !keep.test(text[start])) start++; + while (end > start && !keep.test(text[end - 1])) end--; + return text.slice(start, end); +} + +function isSilenceMarker(text: string): boolean { + const collapsed = text.trim().replace(/\s+/g, " "); + if (!collapsed) return false; + return ( + SILENCE_MARKERS.has(collapsed.toLowerCase()) || + SILENCE_MARKERS.has(stripEdgeSilencePunctuation(collapsed).toLowerCase()) + ); +} + +/** True when the response IS an intentional-silence token — a harness + * delivery-suppression control token, i.e. turn-over BY DESIGN. + * + * Live defect: a recurring cron job's tool result reported a clean sweep with + * zero items needing action; every combo member then answered exactly + * `[SILENT]` — the run's documented "nothing to report" reply. The guard read + * stop+no-tool-call+short narration as a stall, failed over through every + * member, and the client got HTTP 502 on a healthy, deterministic turn (it + * reproduces on every run with nothing to report). Same exemption class as + * BLOCKED_STATE_OPENER: a deliberate terminal token, not summarization drift. + * + * Exempt exactly what the harness itself treats as silence (whole response, or + * its own first/last line, or a bracketed opener) — no wider. */ +export function isIntentionalSilenceNarration(text: string): boolean { + const t = (text || "").trim(); + if (!t) return false; + if (SILENCE_MARKER_OPENER.test(t)) return true; + const lines = t + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + if (lines.length === 0) return false; + return ( + isSilenceMarker(t) || isSilenceMarker(lines[0]) || isSilenceMarker(lines[lines.length - 1]) + ); +} + +/** Max length of a response that can count as an instructed terminal echo. The + * lane briefs that produce these are one-line control replies; anything longer + * is prose, and prose drift is the defect this guard exists to catch. */ +const INSTRUCTED_TERMINAL_MAX_CHARS = 120; + +/** Literal text a request quoted as a required reply, e.g. a + * background-review lane brief: "just say 'Nothing to save.' and stop." + * + * Quotes must be single/double/backtick and contain no sentence punctuation of + * its own, so prose quotes in a brief ("say 'unable to proceed'") cannot turn a + * whole paragraph into an exempt literal. Returns lower-cased candidates. */ +function quotedTerminalLiterals(body: unknown): Set { + const found = new Set(); + if (!isRecord(body)) return found; + const messages = Array.isArray(body.messages) ? body.messages : []; + const push = (raw: string) => { + const lit = raw.trim().replace(/\s+/g, " "); + if (!lit || lit.length > INSTRUCTED_TERMINAL_MAX_CHARS) return; + found.add(lit.toLowerCase()); + }; + for (const msg of messages) { + if (!isRecord(msg)) continue; + const contents = Array.isArray(msg.content) ? msg.content : [msg.content]; + for (const part of contents) { + let text = ""; + if (typeof part === "string") text = part; + else if (isRecord(part)) { + if (typeof part.text === "string") text = part.text; + else if (typeof part.content === "string") text = part.content; + } + if (!text) continue; + // Quoted literals: '...' | "..." | `...`. The 120 cap mirrors + // INSTRUCTED_TERMINAL_MAX_CHARS — longer quotes are never echoed whole. + for (const m of text.matchAll(/['"`]([^'"`\n]{1,120})['"`]/g)) push(m[1]); + } + } + return found; +} + +/** True when the response is EXACTLY a literal the request itself told the model + * to reply with. This is a lane-instructed terminal token — turn-over by design, + * like the silence markers but without a fixed global vocabulary, so new lanes + * ("Nothing to save.", "[No output requested]") are covered without a code + * change per lane. + * + * Live defect: a background-review lane briefs the model "If nothing is worth + * saving, just say 'Nothing to save.' and stop." Every combo member answered + * exactly that; the guard read stop + no-tool-call + short prose as a stall + * and failed the turn over through the whole combo. + * + * Deliberately narrow — requires a verbatim, case-insensitive, whole-response + * match against a literal present in the request, under a length cap. A model + * drifting into a summary cannot satisfy it: summarization text is not quoted + * in the brief. If the brief contains no literal, this returns false and the + * normal short-prose/gated path is unchanged. */ +export function isInstructedTerminalEcho(body: unknown, text: string): boolean { + const t = (text || "").trim().replace(/\s+/g, " "); + if (!t || t.length > INSTRUCTED_TERMINAL_MAX_CHARS) return false; + const literals = quotedTerminalLiterals(body); + if (literals.size === 0) return false; + return literals.has(t.toLowerCase()); +} + +export function contentLooksLikeStallNarration(text: string): boolean { + const t = (text || "").trim(); + if (!t) return false; + if (SUMMARY_PREFIX.test(t)) return true; + if (BLOCKED_STATE_OPENER.test(t)) return false; // terminal answer, not a stall + if (isIntentionalSilenceNarration(t)) return false; // delivery-suppression token, not a stall + if (t.length <= SHORT_NARRATION_MAX_CHARS && !STRUCTURE_MARKERS.test(t) && !t.includes("\n\n")) { + return true; + } + return false; +} + +type StallSignal = { + finishReason: string | null; + hasToolCalls: boolean; + text: string; +}; + +function extractFromOpenAiJson(parsed: unknown): StallSignal | null { + if (!isRecord(parsed) || !Array.isArray(parsed.choices)) return null; + const choice = parsed.choices[0]; + if (!isRecord(choice)) return null; + const message = isRecord(choice.message) ? choice.message : null; + const toolCalls = message && Array.isArray(message.tool_calls) ? message.tool_calls : []; + let text = ""; + if (message) { + if (typeof message.content === "string") text = message.content; + else if (Array.isArray(message.content)) { + text = message.content + .map((part) => (isRecord(part) && typeof part.text === "string" ? part.text : "")) + .join(""); + } + } + return { + finishReason: typeof choice.finish_reason === "string" ? choice.finish_reason : null, + hasToolCalls: toolCalls.length > 0, + text, + }; +} + +function extractFromAnthropicJson(parsed: unknown): StallSignal | null { + if (!isRecord(parsed) || !Array.isArray(parsed.content)) return null; + if (parsed.type !== "message" && typeof parsed.stop_reason !== "string") return null; + let text = ""; + let hasToolCalls = false; + for (const blk of parsed.content) { + if (!isRecord(blk)) continue; + if (blk.type === "tool_use") hasToolCalls = true; + if (blk.type === "text" && typeof blk.text === "string") text += blk.text; + } + return { + finishReason: typeof parsed.stop_reason === "string" ? parsed.stop_reason : null, + hasToolCalls, + text, + }; +} + +/** Assemble the terminal assistant message from a buffered SSE payload, + * handling both OpenAI chat chunks and Anthropic message events. Returns + * null when nothing recognizable was seen (cannot classify). */ +function extractFromSse(payload: string): StallSignal | null { + let finishReason: string | null = null; + let hasToolCalls = false; + let text = ""; + let recognized = false; + for (const rawLine of payload.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line.startsWith("data:")) continue; + const data = line.slice(5).trim(); + if (!data || data === "[DONE]") continue; + let parsed: unknown; + try { + parsed = JSON.parse(data); + } catch { + continue; + } + if (!isRecord(parsed)) continue; + // OpenAI chat chunks. + if (Array.isArray(parsed.choices)) { + const choice = parsed.choices[0]; + if (isRecord(choice)) { + recognized = true; + const delta = isRecord(choice.delta) ? choice.delta : null; + if (delta) { + if (typeof delta.content === "string") text += delta.content; + if (Array.isArray(delta.tool_calls) && delta.tool_calls.length > 0) { + hasToolCalls = true; + } + } + if (typeof choice.finish_reason === "string" && choice.finish_reason) { + finishReason = choice.finish_reason; + } + } + continue; + } + // Anthropic message events. + const type = typeof parsed.type === "string" ? parsed.type : ""; + if (type === "content_block_start") { + recognized = true; + const block = isRecord(parsed.content_block) ? parsed.content_block : null; + if (block && block.type === "tool_use") hasToolCalls = true; + } else if (type === "content_block_delta") { + recognized = true; + const delta = isRecord(parsed.delta) ? parsed.delta : null; + if (delta && delta.type === "text_delta" && typeof delta.text === "string") { + text += delta.text; + } + } else if (type === "message_delta") { + recognized = true; + const delta = isRecord(parsed.delta) ? parsed.delta : null; + if (delta && typeof delta.stop_reason === "string" && delta.stop_reason) { + finishReason = delta.stop_reason; + } + } else if (type === "message_start" || type === "message_stop") { + recognized = true; + } + } + if (!recognized) return null; + return { finishReason, hasToolCalls, text }; +} + +async function readClonedBodyText(clone: Response): Promise { + try { + if (!clone.body) return null; + const reader = clone.body.getReader(); + const decoder = new TextDecoder(); + let text = ""; + let bytes = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (value) { + bytes += value.byteLength; + if (bytes > MAX_STALL_INSPECT_BYTES) { + await reader.cancel().catch(() => {}); + return null; // too large to safely classify — treat as not-a-stall + } + text += decoder.decode(value, { stream: true }); + } + } + text += decoder.decode(); + return text; + } catch { + return null; + } +} + +export type AgenticStallVerdict = { + signature: typeof AGENTIC_STALL_SIGNATURE; + detail: string; +}; + +/** + * Classify a successful (HTTP 200, quality-valid) combo-member response as an + * agentic stall. Returns the verdict (with a log/audit-safe detail string — + * no credentials, no full bodies) or null when the response is fine or cannot + * be classified. Never throws. + */ +export async function classifyAgenticStallResponse(args: { + body: unknown; + response: Response; +}): Promise { + try { + if (!isAgenticStallFailoverEnabled()) return null; + const { body, response } = args; + if (!requestExpectsToolCall(body)) return null; + if (!response || typeof response.clone !== "function") return null; + let clone: Response; + try { + clone = response.clone(); + } catch { + return null; + } + const ct = clone.headers?.get?.("content-type") || ""; + let signal: StallSignal | null = null; + if (ct.includes("text/event-stream")) { + const text = await readClonedBodyText(clone); + if (text == null) return null; + signal = extractFromSse(text); + } else if (ct.includes("application/json")) { + let parsed: unknown; + try { + parsed = await clone.json(); + } catch { + return null; + } + signal = extractFromOpenAiJson(parsed) ?? extractFromAnthropicJson(parsed); + } else { + return null; + } + if (!signal) return null; + const finish = (signal.finishReason || "").toLowerCase(); + if (finish !== "stop" && finish !== "end_turn") return null; + if (signal.hasToolCalls) return null; + if (isInstructedTerminalEcho(body, signal.text)) return null; + if (!contentLooksLikeStallNarration(signal.text)) return null; + const head = signal.text.trim().slice(0, 120).replace(/\s+/g, " "); + return { + signature: AGENTIC_STALL_SIGNATURE, + detail: `finish=${signal.finishReason}, no tool_calls on tool-result tail, narration="${head}"`, + }; + } catch { + return null; + } +} diff --git a/tests/unit/agentic-stall.test.ts b/tests/unit/agentic-stall.test.ts new file mode 100644 index 000000000000..c162eac89842 --- /dev/null +++ b/tests/unit/agentic-stall.test.ts @@ -0,0 +1,605 @@ +/** + * tests/unit/agentic-stall.test.ts + * + * The agentic-stall guard classifies a quality-valid 200 as a mid-turn stall + * (stop + summary/narration, no tool call, tool-result tail) so the combo loop + * fails over instead of returning the stall to the client. These tests pin: + * - the request-shape gate (tools + tool-result tail, both API shapes), + * - the narration heuristic (summary markers + short structureless prose), + * - full classification over OpenAI/Anthropic × JSON/SSE response shapes, + * - the positive controls (no tools pending / real tool call / structured + * answer are NEVER classified), + * - the env kill switch. + */ +import { test, describe, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; + +import { + AGENTIC_STALL_SIGNATURE, + classifyAgenticStallResponse, + contentLooksLikeStallNarration, + isInstructedTerminalEcho, + isIntentionalSilenceNarration, + requestExpectsToolCall, +} from "../../open-sse/services/combo/agenticStall.ts"; + +const TOOLS = [{ type: "function", function: { name: "read_file" } }]; + +function openAiToolTailBody() { + return { + model: "some/model", + tools: TOOLS, + messages: [ + { role: "user", content: "fix the bug" }, + { role: "assistant", content: null, tool_calls: [{ id: "c1" }] }, + { role: "tool", tool_call_id: "c1", content: "file contents…" }, + ], + }; +} + +function anthropicToolTailBody() { + return { + model: "some/model", + tools: TOOLS, + messages: [ + { role: "user", content: "fix the bug" }, + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "t1", content: "file contents…" }], + }, + ], + }; +} + +function jsonResponse(payload: unknown): Response { + return new Response(JSON.stringify(payload), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +function sseResponse(payload: string): Response { + return new Response(payload, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +const STALL_TEXT = + "\n## Overview\nThe session expanded from a changelog review into a full sweep of open PRs…\n"; + +describe("requestExpectsToolCall", () => { + test("OpenAI shape: tools + role:tool tail → true", () => { + assert.equal(requestExpectsToolCall(openAiToolTailBody()), true); + }); + + test("Anthropic shape: tools + tool_result tail → true", () => { + assert.equal(requestExpectsToolCall(anthropicToolTailBody()), true); + }); + + test("no tools → false (positive control: plain question)", () => { + assert.equal(requestExpectsToolCall({ messages: [{ role: "tool", content: "x" }] }), false); + }); + + test("tools but tail is a plain user message → false", () => { + assert.equal( + requestExpectsToolCall({ + tools: TOOLS, + messages: [{ role: "user", content: "what is 2+2?" }], + }), + false + ); + }); + + test("Anthropic tools but tail user text-only → false", () => { + assert.equal( + requestExpectsToolCall({ + tools: TOOLS, + messages: [{ role: "user", content: [{ type: "text", text: "continue" }] }], + }), + false + ); + }); + + test("garbage body → false", () => { + assert.equal(requestExpectsToolCall(null), false); + assert.equal(requestExpectsToolCall("nope"), false); + assert.equal(requestExpectsToolCall({ tools: TOOLS, messages: [] }), false); + }); +}); + +describe("contentLooksLikeStallNarration", () => { + // ── blocked-state exemption: terminal answers, not stalls ── + // Live prod specimens: three combo members each re-verified and reported a + // genuine blocker; the guard failed each over and the next member re-ran the + // same blocked task. + describe("blocked-state openers are NOT stalls", () => { + for (const [name, text] of [ + ["blocked opener", "Blocked: #12345 still OPEN, no merged fix PR (verified via gh)"], + ["still blocked", "Still blocked: #12345 remains OPEN with no merged fix PR"], + [ + "blocked again w/ attempt", + "Blocked again (attempt 5): acme/widget#12345 remains OPEN with no merged fix PR, so there is nothing new to verify.", + ], + ["attempt-prefixed", "Attempt 3: blocked. #12345 is still OPEN with no merged fix PR"], + ["cannot proceed", "Cannot proceed — CI is red on the base branch"], + [ + "unable to proceed", + "Unable to proceed: the required secret is not available in this environment", + ], + ["no path forward", "No path forward until the upstream review lands"], + ] as const) { + test(name, () => { + assert.equal(contentLooksLikeStallNarration(text), false, text); + }); + } + test("blocked mid-sentence does NOT exempt (opener must lead)", () => { + assert.equal( + contentLooksLikeStallNarration("I checked and found it blocked: nothing to do"), + true + ); + }); + }); + + test("summary block → stall shape", () => { + assert.equal(contentLooksLikeStallNarration(STALL_TEXT), true); + }); + + test("markdown summary heading → stall shape", () => { + assert.equal(contentLooksLikeStallNarration("# Summary\nwe did things and more"), true); + }); + + test("'Summary:' prose opener → stall shape", () => { + assert.equal(contentLooksLikeStallNarration("Summary: fixed three PRs today"), true); + }); + + test("short structureless ack ('Nothing to save.') → stall shape", () => { + assert.equal(contentLooksLikeStallNarration("Nothing to save."), true); + }); + + test("structured answer (list) → NOT stall shape", () => { + assert.equal( + contentLooksLikeStallNarration("Done:\n- fixed the parser\n- added a test"), + false + ); + }); + + test("answer with code fence → NOT stall shape", () => { + assert.equal(contentLooksLikeStallNarration("Apply this:\n```ts\nfoo();\n```"), false); + }); + + test("long prose answer → NOT stall shape", () => { + const long = "The root cause is a stale cache entry in the routing layer. ".repeat(6); + assert.equal(long.trim().length > 240, true); + assert.equal(contentLooksLikeStallNarration(long), false); + }); + + test("empty content → NOT stall shape (quality gates own that case)", () => { + assert.equal(contentLooksLikeStallNarration(""), false); + assert.equal(contentLooksLikeStallNarration(" "), false); + }); +}); + +// ── intentional-silence exemption (live 502 cascade) ── +// Live specimen: recurring cron sweep lane — tool result +// "sweep complete: 11 PR(s) examined, 0 unanswered thread(s)", every combo +// member answered exactly "[SILENT]" (the run's documented nothing-to-report +// reply), the guard failed all five over and the customer saw HTTP 502. +describe("intentional-silence tokens are NOT stalls", () => { + test("live specimen: exact '[SILENT]' → NOT stall shape", () => { + assert.equal(contentLooksLikeStallNarration("[SILENT]"), false); + }); + + for (const [name, text] of [ + ["bracketless SILENT", "SILENT"], + ["NO_REPLY", "NO_REPLY"], + ["NO REPLY", "NO REPLY"], + ["lower-case", "[silent]"], + ["padded", "\n [SILENT] \n"], + ["edge punctuation", "[SILENT]."], + ["marker on its own last line", "Sweep finished.\n[SILENT]"], + ["bracketed opener + note", "[SILENT] No changes detected this tick."], + ] as const) { + test(name, () => { + assert.equal(contentLooksLikeStallNarration(text), false, text); + assert.equal(isIntentionalSilenceNarration(text), true, text); + }); + } + + test("prose that merely mentions the token is still narration", () => { + assert.equal(isIntentionalSilenceNarration("Silent retry succeeded"), false); + assert.equal( + isIntentionalSilenceNarration("I told the job to answer [SILENT] next tick."), + false + ); + assert.equal(isIntentionalSilenceNarration("Nothing to save."), false); + }); + + test("malformed bracket is not a marker", () => { + assert.equal(isIntentionalSilenceNarration("[SILENT"), false); + }); + + test("summary narration is still a stall (exemption stays narrow)", () => { + assert.equal(contentLooksLikeStallNarration(STALL_TEXT), true); + assert.equal(contentLooksLikeStallNarration("Nothing to save."), true); + }); + + test("unrelated short prose is still a stall", () => { + assert.equal(contentLooksLikeStallNarration("Silent retry succeeded"), true); + }); +}); + +// ── lane-instructed terminal literals ── +// Live specimen: a background-review lane briefs the model "If nothing is +// worth saving, just say 'Nothing to save.' and stop." Every combo member +// answered exactly that and the guard failed the turn over through the whole +// combo. The response is turn-over BY DESIGN because the REQUEST asked for that +// exact literal — a different class from the drift the guard exists to catch. +describe("lane-instructed terminal literals are NOT stalls", () => { + const brief = (instruction: string) => ({ + model: "some/model", + tools: TOOLS, + messages: [ + // The lane brief sits earlier in the conversation; the TAIL must be the + // tool result or requestExpectsToolCall() gates the guard off entirely. + { role: "system", content: instruction }, + { role: "user", content: "fix the bug" }, + { role: "assistant", content: null, tool_calls: [{ id: "c1" }] }, + { role: "tool", tool_call_id: "c1", content: "file contents…" }, + ], + }); + + test("live specimen: brief quotes the literal, response echoes it → NOT a stall", () => { + const body = brief( + "Review the conversation above and update the skill library. If nothing is worth " + + "saving, just say 'Nothing to save.' and stop." + ); + assert.equal(isInstructedTerminalEcho(body, "Nothing to save."), true); + }); + + test("case-insensitive and whitespace-normalised echo still matches", () => { + const body = brief('Otherwise reply "No output requested" and stop.'); + assert.equal(isInstructedTerminalEcho(body, "no output requested"), true); + assert.equal(isInstructedTerminalEcho(body, " No Output Requested "), true); + }); + + test("the SAME text is still a stall when the brief never asked for it", () => { + const body = brief("Summarise the sweep results."); + assert.equal(isInstructedTerminalEcho(body, "Nothing to save."), false); + assert.equal(contentLooksLikeStallNarration("Nothing to save."), true); + }); + + test("drift prose is never exempt, even when a literal exists in the brief", () => { + const body = brief("If nothing is worth saving, just say 'Nothing to save.' and stop."); + assert.equal(isInstructedTerminalEcho(body, STALL_TEXT), false); + assert.equal(isInstructedTerminalEcho(body, "Silent retry succeeded"), false); + }); + + test("substring is not a match — the whole response must be the literal", () => { + const body = brief("If nothing is worth saving, just say 'Nothing to save.' and stop."); + assert.equal( + isInstructedTerminalEcho(body, "Nothing to save. But I did find one thing."), + false + ); + }); + + test("over-long literals are rejected by the cap (no paragraph exemptions)", () => { + const long = "x".repeat(200); + const body = brief(`If nothing is worth saving, just say '${long}' and stop.`); + assert.equal(isInstructedTerminalEcho(body, long), false); + }); + + test("no quoted literal in the brief → exemption cannot fire", () => { + assert.equal(isInstructedTerminalEcho(brief("Say nothing at all."), "Nothing to save."), false); + assert.equal(isInstructedTerminalEcho(null, "Nothing to save."), false); + assert.equal(isInstructedTerminalEcho({}, "Nothing to save."), false); + }); + + test("Anthropic content-block briefs are scanned too", () => { + const body = { + model: "some/model", + tools: TOOLS, + messages: [ + { role: "user", content: [{ type: "text", text: "just say 'Nothing to save.' and stop" }] }, + { role: "assistant", content: [{ type: "tool_use", id: "t1", name: "read_file" }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "t1", content: "ok" }] }, + ], + }; + assert.equal(isInstructedTerminalEcho(body, "Nothing to save."), true); + }); + + // ── length cap (mutation-driven, 2026-09-17) ── + // Mutation M4 (INSTRUCTED_TERMINAL_MAX_CHARS 120 -> 100000) survived the + // suite: nothing pinned the cap, so a long monologue could be excused as an + // "instructed echo" purely because the brief quoted it. The cap is what keeps + // this exemption to one-line control replies; these tests kill that mutant. + test("instructed echo: over-cap quoted literal is NOT exempt", () => { + // A quoted literal longer than the cap, echoed verbatim. Long-form prose is + // exactly the drift the guard exists to catch, quoted or not. + const long = "z".repeat(200); + assert.equal(isInstructedTerminalEcho(brief(`just say '${long}' and stop.`), long), false); + }); + + test("instructed echo: literal at the cap boundary is still exempt", () => { + // Guards the other direction, so the cap can't be "fixed" by shrinking it + // to zero and trivially passing the test above. + const atCap = "y".repeat(120); + assert.equal(isInstructedTerminalEcho(brief(`just say '${atCap}' and stop.`), atCap), true); + }); + + test("classify: instructed echo over SSE → null (no failover)", async () => { + const verdict = await classifyAgenticStallResponse({ + body: brief("If nothing is worth saving, just say 'Nothing to save.' and stop."), + response: sseResponse( + `data: {"choices":[{"delta":{"content":"Nothing to save."},"finish_reason":null}]}\n\n` + + `data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n` + + `data: [DONE]\n\n` + ), + }); + assert.equal(verdict, null); + }); + + test("classify: instructed echo over JSON → null (no failover)", async () => { + const verdict = await classifyAgenticStallResponse({ + body: brief("If nothing is worth saving, just say 'Nothing to save.' and stop."), + response: jsonResponse({ + choices: [ + { finish_reason: "stop", message: { role: "assistant", content: "Nothing to save." } }, + ], + }), + }); + assert.equal(verdict, null); + }); + + test("classify: same echo WITHOUT the brief → STALL (control)", async () => { + const verdict = await classifyAgenticStallResponse({ + body: brief("Summarise the sweep results."), + response: jsonResponse({ + choices: [ + { finish_reason: "stop", message: { role: "assistant", content: "Nothing to save." } }, + ], + }), + }); + assert.ok(verdict); + assert.equal(verdict.signature, AGENTIC_STALL_SIGNATURE); + }); +}); + +describe("classifyAgenticStallResponse", () => { + beforeEach(() => { + delete process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER; + }); + afterEach(() => { + delete process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER; + }); + + test("OpenAI JSON: stop + summary, no tool_calls, tool tail → STALL", async () => { + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: jsonResponse({ + choices: [{ finish_reason: "stop", message: { role: "assistant", content: STALL_TEXT } }], + }), + }); + assert.ok(verdict); + assert.equal(verdict.signature, AGENTIC_STALL_SIGNATURE); + }); + + test("OpenAI JSON: same shape WITH tool_calls → not a stall", async () => { + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: jsonResponse({ + choices: [ + { + finish_reason: "stop", + message: { + role: "assistant", + content: STALL_TEXT, + tool_calls: [{ id: "c2", function: { name: "read_file", arguments: "{}" } }], + }, + }, + ], + }), + }); + assert.equal(verdict, null); + }); + + test("OpenAI JSON: finish_reason 'length' → not a stall", async () => { + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: jsonResponse({ + choices: [{ finish_reason: "length", message: { role: "assistant", content: STALL_TEXT } }], + }), + }); + assert.equal(verdict, null); + }); + + test("OpenAI JSON: stop + structured answer → not a stall", async () => { + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: jsonResponse({ + choices: [ + { + finish_reason: "stop", + message: { role: "assistant", content: "Fixed:\n- parser null check\n- new unit test" }, + }, + ], + }), + }); + assert.equal(verdict, null); + }); + + test("positive control: legitimate text answer, no tools pending → not a stall", async () => { + const verdict = await classifyAgenticStallResponse({ + body: { messages: [{ role: "user", content: "summarize this chat" }] }, + response: jsonResponse({ + choices: [{ finish_reason: "stop", message: { role: "assistant", content: STALL_TEXT } }], + }), + }); + assert.equal(verdict, null); + }); + + test("Anthropic JSON: end_turn + text-only, tool_result tail → STALL", async () => { + const verdict = await classifyAgenticStallResponse({ + body: anthropicToolTailBody(), + response: jsonResponse({ + type: "message", + stop_reason: "end_turn", + content: [{ type: "text", text: STALL_TEXT }], + }), + }); + assert.ok(verdict); + assert.equal(verdict.signature, AGENTIC_STALL_SIGNATURE); + }); + + test("Anthropic JSON: end_turn WITH tool_use block → not a stall", async () => { + const verdict = await classifyAgenticStallResponse({ + body: anthropicToolTailBody(), + response: jsonResponse({ + type: "message", + stop_reason: "end_turn", + content: [ + { type: "text", text: STALL_TEXT }, + { type: "tool_use", id: "t2", name: "read_file", input: {} }, + ], + }), + }); + assert.equal(verdict, null); + }); + + test("OpenAI SSE: summary chunks + finish stop → STALL", async () => { + const sse = + `data: {"choices":[{"delta":{"role":"assistant"},"finish_reason":null}]}\n\n` + + `data: {"choices":[{"delta":{"content":${JSON.stringify(STALL_TEXT.slice(0, 40))}},"finish_reason":null}]}\n\n` + + `data: {"choices":[{"delta":{"content":${JSON.stringify(STALL_TEXT.slice(40))}},"finish_reason":null}]}\n\n` + + `data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n` + + `data: [DONE]\n\n`; + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: sseResponse(sse), + }); + assert.ok(verdict); + assert.equal(verdict.signature, AGENTIC_STALL_SIGNATURE); + }); + + test("OpenAI SSE: tool_calls delta present → not a stall", async () => { + const sse = + `data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"name":"read_file"}}]},"finish_reason":null}]}\n\n` + + `data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}\n\n` + + `data: [DONE]\n\n`; + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: sseResponse(sse), + }); + assert.equal(verdict, null); + }); + + test("Anthropic SSE: text blocks + end_turn → STALL", async () => { + const sse = + `event: message_start\ndata: {"type":"message_start"}\n\n` + + `event: content_block_start\ndata: {"type":"content_block_start","content_block":{"type":"text","text":""}}\n\n` + + `event: content_block_delta\ndata: {"type":"content_block_delta","delta":{"type":"text_delta","text":${JSON.stringify(STALL_TEXT)}}}\n\n` + + `event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}\n\n` + + `event: message_stop\ndata: {"type":"message_stop"}\n\n`; + const verdict = await classifyAgenticStallResponse({ + body: anthropicToolTailBody(), + response: sseResponse(sse), + }); + assert.ok(verdict); + assert.equal(verdict.signature, AGENTIC_STALL_SIGNATURE); + }); + + test("Anthropic SSE: tool_use block start → not a stall", async () => { + const sse = + `event: content_block_start\ndata: {"type":"content_block_start","content_block":{"type":"tool_use","id":"t3","name":"read_file"}}\n\n` + + `event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"tool_use"}}\n\n`; + const verdict = await classifyAgenticStallResponse({ + body: anthropicToolTailBody(), + response: sseResponse(sse), + }); + assert.equal(verdict, null); + }); + + test("short ack over SSE ('Nothing to save.') → STALL", async () => { + const sse = + `data: {"choices":[{"delta":{"content":"Nothing to save."},"finish_reason":null}]}\n\n` + + `data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n` + + `data: [DONE]\n\n`; + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: sseResponse(sse), + }); + assert.ok(verdict); + assert.equal(verdict.signature, AGENTIC_STALL_SIGNATURE); + }); + + test("OpenAI SSE: exact '[SILENT]' silence token on a tool tail → NOT a stall", async () => { + const sse = + `data: {"choices":[{"delta":{"role":"assistant"},"finish_reason":null}]}\n\n` + + `data: {"choices":[{"delta":{"content":"[SILENT]"},"finish_reason":null}]}\n\n` + + `data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n` + + `data: [DONE]\n\n`; + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: sseResponse(sse), + }); + assert.equal(verdict, null); + }); + + test("OpenAI JSON: exact '[SILENT]' silence token on a tool tail → NOT a stall", async () => { + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: jsonResponse({ + choices: [{ finish_reason: "stop", message: { role: "assistant", content: "[SILENT]" } }], + }), + }); + assert.equal(verdict, null); + }); + + test("Anthropic SSE: '[SILENT]' text block + end_turn → NOT a stall", async () => { + const sse = + `event: message_start\ndata: {"type":"message_start"}\n\n` + + `event: content_block_delta\ndata: {"type":"content_block_delta","delta":{"type":"text_delta","text":"[SILENT]"}}\n\n` + + `event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}\n\n` + + `event: message_stop\ndata: {"type":"message_stop"}\n\n`; + const verdict = await classifyAgenticStallResponse({ + body: anthropicToolTailBody(), + response: sseResponse(sse), + }); + assert.equal(verdict, null); + }); + + test("kill switch: OMNIROUTE_AGENTIC_STALL_FAILOVER=0 → never classifies", async () => { + process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER = "0"; + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: jsonResponse({ + choices: [{ finish_reason: "stop", message: { role: "assistant", content: STALL_TEXT } }], + }), + }); + assert.equal(verdict, null); + }); + + test("unrecognized content-type → null (cannot classify, never breaks path)", async () => { + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: new Response("plain text", { + status: 200, + headers: { "content-type": "text/plain" }, + }), + }); + assert.equal(verdict, null); + }); + + test("original response body is NOT consumed (clone-only contract)", async () => { + const response = jsonResponse({ + choices: [{ finish_reason: "stop", message: { role: "assistant", content: STALL_TEXT } }], + }); + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response, + }); + assert.ok(verdict); + const body = await response.json(); + assert.equal(body.choices[0].message.content, STALL_TEXT); + }); +}); From 74fed18e243f0f21d15a4acb486df08aa2aaac20 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Thu, 17 Sep 2026 09:28:04 +0200 Subject: [PATCH 2/4] chore(ci): retrigger after check:node-runtime install flake The previous run failed at `npm run check:node-runtime` (ERR_MODULE_NOT_FOUND: tsx) even though the install step reported success, and every downstream job was skipped as a result. No other open PR fails that step. Re-running to distinguish a transient install/cache failure from a real one. From 2428309db2cd0619dce32e15cda235661ef1597c Mon Sep 17 00:00:00 2001 From: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:24:48 -0300 Subject: [PATCH 3/4] feat(combo): make the agentic-stall classifier opt-in and document its env var OMNIROUTE_AGENTIC_STALL_FAILOVER now defaults OFF (enabled only by 1/true), matching the project rule that new behavior-changing flags are opt-in. The variable is documented in .env.example and docs/reference/ENVIRONMENT.md so check:env-doc-sync passes, and a changelog fragment is added. Tests: the classify cases enable the flag explicitly; new cases prove the default-OFF and 0/false/empty/1/true behavior (red before the change). --- .env.example | 7 ++++ .../13954-agentic-stall-classifier.md | 1 + docs/reference/ENVIRONMENT.md | 1 + open-sse/services/combo/agenticStall.ts | 8 ++-- tests/unit/agentic-stall.test.ts | 39 +++++++++++++++++-- 5 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 changelog.d/features/13954-agentic-stall-classifier.md diff --git a/.env.example b/.env.example index e6c8f35c8f03..12c332c4913d 100644 --- a/.env.example +++ b/.env.example @@ -567,6 +567,13 @@ ALLOW_API_KEY_REVEAL=false # Validated to >= 1, clamped to <= 32. | Default: 3 # COMBO_CONCURRENCY_PER_MODEL=3 +# Agentic-stall response classifier (opt-in, default OFF). +# Used by: open-sse/services/combo/agenticStall.ts — classifies a combo member's +# "finish: stop + narration + zero tool calls" answer to a tool-result turn as a +# stall. Classification only for now: no combo call site consumes the verdict yet. +# Set to 1 (or true) to enable. | Default: false +# OMNIROUTE_AGENTIC_STALL_FAILOVER=false + # ═══════════════════════════════════════════════════════════════════════════════ # 7. URLS & CLOUD SYNC # ═══════════════════════════════════════════════════════════════════════════════ diff --git a/changelog.d/features/13954-agentic-stall-classifier.md b/changelog.d/features/13954-agentic-stall-classifier.md new file mode 100644 index 000000000000..2e12c09e806f --- /dev/null +++ b/changelog.d/features/13954-agentic-stall-classifier.md @@ -0,0 +1 @@ +- **feat(combo):** add the agentic-stall response classifier (`open-sse/services/combo/agenticStall.ts`) — flags a `finish: stop` + narration + zero-tool-call answer to a tool-result turn. Opt-in via `OMNIROUTE_AGENTIC_STALL_FAILOVER=1` (default OFF); classification only, no combo call site yet ([#13954](https://github.com/cheaper-inference/OmniRoute/pull/13954) - thanks @patrykkopycinski) diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 72db4a2aae1c..c71d2cb4a607 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -286,6 +286,7 @@ OmniRoute provides a two-layer defense: request-side injection scanning and resp | `OMNIROUTE_PAYLOAD_RULES_RELOAD_MS` | `5000` | `open-sse/services/payloadRules.ts` | Reload interval (ms) for hot-reloading the payload rules file. Minimum `1000`. | | `OMNIROUTE_PREFER_CLAUDE_CODE_FOR_UNPREFIXED_CLAUDE_MODELS` | `false` | `open-sse/services/model.ts` | Opt-in: route bare `claude-*` model IDs from Claude Code clients through the Claude Code OAuth account instead of requiring a provider prefix. Explicit provider prefixes still win. Also configurable via a dashboard toggle on the Claude provider page. | | `COMBO_CONCURRENCY_PER_MODEL` | `3` | `open-sse/services/comboConfig.ts` | Per-model concurrency cap for round-robin combos (#9100). The round-robin combo semaphore was hard-capped at 3 concurrent requests per model with no override, serializing higher-concurrency traffic behind that cap. Validated to `>= 1`, clamped to `<= 32`. | +| `OMNIROUTE_AGENTIC_STALL_FAILOVER` | `false` | `open-sse/services/combo/agenticStall.ts` | Opt-in: set to `1` (or `true`) to enable the agentic-stall response classifier, which flags a combo member's `finish: stop` + narration + zero-tool-call answer to a tool-result turn as a stall. Default OFF. Classification only for now — no combo call site consumes the verdict yet. | | `DISABLE_CONTEXT_WINDOW_CHECKS` | `false` | `open-sse/handlers/chatCore.ts` | Dangerous opt-in that skips OmniRoute's local context-window / max-input-token check for direct single-model requests. Upstream providers still enforce their actual limits; prompt compression and the model's own output-token cap remain active. Effective precedence is Feature Flags DB override > environment variable > default; no restart is required. | --- diff --git a/open-sse/services/combo/agenticStall.ts b/open-sse/services/combo/agenticStall.ts index 62af8db09b5c..5ba23124d722 100644 --- a/open-sse/services/combo/agenticStall.ts +++ b/open-sse/services/combo/agenticStall.ts @@ -40,8 +40,9 @@ * applyComboStepResponseGuards). It must NEVER throw or break the request * path: every failure mode returns null (not a stall). * - * Kill switch: OMNIROUTE_AGENTIC_STALL_FAILOVER=0 disables classification - * (default: enabled). + * Opt-in flag: OMNIROUTE_AGENTIC_STALL_FAILOVER=1 (or "true") enables + * classification. Default: OFF — new behavior-changing flags are opt-in. This + * module is classification only; no combo call site consumes the verdict yet. */ export const AGENTIC_STALL_SIGNATURE = "agentic_stall_no_toolcall"; @@ -51,7 +52,8 @@ export const AGENTIC_STALL_SIGNATURE = "agentic_stall_no_toolcall"; const MAX_STALL_INSPECT_BYTES = 1024 * 1024; export function isAgenticStallFailoverEnabled(): boolean { - return process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER !== "0"; + const raw = (process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER ?? "").trim().toLowerCase(); + return raw === "1" || raw === "true"; } function isRecord(value: unknown): value is Record { diff --git a/tests/unit/agentic-stall.test.ts b/tests/unit/agentic-stall.test.ts index c162eac89842..78913e5ba52f 100644 --- a/tests/unit/agentic-stall.test.ts +++ b/tests/unit/agentic-stall.test.ts @@ -18,6 +18,7 @@ import { AGENTIC_STALL_SIGNATURE, classifyAgenticStallResponse, contentLooksLikeStallNarration, + isAgenticStallFailoverEnabled, isInstructedTerminalEcho, isIntentionalSilenceNarration, requestExpectsToolCall, @@ -236,6 +237,15 @@ describe("intentional-silence tokens are NOT stalls", () => { // combo. The response is turn-over BY DESIGN because the REQUEST asked for that // exact literal — a different class from the drift the guard exists to catch. describe("lane-instructed terminal literals are NOT stalls", () => { + // classifyAgenticStallResponse is opt-in (default OFF): enable it so the null + // verdicts below prove the exemption, not a disabled classifier. + beforeEach(() => { + process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER = "1"; + }); + afterEach(() => { + delete process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER; + }); + const brief = (instruction: string) => ({ model: "some/model", tools: TOOLS, @@ -367,7 +377,8 @@ describe("lane-instructed terminal literals are NOT stalls", () => { describe("classifyAgenticStallResponse", () => { beforeEach(() => { - delete process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER; + // Opt-in flag (default OFF): the classification cases below run with it enabled. + process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER = "1"; }); afterEach(() => { delete process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER; @@ -568,8 +579,9 @@ describe("classifyAgenticStallResponse", () => { assert.equal(verdict, null); }); - test("kill switch: OMNIROUTE_AGENTIC_STALL_FAILOVER=0 → never classifies", async () => { - process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER = "0"; + test("default OFF: OMNIROUTE_AGENTIC_STALL_FAILOVER unset → never classifies", async () => { + delete process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER; + assert.equal(isAgenticStallFailoverEnabled(), false); const verdict = await classifyAgenticStallResponse({ body: openAiToolTailBody(), response: jsonResponse({ @@ -579,6 +591,27 @@ describe("classifyAgenticStallResponse", () => { assert.equal(verdict, null); }); + test("OMNIROUTE_AGENTIC_STALL_FAILOVER=0 / false / empty → never classifies", async () => { + for (const value of ["0", "false", ""]) { + process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER = value; + assert.equal(isAgenticStallFailoverEnabled(), false, `value=${JSON.stringify(value)}`); + const verdict = await classifyAgenticStallResponse({ + body: openAiToolTailBody(), + response: jsonResponse({ + choices: [{ finish_reason: "stop", message: { role: "assistant", content: STALL_TEXT } }], + }), + }); + assert.equal(verdict, null, `value=${JSON.stringify(value)}`); + } + }); + + test("OMNIROUTE_AGENTIC_STALL_FAILOVER=1 / true enables classification", () => { + for (const value of ["1", "true", "TRUE"]) { + process.env.OMNIROUTE_AGENTIC_STALL_FAILOVER = value; + assert.equal(isAgenticStallFailoverEnabled(), true, `value=${JSON.stringify(value)}`); + } + }); + test("unrecognized content-type → null (cannot classify, never breaks path)", async () => { const verdict = await classifyAgenticStallResponse({ body: openAiToolTailBody(), From aec89809772cae0fa3c1eabff83e499f00d4872a Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Wed, 28 Oct 2026 21:23:15 -0300 Subject: [PATCH 4/4] refactor(combo): split agentic-stall classifier functions under the complexity ratchet quotedTerminalLiterals, extractFromSse and classifyAgenticStallResponse exceeded the complexity/cognitive-complexity threshold (15) of eslint.complexity-ratchets.config.mjs. Extract message-part text and quoted-literal collection, per-line SSE parsing with OpenAI/Anthropic folding helpers (Anthropic events via a per-type handler map), and response-signal reading plus the stop/no-tool-call checks. Behaviour is unchanged; the existing 68-test suite covers the JSON and SSE paths. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- open-sse/services/combo/agenticStall.ts | 242 ++++++++++++++---------- 1 file changed, 139 insertions(+), 103 deletions(-) diff --git a/open-sse/services/combo/agenticStall.ts b/open-sse/services/combo/agenticStall.ts index 5ba23124d722..61f29c455e62 100644 --- a/open-sse/services/combo/agenticStall.ts +++ b/open-sse/services/combo/agenticStall.ts @@ -189,30 +189,36 @@ function quotedTerminalLiterals(body: unknown): Set { const found = new Set(); if (!isRecord(body)) return found; const messages = Array.isArray(body.messages) ? body.messages : []; - const push = (raw: string) => { - const lit = raw.trim().replace(/\s+/g, " "); - if (!lit || lit.length > INSTRUCTED_TERMINAL_MAX_CHARS) return; - found.add(lit.toLowerCase()); - }; for (const msg of messages) { if (!isRecord(msg)) continue; const contents = Array.isArray(msg.content) ? msg.content : [msg.content]; - for (const part of contents) { - let text = ""; - if (typeof part === "string") text = part; - else if (isRecord(part)) { - if (typeof part.text === "string") text = part.text; - else if (typeof part.content === "string") text = part.content; - } - if (!text) continue; - // Quoted literals: '...' | "..." | `...`. The 120 cap mirrors - // INSTRUCTED_TERMINAL_MAX_CHARS — longer quotes are never echoed whole. - for (const m of text.matchAll(/['"`]([^'"`\n]{1,120})['"`]/g)) push(m[1]); - } + for (const part of contents) addQuotedLiterals(found, messagePartText(part)); } return found; } +/** Text of one message content part: a bare string, or a block's `text` / + * `content` string. Anything else yields "". */ +function messagePartText(part: unknown): string { + if (typeof part === "string") return part; + if (!isRecord(part)) return ""; + if (typeof part.text === "string") return part.text; + if (typeof part.content === "string") return part.content; + return ""; +} + +/** Add every quoted literal ('...' | "..." | `...`) in `text` to `found`. The + * 120 cap mirrors INSTRUCTED_TERMINAL_MAX_CHARS — longer quotes are never + * echoed whole. */ +function addQuotedLiterals(found: Set, text: string): void { + if (!text) return; + for (const m of text.matchAll(/['"`]([^'"`\n]{1,120})['"`]/g)) { + const lit = m[1].trim().replace(/\s+/g, " "); + if (!lit || lit.length > INSTRUCTED_TERMINAL_MAX_CHARS) continue; + found.add(lit.toLowerCase()); + } +} + /** True when the response is EXACTLY a literal the request itself told the model * to reply with. This is a lane-instructed terminal token — turn-over by design, * like the silence markers but without a fixed global vocabulary, so new lanes @@ -294,68 +300,91 @@ function extractFromAnthropicJson(parsed: unknown): StallSignal | null { }; } +type SseAccumulator = StallSignal & { recognized: boolean }; + +/** Parse one SSE line into its JSON `data:` record, or null when the line is + * not a data line, is the [DONE] sentinel, or is not a JSON object. */ +function parseSseDataLine(rawLine: string): Record | null { + const line = rawLine.trim(); + if (!line.startsWith("data:")) return null; + const data = line.slice(5).trim(); + if (!data || data === "[DONE]") return null; + try { + const parsed: unknown = JSON.parse(data); + return isRecord(parsed) ? parsed : null; + } catch { + return null; + } +} + +/** Fold one OpenAI chat chunk (`choices[0].delta`) into the accumulator. */ +function applyOpenAiSseChunk(acc: SseAccumulator, choices: unknown[]): void { + const choice = choices[0]; + if (!isRecord(choice)) return; + acc.recognized = true; + const delta = isRecord(choice.delta) ? choice.delta : null; + if (delta && typeof delta.content === "string") acc.text += delta.content; + if (delta && Array.isArray(delta.tool_calls) && delta.tool_calls.length > 0) { + acc.hasToolCalls = true; + } + if (typeof choice.finish_reason === "string" && choice.finish_reason) { + acc.finishReason = choice.finish_reason; + } +} + +type AnthropicSseHandler = (acc: SseAccumulator, event: Record) => void; + +const noopAnthropicSseHandler: AnthropicSseHandler = () => {}; + +/** Per-event-type folding of Anthropic message events into the accumulator. */ +const ANTHROPIC_SSE_HANDLERS: Record = { + content_block_start(acc, event) { + const block = isRecord(event.content_block) ? event.content_block : null; + if (block && block.type === "tool_use") acc.hasToolCalls = true; + }, + content_block_delta(acc, event) { + const delta = isRecord(event.delta) ? event.delta : null; + if (delta && delta.type === "text_delta" && typeof delta.text === "string") { + acc.text += delta.text; + } + }, + message_delta(acc, event) { + const delta = isRecord(event.delta) ? event.delta : null; + if (delta && typeof delta.stop_reason === "string" && delta.stop_reason) { + acc.finishReason = delta.stop_reason; + } + }, + message_start: noopAnthropicSseHandler, + message_stop: noopAnthropicSseHandler, +}; + +/** Fold one Anthropic message event into the accumulator. Unknown event types + * are ignored and do not mark the payload as recognized. */ +function applyAnthropicSseEvent(acc: SseAccumulator, event: Record): void { + const type = typeof event.type === "string" ? event.type : ""; + if (!Object.hasOwn(ANTHROPIC_SSE_HANDLERS, type)) return; + acc.recognized = true; + ANTHROPIC_SSE_HANDLERS[type](acc, event); +} + /** Assemble the terminal assistant message from a buffered SSE payload, * handling both OpenAI chat chunks and Anthropic message events. Returns * null when nothing recognizable was seen (cannot classify). */ function extractFromSse(payload: string): StallSignal | null { - let finishReason: string | null = null; - let hasToolCalls = false; - let text = ""; - let recognized = false; + const acc: SseAccumulator = { + finishReason: null, + hasToolCalls: false, + text: "", + recognized: false, + }; for (const rawLine of payload.split(/\r?\n/)) { - const line = rawLine.trim(); - if (!line.startsWith("data:")) continue; - const data = line.slice(5).trim(); - if (!data || data === "[DONE]") continue; - let parsed: unknown; - try { - parsed = JSON.parse(data); - } catch { - continue; - } - if (!isRecord(parsed)) continue; - // OpenAI chat chunks. - if (Array.isArray(parsed.choices)) { - const choice = parsed.choices[0]; - if (isRecord(choice)) { - recognized = true; - const delta = isRecord(choice.delta) ? choice.delta : null; - if (delta) { - if (typeof delta.content === "string") text += delta.content; - if (Array.isArray(delta.tool_calls) && delta.tool_calls.length > 0) { - hasToolCalls = true; - } - } - if (typeof choice.finish_reason === "string" && choice.finish_reason) { - finishReason = choice.finish_reason; - } - } - continue; - } - // Anthropic message events. - const type = typeof parsed.type === "string" ? parsed.type : ""; - if (type === "content_block_start") { - recognized = true; - const block = isRecord(parsed.content_block) ? parsed.content_block : null; - if (block && block.type === "tool_use") hasToolCalls = true; - } else if (type === "content_block_delta") { - recognized = true; - const delta = isRecord(parsed.delta) ? parsed.delta : null; - if (delta && delta.type === "text_delta" && typeof delta.text === "string") { - text += delta.text; - } - } else if (type === "message_delta") { - recognized = true; - const delta = isRecord(parsed.delta) ? parsed.delta : null; - if (delta && typeof delta.stop_reason === "string" && delta.stop_reason) { - finishReason = delta.stop_reason; - } - } else if (type === "message_start" || type === "message_stop") { - recognized = true; - } + const parsed = parseSseDataLine(rawLine); + if (!parsed) continue; + if (Array.isArray(parsed.choices)) applyOpenAiSseChunk(acc, parsed.choices); + else applyAnthropicSseEvent(acc, parsed); } - if (!recognized) return null; - return { finishReason, hasToolCalls, text }; + if (!acc.recognized) return null; + return { finishReason: acc.finishReason, hasToolCalls: acc.hasToolCalls, text: acc.text }; } async function readClonedBodyText(clone: Response): Promise { @@ -389,6 +418,41 @@ export type AgenticStallVerdict = { detail: string; }; +/** Clone the response and extract its terminal stall signal (SSE or JSON). + * Returns null when the body cannot be read or classified. */ +async function readStallSignal(response: Response): Promise { + if (!response || typeof response.clone !== "function") return null; + let clone: Response; + try { + clone = response.clone(); + } catch { + return null; + } + const ct = clone.headers?.get?.("content-type") || ""; + if (ct.includes("text/event-stream")) { + const text = await readClonedBodyText(clone); + return text == null ? null : extractFromSse(text); + } + if (!ct.includes("application/json")) return null; + let parsed: unknown; + try { + parsed = await clone.json(); + } catch { + return null; + } + return extractFromOpenAiJson(parsed) ?? extractFromAnthropicJson(parsed); +} + +/** Conditions 2 and 3 on an extracted signal: stop finish, no tool calls, not an + * instructed terminal echo, and stall-shaped narration. */ +function signalIsStall(body: unknown, signal: StallSignal): boolean { + const finish = (signal.finishReason || "").toLowerCase(); + if (finish !== "stop" && finish !== "end_turn") return false; + if (signal.hasToolCalls) return false; + if (isInstructedTerminalEcho(body, signal.text)) return false; + return contentLooksLikeStallNarration(signal.text); +} + /** * Classify a successful (HTTP 200, quality-valid) combo-member response as an * agentic stall. Returns the verdict (with a log/audit-safe detail string — @@ -403,36 +467,8 @@ export async function classifyAgenticStallResponse(args: { if (!isAgenticStallFailoverEnabled()) return null; const { body, response } = args; if (!requestExpectsToolCall(body)) return null; - if (!response || typeof response.clone !== "function") return null; - let clone: Response; - try { - clone = response.clone(); - } catch { - return null; - } - const ct = clone.headers?.get?.("content-type") || ""; - let signal: StallSignal | null = null; - if (ct.includes("text/event-stream")) { - const text = await readClonedBodyText(clone); - if (text == null) return null; - signal = extractFromSse(text); - } else if (ct.includes("application/json")) { - let parsed: unknown; - try { - parsed = await clone.json(); - } catch { - return null; - } - signal = extractFromOpenAiJson(parsed) ?? extractFromAnthropicJson(parsed); - } else { - return null; - } - if (!signal) return null; - const finish = (signal.finishReason || "").toLowerCase(); - if (finish !== "stop" && finish !== "end_turn") return null; - if (signal.hasToolCalls) return null; - if (isInstructedTerminalEcho(body, signal.text)) return null; - if (!contentLooksLikeStallNarration(signal.text)) return null; + const signal = await readStallSignal(response); + if (!signal || !signalIsStall(body, signal)) return null; const head = signal.text.trim().slice(0, 120).replace(/\s+/g, " "); return { signature: AGENTIC_STALL_SIGNATURE,