fix(claude-sdk-oauth): fork stream-start-timeout retries at the pre-turn boundary (fixes #723)

.agents/skills/senpi-qa/scripts/claude-sdk-oauth-stream-stall-retry-probe.mjs

@@ -0,0 +1,201 @@
+#!/usr/bin/env node
+// Issue #723 real-surface probe: a stream-start-stalled headless turn must retry
+// by forking at the pre-turn assistant boundary (delta re-send, prefix-cache
+// read), never by re-attaching and re-sending the full conversation.
+
+import { spawn } from "node:child_process";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { bootHermeticStack, SOURCE_ROOT } from "./lib/claude-sdk-oauth-fullstack-harness.mjs";
+import { classifyPayload } from "./lib/claude-sdk-oauth-fullstack-support.mjs";
+import { safeDetail } from "./lib/output-safety.mjs";
+
+const TURN_TIMEOUT_MS = 120_000;
+const tsxImport = fileURLToPath(import.meta.resolve("tsx"));
+const evidenceArg = process.argv.indexOf("--evidence");
+const evidenceSlug = evidenceArg === -1 ? undefined : process.argv[evidenceArg + 1];
+
+function killTree(child) {
+	if (child.exitCode !== null) return;
+	try {
+		if (process.platform === "win32") child.kill("SIGKILL");
+		else process.kill(-child.pid, "SIGKILL");
+	} catch {
+		child.kill("SIGKILL");
+	}
+}
+
+function runCli(stack, args, onLine) {
+	return new Promise((resolve, reject) => {
+		const child = spawn(process.execPath, ["--import", tsxImport, join(SOURCE_ROOT, "cli.ts"), ...args], {
+			cwd: stack.box.cwd,
+			detached: process.platform !== "win32",
+			env: { ...process.env },
+			stdio: ["ignore", "pipe", "pipe"],
+		});
+		let stdout = "";
+		let stderr = "";
+		child.stdout.setEncoding("utf8");
+		child.stderr.setEncoding("utf8");
+		child.stdout.on("data", (chunk) => {
+			stdout += chunk;
+			if (onLine) for (const line of chunk.split("\n")) onLine(line);
+		});
+		child.stderr.on("data", (chunk) => {
+			stderr += chunk;
+		});
+		const timeout = setTimeout(() => {
+			killTree(child);
+			reject(new Error(`CLI turn exceeded ${TURN_TIMEOUT_MS}ms`));
+		}, TURN_TIMEOUT_MS);
+		child.once("error", (error) => {
+			clearTimeout(timeout);
+			reject(error);
+		});
+		child.once("close", (code) => {
+			clearTimeout(timeout);
+			resolve({ code: code ?? -1, stdout, stderr });
+		});
+	});
+}
+
+function events(stdout, type) {
+	const out = [];
+	for (const line of stdout.split("\n")) {
+		if (!line.trim()) continue;
+		let parsed;
+		try {
+			parsed = JSON.parse(line);
+		} catch {
+			continue;
+		}
+		const visit = (value) => {
+			if (Array.isArray(value)) return value.forEach(visit);
+			if (!value || typeof value !== "object") return;
+			if (value.type === type) out.push(value);
+			for (const nested of Object.values(value)) visit(nested);
+		};
+		visit(parsed);
+	}
+	return out;
+}
+
+function seedFastTimeoutSettings(stack) {
+	writeFileSync(
+		join(stack.box.agentDir, "settings.json"),
+		JSON.stringify({
+			retry: { enabled: true, baseDelayMs: 0, provider: { streamStartTimeoutMs: 3000, streamRetryTimeoutMs: 0 } },
+		}),
+	);
+}
+
+const common = (sessionDir) => [
+	"-p",
+	"--provider", "claude-sdk-oauth",
+	"--model", "claude-haiku-4-5",
+	"--thinking", "off",
+	"--mode", "json",
+	"--session-dir", sessionDir,
+	"--no-tools",
+	"--no-context-files",
+	"--offline",
+	"--no-model-fallback",
+	"--no-recommended-models",
+	"--system-prompt", "Reply with the requested marker only.",
+];
+
+let stack;
+let summary;
+const cleanup = [];
+try {
+	stack = await bootHermeticStack({ sandboxLabel: "issue-723-stream-stall-retry" });
+	seedFastTimeoutSettings(stack);
+
+	// Turn 1: normal — establishes lineage + persisted binding.
+	const first = await runCli(stack, [...common(stack.box.sessionDir), "Reply exactly STALL-1."]);
+	if (first.code !== 0) throw new Error(`turn 1 failed (${first.code}): ${safeDetail(first.stderr.slice(-400))}`);
+	const payloadsBeforeStall = stack.creations.reduce((n, c) => n + c.payloads.length, 0);
+
+	// Turn 2: stall the FIRST provider request, watch for the retry's payload.
+	const releaseStall = stack.stallNextResponse();
+	const retryPayloads = [];
+	const second = await runCli(
+		stack,
+		[...common(stack.box.sessionDir), "-c", "Reply exactly STALL-2."],
+		(line) => {
+			if (line.includes("auto_retry_start")) {
+				// The stalled first attempt timed out and the retry is about to fire:
+				// let the loopback answer normally now.
+				releaseStall();
+			}
+		},
+	);
+	// The stall only held ONE response; the retry streams a fresh body, so no
+	// further release is needed even if the retry-start line never surfaced.
+	releaseStall();
+
+	const secondTurnCreations = stack.creations.slice(1); // everything after the turn-1 resident query
+	const classified = secondTurnCreations.flatMap((c) =>
+		c.payloads.map((message) => ({ path: c.path, lineage: c.lineage, forked: c.forked, resumeAt: c.resumeAt, ...classifyPayload(message) })),
+	);
+	const continuity = events(second.stdout, "claude_sdk_oauth_session_continuity").map((e) => e.details ?? e);
+	const retries = events(second.stdout, "auto_retry_start").length;
+	// The decisive signals (issue #723): after the stall, every continuity
+	// decision resumes lineage (fork|delta|reattach) carrying ONLY the turn's own
+	// message (deltaMessages === 1); a flatten/bootstrap or a larger delta means
+	// the retry re-billed the conversation.
+	const noColdSeedAfterStall = continuity.every((o) => o.kind !== "flatten" && o.kind !== "bootstrap");
+	const deltaOnlyAfterStall = continuity.length > 0 && continuity.every((o) => o.deltaMessages === 1);
+	const resumedWithFork = continuity.some((o) => o.kind === "fork" || o.kind === "reattach");
+	const noFlattenPayload = classified.every((p) => p.kind !== "flatten");
+
+	const passed =
+		second.code === 0 &&
+		retries >= 1 &&
+		resumedWithFork &&
+		noColdSeedAfterStall &&
+		deltaOnlyAfterStall &&
+		noFlattenPayload;
+
+	summary = {
+		passed,
+		turn2: { code: second.code, retries, continuity },
+		classified,
+		providerRequests: stack.providerRequests.length,
+		resumedWithFork,
+		noColdSeedAfterStall,
+		deltaOnlyAfterStall,
+		stderr: safeDetail(second.stderr.split("\n").filter(Boolean).slice(-6).join("\n")),
+	};
+} catch (error) {
+	summary = { passed: false, error: safeDetail(error instanceof Error ? error.stack : String(error)) };
+} finally {
+	if (stack) {
+		await stack.shutdown().catch(() => {});
+		stack.authGuard.assertUnchanged();
+		stack.box.cleanup();
+		cleanup.push("loopback server closed", "sandbox removed", "real auth unchanged");
+	}
+}
+
+if (evidenceSlug) {
+	const directory = join(
+		process.cwd(),
+		"local-ignore",
+		"qa-evidence",
+		`${new Date().toISOString().slice(0, 10).replaceAll("-", "")}-${evidenceSlug}`,
+	);
+	mkdirSync(directory, { recursive: true });
+	writeFileSync(join(directory, "probe-resume.json"), `${JSON.stringify(summary, null, 2)}\n`);
+	writeFileSync(
+		join(directory, "probe-resume.log"),
+		`command: node .agents/skills/senpi-qa/scripts/claude-sdk-oauth-stream-stall-retry-probe.mjs --evidence ${evidenceSlug}\n` +
+			`cleanup: ${cleanup.join("; ")}\n\n${JSON.stringify(summary, null, 2)}\n`,
+	);
+	process.stdout.write(`EVIDENCE ${join(directory, "probe-resume.log")}\n`);
+}
+const finalPassed = summary?.passed === true;
+process.stdout.write(`${JSON.stringify(summary)}\n`);
+process.stdout.write(finalPassed ? "VERDICT: PASS claude-sdk-oauth stream-stall retry continuity\n" : "VERDICT: FAIL claude-sdk-oauth stream-stall retry continuity\n");
+process.exitCode = finalPassed ? 0 : 1;

.agents/skills/senpi-qa/scripts/lib/claude-sdk-oauth-fullstack-harness.mjs

@@ -9,6 +9,7 @@
  */
 
 import { createServer } from "node:http";
+import { mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { pathToFileURL } from "node:url";
 import { guardRealAuth, makeSandbox, repoRoot, track } from "./common.mjs";
@@ -69,6 +70,13 @@ async function startLoopbackServer(onRequest, holdRelease) {
 					response.end(sse);
 					return;
 				}
+				if (hold.stall === true) {
+					// Stream-start stall: headers flushed above, first SSE event withheld
+					// until release — the client stream-start watchdog must fire.
+					response.flushHeaders();
+					void hold.release.then(() => response.end(sse));
+					return;
+				}
 				// Stream the opening events so the turn is genuinely in flight, tell the
 				// phase it may act now, and finish only when it releases the hold.
 				const { head, tail } = splitSseBody(sse);
@@ -113,6 +121,23 @@ export async function bootHermeticStack({ sandboxLabel = "claude-sdk-fullstack-p
 	);
 
 	seedProbeAgentDir(box.agentDir);
+	// Post-#969 the ambient lane is opt-in AND the SDK subprocess validates its own
+	// credential store, so seed the sandbox CLAUDE_CONFIG_DIR with a dummy OAuth blob
+	// in the exact shape `claude auth status` accepts.
+	const claudeConfigDir = join(box.dir, "claude-config");
+	mkdirSync(claudeConfigDir, { recursive: true, mode: 0o700 });
+	writeFileSync(
+		join(claudeConfigDir, ".credentials.json"),
+		JSON.stringify({
+			claudeAiOauth: {
+				accessToken: "fullstack-probe-dummy-access",
+				refreshToken: "fullstack-probe-dummy-refresh",
+				expiresAt: 4102444800000,
+				scopes: ["user:inference", "user:profile", "user:sessions:claude_code"],
+			},
+		}),
+		{ mode: 0o600 },
+	);
 	// The ambient auth lane hands the probe's own environment to the Claude Code
 	// subprocess, so inherited credentials and proxies are scrubbed BEFORE the
 	// hermetic pins are applied, and the result is asserted below.
@@ -126,10 +151,11 @@ export async function bootHermeticStack({ sandboxLabel = "claude-sdk-fullstack-p
 		PI_TELEMETRY: "0",
 		ANTHROPIC_BASE_URL: baseUrl,
 		ANTHROPIC_API_KEY: "fullstack-probe-dummy-key",
+		SENPI_CLAUDE_SDK_OAUTH_TOKEN_INJECTION: "ambient",
+		SENPI_CLAUDE_SDK_OAUTH_ENABLED: "1",
 		CLAUDE_CONFIG_DIR: join(box.dir, "claude-config"),
 		CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
 		CLAUDE_CODE_DISABLE_TELEMETRY: "1",
-		SENPI_CLAUDE_SDK_OAUTH_TOKEN_INJECTION: "ambient",
 		NO_PROXY: "127.0.0.1,localhost",
 		no_proxy: "127.0.0.1,localhost",
 	});
@@ -222,6 +248,21 @@ export async function bootHermeticStack({ sandboxLabel = "claude-sdk-fullstack-p
 			};
 			return () => releaseHold?.();
 		},
+		/**
+		 * Stalls the NEXT loopback response after headers: the first SSE event is
+		 * never written until release(), so the client stream-start watchdog fires.
+		 * issue #723 timeout-retry probe. Release with the returned function.
+		 */
+		stallNextResponse() {
+			let releaseHold;
+			pendingHold = {
+				stall: true,
+				release: new Promise((resolve) => {
+					releaseHold = resolve;
+				}),
+			};
+			return () => releaseHold?.();
+		},
 		createAgentSession: () =>
 			createAgentSession({ cwd: box.cwd, agentDir: box.agentDir, noTools: "all", autoTitleSessions: false }),
 		/**

packages/coding-agent/CHANGELOG.md

@@ -6,6 +6,14 @@
 
 ### Fixed
 
+- claude-sdk-oauth stream-start-timeout retries now fork the SDK conversation at the last assistant
+  boundary before the stalled turn instead of re-attaching and re-sending it, so each retry re-bills
+  only the turn's own message on a prefix cache read instead of re-writing the whole conversation
+  (fixes #723 retry-storm re-billing: cache writes grew ~8K per attempt, $25/6min, $1084/3days on
+  worker dispatch). A stalled first turn with no boundary to fork at re-seeds byte-identically, which
+  the provider serves from prefix cache after the first write. The retry watchdog cap semantics
+  (`streamRetryTimeoutMs` caps the retry continuation, reconciled to the granted stream-start guard)
+  are now documented on the setting itself.
 - The Cursor exec bridge fails closed when a session bridge has no captured owning run, and rechecks
   run ownership after awaited preflight work so a run that ends during an approval prompt cannot start
   a tool side effect afterward.

packages/coding-agent/src/core/changes.md

@@ -1,5 +1,30 @@
 # changes
 
+## 2026-08-20 - streamRetryTimeoutMs docstring aligned with the reconciled watchdog (issue #723 lane)
+
+### What changed
+
+- `core/retry-fallback/settings.ts`: the `streamRetryTimeoutMs` interface comment now states the actual
+  post-2026-08-18 semantics — it caps the retry-CONTINUATION watchdog, reconciled to
+  `max(cap, streamStartTimeoutMs)` — instead of the stale "first-request liveness cap after a provider
+  timeout" wording. Comment-only; no behavior change.
+
+### Why
+
+- Issue #723 diagnosis (M3) read that comment and concluded the setting clamps the stream-start guard
+  itself. It does not: since the 2026-08-18 reconciliation the retry request keeps its full granted
+  guard and only the continuation watchdog takes this cap. A wrong comment on the exact knob a
+  retry-storm investigation reaches first sends the next diagnosis down the same dead end.
+
+### Why an extension could not handle it
+
+- The setting is a core `ProviderRetrySettings` field consumed by `core/provider-timeout-retry.ts`; the
+  doc contract lives with the interface.
+
+### Expected merge conflict zones
+
+- `core/retry-fallback/settings.ts` `ProviderRetrySettings` field list only (comment line).
+
 ## 2026-08-20 - Append-only goal continuations and exponentially floored 429 waits
 
 ### What changed

packages/coding-agent/src/core/extensions/builtin/claude-sdk-oauth/changes.md

@@ -1,5 +1,57 @@
 # claude-sdk-oauth
 
+## 2026-08-20 - Same-turn timeout retries fork at the pre-turn boundary (issue #723)
+
+### What changed
+
+- `session-reattach.ts`: `ContinuityBinding` gained an optional in-memory-only `unansweredTurnDigest`. It
+  rides the existing clone/remember paths but is NEVER persisted: `storedBindingFromEntry`
+  (`session-binding.ts`) builds the sidecar record from an explicit field list, and the strict
+  `schemaVersion: 1` schema (`session-binding-store.ts`) rejects any record carrying it.
+  Tests: `test/claude-sdk-oauth-binding-store.test.ts` (round-trip rejects it).
+- `session-turn-attempt.ts`: an attempt that pushed its user payload but ended aborted, failed, or
+  discarded now remembers a retry checkpoint binding anchored at the PRE-TURN boundary
+  (`bindingFromEntry(entry, hashes.slice(0, entry.sentCount))` + the attempted turn's full sent-stream
+  digest). Covers the `turn.aborted` resolution, the queue-failure (completion rejected) path, and
+  `discard()` before `closeSession`.
+- `session-continuity.ts`: `decideFromBinding` gains a branch ahead of the existing prefix logic — when
+  the binding carries a checkpoint, the FULL current sent stream hashes to it, and the prefix at
+  `binding.sentCount` matches, it returns `fork` at `binding.lastAssistantUuid` (`reason:
+  "timeout_retry"`), or the cold-seed `flatten` with the same reason when no boundary exists (first
+  turn). A digest mismatch falls through to the pre-existing branches unchanged.
+- `session-observability.ts`: `ContinuityReason` union and the sanitizer allowlist admit
+  `timeout_retry`. No new event types; one observation per main turn is preserved.
+- Tests: `test/suite/regressions/723-claude-sdk-oauth-timeout-abort-retry-continuity.test.ts`,
+  `test/claude-sdk-oauth-continuity-decision.test.ts`, `test/claude-sdk-oauth-continuity-retry-checkpoint.test.ts`.
+
+### Why
+
+A stream-start-timeout abort closes the SDK session with the turn's user message already appended and
+  un-answered. The retry then re-attached to that lineage and appended the SAME message again — one
+  duplicate per attempt, ~8K tokens of cache re-billing per attempt, and for a first turn (no assistant
+  boundary, binding absent) a full re-flatten of the whole conversation at full price on every attempt
+  (issue #723: $25 per 6 minutes, $1084 over 3 days on worker dispatch). Forking at the pre-turn
+  boundary rewinds past the orphaned message, so the retry's request byte-layout matches the failed
+  attempt's and the provider serves it from prefix cache; a first turn re-seeds byte-identically
+  (flatten is a deterministic function of context), which is likewise cache-read after the first write.
+
+### Why an extension could not handle it
+
+- The retry checkpoint must be recorded where the attempt's outcome is known (`session-turn-attempt.ts`)
+  and consumed by the resident-lane continuity decision table (`session-continuity.ts`) — both are
+  internal to this builtin's resident session machinery; no extension hook observes attempt outcomes or
+  continuity bindings.
+
+### Expected merge conflict zones
+
+- `session-continuity.ts` in `decideFromBinding` (head of the function) — upstream continuity reworks
+  touch the same function.
+- `session-turn-attempt.ts` attempt-outcome block and `discard()` — same file upstream reworked in the
+  2026-08-01 continuity pass.
+- `session-reattach.ts` `ContinuityBinding` field list.
+- `session-observability.ts` `ContinuityReason` union tail and `SANITIZED_REASONS` set (mechanically
+  duplicated literals; both must gain the member).
+
 ## 2026-08-19 - Kill-switched lane leaves implicit fallback expansion
 
 ### What changed