fix(replay): scope durable thought signatures per credential and bound persist visibility

devlog/_plan/260819_triage_execution/020_remote_reasoning_leak_rca.md

@@ -0,0 +1,25 @@
+# 020 — #2064 RCA: Remote raw-thinking leak (FIXED-ON-DEV)
+
+Reported on 2.24.2: Codex Remote paints raw Grok thinking live
+(response.reasoning_text.delta), then swaps to the progress line leaving
+italic fragments; stored items are summary:[] + content reasoning_text.
+
+RCA (sol lane + main verification, 2026-08-19):
+
+- v2.24.2 bridge emitted the raw channel: git show v2.24.2:src/bridge.ts has
+  response.reasoning_text.delta; fix commit 56752d7c5 (#2007, landed via
+  PR #2016 merge 891c8284b) is NOT an ancestor of v2.24.2, IS on dev.
+- Current dev has no escape path for openai-chat routed models: the bridge
+  emits summary-channel deltas and summary-shaped items only
+  (src/bridge.ts:987,1016,591-606); the native-Responses rewrite covers WS
+  upstream, eager relay, HTTP SSE tee, and JSON reframing legs
+  (src/server/responses/core.ts:2835-3066,
+  src/server/responses-reasoning-summary-rewrite.ts:51-110).
+- Suites: tests/responses-reasoning-summary-rewrite.test.ts +
+  tests/bridge.test.ts = 73 pass / 0 fail (fresh).
+
+Outcome: no new code. Issue #2064 closed as fixed-on-dev with a note asking
+for on-device Remote verification on the next release build. Model-side
+intermittent reasoning exposure (user caution) stays out of scope — our relay
+provably converts the channel on every leg.
+

src/adapters/google.ts

@@ -26,6 +26,7 @@ import { identifyRoutedModel } from "./identity";
 import { antigravityUsesReplayCache, applyAntigravityReplay, clearAntigravityReplay, observeAntigravityReplay } from "./google-antigravity-replay";
 import { resolveAntigravityEffortWireModel } from "../providers/antigravity-models";
 import { googleVertexLocationConfigError } from "../providers/google-vertex-location";
+import { lookupReplayThoughtSignature } from "../responses/thought-signature-replay";
 import {
   isTranslatorBudgetExceededError,
   retainTranslatedEventBatch,
@@ -215,7 +216,13 @@ function messagesToGeminiFormat(
             const part: Record<string, unknown> = { functionCall };
             // Prefer the metadata that travelled with this exact call; fall back to the legacy
             // field for callers that have not been migrated. Never merge or synthesize.
-            const signature = tc.providerMetadata?.google?.thoughtSignature ?? tc.thoughtSignature;
+            // Final fallback (#1926): the durable store, read AT SERIALIZATION TIME. The
+            // Responses parser runs before the route/credential scope is bound, so its
+            // parse-time lookup can never hit; by the time this adapter serializes, the
+            // credential-scoped identity is bound and the durable lookup is meaningful.
+            const signature = tc.providerMetadata?.google?.thoughtSignature
+              ?? tc.thoughtSignature
+              ?? lookupReplayThoughtSignature(tc.id, parsed._reasoningReplayScope);
             if (isLikelyRealThoughtSignature(signature)) part.thoughtSignature = signature;
             parts.push(part);
           }

src/bridge.ts

@@ -15,6 +15,7 @@ import { rememberReasoningForCall } from "./responses/reasoning-replay-cache";
 import {
   rememberAndSerializeExtraContent,
   rememberExtraContentForReplay,
+  awaitThoughtSignatureDurability,
 } from "./responses/thought-signature-replay";
 import { resolveStallTimeoutSec } from "./stall-timeout";
 import { usageDisplayTotalTokens } from "./usage/totals";
@@ -1202,6 +1203,9 @@ export function bridgeToResponsesSSE(
               if (truncationReasonFor(event.stopReason)) {
                 // Upstream stopped before a normal completion. Surface as incomplete so the
                 // client can distinguish a truncated/filtered turn from a finished one.
+                // #1926 gap 2: bound the window in which a handed-out thought signature is
+                // not yet durable before the turn becomes externally terminal.
+                await awaitThoughtSignatureDurability();
                 const response = {
                   ...responseSnapshot("incomplete", finishedItems, event.endTurn),
                   usage: responsesUsage(event.usage),
@@ -1216,6 +1220,7 @@ export function bridgeToResponsesSSE(
                 emit("response.incomplete", { response });
                 reportTerminal("incomplete");
               } else {
+                await awaitThoughtSignatureDurability();
                 const response = { ...responseSnapshot("completed", finishedItems, event.endTurn), usage: responsesUsage(event.usage) };
                 options?.onCompletedResponse?.(response, event.providerState);
                 options?.onUsage?.(event.usage);
@@ -1236,6 +1241,7 @@ export function bridgeToResponsesSSE(
               if (currentWebSearch) closeCurrentWebSearch("failed", []);
               flushHiddenReasoningEnvelope();
               options?.onUsage?.(event.usage);
+              await awaitThoughtSignatureDurability();
               emit("response.incomplete", {
                 response: {
                   ...responseSnapshot("incomplete", finishedItems, event.endTurn),
@@ -1264,6 +1270,7 @@ export function bridgeToResponsesSSE(
               if (currentWebSearch) closeCurrentWebSearch("failed", []);
               const failure = adapterFailureFromEvent(event);
               if (event.usage) options?.onUsage?.(event.usage);
+              await awaitThoughtSignatureDurability();
               emit("response.failed", {
                 response: {
                   ...responseSnapshot("failed", finishedItems),
@@ -1325,6 +1332,7 @@ export function bridgeToResponsesSSE(
         if (currentToolCall) failCurrentToolCall();
         if (currentWebSearch) closeCurrentWebSearch("failed", []);
         options?.onUsage?.(undefined);
+        await awaitThoughtSignatureDurability();
         emit("response.incomplete", {
           response: {
             ...responseSnapshot("incomplete", finishedItems),
@@ -1365,6 +1373,10 @@ export function bridgeToResponsesSSE(
             flushHiddenRawReasoning();
             if (currentToolCall) failCurrentToolCall();
             if (currentWebSearch) closeCurrentWebSearch("failed", []);
+            // #1926 gap 2 residual: this beat callback is synchronous, so the durability
+            // barrier is not awaited on the stall-timeout kill path. The in-memory store is
+            // already updated; only a crash between here and the queued write loses it,
+            // which is the pre-#1926 status quo for an already-abnormal termination.
             emit("response.incomplete", {
               response: {
                 ...responseSnapshot("incomplete", finishedItems),

src/responses/reasoning-replay-cache.ts

@@ -123,6 +123,36 @@ export function durableReplayDestinationIdentity(baseUrl: string | undefined): s
   return `destination:${createHash("sha256").update("destination\0").update(canonical).digest("hex")}`;
 }
 
+/**
+ * Restart-stable credential identity for the DURABLE thought-signature store (#1926).
+ *
+ * Unlike the destination, credential material may be secret (an API key), so a plain
+ * unsalted digest would turn the store file into an offline verifier for candidate keys.
+ * The identity is therefore an HMAC under a random salt persisted NEXT TO the store: the
+ * salt is not a secret escrow (it holds no credential material) but it makes every digest
+ * useless outside this installation. Full 256-bit output — no truncation.
+ *
+ * OAuth accounts use the persisted account-slot id (not the rotating token/generation):
+ * relinking a slot to a different upstream account keeps the id, but the upstream then
+ * validates signatures against the new credential and rejects stale ones — the same
+ * fail-closed backstop the destination identity relies on. Credential-scoped header
+ * overrides participate so two provider entries sharing one key but different
+ * authorization headers stay distinct, mirroring the process-local identity.
+ */
+export function durableReplayCredentialIdentity(
+  kind: "key" | "oauth" | "codex",
+  material: string | undefined,
+  headers: Record<string, string> | undefined,
+  salt: Buffer | undefined,
+): string | undefined {
+  if (!nonEmpty(material) || !salt || salt.length < 16) return undefined;
+  const overrides = credentialHeaderOverrides(headers);
+  return `credential:${createHmac("sha256", salt)
+    .update(`credential\0${kind}\0`)
+    .update(JSON.stringify([material, overrides]))
+    .digest("hex")}`;
+}
+
 /** Produce a non-reversible process-local identity for credential material. */
 export function reasoningReplayCredentialIdentity(
   kind: "key" | "oauth" | "codex",

src/responses/thought-signature-replay.ts

@@ -20,18 +20,23 @@
  * destination, adapter, model and credential — so a signature can only ever be replayed into
  * the turn that produced it.
  */
-import { readFileSync } from "node:fs";
+import { chmodSync, readFileSync, writeFileSync } from "node:fs";
+import { randomBytes } from "node:crypto";
 import { join } from "node:path";
 import { atomicWriteFileAsync, getConfigDir } from "../config";
 import type { OcxProviderOpaqueToolCallMetadata, OcxReasoningReplayScopeRef } from "../types";
 import { isCarryableSignature, responsesExtraContentFromProviderMetadata } from "./provider-opaque-metadata";
 
 const STORE_FILE_NAME = "thought-signature-replay.json";
+const SALT_FILE_NAME = "thought-signature-replay.salt";
 /**
- * Bumped whenever `keyFor` changes shape. v3 added the durable destination identity, so a
- * v2 file's keys can never match and are dropped on load instead of aging out invisibly.
+ * Bumped whenever `keyFor` changes shape. v3 added the durable destination identity; v4
+ * added the salted durable credential identity (#1926), so a v3 file's keys can never
+ * match and are dropped on load instead of aging out invisibly. v3 rows carried no
+ * credential information, so they are not upgradable — the next Gemini turn re-accumulates
+ * its signatures (bounded, best-effort loss identical to the pre-store status quo).
  */
-const STORE_VERSION = 3;
+const STORE_VERSION = 4;
 
 /** Bound on remembered entries; real signatures are a few hundred bytes, so this stays small. */
 const MAX_ENTRIES = 16_384;
@@ -64,6 +69,45 @@ function storePath(): string {
   return join(getConfigDir(), STORE_FILE_NAME);
 }
 
+function saltPath(): string {
+  return join(getConfigDir(), SALT_FILE_NAME);
+}
+
+let cachedSalt: Buffer | undefined;
+let saltLoaded = false;
+
+/**
+ * Installation-local salt for the durable credential identity (#1926). Created once and
+ * persisted beside the store; losing it invalidates every stored key (the entries then
+ * never match and age out), which is safe — signatures re-accumulate per turn.
+ */
+export function thoughtSignatureReplaySalt(): Buffer | undefined {
+  if (saltLoaded) return cachedSalt;
+  saltLoaded = true;
+  try {
+    const raw = readFileSync(saltPath());
+    if (raw.length >= 16) {
+      // Re-assert owner-only permissions on every load: a pre-existing file may have
+      // been created before this guard or loosened by external tooling.
+      try { chmodSync(saltPath(), 0o600); } catch { /* best effort on exotic filesystems */ }
+      cachedSalt = raw;
+      return cachedSalt;
+    }
+  } catch {
+    // fall through to mint
+  }
+  try {
+    const minted = randomBytes(32);
+    writeFileSync(saltPath(), minted, { mode: 0o600 });
+    cachedSalt = minted;
+  } catch {
+    // Unwritable config dir: no durable credential identity this process; the durable
+    // store fails closed (keyFor returns undefined) rather than keying under a shared id.
+    cachedSalt = undefined;
+  }
+  return cachedSalt;
+}
+
 function nonEmpty(value: unknown): value is string {
   return typeof value === "string" && value.trim().length > 0;
 }
@@ -84,6 +128,10 @@ function keyFor(callId: string, scope: OcxReasoningReplayScopeRef | undefined):
     || !nonEmpty(identity?.providerName)
     || !nonEmpty(identity?.adapterName)
     || !nonEmpty(identity?.modelId)
+    // v4 (#1926): a missing durable credential identity means we cannot isolate this
+    // entry per credential across restarts. Refusing the key is the fail-closed choice —
+    // "credential:unknown" would let two different credentials share one durable slot.
+    || !nonEmpty(identity?.credentialDurableIdentity)
   ) return undefined;
   return JSON.stringify([
     scope.clientThreadId,
@@ -93,6 +141,7 @@ function keyFor(callId: string, scope: OcxReasoningReplayScopeRef | undefined):
     // reasoning cache's randomBytes-keyed HMAC cannot. Without it, one provider NAME
     // serving two endpoints shares signatures across both.
     identity.providerDestinationDurableIdentity ?? "destination:unknown",
+    identity.credentialDurableIdentity,
     identity.adapterName,
     identity.modelId,
     callId,
@@ -265,6 +314,8 @@ export function resetThoughtSignatureReplayForTests(): void {
   totalBytes = 0;
   loaded = false;
   persistChain = Promise.resolve();
+  cachedSalt = undefined;
+  saltLoaded = false;
 }
 
 export function thoughtSignatureReplayCountForTests(): number {
@@ -275,3 +326,22 @@ export function thoughtSignatureReplayCountForTests(): number {
 export function flushThoughtSignatureReplayForTests(): Promise<void> {
   return persistChain;
 }
+
+/**
+ * Bounded commit barrier (#1926 gap 2). All durable writes serialize onto
+ * `persistChain`, so awaiting it (with a cap) before a turn's terminal frame becomes
+ * externally visible bounds the restart window in which a handed-out signature could
+ * be lost. Bounded BEST EFFORT: on timeout the turn proceeds — availability wins, and
+ * the miss cost is one turn's re-accumulation (the pre-store status quo). The
+ * in-memory map is updated synchronously at remember() time, so within one process
+ * lifetime replay never races this barrier at all.
+ */
+export function awaitThoughtSignatureDurability(capMs = 250): Promise<void> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const cap = new Promise<void>(resolve => {
+    timer = setTimeout(resolve, capMs);
+  });
+  return Promise.race([persistChain, cap]).then(() => {
+    if (timer !== undefined) clearTimeout(timer);
+  });
+}

src/server/responses/core.ts

@@ -18,9 +18,11 @@ import {
   reasoningReplayCodexCredentialIdentity,
   reasoningReplayDestinationIdentity,
   durableReplayDestinationIdentity,
+  durableReplayCredentialIdentity,
   reasoningReplayKeyCredentialIdentity,
   reasoningReplayOAuthCredentialIdentity,
 } from "../../responses/reasoning-replay-cache";
+import { awaitThoughtSignatureDurability, thoughtSignatureReplaySalt } from "../../responses/thought-signature-replay";
 import { buildCompactV1Output, COMPACT_PROMPT, decodeCompactionSummary, extractCompactUserMessages } from "../../responses/compaction";
 import { FORWARD_HEADERS, sanitizeReasoningInputContent } from "../../adapters/openai-responses";
 import {
@@ -326,11 +328,21 @@ function bindRouteReasoningReplayScope(args: {
 }): void {
   const { parsed, providerName, provider, adapterName } = args;
   let credentialIdentity: string | undefined;
+  let credentialDurableIdentity: string | undefined;
+  const durableSalt = thoughtSignatureReplaySalt();
   if (provider.authMode === "oauth") {
     credentialIdentity = reasoningReplayOAuthCredentialIdentity(
       args.oauthCredentialSnapshot,
       provider.headers,
     );
+    // The persisted account-slot id survives token refresh and restarts; the rotating
+    // generation deliberately does NOT participate (#1926 design: rotation-safe).
+    credentialDurableIdentity = durableReplayCredentialIdentity(
+      "oauth",
+      args.oauthCredentialSnapshot?.accountId,
+      provider.headers,
+      durableSalt,
+    );
   } else if (provider.authMode === "forward") {
     const poolContext = args.codexAuthContext?.kind === "pool"
       || args.codexAuthContext?.kind === "main-pool"
@@ -349,8 +361,28 @@ function bindRouteReasoningReplayScope(args: {
       writerGeneration: poolContext?.writerGeneration,
       headers: provider.headers,
     });
+    // Durable identity requires a STABLE, TRUSTED account handle. Pool context comes from
+    // our own account store; a client-supplied chatgpt-account-id header is attacker
+    // -influenceable bucket selection and a bearer alone is rotating material — both are
+    // refused, so direct-forward turns get no durable scope (fail closed; the in-process
+    // cache still covers same-process replay).
+    const codexDurableHandle = poolContext?.accountId
+      ?? poolContext?.chatgptAccountId
+      ?? undefined;
+    credentialDurableIdentity = durableReplayCredentialIdentity(
+      "codex",
+      codexDurableHandle ?? undefined,
+      provider.headers,
+      durableSalt,
+    );
   } else if (provider.authMode !== "local") {
     credentialIdentity = reasoningReplayKeyCredentialIdentity(provider);
+    credentialDurableIdentity = durableReplayCredentialIdentity(
+      "key",
+      nonEmptyProviderApiKey(provider),
+      provider.headers,
+      durableSalt,
+    );
   }
   const providerDestinationIdentity = reasoningReplayDestinationIdentity(provider.baseUrl);
   bindReasoningReplayScope(
@@ -363,11 +395,18 @@ function bindRouteReasoningReplayScope(args: {
           adapterName,
           modelId: parsed.modelId,
           credentialIdentity,
+          ...(credentialDurableIdentity ? { credentialDurableIdentity } : {}),
         }
       : undefined,
   );
 }
 
+function nonEmptyProviderApiKey(provider: OcxProviderConfig): string | undefined {
+  return typeof provider.apiKey === "string" && provider.apiKey.trim().length > 0
+    ? provider.apiKey
+    : undefined;
+}
+
 function isFixedCodexAccount(authCtx: CodexAuthContext): boolean {
   return (authCtx.kind === "pool" || authCtx.kind === "main-pool")
     && authCtx.fixedAccount === true;
@@ -3616,6 +3655,10 @@ async function handleResponsesInner(
         responseStateOptions(adapterNeedsForcedContinuation(adapter.name)),
       );
     }
+    // #1926 gap 2: the buffered path queued its signature persists inside
+    // buildResponseJSON; bound the durability window before the JSON becomes
+    // externally visible.
+    await awaitThoughtSignatureDurability();
     return new Response(JSON.stringify(json), { headers: { "Content-Type": "application/json" } });
   }
 
@@ -4461,6 +4504,8 @@ async function handleResponsesInner(
         responseStateOptions(activeAdapter.name === "kiro"),
       );
     }
+    // #1926 gap 2: same buffered-path durability bound as the primary branch.
+    await awaitThoughtSignatureDurability();
     return new Response(JSON.stringify(json), { headers: { "Content-Type": "application/json" } });
   }
 

src/types.ts

@@ -14,6 +14,12 @@ export interface OcxReasoningReplayIdentity {
   modelId: string;
   /** Opaque process-local credential identity; never a raw token or API key. */
   credentialIdentity: string;
+  /**
+   * Salted-HMAC credential identity that survives restarts, for the durable
+   * thought-signature store (#1926). Absent when no durable identity could be
+   * derived — the durable store then refuses to key the entry (fail closed).
+   */
+  credentialDurableIdentity?: string;
 }
 
 /**