fix(inference): make prompt caching actually reach the wire (native Anthropic, OpenRouter breakpoints, cache identities)

crates/openhuman-core/src/core/jsonrpc_tests.rs

@@ -195,7 +195,6 @@ fn domain_subscriber_registration_readiness_helper_is_idempotent() {
         true
     ));
 }
-
 /// #5027 — the tool-execution timeout must be seeded on the always-on core boot
 /// path (`register_domain_subscribers`), NOT inside
 /// `channels::runtime::startup::start_channels`, which is skipped for

gitbooks/developing/architecture/agent-harness.md

@@ -150,6 +150,16 @@ Freezing the prompt is only one third of the contract, and the other two are eas
 
 A resumed session's replayed prefix is folded into `Agent::history` rather than spliced into one request, so the request, the following turn and the persisted transcript all read the same sequence. Splicing it cost the conversation twice: the next turn went out without it, and the transcript written afterwards (serialized from `history`) held only the new turn — which the *next* resume then read back, truncating the thread a little further on every restart.
 
+#### A stable prefix only pays out when the wire says so
+
+Keeping the bytes stable is necessary, not sufficient: the provider still has to be *told* to cache, and two providers need telling explicitly.
+
+- **Anthropic caches nothing without `cache_control` markers, and its OpenAI-compatible endpoint cannot carry them** — Anthropic documents prompt caching as unsupported on that path and reports `prompt_tokens_details` as always empty. A `cloud_providers` entry with `auth_style = "anthropic"` is therefore built as the crate's native Messages adapter (`inference::provider::crate_anthropic`), which places markers on the last tool, the last system block, and the final message so a growing tool loop reuses the previous iteration's cache rather than only the system prompt. The one exception is text mode (`native_tools = false`, prompt-guided tools), which only the Chat Completions adapter implements and which keeps the compat client — and so keeps paying full price.
+- **OpenRouter forwards markers to Anthropic and Gemini** but adds none itself; hosted OpenAI rejects unknown content-part fields. The OpenRouter slug (and any endpoint on `openrouter.ai`) enables `OpenAiModel::with_explicit_cache_control`, which marks the last system and last user message; every other Chat Completions endpoint stays unmarked.
+- **The routing hint is separate from the markers.** The harness derives a `prompt_cache_key` from the declared stable prefix and puts it in `provider_options`, so every turn of a thread — and every sub-agent sharing its system prompt and tool set — routes to the same cache shard on providers that shard (OpenAI's `prompt_cache_key`). Adapters that have no such concept drop it. The managed backend receives it as a top-level body field alongside `thread_id`.
+
+All three hang off `RunPolicy::cache.protect_prompt_prefix`, which the host sets unconditionally, and off the `PromptCacheSegmentMiddleware`, which declares the system prompt and tool catalogue as the cacheable prefix on every request. Until tinyagents began stamping that effective policy onto the outgoing request, both readers consulted `request.cache_policy` alone — always `None` here — so the flag was diagnostic only and no marker or key ever reached the wire. The `[cache]` debug line on every model call now reports whether a key was injected and how many cacheable segments were declared; `cache_read_tokens` on the usage that comes back is the number that proves it worked.
+
 Measure this rather than reasoning about it. `CAPTURE_ALL=1 node scripts/debug/capture-first-inference.mjs` records a whole session's requests, `scripts/debug/run-multi-turn-capture.mjs` drives a multi-turn thread through the production RPC, and `scripts/debug/audit-inference-prefix.mjs` reports the first divergence and attributes it. A single-turn capture cannot see any of these — the question is never what turn 1 costs, it is whether turn 2 can reuse it.
 
 ### AGENTS.md project instructions

src/openhuman/agent/dispatcher.rs

@@ -249,7 +249,6 @@ fn from_dialect_message(message: DialectMessage) -> ChatMessage {
         role: message.role.as_str().to_string(),
         content: message.content,
         extra_metadata: message.extra_metadata,
-        cache_breakpoints: Vec::new(),
     }
 }
 
@@ -293,12 +292,18 @@ fn dispatch_format_results(
     dialect: &dyn ToolDialect,
     results: &[ToolExecutionResult],
 ) -> ConversationMessage {
-    dialect
-        .format_results(&to_outcomes(results))
-        .into_iter()
-        .map(from_transcript_entry)
-        .next()
-        .unwrap_or_else(|| ConversationMessage::ToolResults(Vec::new()))
+    let entries = dialect.format_results(&to_outcomes(results));
+    debug_assert_eq!(
+        entries.len(),
+        1,
+        "OpenHuman tool results must remain a single conversation message"
+    );
+    from_transcript_entry(
+        entries
+            .into_iter()
+            .next()
+            .expect("tool result formatting must produce a message"),
+    )
 }
 
 fn dispatch_provider_messages(

src/openhuman/agent/dispatcher_tests.rs

@@ -129,7 +129,8 @@ fn pformat_dispatcher_parses_tool_call_tag() {
     let dispatcher = PFormatToolDispatcher::new(registry);
     let response = ChatResponse {
         text: Some(
-            "Let me check the weather.\n<tool_call>get_weather[London|metric]</tool_call>".into(),
+            "Let me check the weather.\n<tool_call>get_weather[0|London|1|metric]</tool_call>"
+                .into(),
         ),
         tool_calls: vec![],
         usage: None,
@@ -181,7 +182,7 @@ fn pformat_dispatcher_handles_multiple_tags() {
     let dispatcher = PFormatToolDispatcher::new(registry);
     let response = ChatResponse {
         text: Some(
-            "Step 1.\n<tool_call>shell[ls]</tool_call>\nStep 2.\n<tool_call>shell[pwd]</tool_call>"
+            "Step 1.\n<tool_call>shell[0|ls]</tool_call>\nStep 2.\n<tool_call>shell[0|pwd]</tool_call>"
                 .into(),
         ),
         tool_calls: vec![],

src/openhuman/agent/pformat_tests.rs

@@ -54,7 +54,7 @@ fn a_registered_tool_parses_positionally_through_the_adapter() {
     // arguments come back named and coerced.
     let tools: Vec<Box<dyn Tool>> = vec![Box::new(StubTool("echo"))];
     let reg = build_registry(&tools);
-    let (name, args) = parse_call("echo[3|hi]", &reg).expect("known tool parses");
+    let (name, args) = parse_call("echo[0|3|1|hi]", &reg).expect("known tool parses");
     assert_eq!(name, "echo");
     // Schema properties are ordered alphabetically: count, value.
     assert_eq!(args["count"], 3);
@@ -70,5 +70,5 @@ fn signature_rendering_agrees_between_the_tool_and_schema_forms() {
     let from_tool = render_signature_from_tool(&tool);
     let from_schema = render_signature_from_schema("echo", &tool.parameters_schema());
     assert_eq!(from_tool, from_schema);
-    assert_eq!(from_tool, "echo[count|value]");
+    assert_eq!(from_tool, "echo[0|<count>|1|<value>]");
 }

src/openhuman/agent/prompts/mod_tests_part_01_tests.rs

@@ -475,7 +475,7 @@ fn tools_section_pformat_renders_signature_not_schema() {
     let rendered = ToolsSection.build(&ctx).unwrap();
     // Alphabetical: kind, sugar.
     assert!(
-        rendered.contains("Call as: `make_tea[kind|sugar]`"),
+        rendered.contains("Call as: `make_tea[0|<kind>|1|<sugar>]`"),
         "expected p-format signature in tools section, got:\n{rendered}"
     );
     // Should NOT contain the raw JSON schema dump.

src/openhuman/agent/tinyagents/host/model_resolver.rs

@@ -218,6 +218,13 @@ impl<State: Send + Sync> ChatModel<State> for StatelessModel {
         self.inner.profile()
     }
 
+    // Forwarded so the harness response cache scopes on the *real* model. The
+    // trait default declines identity, and `scoped_cache_key` then folds a fixed
+    // "anonymous-model" marker — every wrapped model would share one key space.
+    fn cache_identity(&self) -> Option<String> {
+        self.inner.cache_identity()
+    }
+
     async fn invoke(
         &self,
         _state: &State,

src/openhuman/agent/tinyagents/mod_part_01.rs

@@ -246,10 +246,17 @@ fn run_policy_for(max_iterations: usize, response_cache_enabled: bool) -> RunPol
     // to manufacture valid stub arguments only because this policy was left at
     // its historical fail-fast default.
     policy.invalid_args = InvalidArgsPolicy::ReturnToolError;
-    // Prompt-prefix protection is always on (issue #4249, 03.2): the
-    // `PromptCacheGuardMiddleware` records a `CacheLayoutEvent` whenever volatile
-    // content busts the provider KV-cache prefix. Purely diagnostic — never
-    // mutates the request.
+    // Prompt-prefix protection is always on (issue #4249, 03.2). Two things
+    // ride on it, and both were inert until the harness started stamping this
+    // effective policy onto the outgoing request (tinyagents `model_call`):
+    //   * the `PromptCacheGuardMiddleware` records a `CacheLayoutEvent` whenever
+    //     volatile content busts the provider KV-cache prefix (diagnostic), and
+    //   * the loop injects a `prompt_cache_key` routing hint derived from the
+    //     stable prefix into `provider_options`, and the provider adapters see
+    //     `protect_prompt_prefix` and emit explicit `cache_control` breakpoints
+    //     where the provider needs them (native Anthropic, OpenRouter relays).
+    // The stable prefix itself is declared per request by the host
+    // `PromptCacheSegmentMiddleware`.
     policy.cache.protect_prompt_prefix = true;
     // Response caching is gated: it is enabled only for deterministic internal
     // runs (which additionally attach a `ResponseCache`). Interactive chat turns

src/openhuman/agent/tinyagents/model.rs

@@ -440,6 +440,10 @@ impl ChatModel<()> for RouteRecordingModel {
         self.inner.profile()
     }
 
+    fn cache_identity(&self) -> Option<String> {
+        self.inner.cache_identity()
+    }
+
     async fn invoke(
         &self,
         state: &(),
@@ -496,6 +500,15 @@ impl ChatModel<()> for ProfileOverrideModel {
         Some(&self.profile)
     }
 
+    fn cache_identity(&self) -> Option<String> {
+        self.inner
+            .cache_identity()
+            .map(|identity| match &self.request_model {
+                Some(model) => format!("{identity}:request-model={model}"),
+                None => identity,
+            })
+    }
+
     async fn invoke(
         &self,
         state: &(),
@@ -538,6 +551,14 @@ impl ChatModel<()> for MaxTokensModel {
         self.inner.profile()
     }
 
+    // Every host wrapper forwards `cache_identity`: the trait default declines
+    // identity, and the harness response cache then keys every wrapped model
+    // under one "anonymous-model" marker, so a shared cache could serve a
+    // local model's answer to a hosted one.
+    fn cache_identity(&self) -> Option<String> {
+        self.inner.cache_identity()
+    }
+
     async fn invoke(
         &self,
         state: &(),

src/openhuman/agent/tinyagents/model_g1_usage_tests_tests.rs

@@ -148,7 +148,7 @@ fn prompt_guided_response_uses_tinyagents_xml_parser() {
 #[test]
 fn prompt_guided_response_keeps_legacy_pformat_fallback() {
     let response = prompt_guided_text_response(
-        "<tool_call>lookup[7|needle]</tool_call>".to_string(),
+        "<tool_call>lookup[0|7|1|needle]</tool_call>".to_string(),
         &tool_request(),
     );
 

src/openhuman/agent/tinyagents/model_route_recording_tests_tests.rs

@@ -65,3 +65,63 @@ async fn streamed_model_records_concrete_route_before_stream_consumption() {
         })
     );
 }
+
+/// A model that identifies itself for response-cache scoping.
+struct IdentifiedModel;
+
+#[async_trait]
+impl ChatModel<()> for IdentifiedModel {
+    fn cache_identity(&self) -> Option<String> {
+        Some("test-provider:https://example/v1:model-x".to_string())
+    }
+
+    async fn invoke(
+        &self,
+        _state: &(),
+        _request: ModelRequest,
+    ) -> tinyinference::Result<ModelResponse> {
+        Ok(ModelResponse::assistant("ok"))
+    }
+}
+
+/// The harness scopes its response cache on `ChatModel::cache_identity`, and
+/// the trait default declines identity. Every host wrapper therefore has to
+/// forward it, or every wrapped model collapses into the crate's
+/// "anonymous-model" key space and a shared cache can cross-serve answers
+/// between a hosted and a local model.
+#[test]
+fn every_turn_wrapper_forwards_the_inner_cache_identity() {
+    let expected = Some("test-provider:https://example/v1:model-x".to_string());
+    let inner: Arc<dyn ChatModel<()>> = Arc::new(IdentifiedModel);
+
+    let route = RouteRecordingModel::new(inner.clone(), "p", "m");
+    assert_eq!(route.cache_identity(), expected);
+
+    let profile = ProfileOverrideModel::new(inner.clone(), ModelProfile::default());
+    assert_eq!(profile.cache_identity(), expected);
+
+    let capped = MaxTokensModel::new(inner.clone(), 1024);
+    assert_eq!(capped.cache_identity(), expected);
+
+    // Nested the way the turn builder stacks them.
+    let stacked = RouteRecordingModel::new(
+        Arc::new(MaxTokensModel::new(
+            Arc::new(ProfileOverrideModel::new(inner, ModelProfile::default())),
+            1024,
+        )),
+        "p",
+        "m",
+    );
+    assert_eq!(stacked.cache_identity(), expected);
+}
+
+#[test]
+fn profile_override_cache_identity_includes_request_model() {
+    let inner: Arc<dyn ChatModel<()>> = Arc::new(IdentifiedModel);
+    let first = ProfileOverrideModel::new(inner.clone(), ModelProfile::default())
+        .with_request_model("model-a");
+    let second =
+        ProfileOverrideModel::new(inner, ModelProfile::default()).with_request_model("model-b");
+
+    assert_ne!(first.cache_identity(), second.cache_identity());
+}

src/openhuman/inference/provider/claude_agent_sdk/subprocess.rs

@@ -254,6 +254,19 @@ impl ChatModel<()> for ClaudeAgentSdkProvider {
         Some(&self.profile)
     }
 
+    /// Identity for harness response-cache scoping: the SDK binary and the
+    /// selected model. No credential is involved on this path.
+    fn cache_identity(&self) -> Option<String> {
+        Some(format!(
+            "claude_agent_sdk:{}:{}",
+            self.config.binary,
+            self.profile
+                .model
+                .as_deref()
+                .unwrap_or(&self.config.default_model)
+        ))
+    }
+
     async fn invoke(
         &self,
         _state: &(),

src/openhuman/inference/provider/claude_agent_sdk/subprocess_tests.rs

@@ -213,6 +213,15 @@ printf '%s\n' '{"type":"result","result":"Calling.<tool_call>{\"name\":\"lookup\
     );
 }
 
+#[test]
+fn cache_identity_includes_selected_model() {
+    let config = ClaudeAgentSdkConfig::default();
+    let first = ClaudeAgentSdkProvider::for_model(config.clone(), "model-a");
+    let second = ClaudeAgentSdkProvider::for_model(config, "model-b");
+
+    assert_ne!(first.cache_identity(), second.cache_identity());
+}
+
 /// The CLI takes one `--system-prompt`, so every system message has to be
 /// folded into it. Taking only the first dropped the artifact contents list and
 /// the turn-cap wrap-up, which are appended as system messages exactly so the

src/openhuman/inference/provider/claude_code/mod.rs

@@ -221,6 +221,16 @@ impl ChatModel<()> for ClaudeCodeProvider {
         Some(&self.profile)
     }
 
+    /// Identity for harness response-cache scoping. Names the binary and the
+    /// model, never the API key.
+    fn cache_identity(&self) -> Option<String> {
+        Some(format!(
+            "claude_code:{}:{}",
+            self.bin_path.display(),
+            self.model
+        ))
+    }
+
     async fn invoke(
         &self,
         _state: &(),