diff --git a/Cargo.lock b/Cargo.lock index 2bd0d26b493..6049cf35296 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4731,6 +4731,7 @@ dependencies = [ "tinystoragedrivers", "tinytools", "tinytools-agent", + "tinytools-jev", "tinytools-std", "tinyvoice", "tinyvoice-bus", diff --git a/crates/openhuman-core/Cargo.toml b/crates/openhuman-core/Cargo.toml index cbf7cbcca3e..21e6b32f8b1 100644 --- a/crates/openhuman-core/Cargo.toml +++ b/crates/openhuman-core/Cargo.toml @@ -183,6 +183,7 @@ tinyinference-voice = { path = "../../vendor/tinyagents/vendor/tinyinference/cra # No `[patch.crates-io]` entry: the crate is not published, so the path is the # whole address — the same arrangement as `tinyhumans-sdk` and the `-bus` # crates. After cloning: `git submodule update --init --recursive vendor/`. +tinytools-jev = { path = "../../vendor/tinyagents/vendor/tinytools/crates/tinytools-jev" } tinytools = { path = "../../vendor/tinyagents/vendor/tinytools/crates/tinytools" } tinytools-agent = { path = "../../vendor/tinyagents/vendor/tinytools/crates/tinytools-agent" } tinytools-std = { path = "../../vendor/tinyagents/vendor/tinytools/crates/tinytools-std" } diff --git a/crates/openhuman-core/src/agent/tinyagents/harness_assembly.rs b/crates/openhuman-core/src/agent/tinyagents/harness_assembly.rs index 0c2d02eccfa..c1887e8c71b 100644 --- a/crates/openhuman-core/src/agent/tinyagents/harness_assembly.rs +++ b/crates/openhuman-core/src/agent/tinyagents/harness_assembly.rs @@ -245,6 +245,7 @@ pub(super) fn assemble_turn_harness( } // Capture context settings before `install` consumes `context_mw`. + let recovery_config = context_mw.runtime_config.clone(); let autocompact_enabled = context_mw.autocompact_enabled; let compaction = context_mw.compaction; // Captured for the same reason `autocompact_enabled` is — `install` consumes @@ -313,7 +314,10 @@ pub(super) fn assemble_turn_harness( // Repeated failures stop the run using each tool's declared side effects. let repeated_failure = handle.as_ref().map(|handle| { let (t, halt) = (REPEATED_TOOL_FAILURE_THRESHOLD, halt_summary.clone()); - let mw = middleware::RepeatedToolFailureMiddleware::new(handle.clone(), t, halt); + let mut mw = middleware::RepeatedToolFailureMiddleware::new(handle.clone(), t, halt); + if let Some(config) = &recovery_config { + mw = mw.with_recovery(config); + } Arc::new(mw.with_tool_facts(middleware::tool_sets_lookup(tool_sets.clone()))) }); if let Some(mw) = &repeated_failure { @@ -350,6 +354,10 @@ pub(super) fn assemble_turn_harness( &session_deferred, ); + if let Some(mw) = &repeated_failure { + mw.set_recovery_registry(harness.tools(), tool_policy.as_ref().map(|p| &p.session)); + } + // SHADOW tool-exposure layer (issue #4249, 01.3 — dynamic exposure). Compose // the OpenHuman exposure policy as a crate-native selection layer // (ToolAllowlistMiddleware + a ContextualToolSelectionMiddleware built via diff --git a/crates/openhuman-core/src/agent/tinyagents/middleware.rs b/crates/openhuman-core/src/agent/tinyagents/middleware.rs index c71f37d9020..6207bd17e6c 100644 --- a/crates/openhuman-core/src/agent/tinyagents/middleware.rs +++ b/crates/openhuman-core/src/agent/tinyagents/middleware.rs @@ -36,6 +36,8 @@ mod loop_guards; mod memory_pack; mod nudge_injector; mod packed_tool_route; +mod recovery_advice; +mod recovery_telemetry; mod repeated_failure; mod research_budget; mod shell_turn_budget; diff --git a/crates/openhuman-core/src/agent/tinyagents/middleware/recovery_advice.rs b/crates/openhuman-core/src/agent/tinyagents/middleware/recovery_advice.rs new file mode 100644 index 00000000000..7f5e410f373 --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/middleware/recovery_advice.rs @@ -0,0 +1,382 @@ +//! Bounded host accounting and advice projection. TinyTools owns evaluation logic. +use super::call_effect::CallEffect; +use crate::agent::tinyagents::host::OpenHumanRunContext; +use crate::config::schema::{RecoveryClassifier, RecoveryConfig}; +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::Mutex; +use tinyagents_harness::context::RunContext; +use tinytools_jev::recovery::{ + RecoveryAdvice, RecoveryAdviser, RecoveryAlternateReason, RecoveryAnswer, RecoveryClass, + RecoveryEffect, RecoveryObservation, RecoveryPhase, +}; + +pub(super) struct RunRecovery { + config: RecoveryConfig, + adviser: Option, + evaluations: AtomicU32, + gate: tokio::sync::Semaphore, + ledger: Mutex>, + candidates: Mutex>, + failed_tools: Mutex>, +} +impl RunRecovery { + pub(super) fn new(config: &crate::config::Config) -> Self { + let settings = config.agent.recovery.clone(); + let adviser = settings.validate().ok().and_then(|_| { + if !matches!( + settings.classifier, + RecoveryClassifier::Compare | RecoveryClassifier::Jev + ) { + return None; + } + let factory = + crate::agent::tinyagents::recovery_provider::installed_recovery_provider()?; + RecoveryAdviser::new(factory(config)?, settings.thresholds()).ok() + }); + Self { + config: settings, + adviser, + evaluations: AtomicU32::new(0), + gate: tokio::sync::Semaphore::new(1), + ledger: Mutex::new(HashMap::new()), + candidates: Mutex::default(), + failed_tools: Mutex::default(), + } + } + #[cfg(test)] + pub(super) fn with_evaluator( + config: RecoveryConfig, + evaluator: std::sync::Arc, + ) -> Self { + let adviser = RecoveryAdviser::new(evaluator, config.thresholds()).ok(); + Self { + config, + adviser, + evaluations: AtomicU32::new(0), + gate: tokio::sync::Semaphore::new(1), + ledger: Mutex::default(), + candidates: Mutex::default(), + failed_tools: Mutex::default(), + } + } + pub(super) fn set_candidates(&self, candidates: Vec) { + if let Ok(mut slot) = self.candidates.lock() { + *slot = candidates.into_iter().take(16).collect(); + } + } + pub(super) fn record_failure(&self, tool: &str) { + if !matches!( + self.config.classifier, + RecoveryClassifier::Jev | RecoveryClassifier::Compare + ) { + return; + } + if let Ok(mut failed) = self.failed_tools.lock() { + failed.insert(tool.to_owned()); + } + } + fn take_decision(&self) -> bool { + self.evaluations + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| { + (n < self.config.max_decisions_per_run).then_some(n + 1) + }) + .is_ok() + } + pub(super) fn clear(&self, scope: &str) { + if let Ok(mut ledger) = self.ledger.lock() { + ledger.remove(scope); + } + } + /// Return a lower ceiling or a request-only nudge. Never increases headroom. + /// Unknown failures are always fed to the existing tracker by the caller. + pub(super) async fn advise( + &self, + ctx: &RunContext, + scope: &str, + tool: &str, + shape: String, + failure: &str, + effect: CallEffect, + ) -> (bool, Option) { + let started = std::time::Instant::now(); + let record = |outcome: &str, advice: Option<&RecoveryAdvice>| { + super::recovery_telemetry::record(&self.config, started, outcome, advice) + }; + if self.config.validate().is_err() { + record("invalid_config", None); + return (false, None); + } + if !matches!( + self.config.classifier, + RecoveryClassifier::Compare | RecoveryClassifier::Jev + ) { + return (false, None); + } + let compare = self.config.classifier == RecoveryClassifier::Compare; + // Stable key is operation/scope only, independent of model class and args. + // Compare's private observation ledger never enters either host tracker + // or policy budgets; it only makes the evaluated evidence comparable. + let count = self + .ledger + .lock() + .map(|mut ledger| { + let n = ledger.entry(scope.into()).or_default(); + *n = n.saturating_add(1); + *n + }) + .unwrap_or(0); + let Some(adviser) = &self.adviser else { + record("no_provider", None); + return (false, None); + }; + // Executed writes have uncertain outcomes; do not ask a classifier to + // justify repeating them. The generic ladder remains authoritative. + if effect != CallEffect::ReadOnly { + record("effect_unsafe", None); + return (false, None); + } + let Ok(_permit) = self.gate.try_acquire() else { + record("concurrency_limit", None); + return (false, None); + }; + if ctx + .remaining_wall_clock() + .is_some_and(|remaining| remaining.is_zero()) + { + record("run_deadline", None); + return (false, None); + } + if !self.take_decision() { + record("decision_limit", None); + return (false, None); + } + let ceiling = std::time::Duration::from_millis(self.config.decision_timeout_ms); + let deadline = ctx + .remaining_wall_clock() + .map_or(ceiling, |d| d.min(ceiling)); + let mut observation = RecoveryObservation::new( + RecoveryPhase::Unknown, + format!("Capability: {tool}"), + failure, + ); + observation.argument_shape = shape; + observation.effect = RecoveryEffect::ReadOnly; + observation.attempts = count; + observation.repeated_failures = count; + // Both batches share one wall-clock budget. The first answer can enable + // wrong-tool advice; a repeated blocker can enable it independently. + let expires = tokio::time::Instant::now() + deadline; + let (advice, outcome) = tokio::select! { + biased; + _ = ctx.cancellation.cancelled() => (None, "cancelled"), + result = tokio::time::timeout_at(expires, adviser.advise(observation)) => match result { + Err(_) => (None, "timeout"), + Ok(Err(_)) => (None, "invalid_decision"), + Ok(Ok(advice)) => (Some(advice), "evaluated"), + }, + }; + record(outcome, advice.as_ref()); + let Some(RecoveryAdvice::Classified { + class, decision, .. + }) = advice + else { + return (false, None); + }; + let recoverable = matches!(decision.answers.get("recoverability"), Some(RecoveryAnswer::Noul(value)) if *value >= self.config.recoverability); + let mut alternate = None; + if self.config.alternate_tools + && (class == RecoveryClass::WrongTool || count >= self.config.repeated_blocker) + { + let failed = self + .failed_tools + .lock() + .map(|f| f.clone()) + .unwrap_or_default(); + let candidates = self + .candidates + .lock() + .map(|c| { + c.iter() + .filter(|c| c.key != tool && !failed.contains(&c.key)) + .cloned() + .collect::>() + }) + .unwrap_or_default(); + let expired = tokio::time::Instant::now() >= expires; + let permitted = !candidates.is_empty() && !expired && self.take_decision(); + if !candidates.is_empty() && !permitted { + record( + if expired { + "decision_deadline" + } else { + "decision_limit" + }, + None, + ); + } + if permitted { + let mut observation = RecoveryObservation::new( + RecoveryPhase::Unknown, + format!("Capability: {tool}"), + failure, + ); + observation.effect = RecoveryEffect::ReadOnly; + observation.attempts = count; + observation.repeated_failures = count; + observation.candidates = candidates; + observation.alternate_reason = Some(if class == RecoveryClass::WrongTool { + RecoveryAlternateReason::WrongToolAdvice + } else { + RecoveryAlternateReason::RepeatedBlocker + }); + let (response, outcome) = tokio::select! { + biased; + _ = ctx.cancellation.cancelled() => (None, "cancelled"), + result = tokio::time::timeout_at(expires, adviser.advise(observation)) => match result { + Err(_) => (None, "timeout"), + Ok(Err(_)) => (None, "invalid_decision"), + Ok(Ok(advice)) => (Some(advice), "evaluated"), + }, + }; + record(outcome, response.as_ref()); + if let Some(RecoveryAdvice::Classified { + alternate: choice, .. + }) = response + { + alternate = choice; + } + } + } + if compare { + return (false, None); + } + let optional_refusal = super::failure_policy::is_optional_service(tool) + && matches!( + class, + RecoveryClass::Authentication + | RecoveryClass::Permission + | RecoveryClass::Unavailable + ); + let halt = count >= 3 + || (optional_refusal && count >= 2) + || (!optional_refusal + && matches!( + class, + RecoveryClass::Authentication + | RecoveryClass::Permission + | RecoveryClass::Unsupported + )) + || (!optional_refusal && class == RecoveryClass::Unavailable && !recoverable); + let nudge = (!halt).then(|| if optional_refusal { + format!("Advisory recovery for `{tool}`: this optional service cannot currently be used. Continue with other available capabilities through the normal tool gates; do not resend the failed operation.") + } else { match class { + RecoveryClass::WrongArguments if recoverable => format!("Advisory recovery for `{tool}`: inspect its parameter schema and choose a corrected call through the normal tool gates. Do not resend the failed call unchanged."), + RecoveryClass::WrongTool => format!("Advisory recovery for `{tool}`: reconsider whether this capability addresses the task; choose your next step through the normal tool gates."), + _ => format!("Advisory recovery for `{tool}`: reassess the failed observation before choosing your next step. Existing permissions and retry limits still apply."), + }}); + let nudge = nudge.map(|n| match alternate { + Some(name) => format!("{n} Consider the already available `{name}` capability; any call must pass the normal admission checks."), + None => n, + }); + (halt, nudge) + } +} + +/// Arguments never cross the decision boundary: expose bounded field types only. +pub(super) fn argument_shape(value: &serde_json::Value) -> String { + value + .as_object() + .map(|m| { + m.iter() + .take(16) + .map(|(name, value)| { + let kind = match value { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "boolean", + serde_json::Value::Number(_) => "number", + serde_json::Value::String(_) => "string", + serde_json::Value::Array(_) => "array", + serde_json::Value::Object(_) => "object", + }; + // Keys can themselves be data. Admit schema-like ASCII identifiers only. + let name = if name.len() <= 64 + && name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') + { + name.as_str() + } else { + "field" + }; + format!("{name}:{kind}") + }) + .collect::>() + .join(", ") + }) + .unwrap_or_default() +} + +/// Only a short scrubbed diagnostic sentence is eligible. JSON/HTML/URLs and +/// multiline payloads cannot be reliably minimized here and cause abstention. +pub(super) fn diagnostic(text: &str) -> Option { + let first = text.lines().find(|l| !l.trim().is_empty())?.trim(); + if first.len() > 240 || first.contains(['{', '[', '<', '"', '=']) || first.contains("://") { + return None; + } + let safe = crate::security::scrub::sanitize_text(first).value; + if safe.is_empty() { + None + } else { + Some(safe) + } +} + +/// A conservative snapshot of the registered callable session surface. Only +/// self-contained declared reads need no new credentials, roots or approval. +/// The next model call still enters the ordinary live authorization path. +pub(super) fn admitted_candidates( + registry: &tinyagents_harness::tool::ToolRegistry<(), OpenHumanRunContext>, + session: Option<&crate::tools::agent_policy::ToolPolicySession>, +) -> Vec { + registry + .model_callable_names() + .into_iter() + .filter_map(|name| { + if name == "none" + || name.len() > 128 + || !name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-' || b == b'.') + { + return None; + } + if session.is_some_and(|s| { + !s.is_allowed(&name) + || s.hidden_tool_names.contains(&name) + || !matches!( + s.decision_for(&name).action, + crate::tools::agent_policy::ToolPolicyAction::Allow + ) + }) { + return None; + } + let tool = registry.get(&name)?; + let policy = tool.policy(); + if !policy.classified + || !policy.side_effects.read_only + || policy.side_effects.external_service + || tool.permission_level() != tinytools::PermissionLevel::ReadOnly + || policy.access.approval_required + || !policy.access.credentials.is_empty() + || policy.access.workspace != tinytools::WorkspaceAccess::None + { + return None; + } + let description = diagnostic(tool.description())?; + Some(tinytools_jev::JevOption { + key: name, + description, + }) + }) + .take(16) + .collect() +} diff --git a/crates/openhuman-core/src/agent/tinyagents/middleware/recovery_telemetry.rs b/crates/openhuman-core/src/agent/tinyagents/middleware/recovery_telemetry.rs new file mode 100644 index 00000000000..860e8649e01 --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/middleware/recovery_telemetry.rs @@ -0,0 +1,99 @@ +//! Payload-free evidence for advisory evaluation and observational compare mode. +use crate::config::schema::{RecoveryClassifier, RecoveryConfig}; +use std::time::Instant; +use tinytools_jev::recovery::{RecoveryAdvice, RecoveryAnswer}; + +#[derive(Debug, PartialEq)] +pub(super) struct RecoveryTelemetry<'a> { + pub mode: &'static str, + pub threshold_version: &'a str, + pub outcome: &'a str, + pub class: Option<&'static str>, + pub confidence: Option, + pub recoverability: Option, + pub attempts: Option, + pub alternate: bool, + pub input_tokens: Option, + pub output_tokens: Option, +} + +pub(super) fn metadata<'a>( + config: &'a RecoveryConfig, + outcome: &'a str, + advice: Option<&RecoveryAdvice>, +) -> RecoveryTelemetry<'a> { + let mode = match config.classifier { + RecoveryClassifier::Keywords => "keywords", + RecoveryClassifier::Compare => "compare", + RecoveryClassifier::Jev => "jev", + RecoveryClassifier::Off => "off", + }; + let threshold_version = if config.threshold_version.len() <= 128 + && config + .threshold_version + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) + { + &config.threshold_version + } else { + "invalid" + }; + let mut result = RecoveryTelemetry { + mode, + threshold_version, + outcome, + class: None, + confidence: None, + recoverability: None, + attempts: None, + alternate: false, + input_tokens: None, + output_tokens: None, + }; + if let Some(RecoveryAdvice::Classified { + class, + decision, + alternate, + .. + }) = advice + { + result.class = Some(class.key()); + if let Some(RecoveryAnswer::Choice { confidence, .. }) = decision.answers.get("class") { + result.confidence = Some(*confidence); + } + if let Some(RecoveryAnswer::Noul(value)) = decision.answers.get("recoverability") { + result.recoverability = Some(*value); + } + result.attempts = Some(decision.attempts); + result.alternate = alternate.is_some(); + result.input_tokens = decision.input_tokens; + result.output_tokens = decision.output_tokens; + } + result +} + +pub(super) fn record( + config: &RecoveryConfig, + started: Instant, + outcome: &str, + advice: Option<&RecoveryAdvice>, +) { + let event = metadata(config, outcome, advice); + let abstention = match advice { + Some(RecoveryAdvice::Abstained(reason)) => Some(format!("{reason:?}")), + _ => None, + }; + tracing::info!(target: "openhuman::recovery", + source = "jev", mode = event.mode, request_version = "tool-recovery-v1", + threshold_version = event.threshold_version, outcome = event.outcome, + class = event.class, confidence = event.confidence, recoverability = event.recoverability, + elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), + attempts = event.attempts, alternate = event.alternate, + input_tokens = event.input_tokens, output_tokens = event.output_tokens, + abstention = abstention.as_deref(), + "tool recovery evaluation"); +} + +#[cfg(test)] +#[path = "recovery_telemetry_tests.rs"] +mod tests; diff --git a/crates/openhuman-core/src/agent/tinyagents/middleware/recovery_telemetry_tests.rs b/crates/openhuman-core/src/agent/tinyagents/middleware/recovery_telemetry_tests.rs new file mode 100644 index 00000000000..46dc2f72421 --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/middleware/recovery_telemetry_tests.rs @@ -0,0 +1,71 @@ +use super::*; +use tinytools_jev::recovery::{RecoveryAbstention, RecoveryClass, RecoveryDecision}; + +#[test] +fn comparison_metadata_preserves_validated_numbers_without_payloads() { + let config = RecoveryConfig { + classifier: RecoveryClassifier::Compare, + ..Default::default() + }; + let advice = RecoveryAdvice::Classified { + class: RecoveryClass::WrongTool, + decision: RecoveryDecision { + answers: [ + ( + "class".into(), + RecoveryAnswer::Choice { + probabilities: Default::default(), + confidence: 0.9, + }, + ), + ("recoverability".into(), RecoveryAnswer::Noul(0.8)), + ] + .into_iter() + .collect(), + input_tokens: Some(100), + output_tokens: Some(20), + latency: Default::default(), + attempts: 1, + }, + alternate: Some("private_tool_identifier".into()), + correction_score: None, + }; + let event = metadata(&config, "accepted", Some(&advice)); + assert_eq!(event.mode, "compare"); + assert_eq!(event.class, Some("wrong_tool")); + assert_eq!(event.confidence, Some(0.9)); + assert_eq!(event.recoverability, Some(0.8)); + assert_eq!(event.attempts, Some(1)); + assert!(event.alternate); + assert_eq!(event.input_tokens, Some(100)); + assert!(!format!("{event:?}").contains("private_tool_identifier")); +} +#[test] +fn fallback_metadata_has_no_invented_provider_results() { + let config = RecoveryConfig::default(); + for advice in [ + None, + Some(RecoveryAdvice::Abstained( + RecoveryAbstention::EvaluatorFailure, + )), + ] { + let event = metadata(&config, "fallback", advice.as_ref()); + assert_eq!(event.class, None); + assert_eq!(event.confidence, None); + assert_eq!(event.recoverability, None); + assert_eq!(event.attempts, None); + assert_eq!(event.input_tokens, None); + assert!(!event.alternate); + } +} +#[test] +fn unexpected_version_payload_is_not_logged() { + let config = RecoveryConfig { + threshold_version: "secret value\nsecond line".into(), + ..Default::default() + }; + assert_eq!( + metadata(&config, "fallback", None).threshold_version, + "invalid" + ); +} diff --git a/crates/openhuman-core/src/agent/tinyagents/middleware/repeated_failure.rs b/crates/openhuman-core/src/agent/tinyagents/middleware/repeated_failure.rs index bacfeb7a850..9a4d952f1d1 100644 --- a/crates/openhuman-core/src/agent/tinyagents/middleware/repeated_failure.rs +++ b/crates/openhuman-core/src/agent/tinyagents/middleware/repeated_failure.rs @@ -104,112 +104,12 @@ pub(crate) struct RepeatedToolFailureMiddleware { /// The registered tools' declarations (read-only policy, external effect, /// permission level). Without it only tool names are read. tool_facts: Option, + recovery: Option, + recovery_shapes: Mutex>, } -impl RepeatedToolFailureMiddleware { - /// Build the breaker. `identical_threshold` (the identical-signature retry - /// ceiling) is handed straight to [`NoProgressTracker::new`], which clamps it - /// so a nudge always precedes a halt (a single failure is never a loop). - pub(crate) fn new( - handle: SteeringHandle, - identical_threshold: usize, - halt_summary: crate::agent::tinyagents::HaltSummarySlot, - ) -> Self { - Self { - handle, - halt_summary, - tracker: NoProgressTracker::new(identical_threshold), - classified: ClassifiedFailureTracker::default(), - last_exit_report: std::sync::Mutex::default(), - step: AtomicUsize::new(0), - arg_sigs: std::sync::Mutex::new(std::collections::HashMap::new()), - target_scopes: std::sync::Mutex::new(std::collections::HashMap::new()), - recoverable_sig_counts: std::sync::Mutex::new(std::collections::HashMap::new()), - recoverable_consecutive: AtomicU32::new(0), - pending_nudges: Arc::new(Mutex::new(Vec::new())), - call_effects: std::sync::Mutex::new(std::collections::HashMap::new()), - tool_facts: None, - } - } - - /// Judge each call's side effect from the registered tools' own - /// declarations ([`super::call_effect::tool_sets_lookup`]) rather than - /// from tool names alone. - pub(crate) fn with_tool_facts(mut self, lookup: ToolFactsLookup) -> Self { - self.tool_facts = Some(lookup); - self - } - - /// The request-scoped half of this breaker. Register it **last**: its - /// `before_model` must run after the transcript snapshot (which a failed - /// turn persists) and after every reduction step. - pub(crate) fn nudge_injector(&self) -> PendingNudgeInjector { - PendingNudgeInjector { - pending: self.pending_nudges.clone(), - } - } - - fn queue_nudge(&self, instruction: impl Into) { - if let Ok(mut pending) = self.pending_nudges.lock() { - pending.push(instruction.into()); - } - } - - /// Drain the queued nudges (what the injector does before a request). - #[cfg(test)] - pub(crate) fn take_pending_nudges(&self) -> Vec { - self.pending_nudges - .lock() - .map(|mut pending| std::mem::take(&mut *pending)) - .unwrap_or_default() - } - - /// Clear the consecutive recoverable-failure streak. Called on any success or - /// non-recoverable failure (the per-signature identical counts persist across - /// the turn, matching the legacy guard). Idempotent. - fn reset_recoverable_streak(&self) { - self.recoverable_consecutive.store(0, Ordering::SeqCst); - } - - /// Record one recoverable failure and return a root-cause halt summary once - /// its extended headroom is exhausted (identical `>=` [`RECOVERABLE_REPEAT_FAILURE_THRESHOLD`] - /// or consecutive `>=` [`RECOVERABLE_NO_PROGRESS_FAILURE_THRESHOLD`]). - fn record_recoverable(&self, tool: &str, arg_fp: &str, failure_text: &str) -> Option { - let key = format!("{tool}\u{1f}{arg_fp}"); - let count = self - .recoverable_sig_counts - .lock() - .ok() - .map(|mut counts| { - let c = counts.entry(key).or_insert(0); - *c += 1; - *c - }) - .unwrap_or(0); - let consecutive = self.recoverable_consecutive.fetch_add(1, Ordering::SeqCst) + 1; - tracing::debug!( - tool, - count, - consecutive, - "[tinyagents::mw] recoverable tool failure recorded with extended circuit-breaker headroom" - ); - if count >= RECOVERABLE_REPEAT_FAILURE_THRESHOLD { - return Some(recoverable_identical_halt_summary( - tool, - count, - failure_text, - )); - } - if consecutive >= RECOVERABLE_NO_PROGRESS_FAILURE_THRESHOLD { - return Some(recoverable_no_progress_halt_summary( - consecutive, - tool, - failure_text, - )); - } - None - } -} +#[path = "repeated_failure_state.rs"] +mod state; /// The first line of a failure, plus the first stderr line when the text is a /// command exit report -- that is where a program's own reason tends to be. @@ -357,6 +257,23 @@ impl Middleware<(), crate::agent::tinyagents::host::OpenHumanRunContext> scopes.insert(call.id.clone(), failure_scope(&call.name, &call.arguments)); } let effect = call_effect(self.tool_facts.as_ref(), &call.name, &call.arguments); + // New remote advice trusts only host registry declarations; name guesses + // remain confined to the pre-existing keyword policy. + let advisory_effect = self + .tool_facts + .as_ref() + .and_then(|f| f(&call.name, &call.arguments)) + .filter(|f| !f.external && !f.elevated && f.classified == Some(CallEffect::ReadOnly)) + .map_or(CallEffect::Unknown, |_| CallEffect::ReadOnly); + if let Ok(mut shapes) = self.recovery_shapes.lock() { + shapes.insert( + call.id.clone(), + ( + super::recovery_advice::argument_shape(&call.arguments), + advisory_effect, + ), + ); + } tracing::trace!( tool = %call.name, call_id = %call.id, @@ -377,6 +294,17 @@ impl Middleware<(), crate::agent::tinyagents::host::OpenHumanRunContext> result: &mut TaToolResult, ) -> TaResult<()> { let tool_name = invocation.tool_name(); + let (shape, advisory_effect) = self + .recovery_shapes + .lock() + .ok() + .and_then(|mut m| m.remove(&invocation.call_id().to_string())) + .unwrap_or_else(|| (String::new(), CallEffect::Unknown)); + if result.is_error { + if let Some(recovery) = &self.recovery { + recovery.record_failure(tool_name); + } + } let content = crate::agent::tinyagents::middleware::tool_result_text(result); let arg_fp = self .arg_sigs @@ -427,6 +355,9 @@ impl Middleware<(), crate::agent::tinyagents::host::OpenHumanRunContext> let heuristic_failure_text = heuristic_text(tool_name, &failure_text); if !result.is_error && !body_level_failure { + if let Some(recovery) = &self.recovery { + recovery.clear(&scope); + } // Only a successful observation against this operation and scope // demonstrates that its blocker changed. Unrelated successes do not. for class in [ @@ -671,6 +602,7 @@ impl Middleware<(), crate::agent::tinyagents::host::OpenHumanRunContext> recoverable_miss: false, }; + let mut tracker_halted = false; match self.tracker.record(step, &attempt) { NoProgress::Continue => {} NoProgress::Nudge(instruction) => { @@ -686,6 +618,7 @@ impl Middleware<(), crate::agent::tinyagents::host::OpenHumanRunContext> self.queue_nudge(instruction); } NoProgress::Halt(summary) => { + tracker_halted = true; // #4092: if the blocker is user-actionable (a missing connection), // escalate with a concrete ask instead of the crate's generic // "unreachable environment, report back" summary. @@ -714,6 +647,44 @@ impl Middleware<(), crate::agent::tinyagents::host::OpenHumanRunContext> self.tracker.reset(); } } + // All hard/keyword/terminal/recoverable paths have already returned. + // Count this unknown invocation exactly once in the existing tracker, + // regardless of provider success. Compare never changes visible state. + if !tracker_halted + && attempt_error.is_some() + && !hard_reject + && !exit_report + && !body_level_failure + && !is_repeat_call_exempt(tool_name) + { + if let Some(recovery) = &self.recovery { + let safe = super::recovery_advice::diagnostic(&failure_text); + // Unsafe excerpts abstain while preserving stable accounting. + let (halt, nudge) = recovery + .advise( + _ctx, + &scope, + tool_name, + shape, + safe.as_deref() + .unwrap_or("Tool failed; diagnostic cannot be safely minimized"), + if safe.is_some() { + advisory_effect + } else { + CallEffect::Unknown + }, + ) + .await; + if halt { + if let Ok(mut slot) = self.halt_summary.lock() { + *slot = Some(format!("The `{tool_name}` operation remains blocked. Stop this operation and report the blocker; existing permissions still apply.")); + } + self.handle.send(SteeringCommand::Pause); + } else if let Some(nudge) = nudge { + self.queue_nudge(nudge); + } + } + } Ok(()) } } diff --git a/crates/openhuman-core/src/agent/tinyagents/middleware/repeated_failure_state.rs b/crates/openhuman-core/src/agent/tinyagents/middleware/repeated_failure_state.rs new file mode 100644 index 00000000000..e12ab2809e6 --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/middleware/repeated_failure_state.rs @@ -0,0 +1,150 @@ +//! Construction and run-local state helpers for the failure driver. +use super::*; + +impl RepeatedToolFailureMiddleware { + /// Build the breaker. `identical_threshold` (the identical-signature retry + /// ceiling) is handed straight to [`NoProgressTracker::new`], which clamps it + /// so a nudge always precedes a halt (a single failure is never a loop). + pub(crate) fn new( + handle: SteeringHandle, + identical_threshold: usize, + halt_summary: crate::agent::tinyagents::HaltSummarySlot, + ) -> Self { + Self { + handle, + halt_summary, + tracker: NoProgressTracker::new(identical_threshold), + classified: ClassifiedFailureTracker::default(), + last_exit_report: std::sync::Mutex::default(), + step: AtomicUsize::new(0), + arg_sigs: std::sync::Mutex::new(std::collections::HashMap::new()), + target_scopes: std::sync::Mutex::new(std::collections::HashMap::new()), + recoverable_sig_counts: std::sync::Mutex::new(std::collections::HashMap::new()), + recoverable_consecutive: AtomicU32::new(0), + pending_nudges: Arc::new(Mutex::new(Vec::new())), + call_effects: std::sync::Mutex::new(std::collections::HashMap::new()), + tool_facts: None, + recovery: None, + recovery_shapes: Mutex::default(), + } + } + + /// Snapshot recovery settings and the host evaluator once for this run. + pub(crate) fn with_recovery(mut self, config: &crate::config::Config) -> Self { + self.recovery = Some(super::super::recovery_advice::RunRecovery::new(config)); + self + } + + #[cfg(test)] + pub(crate) fn with_recovery_evaluator( + mut self, + config: crate::config::schema::RecoveryConfig, + evaluator: Arc, + ) -> Self { + self.recovery = Some(super::super::recovery_advice::RunRecovery::with_evaluator( + config, evaluator, + )); + self + } + + pub(crate) fn set_recovery_registry( + &self, + registry: &tinyagents_harness::tool::ToolRegistry< + (), + crate::agent::tinyagents::host::OpenHumanRunContext, + >, + session: Option<&crate::tools::agent_policy::ToolPolicySession>, + ) { + self.set_recovery_candidates(super::super::recovery_advice::admitted_candidates( + registry, session, + )); + } + + /// Attach only the assembly's admitted model-callable candidate snapshot. + pub(crate) fn set_recovery_candidates(&self, candidates: Vec) { + if let Some(recovery) = &self.recovery { + recovery.set_candidates(candidates); + } + } + + /// Judge side effects from host registry declarations. + pub(crate) fn with_tool_facts(mut self, lookup: ToolFactsLookup) -> Self { + self.tool_facts = Some(lookup); + self + } + + /// The request-scoped half of this breaker. Register it **last**: its + /// `before_model` must run after the transcript snapshot (which a failed + /// turn persists) and after every reduction step. + pub(crate) fn nudge_injector(&self) -> PendingNudgeInjector { + PendingNudgeInjector { + pending: self.pending_nudges.clone(), + } + } + + pub(super) fn queue_nudge(&self, instruction: impl Into) { + if let Ok(mut pending) = self.pending_nudges.lock() { + pending.push(instruction.into()); + } + } + + /// Drain the queued nudges (what the injector does before a request). + #[cfg(test)] + pub(crate) fn take_pending_nudges(&self) -> Vec { + self.pending_nudges + .lock() + .map(|mut pending| std::mem::take(&mut *pending)) + .unwrap_or_default() + } + + /// Clear the consecutive recoverable-failure streak. Called on any success or + /// non-recoverable failure (the per-signature identical counts persist across + /// the turn, matching the legacy guard). Idempotent. + pub(super) fn reset_recoverable_streak(&self) { + self.recoverable_consecutive.store(0, Ordering::SeqCst); + } + + /// Record one recoverable failure and return a root-cause halt summary once + /// its extended headroom is exhausted (identical `>=` [`RECOVERABLE_REPEAT_FAILURE_THRESHOLD`] + /// or consecutive `>=` [`RECOVERABLE_NO_PROGRESS_FAILURE_THRESHOLD`]). + pub(super) fn record_recoverable( + &self, + tool: &str, + arg_fp: &str, + failure_text: &str, + ) -> Option { + let key = format!("{tool}\u{1f}{arg_fp}"); + let count = self + .recoverable_sig_counts + .lock() + .ok() + .map(|mut counts| { + let c = counts.entry(key).or_insert(0); + *c += 1; + *c + }) + .unwrap_or(0); + let consecutive = self.recoverable_consecutive.fetch_add(1, Ordering::SeqCst) + 1; + tracing::debug!( + tool, + count, + consecutive, + "[tinyagents::mw] recoverable tool failure recorded with extended circuit-breaker headroom" + ); + if count >= RECOVERABLE_REPEAT_FAILURE_THRESHOLD { + return Some(recoverable_identical_halt_summary( + tool, + count, + failure_text, + )); + } + if consecutive >= RECOVERABLE_NO_PROGRESS_FAILURE_THRESHOLD { + return Some(recoverable_no_progress_halt_summary( + consecutive, + tool, + failure_text, + )); + } + None + } +} diff --git a/crates/openhuman-core/src/agent/tinyagents/middleware_recovery_advice_tests.rs b/crates/openhuman-core/src/agent/tinyagents/middleware_recovery_advice_tests.rs new file mode 100644 index 00000000000..eeedce2e52a --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/middleware_recovery_advice_tests.rs @@ -0,0 +1,656 @@ +use super::*; +use crate::config::schema::{RecoveryClassifier, RecoveryConfig}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Mutex; +use tinytools_jev::recovery::*; + +#[derive(Debug, Default)] +struct Fake { + calls: AtomicUsize, + fail: bool, + pending: bool, + low_confidence: bool, + invalid: bool, + changing_class: bool, + force_class: Option<&'static str>, + recoverability: Option, +} +#[async_trait::async_trait] +impl RecoveryEvaluator for Fake { + async fn evaluate( + &self, + request: &RecoveryRequest, + ) -> Result { + let index = self.calls.fetch_add(1, Ordering::SeqCst); + let class = if let Some(class) = self.force_class { + class + } else if self.changing_class && index % 2 == 1 { + "wrong_arguments" + } else { + "wrong_tool" + }; + assert!(!request.observation.argument_shape.contains("secret-value")); + if self.pending { + std::future::pending::<()>().await; + } + if self.fail { + return Err(tinytools::RankError::backend("failure")); + } + Ok(RecoveryDecision { + answers: request + .questions + .iter() + .map(|q| { + ( + q.id().to_owned(), + match q { + RecoveryQuestion::Choice { options, .. } => RecoveryAnswer::Choice { + probabilities: options + .iter() + .map(|o| { + ( + o.key.clone(), + if (q.id() == "class" && o.key == class) + || (q.id() == "alternate" + && o.key == "lookup_public") + { + 1.0 + } else { + 0.0 + }, + ) + }) + .collect(), + confidence: if self.invalid { + f64::NAN + } else if self.low_confidence { + 0.1 + } else { + 1.0 + }, + }, + RecoveryQuestion::Noul { .. } => { + RecoveryAnswer::Noul(self.recoverability.unwrap_or(1.0)) + } + RecoveryQuestion::Score { rubric, .. } => RecoveryAnswer::Score { + probabilities: vec![1.0 / rubric.len() as f64; rubric.len()], + confidence: 1.0, + }, + }, + ) + }) + .collect(), + input_tokens: None, + output_tokens: None, + latency: std::time::Duration::ZERO, + attempts: 1, + }) + } +} +fn configured(mode: RecoveryClassifier, fake: Arc) -> RepeatedToolFailureMiddleware { + let config = RecoveryConfig { + classifier: mode, + ..Default::default() + }; + RepeatedToolFailureMiddleware::new(SteeringHandle::allow_all(), 3, Arc::new(Mutex::new(None))) + .with_recovery_evaluator(config, fake) + .with_tool_facts(Arc::new(|_, _| { + Some(super::super::call_effect::ToolEffectFacts { + classified: Some(super::super::call_effect::CallEffect::ReadOnly), + external: false, + elevated: false, + }) + })) +} +async fn attempt( + mw: &RepeatedToolFailureMiddleware, + id: &str, + error: Option<&str>, +) -> TaToolResult { + let mut call = TaToolCall::new( + id, + "read_catalog", + serde_json::json!({"query": "secret-value", "limit": id}), + ); + let mut context = ctx(); + mw.before_tool(&mut context, &(), &mut call).await.unwrap(); + let mut result = match error { + Some(e) => failing_result("read_catalog", e), + None => TaToolResult::success("ok"), + }; + mw.after_tool( + &mut context, + &(), + &invocation(id, "read_catalog"), + &mut result, + ) + .await + .unwrap(); + result +} +#[tokio::test] +async fn advisory_unknown_failure_queues_ephemeral_nudge_without_rewriting_result() { + let fake = Arc::new(Fake::default()); + let mw = configured(RecoveryClassifier::Jev, fake.clone()); + let result = attempt(&mw, "one", Some("A capability mismatch occurred")).await; + assert!(result.is_error); + assert_eq!(tool_result_text(&result), "A capability mismatch occurred"); + assert_eq!(fake.calls.load(Ordering::SeqCst), 1); + assert!(mw + .take_pending_nudges() + .iter() + .any(|n| n.contains("Advisory recovery"))); + assert!(mw.take_pending_nudges().is_empty()); +} +#[tokio::test] +async fn success_keywords_hard_facts_and_executed_writes_never_evaluate() { + let fake = Arc::new(Fake::default()); + let mw = configured(RecoveryClassifier::Jev, fake.clone()); + attempt(&mw, "ok", None).await; + attempt(&mw, "policy", Some("403 Forbidden")).await; + attempt(&mw, "schema", Some("schema validation failed")).await; + assert_eq!(fake.calls.load(Ordering::SeqCst), 0); + let mw = + configured(RecoveryClassifier::Jev, fake.clone()).with_tool_facts(Arc::new(|_, _| None)); + attempt(&mw, "write", Some("A capability mismatch occurred")).await; + assert_eq!(fake.calls.load(Ordering::SeqCst), 0); +} +#[tokio::test] +async fn compare_failure_matches_keyword_nudges_and_outputs() { + let fake = Arc::new(Fake::default()); + let compare = configured(RecoveryClassifier::Compare, fake.clone()); + let keyword = configured(RecoveryClassifier::Keywords, Arc::new(Fake::default())); + for id in ["one", "two"] { + let a = attempt(&compare, id, Some("A capability mismatch occurred")).await; + let b = attempt(&keyword, id, Some("A capability mismatch occurred")).await; + assert_eq!(tool_result_text(&a), tool_result_text(&b)); + assert_eq!(compare.take_pending_nudges(), keyword.take_pending_nudges()); + } + assert_eq!(fake.calls.load(Ordering::SeqCst), 2); +} +#[tokio::test] +async fn provider_failure_and_varied_arguments_keep_stable_operation_ceiling() { + let fake = Arc::new(Fake { + fail: true, + ..Default::default() + }); + let handle = SteeringHandle::allow_all(); + // This fixture accesses the middleware's public steering seam via a + // separate instance so a stable resource ceiling can be observed. + let mw = RepeatedToolFailureMiddleware::new(handle.clone(), 8, Arc::new(Mutex::new(None))) + .with_recovery_evaluator( + RecoveryConfig { + classifier: RecoveryClassifier::Jev, + ..Default::default() + }, + fake, + ) + .with_tool_facts(Arc::new(|_, _| { + Some(super::super::call_effect::ToolEffectFacts { + classified: Some(super::super::call_effect::CallEffect::ReadOnly), + external: false, + elevated: false, + }) + })); + for id in ["one", "two", "three", "four", "five", "six"] { + attempt(&mw, id, Some("A capability mismatch occurred")).await; + } + assert_eq!(drain_pause_count(&handle), 1); +} + +#[tokio::test] +async fn cancelled_and_timed_out_advice_preserve_failure_and_fallback() { + let fake = Arc::new(Fake { + pending: true, + ..Default::default() + }); + let config = RecoveryConfig { + classifier: RecoveryClassifier::Jev, + decision_timeout_ms: 1, + ..Default::default() + }; + let mw = configured(RecoveryClassifier::Jev, fake.clone()) + .with_recovery_evaluator(config, fake.clone()); + let mut call = TaToolCall::new("cancel", "read_catalog", serde_json::json!({})); + let mut context = ctx(); + context.cancellation.cancel(); + mw.before_tool(&mut context, &(), &mut call).await.unwrap(); + let mut result = failing_result("read_catalog", "capability mismatch"); + mw.after_tool( + &mut context, + &(), + &invocation("cancel", "read_catalog"), + &mut result, + ) + .await + .unwrap(); + assert_eq!(fake.calls.load(Ordering::SeqCst), 0); + assert!(mw.take_pending_nudges().is_empty()); + attempt(&mw, "timeout", Some("capability mismatch")).await; + assert_eq!(fake.calls.load(Ordering::SeqCst), 1); + assert!(mw.take_pending_nudges().is_empty()); +} + +#[tokio::test] +async fn no_credentials_or_disabled_policy_never_calls_provider() { + let fake = Arc::new(Fake::default()); + for classifier in [RecoveryClassifier::Off, RecoveryClassifier::Keywords] { + let mw = configured(classifier, fake.clone()); + attempt(&mw, "disabled", Some("capability mismatch")).await; + } + assert_eq!(fake.calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn accepted_advice_varied_arguments_remain_bounded_across_unrelated_success() { + let fake = Arc::new(Fake::default()); + let handle = SteeringHandle::allow_all(); + let mw = RepeatedToolFailureMiddleware::new(handle.clone(), 8, Arc::new(Mutex::new(None))) + .with_recovery_evaluator( + RecoveryConfig { + classifier: RecoveryClassifier::Jev, + ..Default::default() + }, + fake.clone(), + ) + .with_tool_facts(Arc::new(|_, _| { + Some(super::super::call_effect::ToolEffectFacts { + classified: Some(super::super::call_effect::CallEffect::ReadOnly), + external: false, + elevated: false, + }) + })); + attempt(&mw, "one", Some("capability mismatch")).await; + let mut other = TaToolCall::new("other", "read_other", serde_json::json!({})); + mw.before_tool(&mut ctx(), &(), &mut other).await.unwrap(); + let mut success = TaToolResult::success("done"); + mw.after_tool( + &mut ctx(), + &(), + &invocation("other", "read_other"), + &mut success, + ) + .await + .unwrap(); + attempt(&mw, "two", Some("different capability mismatch")).await; + attempt(&mw, "three", Some("another capability mismatch")).await; + assert_eq!(drain_pause_count(&handle), 1); + assert_eq!(fake.calls.load(Ordering::SeqCst), 3); +} + +#[tokio::test] +async fn advice_decision_ceiling_never_queues_additional_evaluations() { + let fake = Arc::new(Fake::default()); + let mw = configured(RecoveryClassifier::Jev, fake.clone()).with_recovery_evaluator( + RecoveryConfig { + classifier: RecoveryClassifier::Jev, + max_decisions_per_run: 1, + ..Default::default() + }, + fake.clone(), + ); + attempt(&mw, "one", Some("capability mismatch")).await; + attempt(&mw, "two", Some("different capability mismatch")).await; + assert_eq!(fake.calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn alternate_advice_requires_opt_in_and_counted_second_decision() { + let fake = Arc::new(Fake::default()); + let config = RecoveryConfig { + classifier: RecoveryClassifier::Jev, + alternate_tools: true, + ..Default::default() + }; + let mw = configured(RecoveryClassifier::Jev, fake.clone()) + .with_recovery_evaluator(config, fake.clone()); + mw.set_recovery_candidates(vec![tinytools_jev::JevOption { + key: "lookup_public".into(), + description: "Look up public catalog entries".into(), + }]); + attempt(&mw, "alternate", Some("A capability mismatch occurred")).await; + assert_eq!(fake.calls.load(Ordering::SeqCst), 2); + assert!(mw + .take_pending_nudges() + .iter() + .any(|n| n.contains("lookup_public"))); +} + +struct RecoveryTool { + name: &'static str, + hidden: bool, + fail: bool, + calls: AtomicUsize, +} +#[async_trait::async_trait] +impl tinytools::Tool for RecoveryTool { + fn name(&self) -> &str { + self.name + } + fn description(&self) -> &str { + "Look up public catalog entries" + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type":"object"}) + } + fn policy(&self) -> tinytools::ToolPolicy { + let mut policy = tinytools::ToolPolicy::read_only(); + match self.name { + "scoped_lookup" => policy.access.workspace = tinytools::WorkspaceAccess::Scoped, + "credential_lookup" => policy.access.credentials.push("account".into()), + "approval_lookup" => policy.access.approval_required = true, + "external_lookup" => policy.side_effects.external_service = true, + "unclassified_lookup" => policy.classified = false, + _ => {} + } + policy + } + fn permission_level(&self) -> tinytools::PermissionLevel { + if self.name == "write_lookup" { + tinytools::PermissionLevel::Write + } else { + tinytools::PermissionLevel::ReadOnly + } + } + fn exposure(&self) -> tinytools::ToolExposure { + if self.hidden { + tinytools::ToolExposure::Hidden + } else { + tinytools::ToolExposure::Direct + } + } + async fn execute(&self, _: serde_json::Value) -> anyhow::Result { + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(if self.fail { + TaToolResult::error("A capability mismatch occurred") + } else { + TaToolResult::success("public result") + }) + } +} + +#[tokio::test] +async fn main_model_alternate_call_still_passes_normal_admission_and_hidden_calls_never_execute() { + for (permit_alternate, refused_tool) in [ + (true, None), + (true, Some("denied_lookup")), + (true, Some("hidden_lookup")), + (false, None), + ] { + use tinyagents_harness::middleware::ToolAllowlistMiddleware; + use tinyagents_harness::runtime::AgentHarness; + use tinyagents_harness::testkit::ScriptedModel; + use tinyinference_llm::model::ModelResponse; + let first = Arc::new(RecoveryTool { + name: "read_catalog", + hidden: false, + fail: true, + calls: AtomicUsize::new(0), + }); + let alternate = Arc::new(RecoveryTool { + name: "lookup_public", + hidden: false, + fail: false, + calls: AtomicUsize::new(0), + }); + let hidden = Arc::new(RecoveryTool { + name: "hidden_lookup", + hidden: true, + fail: false, + calls: AtomicUsize::new(0), + }); + let denied = Arc::new(RecoveryTool { + name: "denied_lookup", + hidden: false, + fail: false, + calls: AtomicUsize::new(0), + }); + let fake = Arc::new(Fake::default()); + let mw = Arc::new( + configured(RecoveryClassifier::Jev, fake.clone()).with_recovery_evaluator( + RecoveryConfig { + classifier: RecoveryClassifier::Jev, + alternate_tools: true, + ..Default::default() + }, + fake.clone(), + ), + ); + let response = |id, name| { + let mut r = ModelResponse::assistant(""); + r.message.tool_calls = vec![TaToolCall::new(id, name, serde_json::json!({}))]; + r.finish_reason = Some("tool_calls".into()); + r + }; + let mut responses = vec![ + response("first", "read_catalog"), + response("next", "lookup_public"), + ]; + if let Some(name) = refused_tool { + responses.push(response("refused", name)); + } + responses.push(ModelResponse::assistant("done")); + let model = Arc::new(ScriptedModel::new(responses)); + let mut harness: AgentHarness<(), crate::agent::tinyagents::host::OpenHumanRunContext> = + AgentHarness::new(); + harness.register_model("mock", model.clone()); + for tool in [&first, &alternate, &denied, &hidden] { + harness.register_tool(tool.clone()); + } + let session = crate::tools::agent_policy::ToolPolicyEngine::build_session_from_refs( + "test", + "test", + "test", + &Default::default(), + &[first.as_ref(), alternate.as_ref(), hidden.as_ref()], + &Default::default(), + ); + let candidates = + super::super::recovery_advice::admitted_candidates(harness.tools(), Some(&session)); + assert!(!candidates.iter().any(|c| c.key == "hidden_lookup")); + assert!( + !candidates.iter().any(|c| c.key == "denied_lookup"), + "session-unknown tools default deny" + ); + mw.set_recovery_candidates(candidates); + harness.push_middleware(mw.clone()); + harness.push_middleware(Arc::new(ToolAllowlistMiddleware::new( + if permit_alternate { + vec!["read_catalog", "lookup_public"] + } else { + vec!["read_catalog"] + }, + ))); + harness.push_middleware(Arc::new(mw.nudge_injector())); + let run = harness + .invoke_in_context( + &(), + ctx(), + vec![TaMessage::user("look up the public catalog")], + ) + .await; + if permit_alternate && refused_tool.is_none() { + let run = run.unwrap(); + assert_eq!(run.text().as_deref(), Some("done")); + assert!(!run + .messages + .iter() + .any(|m| m.text().contains("Advisory recovery"))); + } else { + let expected = refused_tool.unwrap_or("lookup_public"); + assert!(matches!( + run, + Err(tinyagents_harness::TinyAgentsError::Validation(message)) + if message == format!("tool `{expected}` is not on the allowlist") + )); + } + assert_eq!(first.calls.load(Ordering::SeqCst), 1); + assert_eq!( + alternate.calls.load(Ordering::SeqCst), + usize::from(permit_alternate) + ); + assert_eq!(denied.calls.load(Ordering::SeqCst), 0); + assert_eq!(hidden.calls.load(Ordering::SeqCst), 0); + assert_eq!(fake.calls.load(Ordering::SeqCst), 2); + assert!(model.requests()[1] + .messages + .iter() + .any(|m| m.text().contains("lookup_public"))); + } +} + +#[tokio::test] +async fn low_confidence_and_malformed_decisions_keep_keyword_behavior() { + for fake in [ + Fake { + low_confidence: true, + ..Default::default() + }, + Fake { + invalid: true, + ..Default::default() + }, + ] { + let fake = Arc::new(fake); + let mw = configured(RecoveryClassifier::Jev, fake.clone()); + let keywords = configured(RecoveryClassifier::Keywords, Arc::new(Fake::default())); + for id in ["one", "two"] { + let result = attempt(&mw, id, Some("A capability mismatch occurred")).await; + let baseline = attempt(&keywords, id, Some("A capability mismatch occurred")).await; + assert_eq!(tool_result_text(&result), tool_result_text(&baseline)); + assert_eq!(mw.take_pending_nudges(), keywords.take_pending_nudges()); + } + assert_eq!(fake.calls.load(Ordering::SeqCst), 2); + } +} + +#[tokio::test] +async fn classification_disagreement_does_not_reset_stable_failure_allowance() { + let handle = SteeringHandle::allow_all(); + let fake = Arc::new(Fake { + changing_class: true, + ..Default::default() + }); + let mw = RepeatedToolFailureMiddleware::new(handle.clone(), 10, Arc::new(Mutex::new(None))) + .with_recovery_evaluator( + RecoveryConfig { + classifier: RecoveryClassifier::Jev, + ..Default::default() + }, + fake.clone(), + ) + .with_tool_facts(Arc::new(|_, _| { + Some(super::super::call_effect::ToolEffectFacts { + classified: Some(super::super::call_effect::CallEffect::ReadOnly), + external: false, + elevated: false, + }) + })); + for id in ["one", "two", "three"] { + attempt(&mw, id, Some("A capability mismatch occurred")).await; + } + assert_eq!(fake.calls.load(Ordering::SeqCst), 3); + assert_eq!(drain_pause_count(&handle), 1); +} + +#[tokio::test] +async fn unavailable_recoverability_controls_permanent_halt_projection() { + for value in [0.0, 1.0] { + let handle = SteeringHandle::allow_all(); + let fake = Arc::new(Fake { + force_class: Some("unavailable"), + recoverability: Some(value), + ..Default::default() + }); + let mw = RepeatedToolFailureMiddleware::new(handle.clone(), 10, Arc::new(Mutex::new(None))) + .with_recovery_evaluator( + RecoveryConfig { + classifier: RecoveryClassifier::Jev, + ..Default::default() + }, + fake, + ) + .with_tool_facts(Arc::new(|_, _| { + Some(super::super::call_effect::ToolEffectFacts { + classified: Some(super::super::call_effect::CallEffect::ReadOnly), + external: false, + elevated: false, + }) + })); + attempt(&mw, "one", Some("A capability mismatch occurred")).await; + assert_eq!(drain_pause_count(&handle), usize::from(value == 0.0)); + } +} +#[tokio::test] +async fn wrong_arguments_without_recoverability_does_not_encourage_a_corrected_retry() { + for value in [0.0, 1.0] { + let fake = Arc::new(Fake { + force_class: Some("wrong_arguments"), + recoverability: Some(value), + ..Default::default() + }); + let mw = configured(RecoveryClassifier::Jev, fake); + attempt(&mw, "one", Some("A capability mismatch occurred")).await; + assert_eq!( + mw.take_pending_nudges() + .iter() + .any(|n| n.contains("choose a corrected call")), + value == 1.0 + ); + } +} + +#[test] +fn candidate_snapshot_excludes_scope_credentials_approval_external_and_unclassified_tools() { + let mut harness: tinyagents_harness::runtime::AgentHarness< + (), + crate::agent::tinyagents::host::OpenHumanRunContext, + > = tinyagents_harness::runtime::AgentHarness::new(); + for name in [ + "lookup_public", + "scoped_lookup", + "credential_lookup", + "approval_lookup", + "external_lookup", + "unclassified_lookup", + "write_lookup", + ] { + harness.register_tool(Arc::new(RecoveryTool { + name, + hidden: false, + fail: false, + calls: AtomicUsize::new(0), + })); + } + let candidates = super::super::recovery_advice::admitted_candidates(harness.tools(), None); + assert_eq!( + candidates + .iter() + .map(|c| c.key.as_str()) + .collect::>(), + vec!["lookup_public"] + ); +} + +#[tokio::test] +async fn compare_evaluates_opt_in_alternates_without_projecting_advice() { + let fake = Arc::new(Fake::default()); + let mw = configured(RecoveryClassifier::Compare, fake.clone()).with_recovery_evaluator( + RecoveryConfig { + classifier: RecoveryClassifier::Compare, + alternate_tools: true, + ..Default::default() + }, + fake.clone(), + ); + mw.set_recovery_candidates(vec![tinytools_jev::JevOption { + key: "lookup_public".into(), + description: "Look up public catalog entries".into(), + }]); + let keywords = configured(RecoveryClassifier::Keywords, Arc::new(Fake::default())); + let result = attempt(&mw, "one", Some("A capability mismatch occurred")).await; + let baseline = attempt(&keywords, "one", Some("A capability mismatch occurred")).await; + assert_eq!(tool_result_text(&result), tool_result_text(&baseline)); + assert_eq!(mw.take_pending_nudges(), keywords.take_pending_nudges()); + assert_eq!(fake.calls.load(Ordering::SeqCst), 2); +} diff --git a/crates/openhuman-core/src/agent/tinyagents/middleware_tests.rs b/crates/openhuman-core/src/agent/tinyagents/middleware_tests.rs index 3ae7013bcf1..5bb991b21b5 100644 --- a/crates/openhuman-core/src/agent/tinyagents/middleware_tests.rs +++ b/crates/openhuman-core/src/agent/tinyagents/middleware_tests.rs @@ -419,3 +419,6 @@ mod tool_output_file_read_tests; mod tool_output_tests; #[path = "middleware_tool_policy_tests.rs"] mod tool_policy_tests; + +#[path = "middleware_recovery_advice_tests.rs"] +mod recovery_advice_tests; diff --git a/crates/openhuman-core/src/agent/tinyagents/mod.rs b/crates/openhuman-core/src/agent/tinyagents/mod.rs index 15dc769f46b..ee218ace9cc 100644 --- a/crates/openhuman-core/src/agent/tinyagents/mod.rs +++ b/crates/openhuman-core/src/agent/tinyagents/mod.rs @@ -30,6 +30,7 @@ pub(crate) mod harness_tool_registration; pub mod host; pub(crate) mod hosted_error; pub(crate) mod journal; +pub mod recovery_provider; // The tool harness behind a live voice session (`voice::live`). #[cfg(feature = "voice")] pub(crate) mod live_harness; diff --git a/crates/openhuman-core/src/agent/tinyagents/recovery_provider.rs b/crates/openhuman-core/src/agent/tinyagents/recovery_provider.rs new file mode 100644 index 00000000000..17ca11e5607 --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/recovery_provider.rs @@ -0,0 +1,29 @@ +//! Host-installed, per-run recovery evaluator factory. Core has no wire client. +use crate::config::Config; +use std::sync::{Arc, OnceLock, RwLock}; +pub use tinytools_jev::recovery::{ + RecoveryAnswer, RecoveryDecision, RecoveryEffect, RecoveryEvaluator, RecoveryObservation, + RecoveryPhase, RecoveryQuestion, RecoveryRequest, +}; + +/// Resolves an evaluator against an immutable turn configuration and freezes +/// its endpoint and credential for that run. No default provider is installed. +pub type RecoveryProviderFactory = + Arc Option> + Send + Sync>; +static PROVIDER: OnceLock>> = OnceLock::new(); +fn slot() -> &'static RwLock> { + PROVIDER.get_or_init(|| RwLock::new(None)) +} +/// Snapshot the process factory without creating an evaluator. +pub fn installed_recovery_provider() -> Option { + slot().read().unwrap_or_else(|e| e.into_inner()).clone() +} +/// Replace the process factory. RuntimeBuilder installations restore the previous +/// factory on drop; direct installations persist until replaced or cleared. +pub fn install_recovery_provider(provider: RecoveryProviderFactory) { + *slot().write().unwrap_or_else(|e| e.into_inner()) = Some(provider); +} +/// Remove the process factory; the core retains its keyword recovery fallback. +pub fn clear_recovery_provider() { + *slot().write().unwrap_or_else(|e| e.into_inner()) = None; +} diff --git a/crates/openhuman-core/src/config/schema/agent.rs b/crates/openhuman-core/src/config/schema/agent.rs index cbd76de64e5..b7309ab69de 100644 --- a/crates/openhuman-core/src/config/schema/agent.rs +++ b/crates/openhuman-core/src/config/schema/agent.rs @@ -135,6 +135,7 @@ impl RequiredOutputContract { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(default)] pub struct AgentConfig { + pub recovery: super::RecoveryConfig, /// When true: bootstrap_max_chars=6000, rag_chunk_limit=2. Use for 13B or smaller models. #[serde(default)] pub compact_context: bool, @@ -511,6 +512,7 @@ impl Default for AgentConfig { required_output: None, agents_md_enabled: default_agents_md_enabled(), tool_search: ToolSearchConfig::default(), + recovery: super::RecoveryConfig::default(), } } } diff --git a/crates/openhuman-core/src/config/schema/mod.rs b/crates/openhuman-core/src/config/schema/mod.rs index 03f4386d2a6..3eb0fda3329 100644 --- a/crates/openhuman-core/src/config/schema/mod.rs +++ b/crates/openhuman-core/src/config/schema/mod.rs @@ -147,3 +147,10 @@ pub use voice_providers::{ }; mod types; pub use types::*; + +#[cfg(test)] +#[path = "recovery_tests.rs"] +mod recovery_tests; + +mod recovery; +pub use recovery::{RecoveryClassifier, RecoveryConfig}; diff --git a/crates/openhuman-core/src/config/schema/recovery.rs b/crates/openhuman-core/src/config/schema/recovery.rs new file mode 100644 index 00000000000..342c6da71c3 --- /dev/null +++ b/crates/openhuman-core/src/config/schema/recovery.rs @@ -0,0 +1,164 @@ +//! Opt-in advisory recovery policy; keywords remain the default. +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Selects whether unresolved failures receive remote advisory evaluation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RecoveryClassifier { + #[default] + /// Preserve the existing deterministic keyword fallback. + Keywords, + /// Evaluate observationally without changing model-facing behavior. + Compare, + /// Apply validated advice within the existing execution limits. + Jev, + /// Disable decision assistance while keeping existing failure guards. + Off, +} + +/// Bounded host policy for the `[agent.recovery]` configuration block. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default, try_from = "RecoveryConfigWire")] +pub struct RecoveryConfig { + /// Evaluation mode; defaults to keywords. + pub classifier: RecoveryClassifier, + /// Optional route override; otherwise inherits the tool-search route. + pub jev_route: Option, + /// Optional endpoint origin override for the selected route. + pub jev_base_url: Option, + /// Total advisory deadline in milliseconds, at most 3000 and remaining run time. + pub decision_timeout_ms: u64, + /// Maximum provider requests during one run, including alternate evaluation. + pub max_decisions_per_run: u32, + /// Allow advisory selection among already admitted read-only capabilities. + pub alternate_tools: bool, + /// Identifies the provisional threshold policy for evaluation records. + pub threshold_version: String, + /// Minimum class distribution concentration; not measured correctness. + pub class_confidence: f64, + /// Minimum positive recovery probability for corrective or alternate advice. + pub recoverability: f64, + /// Minimum concentration for optional correction or alternate answers. + pub advice_confidence: f64, + /// Host-counted failures needed for repeated-blocker alternate eligibility. + pub repeated_blocker: u32, +} + +impl Default for RecoveryConfig { + fn default() -> Self { + Self { + classifier: RecoveryClassifier::Keywords, + jev_route: None, + jev_base_url: None, + decision_timeout_ms: 3000, + max_decisions_per_run: 8, + alternate_tools: false, + threshold_version: "provisional-v1".into(), + class_confidence: 0.75, + recoverability: 0.75, + advice_confidence: 0.75, + repeated_blocker: 2, + } + } +} + +impl RecoveryConfig { + /// Validate both deserialized and directly constructed policy values. + pub fn validate(&self) -> Result<(), String> { + if self.recoverability <= 0.0 + || self.decision_timeout_ms == 0 + || self.decision_timeout_ms > 3000 + || self.max_decisions_per_run == 0 + || self.max_decisions_per_run > 64 + || self.repeated_blocker == 0 + || self.repeated_blocker > 64 + || self.threshold_version.trim().is_empty() + || self.threshold_version.len() > 128 + || [ + self.class_confidence, + self.recoverability, + self.advice_confidence, + ] + .iter() + .any(|p| !p.is_finite() || !(0.0..=1.0).contains(p)) + { + return Err("invalid bounded recovery policy".into()); + } + if self + .jev_route + .as_deref() + .is_some_and(|r| !matches!(r, "auto" | "tinyhumans" | "typesafe" | "openrouter")) + { + return Err("invalid recovery Jev route".into()); + } + Ok(()) + } + + /// Project the host thresholds into the provider-neutral TinyTools policy. + pub fn thresholds(&self) -> tinytools_jev::recovery::RecoveryThresholds { + tinytools_jev::recovery::RecoveryThresholds { + version: self.threshold_version.clone(), + class_confidence: self.class_confidence, + recoverability: self.recoverability, + advice_confidence: self.advice_confidence, + repeated_blocker: self.repeated_blocker, + } + } +} + +// Deserialize through an unvalidated wire type so every config loading path +// enforces the same bounds. Runtime callers still call validate(). +#[derive(Deserialize, JsonSchema)] +#[serde(default)] +struct RecoveryConfigWire { + classifier: RecoveryClassifier, + jev_route: Option, + jev_base_url: Option, + decision_timeout_ms: u64, + max_decisions_per_run: u32, + alternate_tools: bool, + threshold_version: String, + class_confidence: f64, + recoverability: f64, + advice_confidence: f64, + repeated_blocker: u32, +} +impl Default for RecoveryConfigWire { + fn default() -> Self { + let c = RecoveryConfig::default(); + Self { + classifier: c.classifier, + jev_route: c.jev_route, + jev_base_url: c.jev_base_url, + decision_timeout_ms: c.decision_timeout_ms, + max_decisions_per_run: c.max_decisions_per_run, + alternate_tools: c.alternate_tools, + threshold_version: c.threshold_version, + class_confidence: c.class_confidence, + recoverability: c.recoverability, + advice_confidence: c.advice_confidence, + repeated_blocker: c.repeated_blocker, + } + } +} +impl TryFrom for RecoveryConfig { + type Error = String; + fn try_from(c: RecoveryConfigWire) -> Result { + let config = Self { + classifier: c.classifier, + jev_route: c.jev_route, + jev_base_url: c.jev_base_url, + decision_timeout_ms: c.decision_timeout_ms, + max_decisions_per_run: c.max_decisions_per_run, + alternate_tools: c.alternate_tools, + threshold_version: c.threshold_version, + class_confidence: c.class_confidence, + recoverability: c.recoverability, + advice_confidence: c.advice_confidence, + repeated_blocker: c.repeated_blocker, + }; + config.validate()?; + Ok(config) + } +} diff --git a/crates/openhuman-core/src/config/schema/recovery_tests.rs b/crates/openhuman-core/src/config/schema/recovery_tests.rs new file mode 100644 index 00000000000..1af42260283 --- /dev/null +++ b/crates/openhuman-core/src/config/schema/recovery_tests.rs @@ -0,0 +1,22 @@ +use super::*; + +#[test] +fn recovery_defaults_to_keywords_and_rejects_invalid_policy() { + let config = RecoveryConfig::default(); + assert_eq!(config.classifier, RecoveryClassifier::Keywords); + assert!(config.validate().is_ok()); + for input in [ + r#"{"decision_timeout_ms":3001}"#, + r#"{"class_confidence":1.2}"#, + r#"{"jev_route":"typo"}"#, + r#"{"max_decisions_per_run":0}"#, + ] { + assert!( + serde_json::from_str::(input).is_err(), + "{input}" + ); + } + let mut invalid = config; + invalid.recoverability = f64::NAN; + assert!(invalid.validate().is_err()); +} diff --git a/crates/openhuman-embed/examples/README.md b/crates/openhuman-embed/examples/README.md index c24342786b7..9922e16ac64 100644 --- a/crates/openhuman-embed/examples/README.md +++ b/crates/openhuman-embed/examples/README.md @@ -13,6 +13,7 @@ - [A per-agent post-turn hook observes completed turns](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/hooks.rs): A per-agent post-turn hook observes completed turns. (offline; optional live via OPENHUMAN_EXAMPLE_LIVE=1 and BASE_URL/API_KEY/MODEL.) - [Host tools and HostOnly keep the advertised tool catalog exact](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/host_tools.rs): Host tools and HostOnly keep the advertised tool catalog exact. (offline with loopback stubs; no live path.) - [Lean runtime without background services](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/lean_headless.rs): Lean runtime without background services. (offline with loopback stubs; optional live via OPENHUMAN_EXAMPLE_LIVE.) +- [Linux agent fleet memory and latency](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs): Measure retained runtime-owned agents using loopback inference and two worker threads. (offline on Linux; use a fresh constrained cgroup for release measurements.) - [Connect an actual MCP protocol stub over loopback](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/mcp.rs): Connect an actual MCP protocol stub over loopback. (offline with loopback stubs; no live path.; feature: mcp) - [Tenant scoped memory facade with an in-memory engine](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/memory.rs): Tenant scoped memory facade with an in-memory engine. (offline with loopback stubs; no live path.) - [SaaS profiles isolate conversation history](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/profiles.rs): SaaS profiles isolate conversation history. (offline with loopback stubs; no live path.) diff --git a/crates/openhuman-embed/src/config.rs b/crates/openhuman-embed/src/config.rs index 6990e17108f..da4b62e13c2 100644 --- a/crates/openhuman-embed/src/config.rs +++ b/crates/openhuman-embed/src/config.rs @@ -72,7 +72,8 @@ pub use tinytools::{ /// Typed runtime builder policy groups from the core configuration contract. pub use openhuman_core::config::schema::{ - AutonomyConfig, CronConfig, PrivacyConfig, PrivacyMode, SecretsConfig, + AutonomyConfig, CronConfig, PrivacyConfig, PrivacyMode, RecoveryClassifier, RecoveryConfig, + SecretsConfig, }; /// The core's config type, as [`crate::RuntimeConfig`]. pub use openhuman_core::config::Config as RuntimeConfig; diff --git a/crates/openhuman-embed/src/lib.rs b/crates/openhuman-embed/src/lib.rs index de781e73ab2..4755a1739dd 100644 --- a/crates/openhuman-embed/src/lib.rs +++ b/crates/openhuman-embed/src/lib.rs @@ -333,3 +333,12 @@ pub mod providers { } pub use agent_progress::AgentProgress; + +/// Advisory recovery extension types; core has no default remote provider. +pub mod recovery { + pub use openhuman_core::agent::tinyagents::recovery_provider::{ + install_recovery_provider, RecoveryAnswer, RecoveryDecision, RecoveryEffect, + RecoveryEvaluator, RecoveryObservation, RecoveryPhase, RecoveryProviderFactory, + RecoveryQuestion, RecoveryRequest, + }; +} diff --git a/crates/openhuman-embed/src/runtime/seams.rs b/crates/openhuman-embed/src/runtime/seams.rs index be563bfffba..cb2d9299914 100644 --- a/crates/openhuman-embed/src/runtime/seams.rs +++ b/crates/openhuman-embed/src/runtime/seams.rs @@ -10,6 +10,7 @@ //! |---|---|---| //! | [`controller_extension`](RuntimeBuilder::controller_extension) | before boot | kept: the registry has no removal; re-registering the same controllers is a no-op | //! | [`tool_ranker`](RuntimeBuilder::tool_ranker) | before boot | the previous ranker is restored, if ours still holds the slot | +//! | [`recovery_provider`](RuntimeBuilder::recovery_provider) | before boot | the previous factory is restored, if ours still holds the slot | //! | [`post_turn_hook`](RuntimeBuilder::post_turn_hook) / [`tool_hook`](RuntimeBuilder::tool_hook) | before boot, replacing a same-named hook | newest first, while ours still holds the name: ours removed, a replaced same-named hook restored | //! | [`server_launcher`](RuntimeBuilder::server_launcher) | before boot | kept: first install wins for the process | //! | [`live_policy`](RuntimeBuilder::live_policy) | after boot | kept: the next boot (or a config reload) installs its own | @@ -20,6 +21,11 @@ //! later `config.update_autonomy_settings` reload or channel startup rebuilds //! the policy from config again — the override lasts until then. +use openhuman_core::agent::tinyagents::recovery_provider::{ + clear_recovery_provider, install_recovery_provider, installed_recovery_provider, + RecoveryProviderFactory, +}; + use std::sync::Arc; use openhuman_core::agent::hooks::{ @@ -81,6 +87,7 @@ impl From> for StorageSource { pub(crate) struct HostSeams { pub(crate) controller_extensions: Vec, pub(crate) tool_ranker: Option>, + pub(crate) recovery_provider: Option, pub(crate) post_turn_hooks: Vec>, pub(crate) tool_hooks: Vec>, pub(crate) server_launcher: Option, @@ -211,7 +218,13 @@ impl HostSeams { None => None, }; + let recovery_provider = self.recovery_provider.map(|ours| { + let previous = installed_recovery_provider(); + install_recovery_provider(ours.clone()); + (ours, previous) + }); Ok(InstalledSeams { + recovery_provider, storage, ranker, post_turn_hooks, @@ -238,6 +251,7 @@ struct InstalledHook { /// Seams a runtime installed. Dropping it restores the restorable ones; see /// the module docs. pub(crate) struct InstalledSeams { + recovery_provider: Option<(RecoveryProviderFactory, Option)>, storage: Option, ranker: Option, /// Our hooks, in install order, each with the same-named hook it replaced. @@ -340,6 +354,14 @@ impl Drop for InstalledSeams { if !self.restore { return; } + if let Some((ours, previous)) = self.recovery_provider.take() { + if installed_recovery_provider().is_some_and(|now| Arc::ptr_eq(&now, &ours)) { + match previous { + Some(previous) => install_recovery_provider(previous), + None => clear_recovery_provider(), + } + } + } if let Some((ours, previous)) = self.storage.take() { let still_ours = openhuman_core::storage::installed().is_some_and(|now| Arc::ptr_eq(&now, &ours)); @@ -417,6 +439,12 @@ impl RuntimeBuilder { self } + /// Installs an advisory recovery factory, restored when the runtime drops. + pub fn recovery_provider(mut self, provider: RecoveryProviderFactory) -> Self { + self.seams.recovery_provider = Some(provider); + self + } + /// The process-wide `tool_search` ranker. Restored to the previous one /// when the runtime drops. pub fn tool_ranker(mut self, ranker: Arc) -> Self { diff --git a/crates/openhuman-embed/src/runtime/seams_tests.rs b/crates/openhuman-embed/src/runtime/seams_tests.rs index 571f75166a6..958c4264482 100644 --- a/crates/openhuman-embed/src/runtime/seams_tests.rs +++ b/crates/openhuman-embed/src/runtime/seams_tests.rs @@ -366,3 +366,41 @@ async fn an_unopenable_storage_url_fails_the_open() { let error = seams.open_storage().await.expect_err("bad url"); assert!(error.contains("storage backend"), "{error}"); } + +static RECOVERY_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); +#[test] +fn dropping_recovery_seam_restores_previous_factory_and_preserves_replacement() { + let _guard = RECOVERY_LOCK.lock().unwrap(); + let original = installed_recovery_provider(); + let previous: RecoveryProviderFactory = Arc::new(|_| None); + install_recovery_provider(previous.clone()); + let ours: RecoveryProviderFactory = Arc::new(|_| None); + let installed = HostSeams { + recovery_provider: Some(ours), + ..Default::default() + } + .install() + .unwrap(); + drop(installed); + assert!(Arc::ptr_eq( + &installed_recovery_provider().unwrap(), + &previous + )); + let installed = HostSeams { + recovery_provider: Some(Arc::new(|_| None)), + ..Default::default() + } + .install() + .unwrap(); + let replacement: RecoveryProviderFactory = Arc::new(|_| None); + install_recovery_provider(replacement.clone()); + drop(installed); + assert!(Arc::ptr_eq( + &installed_recovery_provider().unwrap(), + &replacement + )); + match original { + Some(p) => install_recovery_provider(p), + None => clear_recovery_provider(), + } +} diff --git a/crates/openhuman-embed/src/runtime/summary.rs b/crates/openhuman-embed/src/runtime/summary.rs index c4934eaf101..85046a2b9f8 100644 --- a/crates/openhuman-embed/src/runtime/summary.rs +++ b/crates/openhuman-embed/src/runtime/summary.rs @@ -51,6 +51,8 @@ pub struct BuilderSummary { pub controller_extensions: Vec, /// The [`kind`](tinytools::ToolRanker::kind) of the tool ranker. pub tool_ranker: Option, + /// Whether an advisory recovery factory was supplied. + pub has_recovery_provider: bool, /// Names of configured post-turn hooks. pub post_turn_hooks: Vec, /// Names of configured tool hooks. @@ -92,6 +94,7 @@ impl RuntimeBuilder { .tool_ranker .as_ref() .map(|ranker| ranker.kind().to_string()), + has_recovery_provider: self.seams.recovery_provider.is_some(), post_turn_hooks: self .seams .post_turn_hooks diff --git a/crates/openhuman-tinyhumans/src/install.rs b/crates/openhuman-tinyhumans/src/install.rs index b789dd379a8..98b1fe38ce2 100644 --- a/crates/openhuman-tinyhumans/src/install.rs +++ b/crates/openhuman-tinyhumans/src/install.rs @@ -199,6 +199,8 @@ pub(crate) fn wiring(options: &InstallOptions) -> Result { /// resolves the process global when a context carries none. pub fn install(options: InstallOptions) -> Result, InstallError> { let (controllers, ranker) = prepare(&options); + #[cfg(feature = "jev")] + openhuman_embed::recovery::install_recovery_provider(crate::jev::recovery::recovery_provider()); if let Some(extension) = controllers { // Idempotent in the core: an identical re-registration is a no-op. diff --git a/crates/openhuman-tinyhumans/src/jev/mod.rs b/crates/openhuman-tinyhumans/src/jev/mod.rs index f08e06d4696..79b5cf47f40 100644 --- a/crates/openhuman-tinyhumans/src/jev/mod.rs +++ b/crates/openhuman-tinyhumans/src/jev/mod.rs @@ -19,6 +19,7 @@ mod evaluator; mod ranker; +pub mod recovery; mod route; pub use evaluator::TinyJevEvaluator; diff --git a/crates/openhuman-tinyhumans/src/jev/recovery.rs b/crates/openhuman-tinyhumans/src/jev/recovery.rs new file mode 100644 index 00000000000..27ae83ad199 --- /dev/null +++ b/crates/openhuman-tinyhumans/src/jev/recovery.rs @@ -0,0 +1,145 @@ +//! Mechanical recovery question transport. Wording and validation are TinyTools-owned. +use openhuman_embed::__host::config::Config; +use openhuman_embed::recovery::RecoveryProviderFactory; +use serde_json::json; +use std::{collections::BTreeMap, sync::Arc, time::Duration}; +use tinyjevclient::{Answer, Choice, Client, EvaluationRequest, Noul, Question, Score}; +use tinytools::RankError; +use tinytools_jev::recovery::{ + RecoveryAnswer, RecoveryDecision, RecoveryEvaluator, RecoveryQuestion, RecoveryRequest, +}; + +/// Transports TinyTools recovery questions through a preconfigured Jev client. +#[derive(Debug)] +pub struct TinyJevRecoveryEvaluator { + client: Client, + deadline: Duration, +} +impl TinyJevRecoveryEvaluator { + /// Use the supplied credential snapshot and bound each evaluation by `deadline`. + pub fn new(client: Client, deadline: Duration) -> Self { + Self { client, deadline } + } + fn build(request: &RecoveryRequest) -> EvaluationRequest { + let o = &request.observation; + EvaluationRequest { + model: "jev-latest".into(), + state: json!({"version": request.version, "tool": o.tool_description, + "diagnostic": o.failure, "argument_shape": o.argument_shape, + "phase": format!("{:?}", o.phase), "effect": format!("{:?}", o.effect), + "attempts": o.attempts, "repeated_failures": o.repeated_failures, + "concrete_correction": o.concrete_correction}), + questions: request + .questions + .iter() + .map(|q| { + let wire = match q { + RecoveryQuestion::Choice { + prompt, options, .. + } => Question::Choice(Choice { + instructions: json!(prompt), + criteria: options + .iter() + .map(|o| (o.key.clone(), Some(json!(o.description)))) + .collect(), + }), + RecoveryQuestion::Noul { prompt, .. } => Question::Noul(Noul { + instructions: json!(prompt), + criteria: None, + }), + RecoveryQuestion::Score { prompt, rubric, .. } => Question::Score(Score { + instructions: json!(prompt), + criteria: rubric.iter().map(|r| json!(r)).collect(), + }), + }; + (q.id().to_owned(), wire) + }) + .collect(), + } + } +} +#[async_trait::async_trait] +impl RecoveryEvaluator for TinyJevRecoveryEvaluator { + async fn evaluate(&self, request: &RecoveryRequest) -> Result { + let result = + tokio::time::timeout(self.deadline, self.client.evaluate(&Self::build(request))) + .await + .map_err(|_| RankError::Timeout)? + .map_err(|_| RankError::backend("recovery provider unavailable"))?; + let mut answers = BTreeMap::new(); + // Preserve exact IDs, types and complete distributions; TinyTools rejects + // missing/extra/type-mismatched answers. Client validates Score legends. + for (id, answer) in result.response.answers { + let mapped = match answer { + Answer::Choice(c) => RecoveryAnswer::Choice { + probabilities: c.probabilities, + confidence: c.confidence, + }, + Answer::Noul(n) => RecoveryAnswer::Noul(n.noul), + Answer::Score(s) => { + let n = s.probabilities.len(); + let probabilities = (0..n) + .map(|i| s.probabilities.get(&i.to_string()).copied()) + .collect::>>() + .ok_or_else(|| RankError::backend("invalid recovery score"))?; + RecoveryAnswer::Score { + probabilities, + confidence: s.confidence, + } + } + }; + answers.insert(id, mapped); + } + Ok(RecoveryDecision { + answers, + input_tokens: result.response.usage.input_tokens, + output_tokens: result.response.usage.output_tokens, + latency: result.latency, + attempts: result.attempts, + }) + } +} + +fn resolved( + config: &Config, + env: super::route::EnvLookup<'_>, +) -> Result { + config + .agent + .recovery + .validate() + .map_err(RankError::invalid_input)?; + let mut config = config.clone(); + if let Some(route) = config.agent.recovery.jev_route.clone() { + config.agent.tool_search.jev_route = route; + } + // Validate inherited routes too: recovery must never silently change route. + if !matches!( + config.agent.tool_search.jev_route.as_str(), + "auto" | "tinyhumans" | "typesafe" | "openrouter" + ) { + return Err(RankError::invalid_input("invalid inherited recovery route")); + } + if let Some(base) = config.agent.recovery.jev_base_url.clone() { + config.agent.tool_search.jev_base_url = Some(base); + } + super::route::resolve(&config, env) +} + +/// Freeze credential and URL at turn assembly, with a single transport attempt. +/// Signed-out or invalid configuration leaves the core's normal fallback intact. +pub fn recovery_provider() -> RecoveryProviderFactory { + Arc::new(|config| { + let mut route = resolved(config, &super::route::process_env).ok()?; + route.client.retry.max_retries = 0; + let client = Client::new(route.client).ok()?; + Some(Arc::new(TinyJevRecoveryEvaluator::new( + client, + Duration::from_millis(config.agent.recovery.decision_timeout_ms), + ))) + }) +} + +#[cfg(test)] +#[path = "recovery_tests.rs"] +mod tests; diff --git a/crates/openhuman-tinyhumans/src/jev/recovery_tests.rs b/crates/openhuman-tinyhumans/src/jev/recovery_tests.rs new file mode 100644 index 00000000000..f2a9252cdf3 --- /dev/null +++ b/crates/openhuman-tinyhumans/src/jev/recovery_tests.rs @@ -0,0 +1,183 @@ +use super::*; +use tinytools_jev::recovery::{RecoveryObservation, RecoveryPhase}; + +#[test] +fn recovery_transport_keeps_tinytools_questions_and_minimal_state() { + let request = RecoveryRequest::new(RecoveryObservation::new( + RecoveryPhase::Execution, + "read catalog", + "unrecognized diagnostic", + )) + .unwrap(); + let wire = TinyJevRecoveryEvaluator::build(&request); + assert_eq!(wire.questions.len(), request.questions.len()); + assert_eq!(wire.state["diagnostic"], "unrecognized diagnostic"); + assert!(wire.state.get("transcript").is_none()); + for q in &request.questions { + assert!(wire.questions.contains_key(q.id())); + } +} + +#[test] +fn recovery_route_inherits_and_explicit_typo_fails_closed() { + let mut config = Config::default(); + config.agent.tool_search.jev_route = "typesafe".into(); + config.agent.tool_search.jev_base_url = Some("http://127.0.0.1:18080".into()); + let env = |_: &str| Some("test-key".to_owned()); + let route = resolved(&config, &env).unwrap(); + assert_eq!(route.client.base_url, "http://127.0.0.1:18080"); + config.agent.recovery.jev_route = Some("typo".into()); + assert!(resolved(&config, &env).is_err()); + config.agent.recovery.jev_route = Some("openrouter".into()); + config.agent.recovery.jev_base_url = Some("http://127.0.0.1:18081".into()); + assert_eq!( + resolved(&config, &env).unwrap().client.base_url, + "http://127.0.0.1:18081" + ); +} + +#[tokio::test] +async fn local_transport_maps_complete_answers_and_rejects_malformed_answers() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + let mut observation = RecoveryObservation::new( + RecoveryPhase::Unknown, + "read catalog", + "capability mismatch", + ); + observation.concrete_correction = Some("Use the documented parameter name".into()); + let request = RecoveryRequest::new(observation).unwrap(); + let answers: BTreeMap<_, _> = request.questions.iter().map(|q| (q.id().to_owned(), match q { + RecoveryQuestion::Choice { options, .. } => json!({"type":"choice", "choice":"wrong_tool", "confidence":1.0, + "probabilities":options.iter().map(|o| (o.key.clone(), if o.key == "wrong_tool" { 1.0 } else { 0.0 })).collect::>() }), + RecoveryQuestion::Noul { .. } => json!({"type":"noul", "noul":1.0}), + RecoveryQuestion::Score { rubric, .. } => json!({"type":"score", "score":1.1, "confidence":0.5, + "probabilities":{"0":0.2,"1":0.5,"2":0.3}, + "legend":rubric.iter().enumerate().map(|(i,r)| (i.to_string(), json!(r))).collect::>() }), + })).collect(); + Mock::given(method("POST")).and(path("/v1/systemone")).respond_with(ResponseTemplate::new(200).set_body_json(json!({"model":"jev-latest", "answers":answers, "usage":{"input_tokens":12,"output_tokens":3}}))).mount(&server).await; + let mut config = tinyjevclient::ClientConfig::new("test-key"); + config.base_url = server.uri(); + config.retry.max_retries = 0; + let evaluator = + TinyJevRecoveryEvaluator::new(Client::new(config.clone()).unwrap(), Duration::from_secs(1)); + let decision = evaluator.evaluate(&request).await.unwrap(); + assert_eq!(decision.answers.len(), request.questions.len()); + assert_eq!(decision.input_tokens, Some(12)); + assert_eq!(decision.output_tokens, Some(3)); + assert_eq!(decision.attempts, 1); + let score_id = request + .questions + .iter() + .find(|q| matches!(q, RecoveryQuestion::Score { .. })) + .unwrap() + .id(); + assert_eq!( + decision.answers[score_id], + RecoveryAnswer::Score { + probabilities: vec![0.2, 0.5, 0.3], + confidence: 0.5 + } + ); + server.reset().await; + let mut malformed = answers.clone(); + malformed.get_mut(score_id).unwrap()["probabilities"] = json!({"0":0.2,"1":0.5,"extra":0.3}); + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({"model":"jev-latest", "answers":malformed, "usage":{}})), + ) + .mount(&server) + .await; + assert!(evaluator.evaluate(&request).await.is_err()); + server.reset().await; + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({"model":"jev-latest", "answers":{}, "usage":{}})), + ) + .mount(&server) + .await; + // Typed client or generic adviser rejects missing answers; no host guesses. + let result = evaluator.evaluate(&request).await; + if let Ok(decision) = result { + assert!(decision.answers.is_empty()); + } + let adviser = tinytools_jev::recovery::RecoveryAdviser::new( + Arc::new(TinyJevRecoveryEvaluator::new( + Client::new(config.clone()).unwrap(), + Duration::from_secs(1), + )), + Config::default().agent.recovery.thresholds(), + ) + .unwrap(); + assert!(matches!( + adviser.advise(request.observation.clone()).await.unwrap(), + tinytools_jev::recovery::RecoveryAdvice::Abstained(_) + )); + server.reset().await; + Mock::given(method("POST")).respond_with(ResponseTemplate::new(200).set_body_json(json!({"model":"jev-latest", "answers":{"class":{"type":"noul","noul":12.0}}, "usage":{}}))).mount(&server).await; + assert!(evaluator.evaluate(&request).await.is_err()); +} + +#[tokio::test] +async fn transport_deadline_is_bounded_and_no_retry_is_started() { + use wiremock::matchers::method; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(503).set_delay(Duration::from_secs(1))) + .mount(&server) + .await; + let mut config = tinyjevclient::ClientConfig::new("test-key"); + config.base_url = server.uri(); + config.retry.max_retries = 0; + let evaluator = + TinyJevRecoveryEvaluator::new(Client::new(config).unwrap(), Duration::from_millis(100)); + let request = RecoveryRequest::new(RecoveryObservation::new( + RecoveryPhase::Unknown, + "read catalog", + "failure", + )) + .unwrap(); + assert!(matches!( + evaluator.evaluate(&request).await, + Err(RankError::Timeout) + )); + assert!(server.received_requests().await.unwrap().len() <= 1); +} + +#[test] +fn score_question_preserves_library_rubric_order_without_interpreting_scores() { + let mut observation = + RecoveryObservation::new(RecoveryPhase::PreExecution, "read catalog", "bad schema"); + observation.concrete_correction = Some("Use the documented parameter name".into()); + let request = RecoveryRequest::new(observation).unwrap(); + let wire = TinyJevRecoveryEvaluator::build(&request); + let score = request + .questions + .iter() + .find_map(|q| { + if let RecoveryQuestion::Score { id, rubric, .. } = q { + Some((id, rubric)) + } else { + None + } + }) + .unwrap(); + let Some(Question::Score(mapped)) = wire.questions.get(score.0) else { + panic!("missing score") + }; + assert_eq!( + mapped.criteria, + score.1.iter().map(|r| json!(r)).collect::>() + ); +} + +#[test] +fn explicitly_missing_recovery_credentials_fail_closed() { + let mut config = Config::default(); + config.agent.recovery.jev_route = Some("typesafe".into()); + assert!(resolved(&config, &|_| None).is_err()); +} diff --git a/crates/openhuman-tinyhumans/src/runtime.rs b/crates/openhuman-tinyhumans/src/runtime.rs index dde8e0b5fa2..7eec631f5c0 100644 --- a/crates/openhuman-tinyhumans/src/runtime.rs +++ b/crates/openhuman-tinyhumans/src/runtime.rs @@ -122,6 +122,16 @@ impl RuntimeBuilder { self } + /// Use a host recovery evaluator factory instead of the connected default. + /// The embed runtime restores the previous factory when it drops. + pub fn recovery_provider( + mut self, + factory: openhuman_embed::recovery::RecoveryProviderFactory, + ) -> Self { + self.inner = self.inner.recovery_provider(factory); + self + } + /// A `tool_search` ranker of the host's own, in place of the Jev ranker. /// See [`openhuman_embed::RuntimeBuilder::tool_ranker`]. pub fn tool_ranker(mut self, ranker: Arc) -> Self { @@ -277,6 +287,12 @@ impl RuntimeBuilder { inner = inner.tool_ranker(ranker); } } + #[cfg(feature = "jev")] + { + if !inner.summary().has_recovery_provider { + inner = inner.recovery_provider(crate::jev::recovery::recovery_provider()); + } + } Ok(inner) } diff --git a/docs/gitbooks/en/developing/embed/api-index.json b/docs/gitbooks/en/developing/embed/api-index.json index 76919ff831f..a1d19f9b920 100644 --- a/docs/gitbooks/en/developing/embed/api-index.json +++ b/docs/gitbooks/en/developing/embed/api-index.json @@ -76,6 +76,7 @@ "ModelDefaults", "OpenError", "PendingApproval", + "PermissionFuture", "PermissionLevel", "PickListenPortError", "ProfileError", @@ -126,6 +127,7 @@ "TrustedAccess", "TrustedAutomationSource", "Turn", + "TurnCancellation", "TurnContext", "TurnOutcome", "TurnRequest", @@ -136,6 +138,8 @@ "absolute", "agent_progress", "artifacts", + "budget", + "cancellation", "channels", "chat_surface", "complete", @@ -143,20 +147,26 @@ "cron", "embeddings", "events", + "fanout", "identity", "install_backend_transport", "installed_backend_transport", "memory", "modules", + "observe", "process", "profiles", "providers", + "recovery", + "repository", + "routing", "run_from_args", "schema_for_rpc_method", "seams", "session_store", "skill_registry", - "stream" + "stream", + "structured" ], "builder_setters": [ { @@ -217,7 +227,7 @@ "name": "controller_extension", "signature": "pub fn controller_extension(mut self, extension: ControllerExtension) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 415 + "line": 437 }, { "name": "cron", @@ -271,7 +281,7 @@ "name": "live_policy", "signature": "pub fn live_policy(mut self, policy: Arc) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 464 + "line": 492 }, { "name": "max_agents", @@ -307,7 +317,7 @@ "name": "post_turn_hook", "signature": "pub fn post_turn_hook(mut self, hook: Arc) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 430 + "line": 458 }, { "name": "privacy", @@ -321,6 +331,12 @@ "source": "crates/openhuman-embed/src/runtime/builder.rs", "line": 336 }, + { + "name": "recovery_provider", + "signature": "pub fn recovery_provider(mut self, provider: RecoveryProviderFactory) -> Self", + "source": "crates/openhuman-embed/src/runtime/seams.rs", + "line": 443 + }, { "name": "sandbox", "signature": "pub fn sandbox(mut self, mode: crate::SandboxModeSpec) -> Self", @@ -337,7 +353,7 @@ "name": "server_launcher", "signature": "pub fn server_launcher(mut self, launcher: ServerLauncher) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 444 + "line": 472 }, { "name": "services", @@ -367,7 +383,7 @@ "name": "storage", "signature": "pub fn storage(mut self, source: impl Into) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 457 + "line": 485 }, { "name": "token", @@ -385,13 +401,13 @@ "name": "tool_hook", "signature": "pub fn tool_hook(mut self, hook: Arc) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 437 + "line": 465 }, { "name": "tool_ranker", "signature": "pub fn tool_ranker(mut self, ranker: Arc) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 422 + "line": 450 }, { "name": "tool_rules", @@ -435,7 +451,7 @@ "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/docs/gitbooks/en/developing/embed/api-index.md b/docs/gitbooks/en/developing/embed/api-index.md index 3e386b0c6d6..598d3ce8f27 100644 --- a/docs/gitbooks/en/developing/embed/api-index.md +++ b/docs/gitbooks/en/developing/embed/api-index.md @@ -79,6 +79,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`ModelDefaults`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`OpenError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PendingApproval`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`PermissionFuture`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PermissionLevel`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PickListenPortError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`ProfileError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -129,6 +130,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`TrustedAccess`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TrustedAutomationSource`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`Turn`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`TurnCancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnContext`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnOutcome`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnRequest`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -139,6 +141,8 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`absolute`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`agent_progress`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`artifacts`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`budget`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`cancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`channels`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`chat_surface`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`complete`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -146,20 +150,26 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`cron`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`embeddings`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`events`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`fanout`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`identity`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`install_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`installed_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`memory`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`modules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`observe`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`process`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`profiles`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`providers`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`recovery`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`repository`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`routing`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`run_from_args`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`schema_for_rpc_method`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`seams`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`session_store`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`skill_registry`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`stream`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`structured`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) The compiled capability report describes this build: @@ -187,7 +197,7 @@ The compiled capability report describes this build: "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/docs/gitbooks/en/developing/embed/builder-setters.md b/docs/gitbooks/en/developing/embed/builder-setters.md index f2cded84d6d..f2e11806b52 100644 --- a/docs/gitbooks/en/developing/embed/builder-setters.md +++ b/docs/gitbooks/en/developing/embed/builder-setters.md @@ -13,7 +13,7 @@ Public consuming methods that return `Self`, extracted from the RuntimeBuilder i - [`backend_url`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L330): `pub fn backend_url(mut self, url: impl Into) -> Self` - [`config`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L444): `pub fn config(mut self, config: Config) -> Self` - [`config_source`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L308): `pub fn config_source(mut self, source: ConfigSource) -> Self` -- [`controller_extension`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L415): `pub fn controller_extension(mut self, extension: ControllerExtension) -> Self` +- [`controller_extension`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L437): `pub fn controller_extension(mut self, extension: ControllerExtension) -> Self` - [`cron`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L211): `pub fn cron(mut self, value: openhuman_core::config::schema::CronConfig) -> Self` - [`definition_base`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L170): `pub fn definition_base(mut self, definition: crate::AgentDefinitionSpec) -> Self` - [`domains`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L374): `pub fn domains(mut self, domains: DomainSet) -> Self` @@ -22,27 +22,28 @@ Public consuming methods that return `Self`, extracted from the RuntimeBuilder i - [`listen`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L413): `pub fn listen(self, host: impl Into, port: u16) -> Self` - [`listen_host`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L418): `pub fn listen_host(mut self, host: impl Into) -> Self` - [`listen_port`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L424): `pub fn listen_port(mut self, port: u16) -> Self` -- [`live_policy`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L464): `pub fn live_policy(mut self, policy: Arc) -> Self` +- [`live_policy`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L492): `pub fn live_policy(mut self, policy: Arc) -> Self` - [`max_agents`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L263): `pub fn max_agents(mut self, limit: usize) -> Self` - [`mcp_baseline`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L181): `pub fn mcp_baseline(mut self, servers: impl IntoIterator) -> Self` - [`memory_engine`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L239): `pub fn memory_engine(mut self, engine: Arc) -> Self` - [`model_defaults`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L160): `pub fn model_defaults(mut self, model: super::ModelDefaults) -> Self` - [`modules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/selection.rs#L156): `pub fn modules(mut self, modules: impl IntoIterator) -> Self` -- [`post_turn_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L430): `pub fn post_turn_hook(mut self, hook: Arc) -> Self` +- [`post_turn_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L458): `pub fn post_turn_hook(mut self, hook: Arc) -> Self` - [`privacy`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L196): `pub fn privacy(mut self, value: openhuman_core::config::schema::PrivacyConfig) -> Self` - [`provider`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L336): `pub fn provider(mut self, provider: Provider) -> Self` +- [`recovery_provider`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L443): `pub fn recovery_provider(mut self, provider: RecoveryProviderFactory) -> Self` - [`sandbox`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L165): `pub fn sandbox(mut self, mode: crate::SandboxModeSpec) -> Self` - [`secrets`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L201): `pub fn secrets(mut self, value: openhuman_core::config::schema::SecretsConfig) -> Self` -- [`server_launcher`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L444): `pub fn server_launcher(mut self, launcher: ServerLauncher) -> Self` +- [`server_launcher`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L472): `pub fn server_launcher(mut self, launcher: ServerLauncher) -> Self` - [`services`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L362): `pub fn services(mut self, services: ServiceSet) -> Self` - [`session`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L434): `pub fn session(mut self, session: Session) -> Self` - [`session_store`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L254): `pub fn session_store(mut self, provider: Arc) -> Self` - [`skills`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L175): `pub fn skills(mut self, policy: super::SkillsPolicy) -> Self` -- [`storage`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L457): `pub fn storage(mut self, source: impl Into) -> Self` +- [`storage`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L485): `pub fn storage(mut self, source: impl Into) -> Self` - [`token`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L401): `pub fn token(mut self, token: TokenSource) -> Self` - [`tool_groups`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L383): `pub fn tool_groups(mut self, tool_groups: ToolGroups) -> Self` -- [`tool_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L437): `pub fn tool_hook(mut self, hook: Arc) -> Self` -- [`tool_ranker`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L422): `pub fn tool_ranker(mut self, ranker: Arc) -> Self` +- [`tool_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L465): `pub fn tool_hook(mut self, hook: Arc) -> Self` +- [`tool_ranker`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L450): `pub fn tool_ranker(mut self, ranker: Arc) -> Self` - [`tool_rules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L191): `pub fn tool_rules(mut self, value: tinytools::ToolRules) -> Self` - [`workspace`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L273): `pub fn workspace(mut self, workspace: Workspace) -> Self` - [`workspace_dir`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L287): `pub fn workspace_dir(mut self, dir: impl Into) -> Self` diff --git a/docs/gitbooks/en/developing/embed/capability-matrix.md b/docs/gitbooks/en/developing/embed/capability-matrix.md index aba183fbf7e..c6219821881 100644 --- a/docs/gitbooks/en/developing/embed/capability-matrix.md +++ b/docs/gitbooks/en/developing/embed/capability-matrix.md @@ -25,7 +25,7 @@ This matrix comes from the default-feature compiled capability-report example. R | `skills` | Yes | | `storage-file` | No | | `storage-mongodb` | No | -| `storage-sqlite` | No | +| `storage-sqlite` | Yes | | `tinymemes` | Yes | | `voice` | No | | `web3` | No | diff --git a/docs/gitbooks/en/developing/embed/cookbook.md b/docs/gitbooks/en/developing/embed/cookbook.md index 6b7caefc55b..3462bef8af5 100644 --- a/docs/gitbooks/en/developing/embed/cookbook.md +++ b/docs/gitbooks/en/developing/embed/cookbook.md @@ -76,6 +76,14 @@ Lean runtime without background services. Run: `cargo run -p openhuman-embed --example lean_headless` +## Linux agent fleet memory and latency + +Measure retained runtime-owned agents using loopback inference and two worker threads. + +[Source](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs) · offline on Linux; use a fresh constrained cgroup for release measurements. + +Run: `cargo run -p openhuman-embed --example linux_fleet` + ## Connect an actual MCP protocol stub over loopback Connect an actual MCP protocol stub over loopback. diff --git a/gitbooks/developing/embed/api-index.json b/gitbooks/developing/embed/api-index.json index 76919ff831f..a1d19f9b920 100644 --- a/gitbooks/developing/embed/api-index.json +++ b/gitbooks/developing/embed/api-index.json @@ -76,6 +76,7 @@ "ModelDefaults", "OpenError", "PendingApproval", + "PermissionFuture", "PermissionLevel", "PickListenPortError", "ProfileError", @@ -126,6 +127,7 @@ "TrustedAccess", "TrustedAutomationSource", "Turn", + "TurnCancellation", "TurnContext", "TurnOutcome", "TurnRequest", @@ -136,6 +138,8 @@ "absolute", "agent_progress", "artifacts", + "budget", + "cancellation", "channels", "chat_surface", "complete", @@ -143,20 +147,26 @@ "cron", "embeddings", "events", + "fanout", "identity", "install_backend_transport", "installed_backend_transport", "memory", "modules", + "observe", "process", "profiles", "providers", + "recovery", + "repository", + "routing", "run_from_args", "schema_for_rpc_method", "seams", "session_store", "skill_registry", - "stream" + "stream", + "structured" ], "builder_setters": [ { @@ -217,7 +227,7 @@ "name": "controller_extension", "signature": "pub fn controller_extension(mut self, extension: ControllerExtension) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 415 + "line": 437 }, { "name": "cron", @@ -271,7 +281,7 @@ "name": "live_policy", "signature": "pub fn live_policy(mut self, policy: Arc) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 464 + "line": 492 }, { "name": "max_agents", @@ -307,7 +317,7 @@ "name": "post_turn_hook", "signature": "pub fn post_turn_hook(mut self, hook: Arc) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 430 + "line": 458 }, { "name": "privacy", @@ -321,6 +331,12 @@ "source": "crates/openhuman-embed/src/runtime/builder.rs", "line": 336 }, + { + "name": "recovery_provider", + "signature": "pub fn recovery_provider(mut self, provider: RecoveryProviderFactory) -> Self", + "source": "crates/openhuman-embed/src/runtime/seams.rs", + "line": 443 + }, { "name": "sandbox", "signature": "pub fn sandbox(mut self, mode: crate::SandboxModeSpec) -> Self", @@ -337,7 +353,7 @@ "name": "server_launcher", "signature": "pub fn server_launcher(mut self, launcher: ServerLauncher) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 444 + "line": 472 }, { "name": "services", @@ -367,7 +383,7 @@ "name": "storage", "signature": "pub fn storage(mut self, source: impl Into) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 457 + "line": 485 }, { "name": "token", @@ -385,13 +401,13 @@ "name": "tool_hook", "signature": "pub fn tool_hook(mut self, hook: Arc) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 437 + "line": 465 }, { "name": "tool_ranker", "signature": "pub fn tool_ranker(mut self, ranker: Arc) -> Self", "source": "crates/openhuman-embed/src/runtime/seams.rs", - "line": 422 + "line": 450 }, { "name": "tool_rules", @@ -435,7 +451,7 @@ "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/gitbooks/developing/embed/api-index.md b/gitbooks/developing/embed/api-index.md index 3e386b0c6d6..598d3ce8f27 100644 --- a/gitbooks/developing/embed/api-index.md +++ b/gitbooks/developing/embed/api-index.md @@ -79,6 +79,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`ModelDefaults`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`OpenError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PendingApproval`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`PermissionFuture`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PermissionLevel`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PickListenPortError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`ProfileError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -129,6 +130,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`TrustedAccess`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TrustedAutomationSource`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`Turn`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`TurnCancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnContext`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnOutcome`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnRequest`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -139,6 +141,8 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`absolute`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`agent_progress`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`artifacts`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`budget`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`cancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`channels`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`chat_surface`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`complete`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -146,20 +150,26 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`cron`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`embeddings`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`events`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`fanout`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`identity`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`install_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`installed_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`memory`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`modules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`observe`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`process`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`profiles`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`providers`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`recovery`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`repository`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`routing`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`run_from_args`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`schema_for_rpc_method`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`seams`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`session_store`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`skill_registry`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`stream`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`structured`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) The compiled capability report describes this build: @@ -187,7 +197,7 @@ The compiled capability report describes this build: "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/gitbooks/developing/embed/builder-setters.md b/gitbooks/developing/embed/builder-setters.md index f2cded84d6d..f2e11806b52 100644 --- a/gitbooks/developing/embed/builder-setters.md +++ b/gitbooks/developing/embed/builder-setters.md @@ -13,7 +13,7 @@ Public consuming methods that return `Self`, extracted from the RuntimeBuilder i - [`backend_url`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L330): `pub fn backend_url(mut self, url: impl Into) -> Self` - [`config`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L444): `pub fn config(mut self, config: Config) -> Self` - [`config_source`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L308): `pub fn config_source(mut self, source: ConfigSource) -> Self` -- [`controller_extension`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L415): `pub fn controller_extension(mut self, extension: ControllerExtension) -> Self` +- [`controller_extension`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L437): `pub fn controller_extension(mut self, extension: ControllerExtension) -> Self` - [`cron`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L211): `pub fn cron(mut self, value: openhuman_core::config::schema::CronConfig) -> Self` - [`definition_base`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L170): `pub fn definition_base(mut self, definition: crate::AgentDefinitionSpec) -> Self` - [`domains`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L374): `pub fn domains(mut self, domains: DomainSet) -> Self` @@ -22,27 +22,28 @@ Public consuming methods that return `Self`, extracted from the RuntimeBuilder i - [`listen`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L413): `pub fn listen(self, host: impl Into, port: u16) -> Self` - [`listen_host`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L418): `pub fn listen_host(mut self, host: impl Into) -> Self` - [`listen_port`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L424): `pub fn listen_port(mut self, port: u16) -> Self` -- [`live_policy`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L464): `pub fn live_policy(mut self, policy: Arc) -> Self` +- [`live_policy`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L492): `pub fn live_policy(mut self, policy: Arc) -> Self` - [`max_agents`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L263): `pub fn max_agents(mut self, limit: usize) -> Self` - [`mcp_baseline`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L181): `pub fn mcp_baseline(mut self, servers: impl IntoIterator) -> Self` - [`memory_engine`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L239): `pub fn memory_engine(mut self, engine: Arc) -> Self` - [`model_defaults`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L160): `pub fn model_defaults(mut self, model: super::ModelDefaults) -> Self` - [`modules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/selection.rs#L156): `pub fn modules(mut self, modules: impl IntoIterator) -> Self` -- [`post_turn_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L430): `pub fn post_turn_hook(mut self, hook: Arc) -> Self` +- [`post_turn_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L458): `pub fn post_turn_hook(mut self, hook: Arc) -> Self` - [`privacy`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L196): `pub fn privacy(mut self, value: openhuman_core::config::schema::PrivacyConfig) -> Self` - [`provider`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L336): `pub fn provider(mut self, provider: Provider) -> Self` +- [`recovery_provider`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L443): `pub fn recovery_provider(mut self, provider: RecoveryProviderFactory) -> Self` - [`sandbox`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L165): `pub fn sandbox(mut self, mode: crate::SandboxModeSpec) -> Self` - [`secrets`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L201): `pub fn secrets(mut self, value: openhuman_core::config::schema::SecretsConfig) -> Self` -- [`server_launcher`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L444): `pub fn server_launcher(mut self, launcher: ServerLauncher) -> Self` +- [`server_launcher`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L472): `pub fn server_launcher(mut self, launcher: ServerLauncher) -> Self` - [`services`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L362): `pub fn services(mut self, services: ServiceSet) -> Self` - [`session`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L434): `pub fn session(mut self, session: Session) -> Self` - [`session_store`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L254): `pub fn session_store(mut self, provider: Arc) -> Self` - [`skills`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L175): `pub fn skills(mut self, policy: super::SkillsPolicy) -> Self` -- [`storage`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L457): `pub fn storage(mut self, source: impl Into) -> Self` +- [`storage`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L485): `pub fn storage(mut self, source: impl Into) -> Self` - [`token`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L401): `pub fn token(mut self, token: TokenSource) -> Self` - [`tool_groups`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L383): `pub fn tool_groups(mut self, tool_groups: ToolGroups) -> Self` -- [`tool_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L437): `pub fn tool_hook(mut self, hook: Arc) -> Self` -- [`tool_ranker`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L422): `pub fn tool_ranker(mut self, ranker: Arc) -> Self` +- [`tool_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L465): `pub fn tool_hook(mut self, hook: Arc) -> Self` +- [`tool_ranker`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L450): `pub fn tool_ranker(mut self, ranker: Arc) -> Self` - [`tool_rules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L191): `pub fn tool_rules(mut self, value: tinytools::ToolRules) -> Self` - [`workspace`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L273): `pub fn workspace(mut self, workspace: Workspace) -> Self` - [`workspace_dir`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L287): `pub fn workspace_dir(mut self, dir: impl Into) -> Self` diff --git a/gitbooks/developing/embed/capability-matrix.md b/gitbooks/developing/embed/capability-matrix.md index aba183fbf7e..c6219821881 100644 --- a/gitbooks/developing/embed/capability-matrix.md +++ b/gitbooks/developing/embed/capability-matrix.md @@ -25,7 +25,7 @@ This matrix comes from the default-feature compiled capability-report example. R | `skills` | Yes | | `storage-file` | No | | `storage-mongodb` | No | -| `storage-sqlite` | No | +| `storage-sqlite` | Yes | | `tinymemes` | Yes | | `voice` | No | | `web3` | No | diff --git a/gitbooks/developing/embed/cookbook.md b/gitbooks/developing/embed/cookbook.md index 6b7caefc55b..3462bef8af5 100644 --- a/gitbooks/developing/embed/cookbook.md +++ b/gitbooks/developing/embed/cookbook.md @@ -76,6 +76,14 @@ Lean runtime without background services. Run: `cargo run -p openhuman-embed --example lean_headless` +## Linux agent fleet memory and latency + +Measure retained runtime-owned agents using loopback inference and two worker threads. + +[Source](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs) · offline on Linux; use a fresh constrained cgroup for release measurements. + +Run: `cargo run -p openhuman-embed --example linux_fleet` + ## Connect an actual MCP protocol stub over loopback Connect an actual MCP protocol stub over loopback. diff --git a/gitbooks/developing/jev.md b/gitbooks/developing/jev.md index e7897060571..99297a68224 100644 --- a/gitbooks/developing/jev.md +++ b/gitbooks/developing/jev.md @@ -7,7 +7,7 @@ icon: scale-balanced # Jev -Jev is a decision model, not a text generator. Give it a request and a handful of labelled options, and it returns a calibrated probability for each one in about 150 milliseconds. OpenHuman uses it as a fast, cheap judge wherever a turn must pick from a short list instead of writing prose. That means tool selection during a search and step-by-step browser control. +Jev is a decision model, not a text generator. Give it a request and a handful of labelled options, and it returns a calibrated probability for each one in about 150 milliseconds. OpenHuman uses it as a fast, cheap judge wherever a turn must pick from a short list instead of writing prose. That means tool selection during a search, step-by-step browser control and optional advice about unresolved tool failures. The models are `jev-1.13` and `jev-latest`, built by TypeSafe AI. OpenHuman reaches them through the TinyHumans System One proxy or, on a bring-your-own-key OpenRouter route, directly. Input costs $0.042 per million tokens and output is free. A typical tool search uses one to two thousand tokens. There is no session, no chain of thought and nothing to stream: one request, one set of probabilities. @@ -16,7 +16,7 @@ The models are `jev-1.13` and `jev-latest`, built by TypeSafe AI. OpenHuman reac Jev answers three kinds of question. They are defined in `tinyjevclient`: - Choice: probabilities over a set of named options. The caller can take the best one, the top three, or drop everything below a confidence floor. -- Score: one calibrated number for one thing, instead of a choice among several. +- Score: a probability distribution over ordered rubric categories. Recovery uses its expectation as a correction-feasibility score; that score is not a probability of success. - Noul: a yes/no probability. OpenHuman uses it to ask "does this apply at all" instead of "which of these applies". Tool search sends a Choice and a Noul in the same request. The Choice picks a tool from the shortlist (plus a `none` option), and the Noul says whether the request needs a tool at all. Browser control calls Jev once per step to decide the next action. @@ -44,6 +44,53 @@ The ranker resolves its Jev route and credential on every search instead of cach If the process has no usable embedding provider, `TinyHumansJevRanker` declines and the harness uses its own BM25 ranking. Running Jev over a lexical shortlist would only add a network round trip for no gain. +### Tool failure recovery + +Tool recovery is opt-in. The default classifier remains `keywords`; `compare` and +`jev` consult a separate decision contract for failures that trusted host facts and +the existing keyword rules have not settled. This recovery request is distinct from +tool search: its Choice classifies a failure, its Noul asks whether a permitted next +attempt can recover without an external prerequisite, and a Score is included only +when a concrete correction is evidenced. + +TinyTools owns the provider-neutral observation, question wording, distributions, +validation and `RecoveryAdviser` in `tinytools-jev::recovery`. OpenHuman supplies the +transport and applies its execution policy. A classification cannot grant permission, +renew approval, establish that a write did not happen, or automatically execute a +tool. Success, denial, expired approval, cancellation, terminal faults and trusted +authentication or permission facts take precedence. The host also excludes executed +writes with uncertain effects from decision evaluation. + +A request contains a minimized, scrubbed diagnostic and bounded argument names and +types. It carries no transcript or raw argument values. Diagnostics that cannot be +safely minimized are skipped. Provider errors, malformed answers, contradictory +advice, low confidence, cancellation and exhausted decision limits retain the +existing keyword fallback and repeated-failure guards. Every failed invocation +still counts against its stable operation and resource scope; advice cannot increase +retry headroom or reset that accounting. + +In `jev` mode, validated advice can lower a failure ceiling or add a nudge for the +next model request. The model still chooses its next call through the normal tool +bridge, permissions and approval gates. Nudges are ephemeral: the original tool +result and durable transcript stay intact, and the session's frozen system prompt +and tool declarations are preserved. `compare` evaluates observationally and +leaves those model-facing results, nudges and failure decisions unchanged. + +With `alternate_tools = true`, a confidently classified wrong-tool failure or a +counted repeated blocker can request an alternate. Candidates come from the turn's +registered, callable, read-only tools and exclude refused capabilities. An alternate +is advice for the main model, and its eventual call still passes ordinary admission. +Classification and alternate evaluation share one total deadline and both consume +the per-run decision limit. + +The host snapshots recovery configuration, route and credential when assembling a +turn. It does not switch credentials during that run. Recovery inherits the tool +search route and origin when its own settings are absent; an explicit route uses +only that route's credential. Each advisory evaluation has one total deadline, +including provider work, capped at three seconds and the run's remaining wall-clock +time. Cancellation and a per-run decision limit also bound it. The built-in transport +makes a single attempt. + ### Browser control `crates/openhuman-core/src/modules/browser_task.rs` hands browser tasks to TinyComputer's task members (`StartTask`, then `AwaitTask` until the task pauses or finishes). Tasks stay on the browser surface and the allowed websites. @@ -96,6 +143,45 @@ top_k = 3 Without an embedding provider, or without a credential for the selected route, tool search falls back to BM25 automatically. `auto` and `jev` therefore cost nothing extra when no credential is set. A setup with no TinyHumans account needs an embedding provider that does not depend on one (for example `memory.embedding_provider = "custom:"`) and a TypeSafe or OpenRouter key. +### Recovery configuration + +`RecoveryConfig` (`crates/openhuman-core/src/config/schema/recovery.rs`) is the +separate `[agent.recovery]` block: + +```toml +[agent.recovery] +classifier = "keywords" # default; "off" | "compare" | "jev" +decision_timeout_ms = 3000 # 1..=3000, also bounded by remaining run time +max_decisions_per_run = 8 # 1..=64 +alternate_tools = false # opt-in advice from admitted read-only tools +# jev_route = "typesafe" # absent: inherit agent.tool_search.jev_route +# jev_base_url = "https://api.example.com" # absent: inherit tool search origin +threshold_version = "provisional-v1" +class_confidence = 0.75 +recoverability = 0.75 +advice_confidence = 0.75 +repeated_blocker = 2 +``` + +| Classifier | Behavior | +| --- | --- | +| `keywords` | Existing keyword classification and recovery guards; no decision request. | +| `off` | No decision request; existing repeated-failure and safety guards remain active. | +| `compare` | Evaluate unresolved eligible failures without changing model-facing behavior. | +| `jev` | Apply validated advisory classification to unresolved eligible failures, with keyword fallback. | + +Thresholds are explicit provisional policy values, not a claim of measured recovery +accuracy. Choice confidence measures distribution concentration, not correctness; +Score expectation measures ordered feasibility, not success probability. The tool +search benchmark below does not measure recovery. Recovery remains opt-in until +separate evaluation and calibration justify changing its default. + +Embedders can install a `RecoveryProviderFactory` through +`RuntimeBuilder::recovery_provider`. The factory receives the immutable turn +configuration and returns a TinyTools `RecoveryEvaluator`, or declines so fallback +continues. The core has no default remote provider. Dropping the runtime restores +the previous factory if its installed factory still owns the slot. + ## Running the benchmark `tool-search-bench` lives in the `profile/` crate of [openhuman-benchmarks](https://github.com/tinyhumansai/openhuman-benchmarks). From a checkout of that repository: @@ -110,8 +196,10 @@ cargo run --manifest-path profile/Cargo.toml --bin tool-search-bench -- --dump-c ## Where the code lives -- `vendor/tinyagents/vendor/tinytools/crates/tinytools-jev/`: the transport-neutral `ToolRanker` implementation, its retrieval strategies and the `JevRankerConfig` / `JevRanking` types. -- `crates/openhuman-tinyhumans/src/jev/`: the TinyHumans evaluator (`TinyJevEvaluator`, over `tinyjevclient`) and the process-installed ranker (`TinyHumansJevRanker`). +- `vendor/tinyagents/vendor/tinytools/crates/tinytools-jev/`: the transport-neutral `ToolRanker` implementation, retrieval strategies and ranking types; `src/recovery/` owns the separate advisory contract and validation. +- `crates/openhuman-tinyhumans/src/jev/`: the TinyHumans evaluator (`TinyJevEvaluator`, over `tinyjevclient`), the process-installed ranker (`TinyHumansJevRanker`) and the mechanical recovery transport (`recovery.rs`). +- `crates/openhuman-core/src/agent/tinyagents/recovery_provider.rs` and `middleware/recovery_advice.rs`: the installed recovery seam, run bounds, minimized observations and advice projection. +- `crates/openhuman-core/src/config/schema/recovery.rs`: the recovery configuration and bounds. - `crates/openhuman-core/src/agent/tinyagents/discovery/`: the host slot that holds the installed ranker and turns `[agent.tool_search]` into the policy a turn runs with. - `crates/openhuman-core/src/modules/browser_task.rs`: the browser-task entry point that runs `JevController`. - [`docs/plans/jev-tool-search-baseline.md`](https://github.com/tinyhumansai/openhuman/blob/main/docs/plans/jev-tool-search-baseline.md): the full benchmark writeup. diff --git a/llms-full.txt b/llms-full.txt index 1658cc4d625..cf664b5ef66 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -2441,6 +2441,85 @@ The [cookbook](https://github.com/tinyhumansai/openhuman/blob/main/gitbooks/deve Hosts supply transport, credentials and application resources. Runtime settings establish shared defaults; agents narrow provider, access, prompt and tool behavior. Use ProfileRuntime when users require separate credentials and workspaces. +## Cancelling one turn + +Acquire `Turn::cancellation_handle()` before sending a turn. The handle is +cloneable and `cancel().await` waits for the turn to stop and for tracked +commands to be reaped. The agent remains available for later turns: + + + +Cancellation is scoped to this turn, including while waiting for inference. +Turns with a cancellation handle use owned interpreter subprocesses rather +than the Node/Python pool, which has no acknowledged per-job abort API. This +trades warm-worker reuse for awaited cleanup; ordinary turns retain pooling. +The `meter` callback fires once on cancellation or a dropped send future, +with `None` when dispatch has not supplied usage yet. + +Before send, cancellation prevents dispatch; after completion it is a no-op. +On Unix, the built-in shell, Node, Python and npm commands kill their process +group, including descendants. Other platforms stop the direct command. Host +tools that spawn independent tasks or processes must provide their own cleanup; +MCP server lifecycles remain owned by the agent. Keep polling `send()` while +awaiting cancellation, for example in a spawned task. + +## Scoped worker hooks + +Hooks can be supplied at three levels: `RuntimeBuilder::tool_hook` / +`post_turn_hook` for all agents, `AgentSpec::tool_hook` / `post_turn_hook` +for one agent, and `Turn::tool_hook` / `post_turn_hook` for one dispatch. +Tool callbacks run in that order. Named agent updates replace only that agent’s callback; per-turn callbacks are additional. +`ToolHookContext` carries the agent/session identity when known, and `cwd` +follows the execution workspace descriptor (including `Turn::cwd`), falling +back to the embedding context's configured action root. +The agent and turn hooks are never installed in the global registry, so +concurrent workers and later turns do not pick up one another's callbacks. +Post-turn callbacks run asynchronously with an owned session snapshot. +Independently spawned tasks that build sessions must explicitly inherit +`openhuman_core::agent::hooks::HookScope` to carry scoped hooks. + +Gateway attribution headers can be attached to `Route::header(name, value)` +and used with `Turn::route`, or with `Provider::routed(route)` on an agent. +They follow only that route's endpoint and are never saved to configuration, +sent to background providers, or included as values in `Route`'s `Debug`. + +### Inline permission and usage policy + +`AgentSpec::can_use_tool` and `Turn::can_use_tool` await a host callback before +executing each tool. The callback can wait for an approval UI and return +`ToolHookDecision::Proceed`, `Deny`, or `ProceedWith`. It owns that wait; +returning `Ask` denies execution. These callbacks add to existing tool policies, +and a turn callback cannot override an agent denial. + +`AgentSpec::stop_hook` and `Turn::stop_hook` receive cumulative usage after each +completed model call. Return `StopDecision::Continue` to observe usage, or +`Stop` to prevent subsequent calls. Completed tool rounds may still execute; +this is an after-call budget boundary, so hosts must refuse an already exhausted +budget before sending a turn. Provider-reported charges remain authoritative, +including known zero; missing charges remain unknown unless pricing is known. +A policy stop uses a deterministic partial summary rather than spending on +final-answer repair calls. + +### Per-turn tools and subprocess environment + +`Turn::tools` replaces this turn's host tool belt, including attached sources. +An empty belt revokes host tools; the next turn returns to the agent's belt. +Builtin tools still follow the agent definition. This supplies dynamic tools for +in-process hosts; statically declared MCP servers retain their creation-time +configuration. + +`Turn::tool_env` supplies the base environment of owned builtin subprocesses. +Variables absent from it are not inherited from the daemon. The builtin command +builders retain their own security/runtime additions, including Git restrictions, +managed interpreter paths and scratch directories. Scoped turns bypass Node and +Python pools, which cannot acknowledge per-job cancellation or swap a job's +process environment. A host tool that spawns a separate Tokio task must explicitly +carry the command environment and cleanup scopes into that task. + +Standalone exact source pins and generated Cargo patches: [consumer setup](CONSUMERS.md). +Ordered fallbacks and required exploration: [routing](ROUTING.md). +Host telemetry and the existing exporter: [observers](OBSERVERS.md). + # Embedding OpenHuman @@ -2529,6 +2608,14 @@ Lean runtime without background services. Run: `cargo run -p openhuman-embed --example lean_headless` +## Linux agent fleet memory and latency + +Measure retained runtime-owned agents using loopback inference and two worker threads. + +[Source](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs) · offline on Linux; use a fresh constrained cgroup for release measurements. + +Run: `cargo run -p openhuman-embed --example linux_fleet` + ## Connect an actual MCP protocol stub over loopback Connect an actual MCP protocol stub over loopback. @@ -2661,7 +2748,7 @@ This matrix comes from the default-feature compiled capability-report example. R | `skills` | Yes | | `storage-file` | No | | `storage-mongodb` | No | -| `storage-sqlite` | No | +| `storage-sqlite` | Yes | | `tinymemes` | Yes | | `voice` | No | | `web3` | No | @@ -2683,7 +2770,7 @@ Public consuming methods that return `Self`, extracted from the RuntimeBuilder i - [`backend_url`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L330): `pub fn backend_url(mut self, url: impl Into) -> Self` - [`config`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L444): `pub fn config(mut self, config: Config) -> Self` - [`config_source`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L308): `pub fn config_source(mut self, source: ConfigSource) -> Self` -- [`controller_extension`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L415): `pub fn controller_extension(mut self, extension: ControllerExtension) -> Self` +- [`controller_extension`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L437): `pub fn controller_extension(mut self, extension: ControllerExtension) -> Self` - [`cron`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L211): `pub fn cron(mut self, value: openhuman_core::config::schema::CronConfig) -> Self` - [`definition_base`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L170): `pub fn definition_base(mut self, definition: crate::AgentDefinitionSpec) -> Self` - [`domains`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L374): `pub fn domains(mut self, domains: DomainSet) -> Self` @@ -2692,27 +2779,28 @@ Public consuming methods that return `Self`, extracted from the RuntimeBuilder i - [`listen`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L413): `pub fn listen(self, host: impl Into, port: u16) -> Self` - [`listen_host`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L418): `pub fn listen_host(mut self, host: impl Into) -> Self` - [`listen_port`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L424): `pub fn listen_port(mut self, port: u16) -> Self` -- [`live_policy`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L464): `pub fn live_policy(mut self, policy: Arc) -> Self` +- [`live_policy`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L492): `pub fn live_policy(mut self, policy: Arc) -> Self` - [`max_agents`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L263): `pub fn max_agents(mut self, limit: usize) -> Self` - [`mcp_baseline`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L181): `pub fn mcp_baseline(mut self, servers: impl IntoIterator) -> Self` - [`memory_engine`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L239): `pub fn memory_engine(mut self, engine: Arc) -> Self` - [`model_defaults`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L160): `pub fn model_defaults(mut self, model: super::ModelDefaults) -> Self` - [`modules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/selection.rs#L156): `pub fn modules(mut self, modules: impl IntoIterator) -> Self` -- [`post_turn_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L430): `pub fn post_turn_hook(mut self, hook: Arc) -> Self` +- [`post_turn_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L458): `pub fn post_turn_hook(mut self, hook: Arc) -> Self` - [`privacy`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L196): `pub fn privacy(mut self, value: openhuman_core::config::schema::PrivacyConfig) -> Self` - [`provider`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L336): `pub fn provider(mut self, provider: Provider) -> Self` +- [`recovery_provider`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L443): `pub fn recovery_provider(mut self, provider: RecoveryProviderFactory) -> Self` - [`sandbox`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L165): `pub fn sandbox(mut self, mode: crate::SandboxModeSpec) -> Self` - [`secrets`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L201): `pub fn secrets(mut self, value: openhuman_core::config::schema::SecretsConfig) -> Self` -- [`server_launcher`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L444): `pub fn server_launcher(mut self, launcher: ServerLauncher) -> Self` +- [`server_launcher`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L472): `pub fn server_launcher(mut self, launcher: ServerLauncher) -> Self` - [`services`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L362): `pub fn services(mut self, services: ServiceSet) -> Self` - [`session`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L434): `pub fn session(mut self, session: Session) -> Self` - [`session_store`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L254): `pub fn session_store(mut self, provider: Arc) -> Self` - [`skills`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L175): `pub fn skills(mut self, policy: super::SkillsPolicy) -> Self` -- [`storage`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L457): `pub fn storage(mut self, source: impl Into) -> Self` +- [`storage`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L485): `pub fn storage(mut self, source: impl Into) -> Self` - [`token`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L401): `pub fn token(mut self, token: TokenSource) -> Self` - [`tool_groups`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L383): `pub fn tool_groups(mut self, tool_groups: ToolGroups) -> Self` -- [`tool_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L437): `pub fn tool_hook(mut self, hook: Arc) -> Self` -- [`tool_ranker`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L422): `pub fn tool_ranker(mut self, ranker: Arc) -> Self` +- [`tool_hook`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L465): `pub fn tool_hook(mut self, hook: Arc) -> Self` +- [`tool_ranker`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/seams.rs#L450): `pub fn tool_ranker(mut self, ranker: Arc) -> Self` - [`tool_rules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L191): `pub fn tool_rules(mut self, value: tinytools::ToolRules) -> Self` - [`workspace`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L273): `pub fn workspace(mut self, workspace: Workspace) -> Self` - [`workspace_dir`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/runtime/builder.rs#L287): `pub fn workspace_dir(mut self, dir: impl Into) -> Self` @@ -2799,6 +2887,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`ModelDefaults`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`OpenError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PendingApproval`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`PermissionFuture`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PermissionLevel`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PickListenPortError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`ProfileError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -2849,6 +2938,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`TrustedAccess`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TrustedAutomationSource`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`Turn`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`TurnCancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnContext`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnOutcome`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnRequest`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -2859,6 +2949,8 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`absolute`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`agent_progress`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`artifacts`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`budget`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`cancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`channels`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`chat_surface`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`complete`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -2866,20 +2958,26 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`cron`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`embeddings`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`events`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`fanout`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`identity`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`install_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`installed_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`memory`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`modules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`observe`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`process`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`profiles`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`providers`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`recovery`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`repository`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`routing`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`run_from_args`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`schema_for_rpc_method`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`seams`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`session_store`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`skill_registry`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`stream`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`structured`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) The compiled capability report describes this build: @@ -2907,7 +3005,7 @@ The compiled capability report describes this build: "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, @@ -3003,6 +3101,7 @@ The compiled capability report describes this build: - [A per-agent post-turn hook observes completed turns](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/hooks.rs): A per-agent post-turn hook observes completed turns. (offline; optional live via OPENHUMAN_EXAMPLE_LIVE=1 and BASE_URL/API_KEY/MODEL.) - [Host tools and HostOnly keep the advertised tool catalog exact](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/host_tools.rs): Host tools and HostOnly keep the advertised tool catalog exact. (offline with loopback stubs; no live path.) - [Lean runtime without background services](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/lean_headless.rs): Lean runtime without background services. (offline with loopback stubs; optional live via OPENHUMAN_EXAMPLE_LIVE.) +- [Linux agent fleet memory and latency](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs): Measure retained runtime-owned agents using loopback inference and two worker threads. (offline on Linux; use a fresh constrained cgroup for release measurements.) - [Connect an actual MCP protocol stub over loopback](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/mcp.rs): Connect an actual MCP protocol stub over loopback. (offline with loopback stubs; no live path.; feature: mcp) - [Tenant scoped memory facade with an in-memory engine](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/memory.rs): Tenant scoped memory facade with an in-memory engine. (offline with loopback stubs; no live path.) - [SaaS profiles isolate conversation history](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/profiles.rs): SaaS profiles isolate conversation history. (offline with loopback stubs; no live path.) diff --git a/vendor/tinyagents b/vendor/tinyagents index 554a20ac623..b156f412f7c 160000 --- a/vendor/tinyagents +++ b/vendor/tinyagents @@ -1 +1 @@ -Subproject commit 554a20ac6230d07c32163a2085a712fab3c1a8b2 +Subproject commit b156f412f7cbdda454e6c764b121099644123f8d