From 6bf29639e91e82a820ec17978262b5953ef8d0d1 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 10 Oct 2026 21:56:12 +0300 Subject: [PATCH 01/18] fix(hooks): forward the resolved working directory to configured hooks Co-authored-by: Medulla --- crates/openhuman-core/src/hooks/bridge.rs | 5 ++++- crates/openhuman-core/src/hooks/bridge_tests.rs | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/openhuman-core/src/hooks/bridge.rs b/crates/openhuman-core/src/hooks/bridge.rs index 254037a4f7a..30565b58611 100644 --- a/crates/openhuman-core/src/hooks/bridge.rs +++ b/crates/openhuman-core/src/hooks/bridge.rs @@ -82,7 +82,10 @@ fn identity_from_tool(context: &ToolHookContext) -> TurnIdentity { TurnIdentity { session_id: context.session_id.clone(), agent_id: context.agent_id.clone(), - cwd: string_field(&context.arguments, "cwd"), + cwd: context + .cwd + .as_ref() + .map(|cwd| cwd.to_string_lossy().into_owned()), ..TurnIdentity::default() } } diff --git a/crates/openhuman-core/src/hooks/bridge_tests.rs b/crates/openhuman-core/src/hooks/bridge_tests.rs index e5f0d3a1e4b..caf2a0ab86d 100644 --- a/crates/openhuman-core/src/hooks/bridge_tests.rs +++ b/crates/openhuman-core/src/hooks/bridge_tests.rs @@ -113,3 +113,18 @@ fn timeout_text_classifies_as_a_timeout_failure() { ); assert_eq!(classify_failure("something else"), "error"); } + +#[test] +fn configured_hook_identity_uses_the_resolved_working_directory() { + let mut ctx = context("shell", serde_json::json!({"command": "pwd"})); + ctx.cwd = Some(std::path::PathBuf::from("/resolved/workspace")); + assert_eq!( + identity_from_tool(&ctx).cwd.as_deref(), + Some("/resolved/workspace") + ); + ctx.arguments["cwd"] = serde_json::json!("relative/argument"); + assert_eq!( + identity_from_tool(&ctx).cwd.as_deref(), + Some("/resolved/workspace") + ); +} From 2b1b2325b45f50a492297ef0ee6fb5ad0389bfd2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 10 Oct 2026 22:04:30 +0300 Subject: [PATCH 02/18] fix(embed): preserve control reasons across cancellation races Co-authored-by: Medulla --- crates/openhuman-embed/src/agent/lifecycle.rs | 6 ++++++ .../openhuman-embed/src/runtime/lifecycle.rs | 6 ++++-- .../openhuman-embed/src/turn_cancellation.rs | 4 ++++ crates/openhuman-embed/src/turn_control.rs | 15 ++++++++++++++- .../tests/turn_cancellation.rs | 19 ++++++++++--------- 5 files changed, 38 insertions(+), 12 deletions(-) diff --git a/crates/openhuman-embed/src/agent/lifecycle.rs b/crates/openhuman-embed/src/agent/lifecycle.rs index 7d89b3c8697..5bc8e22a2ef 100644 --- a/crates/openhuman-embed/src/agent/lifecycle.rs +++ b/crates/openhuman-embed/src/agent/lifecycle.rs @@ -31,9 +31,15 @@ impl Lifecycle { /// Stops admitting turns and ends the ones in flight; returns whether this /// caller claimed removal before another caller. pub(crate) fn mark_removed(&self) -> bool { + self.mark_removed_with(|| {}) + } + + /// Claim removal and settle owned approvals before waking in-flight turns. + pub(crate) fn mark_removed_with(&self, before_notify: impl FnOnce()) -> bool { if self.removal_claimed.swap(true, Ordering::SeqCst) { return false; } + before_notify(); // A cancelled turn can retain the watch read while its future drops. // Only the first claimant writes; repeated teardown never waits on it. self.removed.send_replace(true); diff --git a/crates/openhuman-embed/src/runtime/lifecycle.rs b/crates/openhuman-embed/src/runtime/lifecycle.rs index 7df17da08af..c216451c614 100644 --- a/crates/openhuman-embed/src/runtime/lifecycle.rs +++ b/crates/openhuman-embed/src/runtime/lifecycle.rs @@ -113,7 +113,10 @@ impl Runtime { .and_then(|weak| weak.upgrade()) .ok_or_else(|| AgentError::UnknownId(id.to_string()))? }; - if !inner.lifecycle.mark_removed() { + if !inner + .lifecycle + .mark_removed_with(|| inner.deny_approvals("agent_removed")) + { return Err(AgentError::UnknownId(id.to_string())); } let mut removal = RemovalSlot { @@ -123,7 +126,6 @@ impl Runtime { }; let inner = &removal.inner; log::debug!("[embed][runtime] removing agent id={id} purge={purge}"); - inner.deny_approvals("agent_removed"); if !inner.lifecycle.wait_idle(REMOVE_IDLE_WAIT).await { log::warn!( "[embed][runtime] agent id={id} still had turns unwinding after {}s", diff --git a/crates/openhuman-embed/src/turn_cancellation.rs b/crates/openhuman-embed/src/turn_cancellation.rs index 2b8f97ccef4..d8da22f8173 100644 --- a/crates/openhuman-embed/src/turn_cancellation.rs +++ b/crates/openhuman-embed/src/turn_cancellation.rs @@ -51,6 +51,10 @@ impl TurnCancellation { self.0.cleanup.wait().await; } + pub(crate) fn is_cancelled(&self) -> bool { + *self.0.requested.borrow() + } + pub(crate) async fn cancelled(&self) { let mut requested = self.0.requested.subscribe(); let _ = requested.wait_for(|requested| *requested).await; diff --git a/crates/openhuman-embed/src/turn_control.rs b/crates/openhuman-embed/src/turn_control.rs index caed7bb6224..6976449cd10 100644 --- a/crates/openhuman-embed/src/turn_control.rs +++ b/crates/openhuman-embed/src/turn_control.rs @@ -228,7 +228,20 @@ impl Turn { native.cancel(); Err(CoreError::DeadlineExceeded { method: AGENT_CHAT }) }, - outcome = Box::pin(self.send_inner(&meter.usage)) => outcome, + outcome = Box::pin(self.send_inner(&meter.usage)) => { + // The relay can interrupt a long dispatch poll before + // this select polls its control branches again. Keep + // the public error tied to the triggering control. + if token.as_ref().is_some_and(|token| token.is_cancelled()) + || cancellation.is_cancelled() + { + Err(CoreError::TurnCancelled { method: AGENT_CHAT }) + } else if deadline.is_some_and(|deadline| tokio::time::Instant::now() >= deadline) { + Err(CoreError::DeadlineExceeded { method: AGENT_CHAT }) + } else { + outcome + } + }, } }) .await; diff --git a/crates/openhuman-embed/tests/turn_cancellation.rs b/crates/openhuman-embed/tests/turn_cancellation.rs index e0ca914b68c..9277b3c64bb 100644 --- a/crates/openhuman-embed/tests/turn_cancellation.rs +++ b/crates/openhuman-embed/tests/turn_cancellation.rs @@ -115,15 +115,16 @@ async fn scenario() { let observed = Arc::new(Outcomes::default()); // A whole-turn deadline drops inference and leaves the same agent reusable. - assert!(matches!( - blocked - .turn("deadline") - .timeout(Duration::from_millis(100)) - .observer(observed.clone()) - .send() - .await, - Err(CoreError::DeadlineExceeded { .. }) - )); + let deadline = blocked + .turn("deadline") + .timeout(Duration::from_millis(100)) + .observer(observed.clone()) + .send() + .await; + assert!( + matches!(deadline, Err(CoreError::DeadlineExceeded { .. })), + "{deadline:?}" + ); assert_eq!(agent.run("after deadline").await.unwrap().reply, "finished"); assert_eq!( *observed.0.lock().unwrap(), From 0d2d4f1992e50756736a4f168f0a95db7ae2958a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 10 Oct 2026 22:08:43 +0300 Subject: [PATCH 03/18] fix(embed): retain every agent permission callback Co-authored-by: Medulla --- crates/openhuman-embed/src/agent/spec.rs | 2 +- crates/openhuman-embed/src/permission.rs | 24 ++++++++++++-- crates/openhuman-embed/src/turn.rs | 2 +- .../tests/inline_permissions.rs | 31 +++++++++++++++++++ 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/crates/openhuman-embed/src/agent/spec.rs b/crates/openhuman-embed/src/agent/spec.rs index 42352eb75d2..9b20fa8fa32 100644 --- a/crates/openhuman-embed/src/agent/spec.rs +++ b/crates/openhuman-embed/src/agent/spec.rs @@ -338,7 +338,7 @@ impl AgentSpec { + Sync + 'static, { - self.tool_hook(std::sync::Arc::new(crate::permission::PermissionHook( + self.tool_hook(std::sync::Arc::new(crate::permission::PermissionHook::new( callback, ))) } diff --git a/crates/openhuman-embed/src/permission.rs b/crates/openhuman-embed/src/permission.rs index bd55bf0d0b4..9b9cd321a8e 100644 --- a/crates/openhuman-embed/src/permission.rs +++ b/crates/openhuman-embed/src/permission.rs @@ -5,7 +5,25 @@ use crate::seams::{ToolHook, ToolHookContext, ToolHookDecision}; pub type PermissionFuture<'a> = std::pin::Pin + Send + 'a>>; -pub(crate) struct PermissionHook(pub(crate) F); +pub(crate) struct PermissionHook { + name: String, + callback: F, +} + +impl PermissionHook { + pub(crate) fn new(callback: F) -> Self { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + Self { + // Agent-local overrides replace hooks by name. Each permission + // registration must survive, in registration order. + name: format!( + "embed.can_use_tool.{:020}", + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + ), + callback, + } + } +} #[async_trait::async_trait] impl ToolHook for PermissionHook @@ -13,7 +31,7 @@ where F: for<'a> Fn(&'a ToolHookContext) -> PermissionFuture<'a> + Send + Sync, { fn name(&self) -> &str { - "embed.can_use_tool" + &self.name } async fn before_tool(&self, _: &ToolHookContext) -> anyhow::Result<()> { Ok(()) @@ -22,6 +40,6 @@ where Ok(()) } async fn before_tool_decision(&self, context: &ToolHookContext) -> ToolHookDecision { - (self.0)(context).await + (self.callback)(context).await } } diff --git a/crates/openhuman-embed/src/turn.rs b/crates/openhuman-embed/src/turn.rs index 1567f4736b7..1289895959f 100644 --- a/crates/openhuman-embed/src/turn.rs +++ b/crates/openhuman-embed/src/turn.rs @@ -177,7 +177,7 @@ impl Turn { + Sync + 'static, { - self.tool_hook(std::sync::Arc::new(crate::permission::PermissionHook( + self.tool_hook(std::sync::Arc::new(crate::permission::PermissionHook::new( callback, ))) } diff --git a/crates/openhuman-embed/tests/inline_permissions.rs b/crates/openhuman-embed/tests/inline_permissions.rs index bcfcff253a0..733bb127e36 100644 --- a/crates/openhuman-embed/tests/inline_permissions.rs +++ b/crates/openhuman-embed/tests/inline_permissions.rs @@ -165,4 +165,35 @@ async fn scenario() { 1, "closing the UI granted permission" ); + repeated_permissions(&runtime).await; +} + +async fn repeated_permissions(runtime: &Runtime) { + let provider = scripted_provider(vec![tool_call_completion("probe", "{}")], "done").await; + let calls = Arc::new(AtomicUsize::new(0)); + let tool_calls = calls.clone(); + let agent = runtime + .agent( + AgentSpec::new("repeated-permissions") + .provider(route(&provider, "fixture")) + .definition( + AgentDefinitionSpec::new() + .bare_prompt("Use probe then answer") + .tools(ToolScopeSpec::HostOnly), + ) + .tools(move |_| { + HostTurnTools::advertised(vec![Box::new(Probe(tool_calls.clone()))]) + }) + .can_use_tool(|_| { + Box::pin(async { ToolHookDecision::Deny("first policy refused".into()) }) + }) + .can_use_tool(|_| Box::pin(async { ToolHookDecision::Proceed })), + ) + .unwrap(); + assert!(agent.run("perform probe").await.is_err()); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "later permission erased the earlier denial" + ); } From 0b3c1df99efc8619f1cd82600cbf288d3963c5d2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 10 Oct 2026 22:10:52 +0300 Subject: [PATCH 04/18] docs(embed): refresh Linux measurements and generated API references Co-authored-by: Medulla --- README.md | 4 +- crates/openhuman-embed/examples/README.md | 1 + docs/README.ar.md | 4 +- docs/README.de.md | 4 +- docs/README.ja-JP.md | 4 +- docs/README.ko.md | 4 +- docs/README.tr.md | 4 +- docs/README.ur-pk.md | 4 +- docs/README.zh-CN.md | 4 +- docs/benchmarks/medulla-embed-linux.json | 254 +++++++++--------- .../en/developing/embed/api-index.json | 13 +- .../gitbooks/en/developing/embed/api-index.md | 11 +- .../en/developing/embed/capability-matrix.md | 2 +- docs/gitbooks/en/developing/embed/cookbook.md | 8 + docs/gitbooks/en/developing/performance.md | 66 ++++- gitbooks/developing/embed/api-index.json | 13 +- gitbooks/developing/embed/api-index.md | 11 +- .../developing/embed/capability-matrix.md | 2 +- gitbooks/developing/embed/cookbook.md | 8 + gitbooks/developing/performance.md | 43 +-- llms-full.txt | 101 ++++++- 21 files changed, 390 insertions(+), 175 deletions(-) diff --git a/README.md b/README.md index b7901ee6f33..22858053a2b 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ Most agent harnesses run one heavy process per agent and resend a big prompt on

Built for developers

-

Rust quickstart · Embedding guide · Examples

+

Rust quickstart · Embedding guide · Examples

Use it as a Rust library: call an agent like any other function, or run a whole fleet from one small server.

@@ -288,7 +288,7 @@ let reply = agent.run("Summarize what you can see in this directory.").await?; println!("{}", reply.reply); ``` -Next: the [Rust quickstart](https://tinyhumans.gitbook.io/openhuman/developing/quickstart), the [embedding guide](https://tinyhumans.gitbook.io/openhuman/developing/embedding) and the [developer docs](https://tinyhumans.gitbook.io/openhuman/developing). +Next: the [Rust quickstart](https://tinyhumans.gitbook.io/openhuman/developing/quickstart), the [embedding guide](./gitbooks/developing/embed/README.md) and the [developer docs](https://tinyhumans.gitbook.io/openhuman/developing). --- diff --git a/crates/openhuman-embed/examples/README.md b/crates/openhuman-embed/examples/README.md index c24342786b7..9922e16ac64 100644 --- a/crates/openhuman-embed/examples/README.md +++ b/crates/openhuman-embed/examples/README.md @@ -13,6 +13,7 @@ - [A per-agent post-turn hook observes completed turns](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/hooks.rs): A per-agent post-turn hook observes completed turns. (offline; optional live via OPENHUMAN_EXAMPLE_LIVE=1 and BASE_URL/API_KEY/MODEL.) - [Host tools and HostOnly keep the advertised tool catalog exact](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/host_tools.rs): Host tools and HostOnly keep the advertised tool catalog exact. (offline with loopback stubs; no live path.) - [Lean runtime without background services](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/lean_headless.rs): Lean runtime without background services. (offline with loopback stubs; optional live via OPENHUMAN_EXAMPLE_LIVE.) +- [Linux agent fleet memory and latency](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs): Measure retained runtime-owned agents using loopback inference and two worker threads. (offline on Linux; use a fresh constrained cgroup for release measurements.) - [Connect an actual MCP protocol stub over loopback](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/mcp.rs): Connect an actual MCP protocol stub over loopback. (offline with loopback stubs; no live path.; feature: mcp) - [Tenant scoped memory facade with an in-memory engine](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/memory.rs): Tenant scoped memory facade with an in-memory engine. (offline with loopback stubs; no live path.) - [SaaS profiles isolate conversation history](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/profiles.rs): SaaS profiles isolate conversation history. (offline with loopback stubs; no live path.) diff --git a/docs/README.ar.md b/docs/README.ar.md index adc65d0d85e..525adda76f5 100644 --- a/docs/README.ar.md +++ b/docs/README.ar.md @@ -143,7 +143,7 @@ irm https://raw.githubusercontent.com/tinyhumansai/openhuman/main/scripts/instal

مصمم للمطورين

-

البدء السريع مع Rust · دليل التضمين · أمثلة

+

البدء السريع مع Rust · دليل التضمين · أمثلة

استخدمه كمكتبة Rust: استدعِ وكيلاً كأي دالة أخرى، أو شغّل أسطولاً كاملاً من خادم صغير واحد.

@@ -342,7 +342,7 @@ println!("{}", reply.reply);
-بعد ذلك: [البدء السريع مع Rust](https://tinyhumans.gitbook.io/openhuman/developing/quickstart)، و[دليل التضمين](https://tinyhumans.gitbook.io/openhuman/developing/embedding)، و[مستندات المطورين](https://tinyhumans.gitbook.io/openhuman/developing). +بعد ذلك: [البدء السريع مع Rust](https://tinyhumans.gitbook.io/openhuman/developing/quickstart)، و[دليل التضمين](../gitbooks/developing/embed/README.md)، و[مستندات المطورين](https://tinyhumans.gitbook.io/openhuman/developing). --- diff --git a/docs/README.de.md b/docs/README.de.md index 57475cd8a5d..882d2833dca 100644 --- a/docs/README.de.md +++ b/docs/README.de.md @@ -117,7 +117,7 @@ Die meisten Agent-Harnesses starten pro Agent einen schweren Prozess und senden

Für Entwickler gebaut

-

Rust-Schnellstart · Einbettungsanleitung · Beispiele

+

Rust-Schnellstart · Einbettungsanleitung · Beispiele

Nutze es als Rust-Bibliothek: Rufe einen Agenten wie jede andere Funktion auf oder betreibe eine ganze Flotte auf einem kleinen Server.

@@ -288,7 +288,7 @@ let reply = agent.run("Summarize what you can see in this directory.").await?; println!("{}", reply.reply); ``` -Als Nächstes: der [Rust-Schnellstart](https://tinyhumans.gitbook.io/openhuman/developing/quickstart), die [Einbettungsanleitung](https://tinyhumans.gitbook.io/openhuman/developing/embedding) und die [Entwicklerdokumentation](https://tinyhumans.gitbook.io/openhuman/developing). +Als Nächstes: der [Rust-Schnellstart](https://tinyhumans.gitbook.io/openhuman/developing/quickstart), die [Einbettungsanleitung](../gitbooks/developing/embed/README.md) und die [Entwicklerdokumentation](https://tinyhumans.gitbook.io/openhuman/developing). --- diff --git a/docs/README.ja-JP.md b/docs/README.ja-JP.md index 71b8a49bced..97946c41758 100644 --- a/docs/README.ja-JP.md +++ b/docs/README.ja-JP.md @@ -117,7 +117,7 @@ macOS と Linux のスクリプトが何を行うかを事前に確認するに

開発者のために

-

Rust クイックスタート · 組み込みガイド · サンプル

+

Rust クイックスタート · 組み込みガイド · サンプル

Rust ライブラリとして使えます。エージェントを普通の関数のように呼び出すことも、小さなサーバー1台でエージェント群をまるごと動かすこともできます。

@@ -288,7 +288,7 @@ let reply = agent.run("Summarize what you can see in this directory.").await?; println!("{}", reply.reply); ``` -次は、[Rust クイックスタート](https://tinyhumans.gitbook.io/openhuman/developing/quickstart)、[組み込みガイド](https://tinyhumans.gitbook.io/openhuman/developing/embedding)、[開発者向けドキュメント](https://tinyhumans.gitbook.io/openhuman/developing)をご覧ください。 +次は、[Rust クイックスタート](https://tinyhumans.gitbook.io/openhuman/developing/quickstart)、[組み込みガイド](../gitbooks/developing/embed/README.md)、[開発者向けドキュメント](https://tinyhumans.gitbook.io/openhuman/developing)をご覧ください。 --- diff --git a/docs/README.ko.md b/docs/README.ko.md index bbc290e6738..83c87b2a574 100644 --- a/docs/README.ko.md +++ b/docs/README.ko.md @@ -117,7 +117,7 @@ macOS와 Linux 스크립트가 무엇을 하는지 미리 보려면 명령 끝

개발자를 위해 만들었습니다

-

Rust 퀵스타트 · 임베딩 가이드 · 예제

+

Rust 퀵스타트 · 임베딩 가이드 · 예제

Rust 라이브러리로 사용하세요. 다른 함수처럼 에이전트를 호출하거나, 작은 서버 하나에서 수많은 에이전트를 실행할 수 있습니다.

@@ -288,7 +288,7 @@ let reply = agent.run("Summarize what you can see in this directory.").await?; println!("{}", reply.reply); ``` -다음으로는 [Rust 퀵스타트](https://tinyhumans.gitbook.io/openhuman/developing/quickstart), [임베딩 가이드](https://tinyhumans.gitbook.io/openhuman/developing/embedding), [개발자 문서](https://tinyhumans.gitbook.io/openhuman/developing)를 보세요. +다음으로는 [Rust 퀵스타트](https://tinyhumans.gitbook.io/openhuman/developing/quickstart), [임베딩 가이드](../gitbooks/developing/embed/README.md), [개발자 문서](https://tinyhumans.gitbook.io/openhuman/developing)를 보세요. --- diff --git a/docs/README.tr.md b/docs/README.tr.md index cd63024b1b3..cee26fbf319 100644 --- a/docs/README.tr.md +++ b/docs/README.tr.md @@ -117,7 +117,7 @@ macOS ve Linux betiğinin ne yapacağını önceden görmek için komutun sonuna

Geliştiriciler için tasarlandı

-

Rust hızlı başlangıç · Gömme rehberi · Örnekler

+

Rust hızlı başlangıç · Gömme rehberi · Örnekler

Bir Rust kütüphanesi olarak kullanın: bir ajanı herhangi bir işlev gibi çağırın ya da tüm bir filoyu tek bir küçük sunucudan çalıştırın.

@@ -288,7 +288,7 @@ let reply = agent.run("Summarize what you can see in this directory.").await?; println!("{}", reply.reply); ``` -Sırada: [Rust hızlı başlangıç](https://tinyhumans.gitbook.io/openhuman/developing/quickstart), [gömme rehberi](https://tinyhumans.gitbook.io/openhuman/developing/embedding) ve [geliştirici dokümanları](https://tinyhumans.gitbook.io/openhuman/developing). +Sırada: [Rust hızlı başlangıç](https://tinyhumans.gitbook.io/openhuman/developing/quickstart), [gömme rehberi](../gitbooks/developing/embed/README.md) ve [geliştirici dokümanları](https://tinyhumans.gitbook.io/openhuman/developing). --- diff --git a/docs/README.ur-pk.md b/docs/README.ur-pk.md index b125cdf4ecc..df362d0e420 100644 --- a/docs/README.ur-pk.md +++ b/docs/README.ur-pk.md @@ -139,7 +139,7 @@ macOS اور Linux کی اسکرپٹ کیا کرے گی، یہ پہلے دیکھ

ڈیولپرز کے لیے بنایا گیا

-

Rust کوئیک اسٹارٹ · ایمبیڈنگ گائیڈ · مثالیں

+

Rust کوئیک اسٹارٹ · ایمبیڈنگ گائیڈ · مثالیں

اسے Rust لائبریری کے طور پر استعمال کریں: ایجنٹ کو کسی بھی عام فنکشن کی طرح کال کریں، یا ایک چھوٹے سرور سے پورا بیڑا چلائیں۔

@@ -333,7 +333,7 @@ println!("{}", reply.reply);
-اگلا قدم: [Rust کوئیک اسٹارٹ](https://tinyhumans.gitbook.io/openhuman/developing/quickstart)، [ایمبیڈنگ گائیڈ](https://tinyhumans.gitbook.io/openhuman/developing/embedding) اور [ڈیولپر دستاویزات](https://tinyhumans.gitbook.io/openhuman/developing)۔ +اگلا قدم: [Rust کوئیک اسٹارٹ](https://tinyhumans.gitbook.io/openhuman/developing/quickstart)، [ایمبیڈنگ گائیڈ](../gitbooks/developing/embed/README.md) اور [ڈیولپر دستاویزات](https://tinyhumans.gitbook.io/openhuman/developing)۔ --- diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 66c0f693af6..2a21b11d626 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -117,7 +117,7 @@ irm https://raw.githubusercontent.com/tinyhumansai/openhuman/main/scripts/instal

为开发者而建

-

Rust 快速入门 · 嵌入指南 · 示例

+

Rust 快速入门 · 嵌入指南 · 示例

把它当作 Rust 库来用:像调用普通函数一样调用智能体,或者在一台小服务器上运行整个集群。

@@ -288,7 +288,7 @@ let reply = agent.run("Summarize what you can see in this directory.").await?; println!("{}", reply.reply); ``` -接下来看:[Rust 快速入门](https://tinyhumans.gitbook.io/openhuman/developing/quickstart)、[嵌入指南](https://tinyhumans.gitbook.io/openhuman/developing/embedding)和[开发者文档](https://tinyhumans.gitbook.io/openhuman/developing)。 +接下来看:[Rust 快速入门](https://tinyhumans.gitbook.io/openhuman/developing/quickstart)、[嵌入指南](../gitbooks/developing/embed/README.md)和[开发者文档](https://tinyhumans.gitbook.io/openhuman/developing)。 --- diff --git a/docs/benchmarks/medulla-embed-linux.json b/docs/benchmarks/medulla-embed-linux.json index d7f84503fa7..f57c74ffbc3 100644 --- a/docs/benchmarks/medulla-embed-linux.json +++ b/docs/benchmarks/medulla-embed-linux.json @@ -1,236 +1,236 @@ { - "source_commit": "bbd3d83d80fcaaf3b6284c5b7881d080dcd53a34", + "source_commit": "b783b39e78ce7ff1a04e0ab8c764746fc970aa00", "date": "2026-10-10", "host": "Intel Core i7-14700F; x86_64 Ubuntu Linux 7.0.0-31-generic", "request_recording": false, "runs": [ { - "after_turns_rss_kib": 227952, + "after_turns_rss_kib": 276692, "agents": 50, - "before_boot_rss_kib": 14488, - "boot_ms": 256.195196, - "cold_turn_ms": 16.275062, - "fleet_elapsed_ms": 447.54206, + "before_boot_rss_kib": 14744, + "boot_ms": 262.564262, + "cold_turn_ms": 59.251718000000004, + "fleet_elapsed_ms": 1509.882622, "inference": "loopback-http-mock", - "marginal_after_turns_mib": 3.78296875, - "registered_rss_kib": 36464, - "runtime_rss_kib": 34264, + "marginal_after_turns_mib": 4.70171875, + "registered_rss_kib": 37952, + "runtime_rss_kib": 35964, "tool_scope": [ "shell" ], - "turn_p50_ms": 330.948465, - "turn_p95_ms": 398.109413, - "turn_p99_ms": 402.916892, + "turn_p50_ms": 1243.407862, + "turn_p95_ms": 1462.1273370000001, + "turn_p99_ms": 1487.141623, "worker_threads": 2, - "cgroup_memory_peak": "203931648", + "cgroup_memory_peak": "247562240", "cgroup_memory_max": "2147483648", "cgroup_cpu_max": "200000 100000", "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0" }, { - "after_turns_rss_kib": 226972, + "after_turns_rss_kib": 279496, "agents": 50, - "before_boot_rss_kib": 14396, - "boot_ms": 256.484423, - "cold_turn_ms": 18.921321000000002, - "fleet_elapsed_ms": 420.257983, + "before_boot_rss_kib": 14852, + "boot_ms": 260.539379, + "cold_turn_ms": 51.243393, + "fleet_elapsed_ms": 1753.860216, "inference": "loopback-http-mock", - "marginal_after_turns_mib": 3.75734375, - "registered_rss_kib": 36856, - "runtime_rss_kib": 34596, + "marginal_after_turns_mib": 4.743671875, + "registered_rss_kib": 38688, + "runtime_rss_kib": 36620, "tool_scope": [ "shell" ], - "turn_p50_ms": 326.425821, - "turn_p95_ms": 359.84249600000004, - "turn_p99_ms": 396.121414, + "turn_p50_ms": 1438.1386830000001, + "turn_p95_ms": 1731.4588660000002, + "turn_p99_ms": 1750.0709909999998, "worker_threads": 2, - "cgroup_memory_peak": "202309632", + "cgroup_memory_peak": "257216512", "cgroup_memory_max": "2147483648", "cgroup_cpu_max": "200000 100000", "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0" }, { - "after_turns_rss_kib": 222156, + "after_turns_rss_kib": 275208, "agents": 50, - "before_boot_rss_kib": 14532, - "boot_ms": 258.358991, - "cold_turn_ms": 18.654450999999998, - "fleet_elapsed_ms": 410.89376500000003, + "before_boot_rss_kib": 14644, + "boot_ms": 259.527472, + "cold_turn_ms": 57.718647999999995, + "fleet_elapsed_ms": 1526.67821, "inference": "loopback-http-mock", - "marginal_after_turns_mib": 3.665390625, - "registered_rss_kib": 36640, - "runtime_rss_kib": 34488, + "marginal_after_turns_mib": 4.673984375, + "registered_rss_kib": 38000, + "runtime_rss_kib": 35900, "tool_scope": [ "shell" ], - "turn_p50_ms": 273.595824, - "turn_p95_ms": 326.54888900000003, - "turn_p99_ms": 383.942, + "turn_p50_ms": 1235.450996, + "turn_p95_ms": 1430.5981100000001, + "turn_p99_ms": 1476.318741, "worker_threads": 2, - "cgroup_memory_peak": "198098944", + "cgroup_memory_peak": "248852480", "cgroup_memory_max": "2147483648", "cgroup_cpu_max": "200000 100000", "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0" }, { - "after_turns_rss_kib": 380152, + "after_turns_rss_kib": 443016, "agents": 100, - "before_boot_rss_kib": 14696, - "boot_ms": 261.058476, - "cold_turn_ms": 23.715644, - "fleet_elapsed_ms": 843.2798409999999, + "before_boot_rss_kib": 15048, + "boot_ms": 259.398238, + "cold_turn_ms": 54.594975, + "fleet_elapsed_ms": 3871.8118969999996, "inference": "loopback-http-mock", - "marginal_after_turns_mib": 3.37453125, - "registered_rss_kib": 38836, - "runtime_rss_kib": 34600, + "marginal_after_turns_mib": 3.9723046875, + "registered_rss_kib": 40296, + "runtime_rss_kib": 36252, "tool_scope": [ "shell" ], - "turn_p50_ms": 630.060869, - "turn_p95_ms": 781.8404089999999, - "turn_p99_ms": 807.385043, + "turn_p50_ms": 3356.607077, + "turn_p95_ms": 3677.217686, + "turn_p99_ms": 3753.63024, "worker_threads": 2, - "cgroup_memory_peak": "368861184", + "cgroup_memory_peak": "427536384", "cgroup_memory_max": "2147483648", "cgroup_cpu_max": "200000 100000", - "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0" + "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 302" }, { - "after_turns_rss_kib": 381448, + "after_turns_rss_kib": 452564, "agents": 100, - "before_boot_rss_kib": 14532, - "boot_ms": 260.519274, - "cold_turn_ms": 22.652142, - "fleet_elapsed_ms": 1063.297785, + "before_boot_rss_kib": 14940, + "boot_ms": 260.501723, + "cold_turn_ms": 76.418385, + "fleet_elapsed_ms": 3537.497514, "inference": "loopback-http-mock", - "marginal_after_turns_mib": 3.389453125, - "registered_rss_kib": 38588, - "runtime_rss_kib": 34368, + "marginal_after_turns_mib": 4.0640234375, + "registered_rss_kib": 40600, + "runtime_rss_kib": 36408, "tool_scope": [ "shell" ], - "turn_p50_ms": 780.051866, - "turn_p95_ms": 929.080972, - "turn_p99_ms": 964.185504, + "turn_p50_ms": 2926.264162, + "turn_p95_ms": 3369.911277, + "turn_p99_ms": 3479.3718, "worker_threads": 2, - "cgroup_memory_peak": "370405376", + "cgroup_memory_peak": "438190080", "cgroup_memory_max": "2147483648", "cgroup_cpu_max": "200000 100000", - "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0" + "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 302" }, { - "after_turns_rss_kib": 381500, + "after_turns_rss_kib": 447872, "agents": 100, - "before_boot_rss_kib": 14408, - "boot_ms": 258.39308300000005, - "cold_turn_ms": 19.710221999999998, - "fleet_elapsed_ms": 953.5413149999999, + "before_boot_rss_kib": 14640, + "boot_ms": 261.176896, + "cold_turn_ms": 66.686791, + "fleet_elapsed_ms": 3478.3309299999996, "inference": "loopback-http-mock", - "marginal_after_turns_mib": 3.3948828125, - "registered_rss_kib": 38116, - "runtime_rss_kib": 33864, + "marginal_after_turns_mib": 4.0276953125, + "registered_rss_kib": 39496, + "runtime_rss_kib": 35436, "tool_scope": [ "shell" ], - "turn_p50_ms": 713.8235229999999, - "turn_p95_ms": 839.781121, - "turn_p99_ms": 865.269359, + "turn_p50_ms": 2995.865202, + "turn_p95_ms": 3322.377072, + "turn_p99_ms": 3370.5552860000003, "worker_threads": 2, - "cgroup_memory_peak": "370077696", + "cgroup_memory_peak": "441462784", "cgroup_memory_max": "2147483648", "cgroup_cpu_max": "200000 100000", - "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0" + "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 247" }, { - "after_turns_rss_kib": 1511184, + "after_turns_rss_kib": 1754084, "agents": 500, - "before_boot_rss_kib": 14236, - "boot_ms": 258.86011599999995, - "cold_turn_ms": 22.475146000000002, - "fleet_elapsed_ms": 5451.315839, + "before_boot_rss_kib": 14848, + "boot_ms": 259.876961, + "cold_turn_ms": 87.352638, + "fleet_elapsed_ms": 20784.037358999998, "inference": "loopback-http-mock", - "marginal_after_turns_mib": 2.88403125, - "registered_rss_kib": 55136, - "runtime_rss_kib": 34560, + "marginal_after_turns_mib": 3.3552890625, + "registered_rss_kib": 56720, + "runtime_rss_kib": 36176, "tool_scope": [ "shell" ], - "turn_p50_ms": 4820.459523, - "turn_p95_ms": 5284.533808, - "turn_p99_ms": 5339.48242, + "turn_p50_ms": 17427.23177, + "turn_p95_ms": 19450.306773, + "turn_p99_ms": 20044.679603, "worker_threads": 2, - "cgroup_memory_peak": "1625919488", + "cgroup_memory_peak": "1841565696", "cgroup_memory_max": "2147483648", "cgroup_cpu_max": "200000 100000", - "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0" + "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 1502" }, { - "after_turns_rss_kib": 1540228, + "after_turns_rss_kib": 1870448, "agents": 500, - "before_boot_rss_kib": 14452, - "boot_ms": 258.009249, - "cold_turn_ms": 14.884082, - "fleet_elapsed_ms": 4672.801607, + "before_boot_rss_kib": 14316, + "boot_ms": 260.802439, + "cold_turn_ms": 49.858090000000004, + "fleet_elapsed_ms": 16917.273937, "inference": "loopback-http-mock", - "marginal_after_turns_mib": 2.941359375, - "registered_rss_kib": 54784, - "runtime_rss_kib": 34252, + "marginal_after_turns_mib": 3.58465625, + "registered_rss_kib": 55664, + "runtime_rss_kib": 35104, "tool_scope": [ "shell" ], - "turn_p50_ms": 4113.175878, - "turn_p95_ms": 4447.625557, - "turn_p99_ms": 4513.371248, + "turn_p50_ms": 13937.793855, + "turn_p95_ms": 16386.101451, + "turn_p99_ms": 16622.537773, "worker_threads": 2, - "cgroup_memory_peak": "1648812032", + "cgroup_memory_peak": "1994653696", "cgroup_memory_max": "2147483648", "cgroup_cpu_max": "200000 100000", - "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0" + "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 802" }, { - "after_turns_rss_kib": 1542084, + "after_turns_rss_kib": 1746616, "agents": 500, - "before_boot_rss_kib": 14336, - "boot_ms": 257.69439500000004, - "cold_turn_ms": 16.925666, - "fleet_elapsed_ms": 5055.745514, + "before_boot_rss_kib": 14848, + "boot_ms": 260.099823, + "cold_turn_ms": 54.545508, + "fleet_elapsed_ms": 21441.206899, "inference": "loopback-http-mock", - "marginal_after_turns_mib": 2.94503125, - "registered_rss_kib": 54832, - "runtime_rss_kib": 34228, + "marginal_after_turns_mib": 3.3411796875, + "registered_rss_kib": 56552, + "runtime_rss_kib": 35932, "tool_scope": [ "shell" ], - "turn_p50_ms": 4430.5079940000005, - "turn_p95_ms": 4828.56837, - "turn_p99_ms": 4880.706669, + "turn_p50_ms": 17510.694305, + "turn_p95_ms": 20275.004016, + "turn_p99_ms": 20673.577479, "worker_threads": 2, - "cgroup_memory_peak": "1650745344", + "cgroup_memory_peak": "1845829632", "cgroup_memory_max": "2147483648", "cgroup_cpu_max": "200000 100000", - "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0" + "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 1459" } ], "no_swap_500_agents": { - "after_turns_rss_kib": 1537772, + "after_turns_rss_kib": 1848268, "agents": 500, - "before_boot_rss_kib": 14476, - "boot_ms": 259.344405, - "cold_turn_ms": 17.940296, - "fleet_elapsed_ms": 5070.406826, + "before_boot_rss_kib": 14452, + "boot_ms": 261.838417, + "cold_turn_ms": 77.930197, + "fleet_elapsed_ms": 19090.431395, "inference": "loopback-http-mock", - "marginal_after_turns_mib": 2.9364765625, - "registered_rss_kib": 54900, - "runtime_rss_kib": 34296, + "marginal_after_turns_mib": 3.541109375, + "registered_rss_kib": 55800, + "runtime_rss_kib": 35220, "tool_scope": [ "shell" ], - "turn_p50_ms": 4353.5878170000005, - "turn_p95_ms": 4753.196036, - "turn_p99_ms": 4844.317583000001, + "turn_p50_ms": 15202.780968000001, + "turn_p95_ms": 17822.328674, + "turn_p99_ms": 18165.555849, "worker_threads": 2, - "cgroup_memory_peak": "1647300608", + "cgroup_memory_peak": "1976066048", "cgroup_memory_max": "2147483648", "cgroup_cpu_max": "200000 100000", "cgroup_memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0" diff --git a/docs/gitbooks/en/developing/embed/api-index.json b/docs/gitbooks/en/developing/embed/api-index.json index 76919ff831f..2fa0d37e1ad 100644 --- a/docs/gitbooks/en/developing/embed/api-index.json +++ b/docs/gitbooks/en/developing/embed/api-index.json @@ -76,6 +76,7 @@ "ModelDefaults", "OpenError", "PendingApproval", + "PermissionFuture", "PermissionLevel", "PickListenPortError", "ProfileError", @@ -126,6 +127,7 @@ "TrustedAccess", "TrustedAutomationSource", "Turn", + "TurnCancellation", "TurnContext", "TurnOutcome", "TurnRequest", @@ -136,6 +138,8 @@ "absolute", "agent_progress", "artifacts", + "budget", + "cancellation", "channels", "chat_surface", "complete", @@ -143,20 +147,25 @@ "cron", "embeddings", "events", + "fanout", "identity", "install_backend_transport", "installed_backend_transport", "memory", "modules", + "observe", "process", "profiles", "providers", + "repository", + "routing", "run_from_args", "schema_for_rpc_method", "seams", "session_store", "skill_registry", - "stream" + "stream", + "structured" ], "builder_setters": [ { @@ -435,7 +444,7 @@ "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/docs/gitbooks/en/developing/embed/api-index.md b/docs/gitbooks/en/developing/embed/api-index.md index 3e386b0c6d6..9df96592b9c 100644 --- a/docs/gitbooks/en/developing/embed/api-index.md +++ b/docs/gitbooks/en/developing/embed/api-index.md @@ -79,6 +79,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`ModelDefaults`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`OpenError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PendingApproval`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`PermissionFuture`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PermissionLevel`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PickListenPortError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`ProfileError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -129,6 +130,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`TrustedAccess`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TrustedAutomationSource`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`Turn`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`TurnCancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnContext`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnOutcome`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnRequest`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -139,6 +141,8 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`absolute`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`agent_progress`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`artifacts`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`budget`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`cancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`channels`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`chat_surface`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`complete`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -146,20 +150,25 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`cron`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`embeddings`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`events`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`fanout`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`identity`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`install_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`installed_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`memory`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`modules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`observe`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`process`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`profiles`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`providers`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`repository`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`routing`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`run_from_args`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`schema_for_rpc_method`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`seams`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`session_store`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`skill_registry`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`stream`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`structured`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) The compiled capability report describes this build: @@ -187,7 +196,7 @@ The compiled capability report describes this build: "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/docs/gitbooks/en/developing/embed/capability-matrix.md b/docs/gitbooks/en/developing/embed/capability-matrix.md index aba183fbf7e..c6219821881 100644 --- a/docs/gitbooks/en/developing/embed/capability-matrix.md +++ b/docs/gitbooks/en/developing/embed/capability-matrix.md @@ -25,7 +25,7 @@ This matrix comes from the default-feature compiled capability-report example. R | `skills` | Yes | | `storage-file` | No | | `storage-mongodb` | No | -| `storage-sqlite` | No | +| `storage-sqlite` | Yes | | `tinymemes` | Yes | | `voice` | No | | `web3` | No | diff --git a/docs/gitbooks/en/developing/embed/cookbook.md b/docs/gitbooks/en/developing/embed/cookbook.md index 6b7caefc55b..3462bef8af5 100644 --- a/docs/gitbooks/en/developing/embed/cookbook.md +++ b/docs/gitbooks/en/developing/embed/cookbook.md @@ -76,6 +76,14 @@ Lean runtime without background services. Run: `cargo run -p openhuman-embed --example lean_headless` +## Linux agent fleet memory and latency + +Measure retained runtime-owned agents using loopback inference and two worker threads. + +[Source](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs) · offline on Linux; use a fresh constrained cgroup for release measurements. + +Run: `cargo run -p openhuman-embed --example linux_fleet` + ## Connect an actual MCP protocol stub over loopback Connect an actual MCP protocol stub over loopback. diff --git a/docs/gitbooks/en/developing/performance.md b/docs/gitbooks/en/developing/performance.md index 37f5a3d825b..0d4f8af81f1 100644 --- a/docs/gitbooks/en/developing/performance.md +++ b/docs/gitbooks/en/developing/performance.md @@ -9,7 +9,7 @@ icon: gauge OpenHuman's Rust core runs in-process as a library, not as one OS process per agent. Most of the density numbers on this page come from that one decision. A fixed bootstrap cost (allocator warm-up, code paging, registries, detectors) is paid once per process and shared by every agent inside it. -Everything below was measured on Apple Silicon macOS with a `--release` build and a deterministic mock inference provider (the `rss-bench` feature), not real network calls. Treat the absolute numbers as numbers for that machine. The ratios (density, marginal cost, binary delta) are the parts worth generalizing. The scripts to reproduce them are listed at the end. +The original measurements below were made on Apple Silicon macOS with a `--release` build and a deterministic mock inference provider (the `rss-bench` feature), not real network calls. Treat the absolute numbers as numbers for that machine. The ratios (density, marginal cost, binary delta) are the parts worth generalizing. The scripts to reproduce them are listed at the end. ## Fleet: how many agents fit in one process @@ -97,6 +97,68 @@ scripts/kernel-floor.sh flows ## Measurement conditions -These numbers were gathered on macOS. It has no local cgroup memory limit and no `/proc//smaps_rollup`, so there is no true PSS (proportional shared memory) reading. RSS overcounts shared pages, and the error grows with agent count. The fleet and instances numbers project from measured marginal cost. They are not a live test of surviving an OOM kill at N agents on a 2 GB / 2 vCPU Linux box. Every scenario also replaces network inference with a mock provider at fixed latency, so turn timings measure orchestration overhead, not real model latency. Validation under Linux cgroups has not been done yet. +These numbers were gathered on macOS. It has no local cgroup memory limit and no `/proc//smaps_rollup`, so there is no true PSS (proportional shared memory) reading. RSS overcounts shared pages, and the error grows with agent count. The fleet and instances numbers project from measured marginal cost. They are not a live test of surviving an OOM kill at N agents on a 2 GB / 2 vCPU Linux box. Every scenario also replaces network inference with a mock provider at fixed latency, so turn timings measure orchestration overhead, not real model latency. The Linux cgroup measurements in the following section use a different host and harness. For token cost instead of process footprint, see [Smart token compression](../features/token-compression.md). It is the other half of "cheap": it controls how much of what the harness assembles reaches the model. + +## Linux runtime-owned agents (Medulla integration) + +Measured on 2026-10-10 at OpenHuman commit `b783b39e78`, using the release +`openhuman-embed` example `linux_fleet` with default features disabled. This +exercises one `Runtime` and N `AgentSpec`s, with two Tokio workers, ephemeral +session storage, and a loopback HTTP chat-completions mock. Each agent +advertises the builtin `shell` tool; the mock returns text without executing it. +Conversation memory, local model runtimes, and session dual writes are off. +The host is x86_64 Ubuntu, Linux 7.0.0-31-generic, on an Intel Core i7-14700F. +Other builds were running on the host, so these are shared-host measurements. + +Each run starts in a fresh cgroup with `memory.max = 2147483648` and +`cpu.max = 200000 100000` (2 GiB and a two-CPU quota). Three fresh processes +were measured at each size; the table gives medians. RSS is sampled after every +agent completes one concurrent turn, with all agent handles retained. Marginal +RSS is `(RSS after turns - RSS after Runtime::build) / N`, without allocator +trimming. The cgroup peak includes the HTTP mock and the Python measurement +wrapper. Mock request recording is disabled, so retained HTTP request history +is excluded. RSS and cgroup accounting differ because shared file pages may be +charged outside the new cgroup. + +| Concurrent agents | Process RSS after turns | Marginal RSS/agent | Cgroup peak | Fleet wall time | Turn p95 | +| --- | --- | --- | --- | --- | --- | +| 50 | 270.21 MiB | 4.702 MiB | 237.32 MiB | 1,527 ms | 1,462 ms | +| 100 | 437.38 MiB | 4.028 MiB | 417.89 MiB | 3,537 ms | 3,370 ms | +| 500 | 1,712.97 MiB | 3.355 MiB | 1,760.32 MiB | 20,784 ms | 19,450 ms | + +The 500-agent median is within twice the earlier macOS/mock 1.77 MiB +figure (3.54 MiB); the 50- and 100-agent medians miss that target. The +500-agent runs ranged from 3.341 to 3.585 MiB per agent. These are different +hosts and harness entry points, so the table is a capacity measurement rather +than a controlled comparison of the platforms. This run also incorporates the +upstream embed lifecycle-event and dynamic-agent APIs; results are higher than +the earlier prerequisite-branch measurements recorded before that base update. + +Runtime boot took 259–263 ms. The first turn in each fresh process took +49.9–87.4 ms, before launching the concurrent fleet. Both are within twice the +earlier 476 ms bootstrap and 102 ms first-turn figures. “First turn” includes +session/model/tool initialization, but excludes compiling and loading the +executable; the OS page cache was warm. + +A separate 500-agent run with **swap disabled** (`MemorySwapMax=0`) completed +in 19,090 ms, with 3.541 MiB marginal RSS per agent and a +1,884.52 MiB cgroup peak. Its marginal RSS narrowly exceeds the 3.54 MiB +target. Every run recorded zero `max`, `oom`, and `oom_kill` memory events. +This does not establish capacity for real providers, tool subprocesses, MCP +servers, or 1,000 simultaneously active turns. Measure those workloads before +sizing a production fleet. + +To reproduce from the OpenHuman repository root: + +```sh +cargo build -p openhuman-embed --locked --release --no-default-features --example linux_fleet +systemd-run --user --scope -p MemoryMax=2G -p MemorySwapMax=0 -p CPUQuota=200% --quiet \ + python3 crates/openhuman-embed/examples/linux_fleet_cgroup.py 500 +``` + +Use `50` or `100` for the other sizes and repeat each command in a fresh scope. +The JSON output records process RSS, bootstrap/turn timings, cgroup limits, +peak memory, and OOM counters. The raw runs are checked in as +[`docs/benchmarks/medulla-embed-linux.json`](../../../benchmarks/medulla-embed-linux.json). diff --git a/gitbooks/developing/embed/api-index.json b/gitbooks/developing/embed/api-index.json index 76919ff831f..2fa0d37e1ad 100644 --- a/gitbooks/developing/embed/api-index.json +++ b/gitbooks/developing/embed/api-index.json @@ -76,6 +76,7 @@ "ModelDefaults", "OpenError", "PendingApproval", + "PermissionFuture", "PermissionLevel", "PickListenPortError", "ProfileError", @@ -126,6 +127,7 @@ "TrustedAccess", "TrustedAutomationSource", "Turn", + "TurnCancellation", "TurnContext", "TurnOutcome", "TurnRequest", @@ -136,6 +138,8 @@ "absolute", "agent_progress", "artifacts", + "budget", + "cancellation", "channels", "chat_surface", "complete", @@ -143,20 +147,25 @@ "cron", "embeddings", "events", + "fanout", "identity", "install_backend_transport", "installed_backend_transport", "memory", "modules", + "observe", "process", "profiles", "providers", + "repository", + "routing", "run_from_args", "schema_for_rpc_method", "seams", "session_store", "skill_registry", - "stream" + "stream", + "structured" ], "builder_setters": [ { @@ -435,7 +444,7 @@ "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/gitbooks/developing/embed/api-index.md b/gitbooks/developing/embed/api-index.md index 3e386b0c6d6..9df96592b9c 100644 --- a/gitbooks/developing/embed/api-index.md +++ b/gitbooks/developing/embed/api-index.md @@ -79,6 +79,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`ModelDefaults`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`OpenError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PendingApproval`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`PermissionFuture`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PermissionLevel`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PickListenPortError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`ProfileError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -129,6 +130,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`TrustedAccess`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TrustedAutomationSource`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`Turn`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`TurnCancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnContext`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnOutcome`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnRequest`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -139,6 +141,8 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`absolute`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`agent_progress`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`artifacts`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`budget`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`cancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`channels`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`chat_surface`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`complete`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -146,20 +150,25 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`cron`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`embeddings`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`events`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`fanout`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`identity`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`install_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`installed_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`memory`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`modules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`observe`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`process`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`profiles`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`providers`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`repository`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`routing`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`run_from_args`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`schema_for_rpc_method`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`seams`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`session_store`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`skill_registry`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`stream`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`structured`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) The compiled capability report describes this build: @@ -187,7 +196,7 @@ The compiled capability report describes this build: "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/gitbooks/developing/embed/capability-matrix.md b/gitbooks/developing/embed/capability-matrix.md index aba183fbf7e..c6219821881 100644 --- a/gitbooks/developing/embed/capability-matrix.md +++ b/gitbooks/developing/embed/capability-matrix.md @@ -25,7 +25,7 @@ This matrix comes from the default-feature compiled capability-report example. R | `skills` | Yes | | `storage-file` | No | | `storage-mongodb` | No | -| `storage-sqlite` | No | +| `storage-sqlite` | Yes | | `tinymemes` | Yes | | `voice` | No | | `web3` | No | diff --git a/gitbooks/developing/embed/cookbook.md b/gitbooks/developing/embed/cookbook.md index 6b7caefc55b..3462bef8af5 100644 --- a/gitbooks/developing/embed/cookbook.md +++ b/gitbooks/developing/embed/cookbook.md @@ -76,6 +76,14 @@ Lean runtime without background services. Run: `cargo run -p openhuman-embed --example lean_headless` +## Linux agent fleet memory and latency + +Measure retained runtime-owned agents using loopback inference and two worker threads. + +[Source](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs) · offline on Linux; use a fresh constrained cgroup for release measurements. + +Run: `cargo run -p openhuman-embed --example linux_fleet` + ## Connect an actual MCP protocol stub over loopback Connect an actual MCP protocol stub over loopback. diff --git a/gitbooks/developing/performance.md b/gitbooks/developing/performance.md index 557e2bf9951..1303d625f95 100644 --- a/gitbooks/developing/performance.md +++ b/gitbooks/developing/performance.md @@ -9,7 +9,7 @@ icon: gauge OpenHuman's Rust core runs in-process as a library, not as one OS process per agent. Most of the density numbers on this page come from that one decision. A fixed bootstrap cost (allocator warm-up, code paging, registries, detectors) is paid once per process and shared by every agent inside it. -Everything below was measured on Apple Silicon macOS with a `--release` build and a deterministic mock inference provider (the `rss-bench` feature), not real network calls. Treat the absolute numbers as numbers for that machine. The ratios (density, marginal cost, binary delta) are the parts worth generalizing. The scripts to reproduce them are listed at the end. +The original measurements below were made on Apple Silicon macOS with a `--release` build and a deterministic mock inference provider (the `rss-bench` feature), not real network calls. Treat the absolute numbers as numbers for that machine. The ratios (density, marginal cost, binary delta) are the parts worth generalizing. The scripts to reproduce them are listed at the end. ## Fleet: how many agents fit in one process @@ -17,9 +17,9 @@ Everything below was measured on Apple Silicon macOS with a `--release` build an | N agents | Marginal KiB/agent | Settled MiB | Idle CPU ms/10s | Threads | FDs | | ---: | ---: | ---: | ---: | ---: | ---: | -| 50 | 221.65 MiB | 3.757 MiB | 192.94 MiB | 420 ms | 360 ms | -| 100 | 372.51 MiB | 3.389 MiB | 352.93 MiB | 954 ms | 840 ms | -| 500 | 1,504.13 MiB | 2.941 MiB | 1,572.43 MiB | 5,056 ms | 4,829 ms | +| 50 | 1,985 | 223 | 3 | 71 | 420 | +| 100 | 1,866 | 356 | 3 | 123 | 820 | +| 500 | 1,770 | 1,393 | 3 | 211 | 3,220 | 500 agents fit in one process at roughly 1.77 MiB marginal cost each. Idle CPU stays flat as N grows, which matters because an agent that is not mid-turn should not spend cycles. Thread count grows by about 0.35 per agent. Watch that line before pushing past 500 in production. @@ -97,13 +97,13 @@ scripts/kernel-floor.sh flows ## Measurement conditions -These numbers were gathered on macOS. It has no local cgroup memory limit and no `/proc//smaps_rollup`, so there is no true PSS (proportional shared memory) reading. RSS overcounts shared pages, and the error grows with agent count. The fleet and instances numbers project from measured marginal cost. They are not a live test of surviving an OOM kill at N agents on a 2 GB / 2 vCPU Linux box. Every scenario also replaces network inference with a mock provider at fixed latency, so turn timings measure orchestration overhead, not real model latency. Validation under Linux cgroups has not been done yet. +These numbers were gathered on macOS. It has no local cgroup memory limit and no `/proc//smaps_rollup`, so there is no true PSS (proportional shared memory) reading. RSS overcounts shared pages, and the error grows with agent count. The fleet and instances numbers project from measured marginal cost. They are not a live test of surviving an OOM kill at N agents on a 2 GB / 2 vCPU Linux box. Every scenario also replaces network inference with a mock provider at fixed latency, so turn timings measure orchestration overhead, not real model latency. The Linux cgroup measurements in the following section use a different host and harness. For token cost instead of process footprint, see [Smart token compression](../features/token-compression.md). It is the other half of "cheap": it controls how much of what the harness assembles reaches the model. ## Linux runtime-owned agents (Medulla integration) -Measured on 2026-10-10 at OpenHuman commit `e2224a22a3`, using the release +Measured on 2026-10-10 at OpenHuman commit `b783b39e78`, using the release `openhuman-embed` example `linux_fleet` with default features disabled. This exercises one `Runtime` and N `AgentSpec`s, with two Tokio workers, ephemeral session storage, and a loopback HTTP chat-completions mock. Each agent @@ -124,25 +124,28 @@ charged outside the new cgroup. | Concurrent agents | Process RSS after turns | Marginal RSS/agent | Cgroup peak | Fleet wall time | Turn p95 | | --- | --- | --- | --- | --- | --- | -| 50 | 221.65 MiB | 3.757 MiB | 192.94 MiB | 420 ms | 360 ms | -| 100 | 372.51 MiB | 3.389 MiB | 352.93 MiB | 954 ms | 840 ms | -| 500 | 1,504.13 MiB | 2.941 MiB | 1,572.43 MiB | 5,056 ms | 4,829 ms | - -The 100- and 500-agent results are within twice the earlier macOS/mock -1.77 MiB figure (3.54 MiB). The 50-agent result misses that target slightly: -its three runs used 3.665–3.783 MiB per agent. These are different hosts and -harness entry points, so the table is a capacity measurement rather than a -controlled comparison of the platforms. - -Runtime boot took 256–262 ms. The first turn in each fresh process took -14.9–23.8 ms, before launching the concurrent fleet. Both are within twice the +| 50 | 270.21 MiB | 4.702 MiB | 237.32 MiB | 1,527 ms | 1,462 ms | +| 100 | 437.38 MiB | 4.028 MiB | 417.89 MiB | 3,537 ms | 3,370 ms | +| 500 | 1,712.97 MiB | 3.355 MiB | 1,760.32 MiB | 20,784 ms | 19,450 ms | + +The 500-agent median is within twice the earlier macOS/mock 1.77 MiB +figure (3.54 MiB); the 50- and 100-agent medians miss that target. The +500-agent runs ranged from 3.341 to 3.585 MiB per agent. These are different +hosts and harness entry points, so the table is a capacity measurement rather +than a controlled comparison of the platforms. This run also incorporates the +upstream embed lifecycle-event and dynamic-agent APIs; results are higher than +the earlier prerequisite-branch measurements recorded before that base update. + +Runtime boot took 259–263 ms. The first turn in each fresh process took +49.9–87.4 ms, before launching the concurrent fleet. Both are within twice the earlier 476 ms bootstrap and 102 ms first-turn figures. “First turn” includes session/model/tool initialization, but excludes compiling and loading the executable; the OS page cache was warm. A separate 500-agent run with **swap disabled** (`MemorySwapMax=0`) completed -in 5,070 ms, with 2.936 MiB marginal RSS per agent and a 1,570.99 MiB cgroup -peak. Every run recorded zero `max`, `oom`, and `oom_kill` memory events. +in 19,090 ms, with 3.541 MiB marginal RSS per agent and a +1,884.52 MiB cgroup peak. Its marginal RSS narrowly exceeds the 3.54 MiB +target. Every run recorded zero `max`, `oom`, and `oom_kill` memory events. This does not establish capacity for real providers, tool subprocesses, MCP servers, or 1,000 simultaneously active turns. Measure those workloads before sizing a production fleet. diff --git a/llms-full.txt b/llms-full.txt index 1658cc4d625..760b0e639c2 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -2441,6 +2441,85 @@ The [cookbook](https://github.com/tinyhumansai/openhuman/blob/main/gitbooks/deve Hosts supply transport, credentials and application resources. Runtime settings establish shared defaults; agents narrow provider, access, prompt and tool behavior. Use ProfileRuntime when users require separate credentials and workspaces. +## Cancelling one turn + +Acquire `Turn::cancellation_handle()` before sending a turn. The handle is +cloneable and `cancel().await` waits for the turn to stop and for tracked +commands to be reaped. The agent remains available for later turns: + + + +Cancellation is scoped to this turn, including while waiting for inference. +Turns with a cancellation handle use owned interpreter subprocesses rather +than the Node/Python pool, which has no acknowledged per-job abort API. This +trades warm-worker reuse for awaited cleanup; ordinary turns retain pooling. +The `meter` callback fires once on cancellation or a dropped send future, +with `None` when dispatch has not supplied usage yet. + +Before send, cancellation prevents dispatch; after completion it is a no-op. +On Unix, the built-in shell, Node, Python and npm commands kill their process +group, including descendants. Other platforms stop the direct command. Host +tools that spawn independent tasks or processes must provide their own cleanup; +MCP server lifecycles remain owned by the agent. Keep polling `send()` while +awaiting cancellation, for example in a spawned task. + +## Scoped worker hooks + +Hooks can be supplied at three levels: `RuntimeBuilder::tool_hook` / +`post_turn_hook` for all agents, `AgentSpec::tool_hook` / `post_turn_hook` +for one agent, and `Turn::tool_hook` / `post_turn_hook` for one dispatch. +Tool callbacks run in that order. Named agent updates replace only that agent’s callback; per-turn callbacks are additional. +`ToolHookContext` carries the agent/session identity when known, and `cwd` +follows the execution workspace descriptor (including `Turn::cwd`), falling +back to the embedding context's configured action root. +The agent and turn hooks are never installed in the global registry, so +concurrent workers and later turns do not pick up one another's callbacks. +Post-turn callbacks run asynchronously with an owned session snapshot. +Independently spawned tasks that build sessions must explicitly inherit +`openhuman_core::agent::hooks::HookScope` to carry scoped hooks. + +Gateway attribution headers can be attached to `Route::header(name, value)` +and used with `Turn::route`, or with `Provider::routed(route)` on an agent. +They follow only that route's endpoint and are never saved to configuration, +sent to background providers, or included as values in `Route`'s `Debug`. + +### Inline permission and usage policy + +`AgentSpec::can_use_tool` and `Turn::can_use_tool` await a host callback before +executing each tool. The callback can wait for an approval UI and return +`ToolHookDecision::Proceed`, `Deny`, or `ProceedWith`. It owns that wait; +returning `Ask` denies execution. These callbacks add to existing tool policies, +and a turn callback cannot override an agent denial. + +`AgentSpec::stop_hook` and `Turn::stop_hook` receive cumulative usage after each +completed model call. Return `StopDecision::Continue` to observe usage, or +`Stop` to prevent subsequent calls. Completed tool rounds may still execute; +this is an after-call budget boundary, so hosts must refuse an already exhausted +budget before sending a turn. Provider-reported charges remain authoritative, +including known zero; missing charges remain unknown unless pricing is known. +A policy stop uses a deterministic partial summary rather than spending on +final-answer repair calls. + +### Per-turn tools and subprocess environment + +`Turn::tools` replaces this turn's host tool belt, including attached sources. +An empty belt revokes host tools; the next turn returns to the agent's belt. +Builtin tools still follow the agent definition. This supplies dynamic tools for +in-process hosts; statically declared MCP servers retain their creation-time +configuration. + +`Turn::tool_env` supplies the base environment of owned builtin subprocesses. +Variables absent from it are not inherited from the daemon. The builtin command +builders retain their own security/runtime additions, including Git restrictions, +managed interpreter paths and scratch directories. Scoped turns bypass Node and +Python pools, which cannot acknowledge per-job cancellation or swap a job's +process environment. A host tool that spawns a separate Tokio task must explicitly +carry the command environment and cleanup scopes into that task. + +Standalone exact source pins and generated Cargo patches: [consumer setup](CONSUMERS.md). +Ordered fallbacks and required exploration: [routing](ROUTING.md). +Host telemetry and the existing exporter: [observers](OBSERVERS.md). + # Embedding OpenHuman @@ -2529,6 +2608,14 @@ Lean runtime without background services. Run: `cargo run -p openhuman-embed --example lean_headless` +## Linux agent fleet memory and latency + +Measure retained runtime-owned agents using loopback inference and two worker threads. + +[Source](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs) · offline on Linux; use a fresh constrained cgroup for release measurements. + +Run: `cargo run -p openhuman-embed --example linux_fleet` + ## Connect an actual MCP protocol stub over loopback Connect an actual MCP protocol stub over loopback. @@ -2661,7 +2748,7 @@ This matrix comes from the default-feature compiled capability-report example. R | `skills` | Yes | | `storage-file` | No | | `storage-mongodb` | No | -| `storage-sqlite` | No | +| `storage-sqlite` | Yes | | `tinymemes` | Yes | | `voice` | No | | `web3` | No | @@ -2799,6 +2886,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`ModelDefaults`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`OpenError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PendingApproval`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`PermissionFuture`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PermissionLevel`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`PickListenPortError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`ProfileError`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -2849,6 +2937,7 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`TrustedAccess`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TrustedAutomationSource`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`Turn`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`TurnCancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnContext`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnOutcome`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`TurnRequest`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -2859,6 +2948,8 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`absolute`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`agent_progress`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`artifacts`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`budget`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`cancellation`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`channels`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`chat_surface`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`complete`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) @@ -2866,20 +2957,25 @@ Generate the complete local Rust API reference with `cargo doc -p openhuman-embe - [`cron`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`embeddings`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`events`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`fanout`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`identity`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`install_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`installed_backend_transport`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`memory`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`modules`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`observe`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`process`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`profiles`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`providers`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`repository`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`routing`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`run_from_args`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`schema_for_rpc_method`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`seams`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`session_store`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`skill_registry`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) - [`stream`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) +- [`structured`](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/src/lib.rs) The compiled capability report describes this build: @@ -2907,7 +3003,7 @@ The compiled capability report describes this build: "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, @@ -3003,6 +3099,7 @@ The compiled capability report describes this build: - [A per-agent post-turn hook observes completed turns](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/hooks.rs): A per-agent post-turn hook observes completed turns. (offline; optional live via OPENHUMAN_EXAMPLE_LIVE=1 and BASE_URL/API_KEY/MODEL.) - [Host tools and HostOnly keep the advertised tool catalog exact](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/host_tools.rs): Host tools and HostOnly keep the advertised tool catalog exact. (offline with loopback stubs; no live path.) - [Lean runtime without background services](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/lean_headless.rs): Lean runtime without background services. (offline with loopback stubs; optional live via OPENHUMAN_EXAMPLE_LIVE.) +- [Linux agent fleet memory and latency](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs): Measure retained runtime-owned agents using loopback inference and two worker threads. (offline on Linux; use a fresh constrained cgroup for release measurements.) - [Connect an actual MCP protocol stub over loopback](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/mcp.rs): Connect an actual MCP protocol stub over loopback. (offline with loopback stubs; no live path.; feature: mcp) - [Tenant scoped memory facade with an in-memory engine](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/memory.rs): Tenant scoped memory facade with an in-memory engine. (offline with loopback stubs; no live path.) - [SaaS profiles isolate conversation history](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/profiles.rs): SaaS profiles isolate conversation history. (offline with loopback stubs; no live path.) From a03bc519ae9246959f41942d78e4384716a9effa Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 10 Oct 2026 22:17:30 +0300 Subject: [PATCH 05/18] docs(test): map native-worker regression coverage Co-authored-by: Medulla --- docs/TEST-COVERAGE-MATRIX.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/TEST-COVERAGE-MATRIX.md b/docs/TEST-COVERAGE-MATRIX.md index 96872f4804e..6680585297c 100644 --- a/docs/TEST-COVERAGE-MATRIX.md +++ b/docs/TEST-COVERAGE-MATRIX.md @@ -659,7 +659,7 @@ The thread JSONL store moved to `tinyagents_session::threads` (`vendor/tinyagent | 16.1.8 | MCP servers declared programmatically | RU | `crates/openhuman-embed/src/harness/mcp_tests.rs` | ✅ | stdio/HTTP transports mutually exclusive, enabled by default, env accumulation, allow/deny filters, auth carried through | | 16.1.9 | Skill bundles installed into the workspace | RU | `crates/openhuman-embed/src/harness/skills_tests.rs` | ✅ | Copied not linked (discovery rejects symlinked bundles and manifests by design); symlinks skipped both as bundles and inside them; `Inherit` refuses the copy | | 16.1.10 | One runtime per process | RU+RI | `crates/openhuman-embed/src/runtime/builder_tests.rs`, `crates/openhuman-embed/tests/harness_embed.rs` | ✅ | A second `Runtime` (or `Harness`) returns `AlreadyRunning` rather than sharing process-global keyring/event-bus/subscribers; a failed build releases the slot so a retry is possible | -| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused | +| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs`, `crates/openhuman-embed/tests/agent_lifecycle.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused; removal settles pending approvals with `agent_removed` before cancelling turns | | 16.1.12 | API-key credential for library mode | RU+RI | `crates/openhuman-core/src/security/credentials/api_key_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs` | ✅ | Key stored as an `api-key` auth profile; wins over an expired session; managed inference sends it as a bearer with no `x-api-key`; auth state reports `credential = "api-key"` | | 16.1.14 | Host-backed read-only repository toolset | RI | `crates/openhuman-embed/tests/repository_tools.rs`, `crates/openhuman-embed/tests/repository_host_only.rs` | ✅ | Validated tree/range/search/symbol/git queries delegate only to the host; mandatory redaction, fenced bounded output, sanitized failures, and real HostOnly + readonly + untrusted-input refusal of shell/write/network. | | 16.1.15 | Strict structured output and bounded repair | RI | `crates/openhuman-embed/tests/structured_validation.rs`, `crates/openhuman-embed/tests/structured_turns.rs`, `crates/openhuman-embed/tests/unit/turn_usage.rs` | ✅ | Wrong types and numeric combinators refused, invalid/external schemas make no call, repair bounded with usage, terminal length rejected even for complete JSON; successful charged usage reaches outcome/meter, invalid buyer charges stay unknown, root overlays retain child spend and session context. | @@ -667,11 +667,11 @@ The thread JSONL store moved to `tinyagents_session::threads` (`vendor/tinyagent | 16.1.19 | Pinned standalone Embed source consumer | Script+RI | `scripts/__tests__/embed-consumer-bootstrap.test.mjs` | ✅ | Exact SHA verification, inherited workspace edition, generated active patches, stable locked offline builds with optional Embed enabled/disabled, default dependency tree without HTTP, no untracked secrets or Git metadata, refused malformed/escaping patches and existing destinations | | 16.1.17 | Enforced shared turn/run budgets | RU+RI | `crates/openhuman-embed/tests/budget_fanout.rs`, `vendor/tinyagents/vendor/tinyinference/crates/tinyinference-llm/src/model/budget_tests.rs` | ✅ | Atomic parent/child reservations, physical provider admission, conservative unknown spend, cancellation and output cap | | 16.1.18 | Ordered bounded fanout | RI | `crates/openhuman-embed/tests/budget_fanout.rs` | ✅ | Input-order results, per-branch and child error isolation, branch ceilings, shared-budget concurrent refusal and empty fanout | -| 16.1.13 | Awaited per-turn cancellation | RI | `crates/openhuman-embed/tests/turn_cancellation.rs`, `crates/openhuman-embed/tests/process_cancellation.rs` | ✅ | Before send, during inference and during a builtin shell command; agent reuse, concurrent cancellation handles, failure/drop acknowledgement, bounded and unbounded descendant termination plus direct-child reaping on Linux | +| 16.1.13 | Awaited per-turn cancellation | RI | `crates/openhuman-embed/tests/turn_cancellation.rs`, `crates/openhuman-embed/tests/process_cancellation.rs` | ✅ | Before send, during inference and during a builtin shell command; agent reuse, concurrent cancellation handles, failure/drop acknowledgement, bounded and unbounded descendant termination plus direct-child reaping on Linux; deadline errors stay typed when the native cancellation relay interrupts dispatch | | 16.1.21 | Agent and turn hook isolation | RI | `crates/openhuman-embed/tests/scoped_hooks.rs` | ✅ | Additive same-named callbacks, overlapping worker turns, one-turn callback lifetime, resumed-session snapshots, core-scheduled turn callbacks and reuse of a removed agent id without registry leakage | | 16.1.22 | Agent-turn route attribution headers | RU+RI | `crates/openhuman-embed/tests/route_headers.rs`, `crates/openhuman-embed/src/turn_tests.rs` | ✅ | Per-route headers reach only the named endpoint and turn, remain absent on other agents and backend requests, and are redacted in Debug; wire fields pinned against the registered controller schema | -| 16.1.23 | Tool-hook turn identity and cwd | RI | `crates/openhuman-embed/tests/tool_hook_context.rs` | ✅ | Before and after hooks report the worker/session and working root used by builtin shell; a later turn returns to its agent default | -| 16.1.24 | Inline agent/turn permission callback | RI | `crates/openhuman-embed/tests/inline_permissions.rs` | ✅ | UI decisions precede execution; denials narrow authority, concurrent agents remain independent, cancellation closes a pending receiver and a closed UI refuses execution | +| 16.1.23 | Tool-hook turn identity and cwd | RU+RI | `crates/openhuman-embed/tests/tool_hook_context.rs`, `crates/openhuman-core/src/hooks/bridge_tests.rs` | ✅ | Before and after hooks report the worker/session and working root used by builtin shell; a later turn returns to its agent default; configured hooks use resolved cwd even when tool arguments omit it | +| 16.1.24 | Inline agent/turn permission callback | RI | `crates/openhuman-embed/tests/inline_permissions.rs` | ✅ | UI decisions precede execution; denials narrow authority, concurrent agents remain independent, cancellation closes a pending receiver and a closed UI refuses execution; repeated agent callbacks preserve earlier denials | | 16.1.25 | Per-turn host-tool replacement | RI | `crates/openhuman-embed/tests/turn_tools.rs` | ✅ | Replacement and empty-belt revocation apply to resumed HostOnly and mixed definitions, without restoring historical declarations or affecting other agents and later turns | | 16.1.26 | Scoped usage observation and budget stop | RI | `crates/openhuman-embed/tests/usage_hooks.rs` | ✅ | Cumulative charged cost and tokens, known-zero versus unpriced calls, per-agent policy and per-turn observer isolation; no summary-repair model calls after a stop | | 16.1.27 | Scoped builtin command environments | RI | `crates/openhuman-embed/tests/tool_environment.rs`, `crates/openhuman-embed/tests/tool_hook_context.rs` | ✅ | Overlapping subprocess maps exclude host inheritance on Unix; a real shell receives its turn's variables while excluding host HOME and preserving command-owned runtime/security additions | From d73443617e75606a70dc6ff8c9b206b71baec9f7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 10 Oct 2026 22:38:28 +0300 Subject: [PATCH 06/18] fix(embed): preserve completed dispatches and close removal admission Co-authored-by: Medulla --- crates/openhuman-embed/src/agent/lifecycle.rs | 4 +- .../src/agent/lifecycle_tests.rs | 29 +++++++++++ crates/openhuman-embed/src/turn_control.rs | 43 ++++++++++++----- .../openhuman-embed/src/turn_control_tests.rs | 48 +++++++++++++++++++ docs/TEST-COVERAGE-MATRIX.md | 4 +- 5 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 crates/openhuman-embed/src/turn_control_tests.rs diff --git a/crates/openhuman-embed/src/agent/lifecycle.rs b/crates/openhuman-embed/src/agent/lifecycle.rs index 5bc8e22a2ef..1c4eccdc96f 100644 --- a/crates/openhuman-embed/src/agent/lifecycle.rs +++ b/crates/openhuman-embed/src/agent/lifecycle.rs @@ -64,7 +64,9 @@ impl Lifecycle { agent_id: agent_id.to_string(), }; let mut watcher = self.removed.subscribe(); - if *watcher.borrow_and_update() { + // Claiming removal closes admission immediately; waking existing turns + // waits until their approvals have been settled with the removal reason. + if self.removal_claimed.load(Ordering::SeqCst) || *watcher.borrow_and_update() { log::debug!("[embed][agent] turn refused: agent removed id={agent_id}"); return Err(removed()); } diff --git a/crates/openhuman-embed/src/agent/lifecycle_tests.rs b/crates/openhuman-embed/src/agent/lifecycle_tests.rs index be547d2a09b..2505ec4b8fd 100644 --- a/crates/openhuman-embed/src/agent/lifecycle_tests.rs +++ b/crates/openhuman-embed/src/agent/lifecycle_tests.rs @@ -55,3 +55,32 @@ fn teardown_is_claimed_once() { assert!(lifecycle.begin_teardown()); assert!(!lifecycle.begin_teardown()); } + +#[tokio::test] +async fn a_claimed_removal_refuses_turns_before_approvals_are_settled() { + let lifecycle = Arc::new(Lifecycle::new()); + let removing = lifecycle.clone(); + let (entered, callback_entered) = std::sync::mpsc::channel(); + let (release, callback_release) = std::sync::mpsc::channel(); + let removal = std::thread::spawn(move || { + removing.mark_removed_with(|| { + entered.send(()).unwrap(); + callback_release.recv().unwrap(); + }) + }); + callback_entered.recv().unwrap(); + let ran = AtomicBool::new(false); + let outcome = lifecycle + .admit("a", "test", async { + ran.store(true, Ordering::SeqCst); + Ok(()) + }) + .await; + release.send(()).unwrap(); + assert!(removal.join().unwrap()); + assert!(removed(outcome)); + assert!( + !ran.load(Ordering::SeqCst), + "removal admitted a new turn while settling approvals" + ); +} diff --git a/crates/openhuman-embed/src/turn_control.rs b/crates/openhuman-embed/src/turn_control.rs index 6976449cd10..bcfa0aca633 100644 --- a/crates/openhuman-embed/src/turn_control.rs +++ b/crates/openhuman-embed/src/turn_control.rs @@ -229,18 +229,7 @@ impl Turn { Err(CoreError::DeadlineExceeded { method: AGENT_CHAT }) }, outcome = Box::pin(self.send_inner(&meter.usage)) => { - // The relay can interrupt a long dispatch poll before - // this select polls its control branches again. Keep - // the public error tied to the triggering control. - if token.as_ref().is_some_and(|token| token.is_cancelled()) - || cancellation.is_cancelled() - { - Err(CoreError::TurnCancelled { method: AGENT_CHAT }) - } else if deadline.is_some_and(|deadline| tokio::time::Instant::now() >= deadline) { - Err(CoreError::DeadlineExceeded { method: AGENT_CHAT }) - } else { - outcome - } + controlled_outcome(outcome, &native, token.as_ref(), &cancellation, deadline) }, } }) @@ -525,3 +514,33 @@ impl Turn { Ok(()) } } + +// Classify the result at the control boundary after a native dispatch poll. +fn controlled_outcome( + outcome: Result, + native: &crate::CancellationToken, + token: Option<&crate::CancellationToken>, + cancellation: &crate::TurnCancellation, + deadline: Option, +) -> Result { + // The native session runtime currently crosses the RPC boundary as this + // cancellation error. Only that interrupted dispatch needs classification; + // completed replies and unrelated failures retain their original result. + let interrupted = native.is_cancelled() + && matches!(&outcome, Err(CoreError::Rpc { method, message }) + if *method == AGENT_CHAT && message == "session turn cancelled"); + if !interrupted { + return outcome; + } + if token.is_some_and(|token| token.is_cancelled()) || cancellation.is_cancelled() { + Err(CoreError::TurnCancelled { method: AGENT_CHAT }) + } else if deadline.is_some_and(|deadline| tokio::time::Instant::now() >= deadline) { + Err(CoreError::DeadlineExceeded { method: AGENT_CHAT }) + } else { + outcome + } +} + +#[cfg(test)] +#[path = "turn_control_tests.rs"] +mod tests; diff --git a/crates/openhuman-embed/src/turn_control_tests.rs b/crates/openhuman-embed/src/turn_control_tests.rs new file mode 100644 index 00000000000..4d95009fe87 --- /dev/null +++ b/crates/openhuman-embed/src/turn_control_tests.rs @@ -0,0 +1,48 @@ +//! Completed dispatches retain their outcome when control arrives afterward. +use super::*; + +#[tokio::test] +async fn completed_replies_survive_later_cancellation() { + let native = crate::CancellationToken::new(); + let token = crate::CancellationToken::new(); + let handle = crate::TurnCancellation::default(); + let reply = Ok("completed reply"); + token.cancel(); + native.cancel(); + handle.cancel().await; + assert_eq!( + controlled_outcome(reply, &native, Some(&token), &handle, None).unwrap(), + "completed reply" + ); +} + +#[test] +fn completed_failures_survive_later_deadlines() { + let native = crate::CancellationToken::new(); + native.cancel(); + let outcome = controlled_outcome::<()>( + Err(CoreError::InvalidRoute { method: AGENT_CHAT }), + &native, + None, + &crate::TurnCancellation::default(), + Some(tokio::time::Instant::now()), + ); + assert!(matches!(outcome, Err(CoreError::InvalidRoute { .. }))); +} + +#[test] +fn a_native_cancelled_dispatch_keeps_the_deadline_reason() { + let native = crate::CancellationToken::new(); + native.cancel(); + let outcome = controlled_outcome::<()>( + Err(CoreError::Rpc { + method: AGENT_CHAT, + message: "session turn cancelled".into(), + }), + &native, + None, + &crate::TurnCancellation::default(), + Some(tokio::time::Instant::now()), + ); + assert!(matches!(outcome, Err(CoreError::DeadlineExceeded { .. }))); +} diff --git a/docs/TEST-COVERAGE-MATRIX.md b/docs/TEST-COVERAGE-MATRIX.md index 6680585297c..e827603f76f 100644 --- a/docs/TEST-COVERAGE-MATRIX.md +++ b/docs/TEST-COVERAGE-MATRIX.md @@ -659,7 +659,7 @@ The thread JSONL store moved to `tinyagents_session::threads` (`vendor/tinyagent | 16.1.8 | MCP servers declared programmatically | RU | `crates/openhuman-embed/src/harness/mcp_tests.rs` | ✅ | stdio/HTTP transports mutually exclusive, enabled by default, env accumulation, allow/deny filters, auth carried through | | 16.1.9 | Skill bundles installed into the workspace | RU | `crates/openhuman-embed/src/harness/skills_tests.rs` | ✅ | Copied not linked (discovery rejects symlinked bundles and manifests by design); symlinks skipped both as bundles and inside them; `Inherit` refuses the copy | | 16.1.10 | One runtime per process | RU+RI | `crates/openhuman-embed/src/runtime/builder_tests.rs`, `crates/openhuman-embed/tests/harness_embed.rs` | ✅ | A second `Runtime` (or `Harness`) returns `AlreadyRunning` rather than sharing process-global keyring/event-bus/subscribers; a failed build releases the slot so a retry is possible | -| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs`, `crates/openhuman-embed/tests/agent_lifecycle.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused; removal settles pending approvals with `agent_removed` before cancelling turns | +| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs`, `crates/openhuman-embed/tests/agent_lifecycle.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused; removal settles pending approvals with `agent_removed` before cancelling turns; a claimed removal immediately refuses new admission | | 16.1.12 | API-key credential for library mode | RU+RI | `crates/openhuman-core/src/security/credentials/api_key_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs` | ✅ | Key stored as an `api-key` auth profile; wins over an expired session; managed inference sends it as a bearer with no `x-api-key`; auth state reports `credential = "api-key"` | | 16.1.14 | Host-backed read-only repository toolset | RI | `crates/openhuman-embed/tests/repository_tools.rs`, `crates/openhuman-embed/tests/repository_host_only.rs` | ✅ | Validated tree/range/search/symbol/git queries delegate only to the host; mandatory redaction, fenced bounded output, sanitized failures, and real HostOnly + readonly + untrusted-input refusal of shell/write/network. | | 16.1.15 | Strict structured output and bounded repair | RI | `crates/openhuman-embed/tests/structured_validation.rs`, `crates/openhuman-embed/tests/structured_turns.rs`, `crates/openhuman-embed/tests/unit/turn_usage.rs` | ✅ | Wrong types and numeric combinators refused, invalid/external schemas make no call, repair bounded with usage, terminal length rejected even for complete JSON; successful charged usage reaches outcome/meter, invalid buyer charges stay unknown, root overlays retain child spend and session context. | @@ -667,7 +667,7 @@ The thread JSONL store moved to `tinyagents_session::threads` (`vendor/tinyagent | 16.1.19 | Pinned standalone Embed source consumer | Script+RI | `scripts/__tests__/embed-consumer-bootstrap.test.mjs` | ✅ | Exact SHA verification, inherited workspace edition, generated active patches, stable locked offline builds with optional Embed enabled/disabled, default dependency tree without HTTP, no untracked secrets or Git metadata, refused malformed/escaping patches and existing destinations | | 16.1.17 | Enforced shared turn/run budgets | RU+RI | `crates/openhuman-embed/tests/budget_fanout.rs`, `vendor/tinyagents/vendor/tinyinference/crates/tinyinference-llm/src/model/budget_tests.rs` | ✅ | Atomic parent/child reservations, physical provider admission, conservative unknown spend, cancellation and output cap | | 16.1.18 | Ordered bounded fanout | RI | `crates/openhuman-embed/tests/budget_fanout.rs` | ✅ | Input-order results, per-branch and child error isolation, branch ceilings, shared-budget concurrent refusal and empty fanout | -| 16.1.13 | Awaited per-turn cancellation | RI | `crates/openhuman-embed/tests/turn_cancellation.rs`, `crates/openhuman-embed/tests/process_cancellation.rs` | ✅ | Before send, during inference and during a builtin shell command; agent reuse, concurrent cancellation handles, failure/drop acknowledgement, bounded and unbounded descendant termination plus direct-child reaping on Linux; deadline errors stay typed when the native cancellation relay interrupts dispatch | +| 16.1.13 | Awaited per-turn cancellation | RU+RI | `crates/openhuman-embed/tests/turn_cancellation.rs`, `crates/openhuman-embed/tests/process_cancellation.rs`, `crates/openhuman-embed/src/turn_control_tests.rs` | ✅ | Before send, during inference and during a builtin shell command; agent reuse, concurrent cancellation handles, failure/drop acknowledgement, bounded and unbounded descendant termination plus direct-child reaping on Linux; deadline errors stay typed when the native cancellation relay interrupts dispatch; later cancellation/deadlines do not replace completed outcomes | | 16.1.21 | Agent and turn hook isolation | RI | `crates/openhuman-embed/tests/scoped_hooks.rs` | ✅ | Additive same-named callbacks, overlapping worker turns, one-turn callback lifetime, resumed-session snapshots, core-scheduled turn callbacks and reuse of a removed agent id without registry leakage | | 16.1.22 | Agent-turn route attribution headers | RU+RI | `crates/openhuman-embed/tests/route_headers.rs`, `crates/openhuman-embed/src/turn_tests.rs` | ✅ | Per-route headers reach only the named endpoint and turn, remain absent on other agents and backend requests, and are redacted in Debug; wire fields pinned against the registered controller schema | | 16.1.23 | Tool-hook turn identity and cwd | RU+RI | `crates/openhuman-embed/tests/tool_hook_context.rs`, `crates/openhuman-core/src/hooks/bridge_tests.rs` | ✅ | Before and after hooks report the worker/session and working root used by builtin shell; a later turn returns to its agent default; configured hooks use resolved cwd even when tool arguments omit it | From 01457297be06e03ce79f968a8e30f27716eb9201 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 10 Oct 2026 23:03:41 +0300 Subject: [PATCH 07/18] fix(approval): close instance registration before removal snapshot Co-authored-by: Medulla --- .../src/agent/host_overrides.rs | 9 ++ .../src/security/approval/gate_intercept.rs | 138 ++++++++++-------- .../src/security/approval/mod.rs | 2 + .../security/approval/registration_scope.rs | 40 +++++ .../approval/registration_scope_tests.rs | 52 +++++++ crates/openhuman-embed/src/agent/build.rs | 3 + crates/openhuman-embed/src/agent/mod.rs | 3 + .../openhuman-embed/tests/agent_lifecycle.rs | 32 +++- docs/TEST-COVERAGE-MATRIX.md | 2 +- 9 files changed, 217 insertions(+), 64 deletions(-) create mode 100644 crates/openhuman-core/src/security/approval/registration_scope.rs create mode 100644 crates/openhuman-core/src/security/approval/registration_scope_tests.rs diff --git a/crates/openhuman-core/src/agent/host_overrides.rs b/crates/openhuman-core/src/agent/host_overrides.rs index f87001346a5..30c8d732b2d 100644 --- a/crates/openhuman-core/src/agent/host_overrides.rs +++ b/crates/openhuman-core/src/agent/host_overrides.rs @@ -17,6 +17,8 @@ pub use tinyagents_harness::cancel::CancellationToken; pub struct HostOverrides { /// Parent runtime adapters; local hook registration never edits this parent. pub parent: Option>, + /// Approval registration barrier owned by this agent instance. + pub approval_scope: Option>, post_turn_hooks: RwLock>>, tool_hooks: RwLock>>, /// Native inference used by this agent instead of config-routed inference. @@ -38,6 +40,13 @@ impl std::fmt::Debug for HostOverrides { } impl HostOverrides { + /// Resolve the instance barrier inherited by derived turn contexts. + pub fn approval_scope(&self) -> Option> { + self.approval_scope + .clone() + .or_else(|| self.parent.as_ref().and_then(|p| p.approval_scope())) + } + /// Add or replace an agent-local post-turn hook by name; `None` removes it. pub fn post_turn_hook(&self, name: &str, hook: Option>) { let mut hooks = self diff --git a/crates/openhuman-core/src/security/approval/gate_intercept.rs b/crates/openhuman-core/src/security/approval/gate_intercept.rs index 7e0b91d0dd9..d15d9455d4b 100644 --- a/crates/openhuman-core/src/security/approval/gate_intercept.rs +++ b/crates/openhuman-core/src/security/approval/gate_intercept.rs @@ -501,74 +501,88 @@ impl ApprovalGate { // to time out and return `Deny` incorrectly. (CodeRabbit // review on PR #2149.) let (tx, rx) = oneshot::channel::(); - { - let mut waiters = self.waiters.lock(); - waiters.insert(request_id.clone(), tx); - } - // Record the thread → request mapping so an inbound chat reply on this - // thread can be routed to `approval_decide` (see web channel ingress). - if let Some(thread_key) = thread_key.as_ref() { - self.thread_to_request - .lock() - .insert(thread_key.clone(), request_id.clone()); - } - // Record the full routing correlation (thread/client/tool_call_id) so - // whichever path resolves this request's decision — `decide()`, the - // TTL timeout, or a dropped decision channel, all below — can mirror - // it onto `ApprovalDecided` without re-deriving it from ambient - // task-locals that may no longer be in scope by then. - self.insert_request_route( - &request_id, - RequestRoute { + let register = || -> anyhow::Result<()> { + { + let mut waiters = self.waiters.lock(); + waiters.insert(request_id.clone(), tx); + } + // Record the thread → request mapping so an inbound chat reply on this + // thread can be routed to `approval_decide` (see web channel ingress). + if let Some(thread_key) = thread_key.as_ref() { + self.thread_to_request + .lock() + .insert(thread_key.clone(), request_id.clone()); + } + // Record the full routing correlation (thread/client/tool_call_id) so + // whichever path resolves this request's decision — `decide()`, the + // TTL timeout, or a dropped decision channel, all below — can mirror + // it onto `ApprovalDecided` without re-deriving it from ambient + // task-locals that may no longer be in scope by then. + self.insert_request_route( + &request_id, + RequestRoute { + thread_id: chat_thread_id.clone(), + client_id: chat_client_id.clone(), + tool_call_id: tool_call_id.map(str::to_string), + forced, + agent_id: agent_id.clone(), + thread_key: thread_key.clone(), + detached, + }, + ); + store::insert_pending(&self.config, &pending, &self.session_id)?; + + tracing::info!( + request_id = %request_id, + tool = tool_name, + thread_id = chat_thread_id.as_deref().unwrap_or(""), + client_id = chat_client_id.as_deref().unwrap_or(""), + agent_id = agent_id.as_deref().unwrap_or(""), + "[approval::gate] publishing ApprovalRequested (surface fires only if thread_id+client_id are both set)" + ); + BUS.publish(DomainEvent::ApprovalRequested { + request_id: request_id.clone(), + tool_name: tool_name.to_string(), + action_summary: action_summary.to_string(), + args_redacted, thread_id: chat_thread_id.clone(), client_id: chat_client_id.clone(), tool_call_id: tool_call_id.map(str::to_string), - forced, + expires_at: expires_at.map(|t| t.to_rfc3339()), agent_id: agent_id.clone(), - thread_key: thread_key.clone(), - detached, - }, - ); - if let Err(err) = store::insert_pending(&self.config, &pending, &self.session_id) { - self.evict_waiter(&request_id); - self.clear_thread(&thread_key, &request_id); - self.take_request_route(&request_id); - tracing::error!( - error = %err, - tool = tool_name, - "[approval::gate] failed to persist pending row — failing closed" - ); - return ( - GateOutcome::Deny { - reason: format!( - "{POLICY_DENIED_MARKER} Approval gate could not persist the request — \ - denying for safety: {err}" - ), - }, - None, - ); + }); + Ok(()) + }; + let scope = crate::core::runtime::CoreContext::current_host_overrides() + .and_then(|overrides| overrides.approval_scope()); + let registered = match scope { + Some(scope) => scope.register(register), + None => Ok(register()), + }; + match registered { + Err(resolution) => { + return ( + GateOutcome::Deny { + reason: format!("{POLICY_DENIED_MARKER} Agent approval registration closed: {resolution}"), + }, + None, + ); + } + Ok(Err(err)) => { + self.evict_waiter(&request_id); + self.clear_thread(&thread_key, &request_id); + self.take_request_route(&request_id); + tracing::error!(error = %err, tool = tool_name, "[approval::gate] failed to persist pending row — failing closed"); + return ( + GateOutcome::Deny { + reason: format!("{POLICY_DENIED_MARKER} Approval gate could not persist the request — denying for safety: {err}"), + }, + None, + ); + } + Ok(Ok(())) => {} } - tracing::info!( - request_id = %request_id, - tool = tool_name, - thread_id = chat_thread_id.as_deref().unwrap_or(""), - client_id = chat_client_id.as_deref().unwrap_or(""), - agent_id = agent_id.as_deref().unwrap_or(""), - "[approval::gate] publishing ApprovalRequested (surface fires only if thread_id+client_id are both set)" - ); - BUS.publish(DomainEvent::ApprovalRequested { - request_id: request_id.clone(), - tool_name: tool_name.to_string(), - action_summary: action_summary.to_string(), - args_redacted, - thread_id: chat_thread_id.clone(), - client_id: chat_client_id.clone(), - tool_call_id: tool_call_id.map(str::to_string), - expires_at: expires_at.map(|t| t.to_rfc3339()), - agent_id: agent_id.clone(), - }); - // Flow-origin surface bridge (flow-approval-surface, PR3): a flow run // has no chat thread/client to route the generic `ApprovalRequested` // through (both are `None` above, so the web-channel bridge silently diff --git a/crates/openhuman-core/src/security/approval/mod.rs b/crates/openhuman-core/src/security/approval/mod.rs index 188c6a94d51..030be04e7a4 100644 --- a/crates/openhuman-core/src/security/approval/mod.rs +++ b/crates/openhuman-core/src/security/approval/mod.rs @@ -15,6 +15,8 @@ pub mod gate; pub mod redact; +mod registration_scope; +pub use registration_scope::ApprovalScope; pub mod rpc; pub mod schemas; pub mod store; diff --git a/crates/openhuman-core/src/security/approval/registration_scope.rs b/crates/openhuman-core/src/security/approval/registration_scope.rs new file mode 100644 index 00000000000..bf0c0e47fb5 --- /dev/null +++ b/crates/openhuman-core/src/security/approval/registration_scope.rs @@ -0,0 +1,40 @@ +//! Serializes an agent instance's approval registration with its removal. + +use std::sync::Mutex; + +/// An instance-owned barrier; reusing an agent ID requires a fresh scope. +#[derive(Default)] +pub struct ApprovalScope { + closed: Mutex>, +} + +impl ApprovalScope { + /// Stop registration and wait for any accepted registration to finish. + /// The first teardown reason remains authoritative. + pub fn close(&self, reason: &str) { + let mut closed = self + .closed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if closed.is_none() { + *closed = Some(reason.to_owned()); + } + } + + pub(crate) fn register(&self, register: impl FnOnce() -> T) -> Result { + let closed = self + .closed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(reason) = closed.as_ref() { + return Err(reason.clone()); + } + // Keep the barrier through persistence and ApprovalRequested publication. + // No asynchronous work or user callbacks run in this critical section. + Ok(register()) + } +} + +#[cfg(test)] +#[path = "registration_scope_tests.rs"] +mod tests; diff --git a/crates/openhuman-core/src/security/approval/registration_scope_tests.rs b/crates/openhuman-core/src/security/approval/registration_scope_tests.rs new file mode 100644 index 00000000000..c2e2c11a7b6 --- /dev/null +++ b/crates/openhuman-core/src/security/approval/registration_scope_tests.rs @@ -0,0 +1,52 @@ +use super::ApprovalScope; +use std::sync::{mpsc, Arc}; + +#[test] +fn removal_refuses_late_registration_without_creating_a_pending_row() { + let scope = ApprovalScope::default(); + scope.close("agent_removed"); + let mut created = false; + let result = scope.register(|| created = true); + assert_eq!(result, Err("agent_removed".to_owned())); + assert!(!created); +} + +#[test] +fn removal_waits_for_accepted_registration_before_taking_its_snapshot() { + let scope = Arc::new(ApprovalScope::default()); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let (closed_tx, closed_rx) = mpsc::channel(); + let registering = scope.clone(); + let registration = std::thread::spawn(move || { + registering.register(|| { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + }) + }); + entered_rx.recv().unwrap(); + let removing = scope.clone(); + let removal = std::thread::spawn(move || { + removing.close("agent_removed"); + closed_tx.send(()).unwrap(); + }); + let closed_early = closed_rx + .recv_timeout(std::time::Duration::from_millis(100)) + .is_ok(); + release_tx.send(()).unwrap(); + assert_eq!(registration.join().unwrap(), Ok(())); + removal.join().unwrap(); + assert!( + !closed_early, + "removal snapshot raced an accepted registration" + ); +} + +#[test] +fn a_reused_agent_id_has_a_fresh_scope_and_preserves_the_old_removal_reason() { + let old = ApprovalScope::default(); + old.close("agent_removed"); + old.close("agent_dropped"); + assert_eq!(old.register(|| ()), Err("agent_removed".to_owned())); + assert_eq!(ApprovalScope::default().register(|| 42), Ok(42)); +} diff --git a/crates/openhuman-embed/src/agent/build.rs b/crates/openhuman-embed/src/agent/build.rs index 4b7786e8673..72799436541 100644 --- a/crates/openhuman-embed/src/agent/build.rs +++ b/crates/openhuman-embed/src/agent/build.rs @@ -265,6 +265,9 @@ pub(crate) fn instantiate(runtime: &Runtime, spec: AgentSpec) -> Result Date: Sat, 10 Oct 2026 23:06:56 +0300 Subject: [PATCH 08/18] docs(embed): make fleet commands and memory comparisons reproducible Co-authored-by: Medulla --- .../openhuman-embed/examples/linux_fleet.rs | 2 ++ docs/gitbooks/en/developing/embed/cookbook.md | 2 +- docs/gitbooks/en/developing/performance.md | 27 +++++++++++++------ gitbooks/developing/embed/cookbook.md | 2 +- gitbooks/developing/performance.md | 27 +++++++++++++------ llms-full.txt | 2 +- .../__tests__/generate-embed-docs.test.mjs | 19 +++++++++++++ scripts/generate-embed-docs.mjs | 11 ++++++-- 8 files changed, 71 insertions(+), 21 deletions(-) diff --git a/crates/openhuman-embed/examples/linux_fleet.rs b/crates/openhuman-embed/examples/linux_fleet.rs index 664297fc4f2..469259f6493 100644 --- a/crates/openhuman-embed/examples/linux_fleet.rs +++ b/crates/openhuman-embed/examples/linux_fleet.rs @@ -1,7 +1,9 @@ //! Title: Linux agent fleet memory and latency //! Summary: Measure retained runtime-owned agents using loopback inference and two worker threads. //! Run: offline on Linux; use a fresh constrained cgroup for release measurements. +//! Profile: release //! Feature: default +//! Default features: disabled #![recursion_limit = "512"] diff --git a/docs/gitbooks/en/developing/embed/cookbook.md b/docs/gitbooks/en/developing/embed/cookbook.md index 3462bef8af5..04b708f7de1 100644 --- a/docs/gitbooks/en/developing/embed/cookbook.md +++ b/docs/gitbooks/en/developing/embed/cookbook.md @@ -82,7 +82,7 @@ Measure retained runtime-owned agents using loopback inference and two worker th [Source](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs) · offline on Linux; use a fresh constrained cgroup for release measurements. -Run: `cargo run -p openhuman-embed --example linux_fleet` +Run: `cargo run -p openhuman-embed --release --example linux_fleet --no-default-features` ## Connect an actual MCP protocol stub over loopback diff --git a/docs/gitbooks/en/developing/performance.md b/docs/gitbooks/en/developing/performance.md index 0d4f8af81f1..c407bd491cd 100644 --- a/docs/gitbooks/en/developing/performance.md +++ b/docs/gitbooks/en/developing/performance.md @@ -21,7 +21,7 @@ The original measurements below were made on Apple Silicon macOS with a `--relea | 100 | 1,866 | 356 | 3 | 123 | 820 | | 500 | 1,770 | 1,393 | 3 | 211 | 3,220 | -500 agents fit in one process at roughly 1.77 MiB marginal cost each. Idle CPU stays flat as N grows, which matters because an agent that is not mid-turn should not spend cycles. Thread count grows by about 0.35 per agent. Watch that line before pushing past 500 in production. +500 agents fit in one process at 1,770 KiB (about 1.73 MiB) marginal cost each. Idle CPU stays flat as N grows, which matters because an agent that is not mid-turn should not spend cycles. Thread count grows by about 0.35 per agent. Watch that line before pushing past 500 in production. Thousands of agents on one box is a goal, not a measured result. The marginal cost settles across 50, 100 and 500 agents instead of rising, which makes the goal plausible. @@ -114,8 +114,8 @@ Other builds were running on the host, so these are shared-host measurements. Each run starts in a fresh cgroup with `memory.max = 2147483648` and `cpu.max = 200000 100000` (2 GiB and a two-CPU quota). Three fresh processes -were measured at each size; the table gives medians. RSS is sampled after every -agent completes one concurrent turn, with all agent handles retained. Marginal +were measured at each size with the default swap limit; the table gives medians. +RSS is sampled after every agent completes one concurrent turn, with all agent handles retained. Marginal RSS is `(RSS after turns - RSS after Runtime::build) / N`, without allocator trimming. The cgroup peak includes the HTTP mock and the Python measurement wrapper. Mock request recording is disabled, so retained HTTP request history @@ -128,8 +128,9 @@ charged outside the new cgroup. | 100 | 437.38 MiB | 4.028 MiB | 417.89 MiB | 3,537 ms | 3,370 ms | | 500 | 1,712.97 MiB | 3.355 MiB | 1,760.32 MiB | 20,784 ms | 19,450 ms | -The 500-agent median is within twice the earlier macOS/mock 1.77 MiB -figure (3.54 MiB); the 50- and 100-agent medians miss that target. The +The 500-agent median is within twice the earlier macOS/mock 1,770 KiB +figure (1.728515625 MiB per agent, doubled to 3.45703125 MiB, or about +3.46 MiB); the 50- and 100-agent medians miss that target. The 500-agent runs ranged from 3.341 to 3.585 MiB per agent. These are different hosts and harness entry points, so the table is a capacity measurement rather than a controlled comparison of the platforms. This run also incorporates the @@ -144,8 +145,10 @@ executable; the OS page cache was warm. A separate 500-agent run with **swap disabled** (`MemorySwapMax=0`) completed in 19,090 ms, with 3.541 MiB marginal RSS per agent and a -1,884.52 MiB cgroup peak. Its marginal RSS narrowly exceeds the 3.54 MiB -target. Every run recorded zero `max`, `oom`, and `oom_kill` memory events. +1,884.52 MiB cgroup peak. Its marginal RSS exceeds the approximately +3.46 MiB target. The integration ticket used a nominal 1.77 MiB baseline +and 3.54 MiB doubled target; this run also narrowly exceeds that nominal target. +Every run recorded zero `max`, `oom`, and `oom_kill` memory events. This does not establish capacity for real providers, tool subprocesses, MCP servers, or 1,000 simultaneously active turns. Measure those workloads before sizing a production fleet. @@ -154,11 +157,19 @@ To reproduce from the OpenHuman repository root: ```sh cargo build -p openhuman-embed --locked --release --no-default-features --example linux_fleet +systemd-run --user --scope -p MemoryMax=2G -p CPUQuota=200% --quiet \ + python3 crates/openhuman-embed/examples/linux_fleet_cgroup.py 500 +``` + +Use `50` or `100` for the other sizes and repeat each command in a fresh scope +three times to reproduce the main table medians. Leave swap at its default +limit for those runs. To reproduce the separate no-swap 500-agent run: + +```sh systemd-run --user --scope -p MemoryMax=2G -p MemorySwapMax=0 -p CPUQuota=200% --quiet \ python3 crates/openhuman-embed/examples/linux_fleet_cgroup.py 500 ``` -Use `50` or `100` for the other sizes and repeat each command in a fresh scope. The JSON output records process RSS, bootstrap/turn timings, cgroup limits, peak memory, and OOM counters. The raw runs are checked in as [`docs/benchmarks/medulla-embed-linux.json`](../../../benchmarks/medulla-embed-linux.json). diff --git a/gitbooks/developing/embed/cookbook.md b/gitbooks/developing/embed/cookbook.md index 3462bef8af5..04b708f7de1 100644 --- a/gitbooks/developing/embed/cookbook.md +++ b/gitbooks/developing/embed/cookbook.md @@ -82,7 +82,7 @@ Measure retained runtime-owned agents using loopback inference and two worker th [Source](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs) · offline on Linux; use a fresh constrained cgroup for release measurements. -Run: `cargo run -p openhuman-embed --example linux_fleet` +Run: `cargo run -p openhuman-embed --release --example linux_fleet --no-default-features` ## Connect an actual MCP protocol stub over loopback diff --git a/gitbooks/developing/performance.md b/gitbooks/developing/performance.md index 1303d625f95..ec69bea4ae9 100644 --- a/gitbooks/developing/performance.md +++ b/gitbooks/developing/performance.md @@ -21,7 +21,7 @@ The original measurements below were made on Apple Silicon macOS with a `--relea | 100 | 1,866 | 356 | 3 | 123 | 820 | | 500 | 1,770 | 1,393 | 3 | 211 | 3,220 | -500 agents fit in one process at roughly 1.77 MiB marginal cost each. Idle CPU stays flat as N grows, which matters because an agent that is not mid-turn should not spend cycles. Thread count grows by about 0.35 per agent. Watch that line before pushing past 500 in production. +500 agents fit in one process at 1,770 KiB (about 1.73 MiB) marginal cost each. Idle CPU stays flat as N grows, which matters because an agent that is not mid-turn should not spend cycles. Thread count grows by about 0.35 per agent. Watch that line before pushing past 500 in production. Thousands of agents on one box is a goal, not a measured result. The marginal cost settles across 50, 100 and 500 agents instead of rising, which makes the goal plausible. @@ -114,8 +114,8 @@ Other builds were running on the host, so these are shared-host measurements. Each run starts in a fresh cgroup with `memory.max = 2147483648` and `cpu.max = 200000 100000` (2 GiB and a two-CPU quota). Three fresh processes -were measured at each size; the table gives medians. RSS is sampled after every -agent completes one concurrent turn, with all agent handles retained. Marginal +were measured at each size with the default swap limit; the table gives medians. +RSS is sampled after every agent completes one concurrent turn, with all agent handles retained. Marginal RSS is `(RSS after turns - RSS after Runtime::build) / N`, without allocator trimming. The cgroup peak includes the HTTP mock and the Python measurement wrapper. Mock request recording is disabled, so retained HTTP request history @@ -128,8 +128,9 @@ charged outside the new cgroup. | 100 | 437.38 MiB | 4.028 MiB | 417.89 MiB | 3,537 ms | 3,370 ms | | 500 | 1,712.97 MiB | 3.355 MiB | 1,760.32 MiB | 20,784 ms | 19,450 ms | -The 500-agent median is within twice the earlier macOS/mock 1.77 MiB -figure (3.54 MiB); the 50- and 100-agent medians miss that target. The +The 500-agent median is within twice the earlier macOS/mock 1,770 KiB +figure (1.728515625 MiB per agent, doubled to 3.45703125 MiB, or about +3.46 MiB); the 50- and 100-agent medians miss that target. The 500-agent runs ranged from 3.341 to 3.585 MiB per agent. These are different hosts and harness entry points, so the table is a capacity measurement rather than a controlled comparison of the platforms. This run also incorporates the @@ -144,8 +145,10 @@ executable; the OS page cache was warm. A separate 500-agent run with **swap disabled** (`MemorySwapMax=0`) completed in 19,090 ms, with 3.541 MiB marginal RSS per agent and a -1,884.52 MiB cgroup peak. Its marginal RSS narrowly exceeds the 3.54 MiB -target. Every run recorded zero `max`, `oom`, and `oom_kill` memory events. +1,884.52 MiB cgroup peak. Its marginal RSS exceeds the approximately +3.46 MiB target. The integration ticket used a nominal 1.77 MiB baseline +and 3.54 MiB doubled target; this run also narrowly exceeds that nominal target. +Every run recorded zero `max`, `oom`, and `oom_kill` memory events. This does not establish capacity for real providers, tool subprocesses, MCP servers, or 1,000 simultaneously active turns. Measure those workloads before sizing a production fleet. @@ -154,11 +157,19 @@ To reproduce from the OpenHuman repository root: ```sh cargo build -p openhuman-embed --locked --release --no-default-features --example linux_fleet +systemd-run --user --scope -p MemoryMax=2G -p CPUQuota=200% --quiet \ + python3 crates/openhuman-embed/examples/linux_fleet_cgroup.py 500 +``` + +Use `50` or `100` for the other sizes and repeat each command in a fresh scope +three times to reproduce the main table medians. Leave swap at its default +limit for those runs. To reproduce the separate no-swap 500-agent run: + +```sh systemd-run --user --scope -p MemoryMax=2G -p MemorySwapMax=0 -p CPUQuota=200% --quiet \ python3 crates/openhuman-embed/examples/linux_fleet_cgroup.py 500 ``` -Use `50` or `100` for the other sizes and repeat each command in a fresh scope. The JSON output records process RSS, bootstrap/turn timings, cgroup limits, peak memory, and OOM counters. The raw runs are checked in as [`docs/benchmarks/medulla-embed-linux.json`](../../docs/benchmarks/medulla-embed-linux.json). diff --git a/llms-full.txt b/llms-full.txt index 760b0e639c2..03a61a1492b 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -2614,7 +2614,7 @@ Measure retained runtime-owned agents using loopback inference and two worker th [Source](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/linux_fleet.rs) · offline on Linux; use a fresh constrained cgroup for release measurements. -Run: `cargo run -p openhuman-embed --example linux_fleet` +Run: `cargo run -p openhuman-embed --release --example linux_fleet --no-default-features` ## Connect an actual MCP protocol stub over loopback diff --git a/scripts/__tests__/generate-embed-docs.test.mjs b/scripts/__tests__/generate-embed-docs.test.mjs index abf8a3df7a2..52e45e7d00b 100644 --- a/scripts/__tests__/generate-embed-docs.test.mjs +++ b/scripts/__tests__/generate-embed-docs.test.mjs @@ -36,6 +36,25 @@ test('cookbook metadata must come from example headers', () => { assert.deepEqual(exampleMetadata(example, 'hello'), { name: 'hello', title: 'Hello', summary: 'Builds an offline agent.', run: 'offline', feature: undefined }); assert.throws(() => exampleMetadata('fn main() {}', 'bad'), /headers/); }); +test('cookbook benchmark commands use the declared release profile without default features', () => { + const root = mkdtempSync(resolve(tmpdir(), 'embed-benchmark-docs-')); + try { + mkdirSync(resolve(root, 'gitbooks/developing/embed'), { recursive: true }); + mkdirSync(resolve(root, 'crates/openhuman-embed/examples'), { recursive: true }); + mkdirSync(resolve(root, 'crates/openhuman-embed/src'), { recursive: true }); + writeFileSync(resolve(root, 'gitbooks/developing/embed/index.md'), '# Benchmarks\n'); + writeFileSync(resolve(root, 'crates/openhuman-embed/src/lib.rs'), 'pub use runtime::Runtime;'); + writeFileSync(resolve(root, 'crates/openhuman-embed/examples/retained_fleet.rs'), example + '//! Profile: release\n//! Feature: default\n//! Default features: disabled\n'); + writeFileSync(resolve(root, 'crates/openhuman-embed/examples/hello.rs'), example); + const capabilities = resolve(root, 'capabilities.json'); + writeFileSync(capabilities, '{"schema_version":1,"compiled_features":{}}'); + generateEmbedDocs({ root, capabilities }); + const cookbook = readFileSync(resolve(root, 'gitbooks/developing/embed/cookbook.md'), 'utf8'); + assert.match(cookbook, /Run: `cargo run -p openhuman-embed --release --example retained_fleet --no-default-features`/); + assert.match(cookbook, /Run: `cargo run -p openhuman-embed --example hello`/); + assert.doesNotMatch(cookbook, /--features none/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); test('generate then check is stable, edited source fails check, no prose authored by tooling', () => { const root = mkdtempSync(resolve(tmpdir(), 'embed-docs-')); try { diff --git a/scripts/generate-embed-docs.mjs b/scripts/generate-embed-docs.mjs index 2b5ee933408..78949a29991 100644 --- a/scripts/generate-embed-docs.mjs +++ b/scripts/generate-embed-docs.mjs @@ -106,7 +106,14 @@ export function exampleMetadata(source, name) { const summary = field('Summary'); if (!title || !summary) throw new Error(`Example ${name} needs //! Title: and //! Summary: headers`); const feature = field('Feature'); - return { name, title, summary, run: field('Run') ?? 'offline', feature: feature === 'default' ? undefined : feature }; + const profile = field('Profile'); + if (profile && !['dev', 'release'].includes(profile)) throw new Error(`Example ${name} has unsupported Profile: ${profile}`); + return { + name, title, summary, run: field('Run') ?? 'offline', + feature: feature === 'default' ? undefined : feature, + ...(profile ? { profile } : {}), + ...(field('Default features') === 'disabled' ? { noDefaultFeatures: true } : {}), + }; } /** Public facade exports, suitable for rustdoc search links (including aliases). */ @@ -197,7 +204,7 @@ export function generateEmbedDocs({ root = ROOT, check = false, capabilities } = desired.set(path, updated); } const examples = readdirSync(resolve(root, 'crates/openhuman-embed/examples')).filter(name => name.endsWith('.rs')).sort().map(file => exampleMetadata(readFileSync(resolve(root, 'crates/openhuman-embed/examples', file), 'utf8'), file.slice(0, -3))); - desired.set(resolve(directory, 'cookbook.md'), '# Cookbook\n\n\n\nExecutable recipes are indexed from their source headers. Each entry records the declared run mode and any named Embed feature required by the example. Full source includes the surrounding setup and assertions.\n\n' + examples.map(example => `## ${example.title}\n\n${example.summary}\n\n[Source](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/${example.name}.rs) · ${example.run}${example.feature ? ` · feature: ${example.feature}` : ''}\n\nRun: \`cargo run -p openhuman-embed --example ${example.name}${example.feature ? ` --features ${example.feature}` : ''}\`\n`).join('\n')); + desired.set(resolve(directory, 'cookbook.md'), '# Cookbook\n\n\n\nExecutable recipes are indexed from their source headers. Each entry records the declared run mode and any named Embed feature required by the example. Full source includes the surrounding setup and assertions.\n\n' + examples.map(example => `## ${example.title}\n\n${example.summary}\n\n[Source](https://github.com/tinyhumansai/openhuman/blob/main/crates/openhuman-embed/examples/${example.name}.rs) · ${example.run}${example.feature ? ` · feature: ${example.feature}` : ''}\n\nRun: \`cargo run -p openhuman-embed${example.profile === 'release' ? ' --release' : ''} --example ${example.name}${example.noDefaultFeatures ? ' --no-default-features' : ''}${example.feature ? ` --features ${example.feature}` : ''}\`\n`).join('\n')); const report = capabilities ? JSON.parse(readFileSync(capabilities, 'utf8')) : JSON.parse(execFileSync('bash', ['scripts/ci-cancel-aware.sh', 'cargo', 'run', '--quiet', '-p', 'openhuman-embed', '--example', 'capability_report', '--', '--json'], { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'inherit'] })); const exports = exportedNames(readFileSync(resolve(root, 'crates/openhuman-embed/src/lib.rs'), 'utf8')); if (!report || typeof report !== 'object' || !Number.isInteger(report.schema_version) || !report.compiled_features || typeof report.compiled_features !== 'object') throw new Error('Capability report must be versioned RuntimeInfo JSON'); From 7b3c27535c91e5dcc5a62b96622346a974e9c7fe Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 10 Oct 2026 23:23:31 +0300 Subject: [PATCH 09/18] fix(embed): serialize approval decisions with removal claims Co-authored-by: Medulla --- .../approval/registration_scope_tests.rs | 24 ++++----- .../src/agent/approval_handler.rs | 6 ++- crates/openhuman-embed/src/agent/approvals.rs | 18 ++++++- crates/openhuman-embed/src/agent/build.rs | 7 ++- crates/openhuman-embed/src/agent/lifecycle.rs | 42 +++++++++++++-- .../src/agent/lifecycle_tests.rs | 53 +++++++++++++++++++ crates/openhuman-embed/src/agent/mod.rs | 13 ++++- .../openhuman-embed/tests/agent_lifecycle.rs | 2 +- docs/TEST-COVERAGE-MATRIX.md | 2 +- 9 files changed, 143 insertions(+), 24 deletions(-) diff --git a/crates/openhuman-core/src/security/approval/registration_scope_tests.rs b/crates/openhuman-core/src/security/approval/registration_scope_tests.rs index c2e2c11a7b6..06778f54710 100644 --- a/crates/openhuman-core/src/security/approval/registration_scope_tests.rs +++ b/crates/openhuman-core/src/security/approval/registration_scope_tests.rs @@ -12,11 +12,10 @@ fn removal_refuses_late_registration_without_creating_a_pending_row() { } #[test] -fn removal_waits_for_accepted_registration_before_taking_its_snapshot() { +fn accepted_registration_holds_the_removal_barrier_until_it_finishes() { let scope = Arc::new(ApprovalScope::default()); let (entered_tx, entered_rx) = mpsc::channel(); let (release_tx, release_rx) = mpsc::channel(); - let (closed_tx, closed_rx) = mpsc::channel(); let registering = scope.clone(); let registration = std::thread::spawn(move || { registering.register(|| { @@ -25,21 +24,20 @@ fn removal_waits_for_accepted_registration_before_taking_its_snapshot() { }) }); entered_rx.recv().unwrap(); - let removing = scope.clone(); - let removal = std::thread::spawn(move || { - removing.close("agent_removed"); - closed_tx.send(()).unwrap(); - }); - let closed_early = closed_rx - .recv_timeout(std::time::Duration::from_millis(100)) - .is_ok(); + // Observe ownership directly while the accepted registration is suspended; + // no scheduler delay or elapsed-time assertion can hide an unlocked barrier. + let held = matches!( + scope.closed.try_lock(), + Err(std::sync::TryLockError::WouldBlock) + ); release_tx.send(()).unwrap(); assert_eq!(registration.join().unwrap(), Ok(())); - removal.join().unwrap(); assert!( - !closed_early, - "removal snapshot raced an accepted registration" + held, + "registration released the removal barrier before finishing" ); + scope.close("agent_removed"); + assert_eq!(scope.register(|| ()), Err("agent_removed".to_owned())); } #[test] diff --git a/crates/openhuman-embed/src/agent/approval_handler.rs b/crates/openhuman-embed/src/agent/approval_handler.rs index 1f888de68c9..68e4174f383 100644 --- a/crates/openhuman-embed/src/agent/approval_handler.rs +++ b/crates/openhuman-embed/src/agent/approval_handler.rs @@ -26,6 +26,7 @@ impl ApprovalSubscription { agent_id: &str, handler: Arc, removed: tokio::sync::watch::Receiver, + lifecycle: Arc, ) -> Self { let cancellation = CancellationToken::new(); let subscriber = Arc::new(Handler { @@ -33,6 +34,7 @@ impl ApprovalSubscription { handler, cancellation: cancellation.clone(), removed, + lifecycle, }); Self { _subscription: BUS.subscribe(subscriber), @@ -45,6 +47,7 @@ struct Handler { handler: Arc, cancellation: CancellationToken, removed: tokio::sync::watch::Receiver, + lifecycle: Arc, } #[async_trait::async_trait] impl EventHandler for Handler { @@ -63,7 +66,8 @@ impl EventHandler for Handler { if agent_id.as_deref() != Some(&self.agent_id) || *self.removed.borrow() { return; } - let approvals = Approvals::new(&self.agent_id, self.removed.clone()); + let approvals = + Approvals::new(&self.agent_id, self.removed.clone(), self.lifecycle.clone()); let Ok(pending) = approvals.pending() else { return; }; diff --git a/crates/openhuman-embed/src/agent/approvals.rs b/crates/openhuman-embed/src/agent/approvals.rs index 8816fcde743..822dd284c4c 100644 --- a/crates/openhuman-embed/src/agent/approvals.rs +++ b/crates/openhuman-embed/src/agent/approvals.rs @@ -32,13 +32,19 @@ pub enum ApprovalsError { pub struct Approvals { agent_id: String, removed: tokio::sync::watch::Receiver, + lifecycle: std::sync::Arc, } impl Approvals { - pub(crate) fn new(agent_id: &str, removed: tokio::sync::watch::Receiver) -> Self { + pub(crate) fn new( + agent_id: &str, + removed: tokio::sync::watch::Receiver, + lifecycle: std::sync::Arc, + ) -> Self { Self { agent_id: agent_id.to_string(), removed, + lifecycle, } } @@ -69,6 +75,16 @@ impl Approvals { &self, request_id: &str, decision: ApprovalDecision, + ) -> Result { + self.lifecycle + .with_live(|| self.decide_live(request_id, decision)) + .unwrap_or_else(|| Err(ApprovalsError::NotFound(request_id.to_owned()))) + } + + fn decide_live( + &self, + request_id: &str, + decision: ApprovalDecision, ) -> Result { let removed = self.removed.borrow(); if *removed { diff --git a/crates/openhuman-embed/src/agent/build.rs b/crates/openhuman-embed/src/agent/build.rs index 72799436541..e6b5a87a6e7 100644 --- a/crates/openhuman-embed/src/agent/build.rs +++ b/crates/openhuman-embed/src/agent/build.rs @@ -280,7 +280,12 @@ pub(crate) fn instantiate(runtime: &Runtime, spec: AgentSpec) -> Result, - removal_claimed: AtomicBool, + approvals: Arc, in_flight: AtomicUsize, idle: Notify, torn_down: AtomicBool, } +/// Serializes live approval decisions with the instance's removal claim. +#[derive(Debug, Default)] +pub(crate) struct ApprovalState { + claimed: AtomicBool, + decisions: Mutex<()>, +} + +impl ApprovalState { + pub(crate) fn with_live(&self, decide: impl FnOnce() -> T) -> Option { + let _decision = self + .decisions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.claimed.load(Ordering::SeqCst) { + None + } else { + Some(decide()) + } + } + + fn claim_removal(&self) -> bool { + let _decision = self + .decisions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + !self.claimed.swap(true, Ordering::SeqCst) + } +} + impl Lifecycle { pub(crate) fn new() -> Self { Self { removed: watch::Sender::new(false), - removal_claimed: AtomicBool::new(false), + approvals: Arc::new(ApprovalState::default()), in_flight: AtomicUsize::new(0), idle: Notify::new(), torn_down: AtomicBool::new(false), @@ -36,7 +66,7 @@ impl Lifecycle { /// Claim removal and settle owned approvals before waking in-flight turns. pub(crate) fn mark_removed_with(&self, before_notify: impl FnOnce()) -> bool { - if self.removal_claimed.swap(true, Ordering::SeqCst) { + if !self.approvals.claim_removal() { return false; } before_notify(); @@ -46,6 +76,10 @@ impl Lifecycle { true } + pub(crate) fn approval_state(&self) -> Arc { + self.approvals.clone() + } + /// Bind handles to this agent instance, even after its public id is reused. pub(crate) fn removed(&self) -> watch::Receiver { self.removed.subscribe() @@ -66,7 +100,7 @@ impl Lifecycle { let mut watcher = self.removed.subscribe(); // Claiming removal closes admission immediately; waking existing turns // waits until their approvals have been settled with the removal reason. - if self.removal_claimed.load(Ordering::SeqCst) || *watcher.borrow_and_update() { + if self.approvals.claimed.load(Ordering::SeqCst) || *watcher.borrow_and_update() { log::debug!("[embed][agent] turn refused: agent removed id={agent_id}"); return Err(removed()); } diff --git a/crates/openhuman-embed/src/agent/lifecycle_tests.rs b/crates/openhuman-embed/src/agent/lifecycle_tests.rs index 2505ec4b8fd..2890887c3ad 100644 --- a/crates/openhuman-embed/src/agent/lifecycle_tests.rs +++ b/crates/openhuman-embed/src/agent/lifecycle_tests.rs @@ -84,3 +84,56 @@ async fn a_claimed_removal_refuses_turns_before_approvals_are_settled() { "removal admitted a new turn while settling approvals" ); } + +#[test] +fn a_claimed_removal_refuses_approval_decisions_before_denial_finishes() { + let lifecycle = Arc::new(Lifecycle::new()); + let approvals = crate::Approvals::new("a", lifecycle.removed(), lifecycle.approval_state()); + let removing = lifecycle.clone(); + let (entered, callback_entered) = std::sync::mpsc::channel(); + let (release, callback_release) = std::sync::mpsc::channel(); + let removal = std::thread::spawn(move || { + removing.mark_removed_with(|| { + entered.send(()).unwrap(); + callback_release.recv().unwrap(); + }) + }); + callback_entered.recv().unwrap(); + let outcome = approvals.decide("pending-request", crate::ApprovalDecision::ApproveOnce); + release.send(()).unwrap(); + assert!(removal.join().unwrap()); + assert!( + matches!(outcome, Err(crate::ApprovalsError::NotFound(_))), + "{outcome:?}" + ); +} + +#[test] +fn accepted_approval_decisions_hold_the_removal_claim_barrier() { + let state = Arc::new(ApprovalState::default()); + let deciding = state.clone(); + let (entered, decision_entered) = std::sync::mpsc::channel(); + let (release, decision_release) = std::sync::mpsc::channel(); + let decision = std::thread::spawn(move || { + deciding.with_live(|| { + entered.send(()).unwrap(); + decision_release.recv().unwrap(); + 7 + }) + }); + decision_entered.recv().unwrap(); + let held = matches!( + state.decisions.try_lock(), + Err(std::sync::TryLockError::WouldBlock) + ); + release.send(()).unwrap(); + assert_eq!(decision.join().unwrap(), Some(7)); + assert!( + held, + "removal could claim while a decision was still being applied" + ); + assert!(state.claim_removal()); + let mut ran = false; + assert_eq!(state.with_live(|| ran = true), None); + assert!(!ran); +} diff --git a/crates/openhuman-embed/src/agent/mod.rs b/crates/openhuman-embed/src/agent/mod.rs index 9d3c52e064a..61d178dfd6d 100644 --- a/crates/openhuman-embed/src/agent/mod.rs +++ b/crates/openhuman-embed/src/agent/mod.rs @@ -264,7 +264,12 @@ impl Agent { /// Subscribe a host callback to this agent’s pending approvals. /// Removing the agent cancels its callbacks, even if its id is reused. pub fn handle_approvals(&self, handler: Arc) -> ApprovalSubscription { - ApprovalSubscription::new(self.id(), handler, self.inner.lifecycle.removed()) + ApprovalSubscription::new( + self.id(), + handler, + self.inner.lifecycle.removed(), + self.inner.lifecycle.approval_state(), + ) } /// Add, replace, or remove an agent-local post-turn hook by name. @@ -280,7 +285,11 @@ impl Agent { /// This agent's pending approvals: the requests its turns parked, and /// only those. pub fn approvals(&self) -> Approvals { - Approvals::new(&self.inner.id, self.inner.lifecycle.removed()) + Approvals::new( + &self.inner.id, + self.inner.lifecycle.removed(), + self.inner.lifecycle.approval_state(), + ) } /// The agent's read/write root for acting tools. diff --git a/crates/openhuman-embed/tests/agent_lifecycle.rs b/crates/openhuman-embed/tests/agent_lifecycle.rs index 408b7f94f10..7e1169153c5 100644 --- a/crates/openhuman-embed/tests/agent_lifecycle.rs +++ b/crates/openhuman-embed/tests/agent_lifecycle.rs @@ -140,7 +140,7 @@ fn removing_an_agent_releases_everything_it_held() { // snapshot. Its retained context must refuse registration entirely. let gate = openhuman_core::security::approval::ApprovalGate::try_global().unwrap(); let late = tokio::time::timeout( - Duration::from_millis(500), + Duration::from_secs(10), openhuman_core::core::runtime::CoreContext::scope( old_context, openhuman_core::agent::turn_origin::with_origin( diff --git a/docs/TEST-COVERAGE-MATRIX.md b/docs/TEST-COVERAGE-MATRIX.md index 89398dc9ba0..565349f5a3c 100644 --- a/docs/TEST-COVERAGE-MATRIX.md +++ b/docs/TEST-COVERAGE-MATRIX.md @@ -659,7 +659,7 @@ The thread JSONL store moved to `tinyagents_session::threads` (`vendor/tinyagent | 16.1.8 | MCP servers declared programmatically | RU | `crates/openhuman-embed/src/harness/mcp_tests.rs` | ✅ | stdio/HTTP transports mutually exclusive, enabled by default, env accumulation, allow/deny filters, auth carried through | | 16.1.9 | Skill bundles installed into the workspace | RU | `crates/openhuman-embed/src/harness/skills_tests.rs` | ✅ | Copied not linked (discovery rejects symlinked bundles and manifests by design); symlinks skipped both as bundles and inside them; `Inherit` refuses the copy | | 16.1.10 | One runtime per process | RU+RI | `crates/openhuman-embed/src/runtime/builder_tests.rs`, `crates/openhuman-embed/tests/harness_embed.rs` | ✅ | A second `Runtime` (or `Harness`) returns `AlreadyRunning` rather than sharing process-global keyring/event-bus/subscribers; a failed build releases the slot so a retry is possible | -| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs`, `crates/openhuman-embed/tests/agent_lifecycle.rs`, `crates/openhuman-core/src/security/approval/registration_scope_tests.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused; removal settles pending approvals with `agent_removed` before cancelling turns; a claimed removal immediately refuses new admission; instance-owned approval registration closes before the denial snapshot and rejects late registrations without persisting rows | +| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs`, `crates/openhuman-embed/tests/agent_lifecycle.rs`, `crates/openhuman-core/src/security/approval/registration_scope_tests.rs`, `crates/openhuman-embed/src/agent/lifecycle_tests.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused; removal settles pending approvals with `agent_removed` before cancelling turns; a claimed removal immediately refuses new admission; instance-owned approval registration closes before the denial snapshot and rejects late registrations without persisting rows; polling and callback approval decisions serialize with the removal claim | | 16.1.12 | API-key credential for library mode | RU+RI | `crates/openhuman-core/src/security/credentials/api_key_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs` | ✅ | Key stored as an `api-key` auth profile; wins over an expired session; managed inference sends it as a bearer with no `x-api-key`; auth state reports `credential = "api-key"` | | 16.1.14 | Host-backed read-only repository toolset | RI | `crates/openhuman-embed/tests/repository_tools.rs`, `crates/openhuman-embed/tests/repository_host_only.rs` | ✅ | Validated tree/range/search/symbol/git queries delegate only to the host; mandatory redaction, fenced bounded output, sanitized failures, and real HostOnly + readonly + untrusted-input refusal of shell/write/network. | | 16.1.15 | Strict structured output and bounded repair | RI | `crates/openhuman-embed/tests/structured_validation.rs`, `crates/openhuman-embed/tests/structured_turns.rs`, `crates/openhuman-embed/tests/unit/turn_usage.rs` | ✅ | Wrong types and numeric combinators refused, invalid/external schemas make no call, repair bounded with usage, terminal length rejected even for complete JSON; successful charged usage reaches outcome/meter, invalid buyer charges stay unknown, root overlays retain child spend and session context. | From f5ec62f9d202fbfeafd0fe51c947046b8d69f88d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 10 Oct 2026 23:56:24 +0300 Subject: [PATCH 10/18] fix(process): await stopped descendants before cancellation acknowledgement Co-authored-by: Medulla --- .../src/tools/timeout/group_exit.rs | 59 +++++++++++++++++++ .../src/tools/timeout/group_exit_tests.rs | 19 ++++++ .../openhuman-core/src/tools/timeout/mod.rs | 14 ++++- .../tests/process_cancellation.rs | 47 +++++++++++++++ 4 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 crates/openhuman-core/src/tools/timeout/group_exit.rs create mode 100644 crates/openhuman-core/src/tools/timeout/group_exit_tests.rs diff --git a/crates/openhuman-core/src/tools/timeout/group_exit.rs b/crates/openhuman-core/src/tools/timeout/group_exit.rs new file mode 100644 index 00000000000..983a7dfd611 --- /dev/null +++ b/crates/openhuman-core/src/tools/timeout/group_exit.rs @@ -0,0 +1,59 @@ +//! Wait for signalled Linux process-group members to stop executing. +//! +//! The caller keeps the group leader unreaped during this check, reserving +//! its PID and preventing a new process group from reusing the identifier. + +/// Wait until the reserved group contains no executing Linux processes. +#[cfg(target_os = "linux")] +pub(super) async fn wait(pid: u32) -> std::io::Result<()> { + loop { + let active = tokio::task::spawn_blocking(move || active_members(pid)) + .await + .map_err(std::io::Error::other)??; + if !active { + return Ok(()); + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } +} + +/// Other platforms retain their existing direct-child acknowledgement. +#[cfg(not(target_os = "linux"))] +pub(super) async fn wait(_: u32) -> std::io::Result<()> { + Ok(()) +} + +/// Inspect kernel process states without reaping the reserved group leader. +#[cfg(target_os = "linux")] +fn active_members(group: u32) -> std::io::Result { + for entry in std::fs::read_dir("/proc")? { + let entry = entry?; + if entry.file_name().to_string_lossy().parse::().is_err() { + continue; + } + // Process entries can disappear or become unreadable between + // enumeration and reading. + if let Ok(stat) = std::fs::read_to_string(entry.path().join("stat")) { + if is_active_member(&stat, group) { + return Ok(true); + } + } + } + Ok(false) +} + +/// Zombies and dead tasks cannot execute further side effects. +#[cfg(target_os = "linux")] +fn is_active_member(stat: &str, group: u32) -> bool { + let Some((_, fields)) = stat.rsplit_once(") ") else { + return false; + }; + let mut fields = fields.split_whitespace(); + let state = fields.next(); + let process_group = fields.nth(1).and_then(|value| value.parse::().ok()); + process_group == Some(group) && !matches!(state, Some("Z" | "X" | "x")) +} + +#[cfg(all(test, target_os = "linux"))] +#[path = "group_exit_tests.rs"] +mod tests; diff --git a/crates/openhuman-core/src/tools/timeout/group_exit_tests.rs b/crates/openhuman-core/src/tools/timeout/group_exit_tests.rs new file mode 100644 index 00000000000..4dd36b59f09 --- /dev/null +++ b/crates/openhuman-core/src/tools/timeout/group_exit_tests.rs @@ -0,0 +1,19 @@ +use super::*; + +#[test] +fn exited_members_do_not_block_cleanup_but_running_members_do() { + for state in ["R", "S", "D", "T", "I"] { + assert!(is_active_member( + &format!("12 (name with ) spaces) {state} 1 42 42"), + 42 + )); + } + for state in ["Z", "X", "x"] { + assert!(!is_active_member( + &format!("12 (sleep) {state} 1 42 42"), + 42 + )); + } + assert!(!is_active_member("12 (sleep) R 1 43 43", 42)); + assert!(!is_active_member("malformed", 42)); +} diff --git a/crates/openhuman-core/src/tools/timeout/mod.rs b/crates/openhuman-core/src/tools/timeout/mod.rs index c1378afc328..c689a719728 100644 --- a/crates/openhuman-core/src/tools/timeout/mod.rs +++ b/crates/openhuman-core/src/tools/timeout/mod.rs @@ -20,6 +20,7 @@ use tinytools::ToolTimeout; mod command_environment; pub use command_environment::CommandEnvironment; +mod group_exit; mod process_cleanup; pub use process_cleanup::ProcessCleanup; @@ -344,7 +345,18 @@ async fn collect_command_output( _ = async { let _ = cancellation.wait_for(|cancelled| *cancelled).await; } => { group.kill(); child.start_kill()?; - child.wait().await? + // Pipe EOF and the shell's exit can precede a descendant's exit. + // Keep the leader unreaped while checking the reserved group. + let stopped = match group.0 { + Some(pid) => group_exit::wait(pid).await, + None => Ok(()), + }; + let status = child.wait().await?; + // Disarm before propagating a check error after reaping: its PID + // could now be reused, including while unwinding this function. + group.0 = None; + stopped?; + status } result = child.wait() => result?, }; diff --git a/crates/openhuman-embed/tests/process_cancellation.rs b/crates/openhuman-embed/tests/process_cancellation.rs index 479c9e42145..f5d79a2ed82 100644 --- a/crates/openhuman-embed/tests/process_cancellation.rs +++ b/crates/openhuman-embed/tests/process_cancellation.rs @@ -259,3 +259,50 @@ async fn cancellation_reaps_the_leader_when_an_escaped_descendant_holds_its_pipe "escaped descendant blocked cancellation acknowledgement" ); } + +/// Closed output pipes do not prove that a signalled descendant has exited. +#[tokio::test] +async fn cleanup_acknowledgement_waits_for_descendants_that_closed_their_output() { + for _ in 0..16 { + let scratch = tempfile::tempdir().unwrap(); + let pidfile = scratch.path().join("closed-output.pid"); + let mut cmd = tokio::process::Command::new("/bin/sh"); + cmd.args(["-c", &format!( + "pids=''; for i in 1 2 3 4 5 6 7 8; do sleep 30 >/dev/null 2>&1 & pids=\"$pids $!\"; done; echo \"$pids\" > {}; wait", pidfile.display() + )]); + let cleanup = ProcessCleanup::default(); + let mut run = Box::pin(cleanup.scope(output_unbounded(&mut cmd))); + let pids = tokio::time::timeout(Duration::from_secs(5), async { + loop { + tokio::select! { + result = &mut run => panic!("command ended before cancellation: {result:?}"), + _ = tokio::time::sleep(Duration::from_millis(1)) => {} + } + if let Ok(text) = std::fs::read_to_string(&pidfile) { + let pids: Vec = text + .split_whitespace() + .filter_map(|pid| pid.parse().ok()) + .collect(); + if pids.len() == 8 { + break pids; + } + } + } + }) + .await + .unwrap(); + drop(run); + tokio::time::timeout(Duration::from_secs(5), cleanup.wait()) + .await + .unwrap(); + for pid in pids { + if let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) { + let (_, fields) = stat.rsplit_once(") ").unwrap(); + assert!( + fields.starts_with('Z') || fields.starts_with('X'), + "descendant still active after cleanup acknowledgement: {stat}" + ); + } + } + } +} From 3c9d383f5b612c62571142b5a359aff778f345f6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 10 Oct 2026 23:58:37 +0300 Subject: [PATCH 11/18] fix(approval): guard shared gate decisions during instance removal Co-authored-by: Medulla --- .../src/security/approval/gate.rs | 2 + .../src/security/approval/gate_intercept.rs | 9 +- .../src/security/approval/gate_state.rs | 43 ++++++++-- .../security/approval/registration_scope.rs | 19 +++-- .../approval/registration_scope_tests.rs | 12 +-- crates/openhuman-embed/src/agent/build.rs | 6 +- crates/openhuman-embed/src/agent/lifecycle.rs | 19 +++-- .../src/agent/lifecycle_tests.rs | 16 ++-- crates/openhuman-embed/src/agent/mod.rs | 2 +- .../openhuman-embed/src/runtime/lifecycle.rs | 2 +- .../openhuman-embed/tests/agent_lifecycle.rs | 84 +++++++++++++++++++ 11 files changed, 179 insertions(+), 35 deletions(-) diff --git a/crates/openhuman-core/src/security/approval/gate.rs b/crates/openhuman-core/src/security/approval/gate.rs index 8563e72346e..d413d505552 100644 --- a/crates/openhuman-core/src/security/approval/gate.rs +++ b/crates/openhuman-core/src/security/approval/gate.rs @@ -268,6 +268,8 @@ pub(crate) struct RequestRoute { pub(crate) tool_call_id: Option, pub(crate) forced: bool, pub(crate) agent_id: Option, + /// Instance barrier captured at registration, independent of decision context. + pub(crate) approval_scope: Option>, /// The [`thread_route_key`] the request was parked under, so a decision /// made outside the parking task's scope clears the right route. pub(crate) thread_key: Option, diff --git a/crates/openhuman-core/src/security/approval/gate_intercept.rs b/crates/openhuman-core/src/security/approval/gate_intercept.rs index d15d9455d4b..d4d95a3ec9a 100644 --- a/crates/openhuman-core/src/security/approval/gate_intercept.rs +++ b/crates/openhuman-core/src/security/approval/gate_intercept.rs @@ -501,6 +501,8 @@ impl ApprovalGate { // to time out and return `Deny` incorrectly. (CodeRabbit // review on PR #2149.) let (tx, rx) = oneshot::channel::(); + let scope = crate::core::runtime::CoreContext::current_host_overrides() + .and_then(|overrides| overrides.approval_scope()); let register = || -> anyhow::Result<()> { { let mut waiters = self.waiters.lock(); @@ -526,6 +528,7 @@ impl ApprovalGate { tool_call_id: tool_call_id.map(str::to_string), forced, agent_id: agent_id.clone(), + approval_scope: scope.clone(), thread_key: thread_key.clone(), detached, }, @@ -553,10 +556,8 @@ impl ApprovalGate { }); Ok(()) }; - let scope = crate::core::runtime::CoreContext::current_host_overrides() - .and_then(|overrides| overrides.approval_scope()); - let registered = match scope { - Some(scope) => scope.register(register), + let registered = match scope.as_ref() { + Some(scope) => scope.with_open(register), None => Ok(register()), }; match registered { diff --git a/crates/openhuman-core/src/security/approval/gate_state.rs b/crates/openhuman-core/src/security/approval/gate_state.rs index 4538cc4b3c4..2987561b1dc 100644 --- a/crates/openhuman-core/src/security/approval/gate_state.rs +++ b/crates/openhuman-core/src/security/approval/gate_state.rs @@ -39,6 +39,23 @@ impl ApprovalGate { &self, request_id: &str, decision: ApprovalDecision, + ) -> anyhow::Result> { + // Release the routing lock before acquiring the instance barrier: + // registration holds that barrier while inserting its route. + let scope = self.request_scope(request_id); + match scope { + Some(scope) => scope + .with_open(|| self.decide_open(request_id, decision)) + .unwrap_or(Ok(None)), + None => self.decide_open(request_id, decision), + } + } + + /// Commit a decision and release its waiter while the instance is still open. + fn decide_open( + &self, + request_id: &str, + decision: ApprovalDecision, ) -> anyhow::Result> { if !matches!( decision, @@ -159,11 +176,12 @@ impl ApprovalGate { } /// Classify a [`Self::decide`] miss — i.e. when `decide` returned - /// `Ok(None)` because its conditional `UPDATE ... WHERE decided_at IS NULL` - /// matched 0 rows. Two very different states collapse into that `None`: + /// `Ok(None)` because the instance was closed or its conditional + /// `UPDATE ... WHERE decided_at IS NULL` matched 0 rows. /// /// - [`DecideMiss::AlreadyResolved`] — the row exists but was **already - /// decided, lazily expired (denied), or superseded**. This is the benign + /// decided, lazily expired (denied), superseded, or closed for removal**. + /// This is the benign /// double-tap / two-operator / expiry-while-live race the inline-approvals /// design spec classifies as benign (TAURI-RUST-5EH). /// - [`DecideMiss::NeverRegistered`] — no row was ever persisted for this @@ -174,10 +192,17 @@ impl ApprovalGate { /// We disambiguate by consulting [`store::get_decision`], which returns a /// decision only when `decided_at IS NOT NULL` — exactly the already-resolved /// case (expiry writes a `Deny` decision, so expired rows report here too). - /// A `decide` miss can't be an undecided-but-present row: that row would have - /// matched the `UPDATE`. If the lookup itself errors we conservatively keep + /// A closed instance can still have an undecided row while removal takes + /// its denial snapshot; this is also a benign refusal. If the lookup itself + /// errors we conservatively keep /// the event visible (`NeverRegistered`) rather than silently demoting. pub fn classify_decide_miss(&self, request_id: &str) -> DecideMiss { + if self + .request_scope(request_id) + .is_some_and(|scope| scope.is_closed()) + { + return DecideMiss::AlreadyResolved; + } match store::get_decision(&self.config, request_id) { Ok(Some(_)) => DecideMiss::AlreadyResolved, Ok(None) => DecideMiss::NeverRegistered, @@ -279,6 +304,14 @@ impl ApprovalGate { &self.session_id } + /// Clone the instance barrier without retaining the routing-map lock. + fn request_scope(&self, request_id: &str) -> Option> { + self.request_routes + .lock() + .get(request_id) + .and_then(|route| route.approval_scope.clone()) + } + fn take_waiter(&self, request_id: &str) -> Option> { let mut waiters = self.waiters.lock(); waiters.remove(request_id) diff --git a/crates/openhuman-core/src/security/approval/registration_scope.rs b/crates/openhuman-core/src/security/approval/registration_scope.rs index bf0c0e47fb5..3017d2f70d9 100644 --- a/crates/openhuman-core/src/security/approval/registration_scope.rs +++ b/crates/openhuman-core/src/security/approval/registration_scope.rs @@ -1,15 +1,15 @@ -//! Serializes an agent instance's approval registration with its removal. +//! Serializes an agent instance's approval registration and decisions with removal. use std::sync::Mutex; /// An instance-owned barrier; reusing an agent ID requires a fresh scope. -#[derive(Default)] +#[derive(Debug, Default)] pub struct ApprovalScope { closed: Mutex>, } impl ApprovalScope { - /// Stop registration and wait for any accepted registration to finish. + /// Stop registration and decisions; wait for accepted work to finish. /// The first teardown reason remains authoritative. pub fn close(&self, reason: &str) { let mut closed = self @@ -21,7 +21,16 @@ impl ApprovalScope { } } - pub(crate) fn register(&self, register: impl FnOnce() -> T) -> Result { + /// Whether removal has closed this instance's registration and decision barrier. + pub fn is_closed(&self) -> bool { + self.closed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .is_some() + } + + /// Run synchronous registration or a decision while holding the removal barrier. + pub(crate) fn with_open(&self, register: impl FnOnce() -> T) -> Result { let closed = self .closed .lock() @@ -29,7 +38,7 @@ impl ApprovalScope { if let Some(reason) = closed.as_ref() { return Err(reason.clone()); } - // Keep the barrier through persistence and ApprovalRequested publication. + // Keep the barrier through persistence and approval event/waiter publication. // No asynchronous work or user callbacks run in this critical section. Ok(register()) } diff --git a/crates/openhuman-core/src/security/approval/registration_scope_tests.rs b/crates/openhuman-core/src/security/approval/registration_scope_tests.rs index 06778f54710..7436514b9a0 100644 --- a/crates/openhuman-core/src/security/approval/registration_scope_tests.rs +++ b/crates/openhuman-core/src/security/approval/registration_scope_tests.rs @@ -4,9 +4,11 @@ use std::sync::{mpsc, Arc}; #[test] fn removal_refuses_late_registration_without_creating_a_pending_row() { let scope = ApprovalScope::default(); + assert!(!scope.is_closed()); scope.close("agent_removed"); + assert!(scope.is_closed()); let mut created = false; - let result = scope.register(|| created = true); + let result = scope.with_open(|| created = true); assert_eq!(result, Err("agent_removed".to_owned())); assert!(!created); } @@ -18,7 +20,7 @@ fn accepted_registration_holds_the_removal_barrier_until_it_finishes() { let (release_tx, release_rx) = mpsc::channel(); let registering = scope.clone(); let registration = std::thread::spawn(move || { - registering.register(|| { + registering.with_open(|| { entered_tx.send(()).unwrap(); release_rx.recv().unwrap(); }) @@ -37,7 +39,7 @@ fn accepted_registration_holds_the_removal_barrier_until_it_finishes() { "registration released the removal barrier before finishing" ); scope.close("agent_removed"); - assert_eq!(scope.register(|| ()), Err("agent_removed".to_owned())); + assert_eq!(scope.with_open(|| ()), Err("agent_removed".to_owned())); } #[test] @@ -45,6 +47,6 @@ fn a_reused_agent_id_has_a_fresh_scope_and_preserves_the_old_removal_reason() { let old = ApprovalScope::default(); old.close("agent_removed"); old.close("agent_dropped"); - assert_eq!(old.register(|| ()), Err("agent_removed".to_owned())); - assert_eq!(ApprovalScope::default().register(|| 42), Ok(42)); + assert_eq!(old.with_open(|| ()), Err("agent_removed".to_owned())); + assert_eq!(ApprovalScope::default().with_open(|| 42), Ok(42)); } diff --git a/crates/openhuman-embed/src/agent/build.rs b/crates/openhuman-embed/src/agent/build.rs index e6b5a87a6e7..cfc609e42f6 100644 --- a/crates/openhuman-embed/src/agent/build.rs +++ b/crates/openhuman-embed/src/agent/build.rs @@ -265,9 +265,8 @@ pub(crate) fn instantiate(runtime: &Runtime, spec: AgentSpec) -> Result Result, + scope: Arc, } impl ApprovalState { @@ -38,11 +39,14 @@ impl ApprovalState { } } - fn claim_removal(&self) -> bool { + fn claim_removal(&self, reason: &str) -> bool { let _decision = self .decisions .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + // The shared gate closes before this claim becomes observable, so + // RPC/chat decisions cannot approve during the later denial snapshot. + self.scope.close(reason); !self.claimed.swap(true, Ordering::SeqCst) } } @@ -60,13 +64,13 @@ impl Lifecycle { /// Stops admitting turns and ends the ones in flight; returns whether this /// caller claimed removal before another caller. - pub(crate) fn mark_removed(&self) -> bool { - self.mark_removed_with(|| {}) + pub(crate) fn mark_removed(&self, reason: &str) -> bool { + self.mark_removed_with(reason, || {}) } /// Claim removal and settle owned approvals before waking in-flight turns. - pub(crate) fn mark_removed_with(&self, before_notify: impl FnOnce()) -> bool { - if !self.approvals.claim_removal() { + pub(crate) fn mark_removed_with(&self, reason: &str, before_notify: impl FnOnce()) -> bool { + if !self.approvals.claim_removal(reason) { return false; } before_notify(); @@ -80,6 +84,11 @@ impl Lifecycle { self.approvals.clone() } + /// The same instance barrier captured by every request at the shared gate. + pub(crate) fn approval_scope(&self) -> Arc { + self.approvals.scope.clone() + } + /// Bind handles to this agent instance, even after its public id is reused. pub(crate) fn removed(&self) -> watch::Receiver { self.removed.subscribe() diff --git a/crates/openhuman-embed/src/agent/lifecycle_tests.rs b/crates/openhuman-embed/src/agent/lifecycle_tests.rs index 2890887c3ad..85ce46721a0 100644 --- a/crates/openhuman-embed/src/agent/lifecycle_tests.rs +++ b/crates/openhuman-embed/src/agent/lifecycle_tests.rs @@ -16,7 +16,7 @@ async fn a_live_agent_runs_its_turns() { #[tokio::test] async fn a_removed_agent_refuses_new_turns() { let lifecycle = Lifecycle::new(); - lifecycle.mark_removed(); + lifecycle.mark_removed("agent_removed"); let ran = Arc::new(AtomicBool::new(false)); let flag = Arc::clone(&ran); let outcome = lifecycle @@ -43,7 +43,7 @@ async fn removal_ends_a_turn_in_flight_and_the_agent_goes_idle() { } assert!(!lifecycle.wait_idle(Duration::from_millis(10)).await); - lifecycle.mark_removed(); + lifecycle.mark_removed("agent_removed"); assert!(removed(turn.await.expect("turn task"))); assert!(lifecycle.wait_idle(Duration::from_secs(1)).await); @@ -63,7 +63,7 @@ async fn a_claimed_removal_refuses_turns_before_approvals_are_settled() { let (entered, callback_entered) = std::sync::mpsc::channel(); let (release, callback_release) = std::sync::mpsc::channel(); let removal = std::thread::spawn(move || { - removing.mark_removed_with(|| { + removing.mark_removed_with("agent_removed", || { entered.send(()).unwrap(); callback_release.recv().unwrap(); }) @@ -88,20 +88,26 @@ async fn a_claimed_removal_refuses_turns_before_approvals_are_settled() { #[test] fn a_claimed_removal_refuses_approval_decisions_before_denial_finishes() { let lifecycle = Arc::new(Lifecycle::new()); + let shared_scope = lifecycle.approval_scope(); let approvals = crate::Approvals::new("a", lifecycle.removed(), lifecycle.approval_state()); let removing = lifecycle.clone(); let (entered, callback_entered) = std::sync::mpsc::channel(); let (release, callback_release) = std::sync::mpsc::channel(); let removal = std::thread::spawn(move || { - removing.mark_removed_with(|| { + removing.mark_removed_with("agent_removed", || { entered.send(()).unwrap(); callback_release.recv().unwrap(); }) }); callback_entered.recv().unwrap(); + let core_closed = shared_scope.is_closed(); let outcome = approvals.decide("pending-request", crate::ApprovalDecision::ApproveOnce); release.send(()).unwrap(); assert!(removal.join().unwrap()); + assert!( + core_closed, + "the shared gate remained open after removal was claimed" + ); assert!( matches!(outcome, Err(crate::ApprovalsError::NotFound(_))), "{outcome:?}" @@ -132,7 +138,7 @@ fn accepted_approval_decisions_hold_the_removal_claim_barrier() { held, "removal could claim while a decision was still being applied" ); - assert!(state.claim_removal()); + assert!(state.claim_removal("agent_removed")); let mut ran = false; assert_eq!(state.with_live(|| ran = true), None); assert!(!ran); diff --git a/crates/openhuman-embed/src/agent/mod.rs b/crates/openhuman-embed/src/agent/mod.rs index 61d178dfd6d..13ea47e0151 100644 --- a/crates/openhuman-embed/src/agent/mod.rs +++ b/crates/openhuman-embed/src/agent/mod.rs @@ -145,7 +145,7 @@ impl AgentInner { if !self.lifecycle.begin_teardown() { return; } - self.lifecycle.mark_removed(); + self.lifecycle.mark_removed(resolution); self._runtime_guard.events.emit( Some(self.id.clone()), None, diff --git a/crates/openhuman-embed/src/runtime/lifecycle.rs b/crates/openhuman-embed/src/runtime/lifecycle.rs index c216451c614..13587a4c90e 100644 --- a/crates/openhuman-embed/src/runtime/lifecycle.rs +++ b/crates/openhuman-embed/src/runtime/lifecycle.rs @@ -115,7 +115,7 @@ impl Runtime { }; if !inner .lifecycle - .mark_removed_with(|| inner.deny_approvals("agent_removed")) + .mark_removed_with("agent_removed", || inner.deny_approvals("agent_removed")) { return Err(AgentError::UnknownId(id.to_string())); } diff --git a/crates/openhuman-embed/tests/agent_lifecycle.rs b/crates/openhuman-embed/tests/agent_lifecycle.rs index 7e1169153c5..e491275512d 100644 --- a/crates/openhuman-embed/tests/agent_lifecycle.rs +++ b/crates/openhuman-embed/tests/agent_lifecycle.rs @@ -101,6 +101,90 @@ fn removing_an_agent_releases_everything_it_held() { .receiver(); let scratch = tempfile::tempdir().expect("scratch dir"); + // The removal barrier must govern every shared-gate entry point, + // including RPC/chat calls made outside the Embed approvals facade. + // Close the exact instance scope before its denial snapshot runs. + let (closing, _closing_provider) = + parking_agent(&runtime, "closing", scratch.path()).await; + let closing_turn = { + let closing = closing.clone(); + tokio::spawn(async move { closing.turn("write the marker").send().await }) + }; + let closing_request = eventually("closing agent to park", || { + closing.approvals().pending().ok()?.into_iter().next() + }) + .await; + let closing_context = AgentContextRegistry::get("closing").unwrap(); + closing_context + .host_overrides() + .unwrap() + .approval_scope() + .unwrap() + .close("agent_removed"); + let gate = openhuman_core::security::approval::ApprovalGate::try_global().unwrap(); + for decision in [ + openhuman_embed::ApprovalDecision::ApproveOnce, + openhuman_embed::ApprovalDecision::ApproveAlwaysForTool, + openhuman_embed::ApprovalDecision::Deny, + ] { + assert!( + gate.decide(&closing_request.request_id, decision) + .unwrap() + .is_none(), + "a closed instance must refuse direct gate decisions" + ); + assert!(gate + .decide_for_agent("closing", &closing_request.request_id, decision) + .unwrap() + .is_none()); + } + assert_eq!( + gate.classify_decide_miss(&closing_request.request_id), + openhuman_core::security::approval::gate::DecideMiss::AlreadyResolved, + "a removal barrier is a benign decision miss, not a lost registration" + ); + for owner in [None, Some("closing")] { + assert!(openhuman_core::security::approval::rpc::approval_decide( + &closing_request.request_id, + openhuman_embed::ApprovalDecision::ApproveOnce, + owner, + ) + .await + .is_err()); + } + assert_eq!(closing.approvals().pending().unwrap().len(), 1); + runtime.remove_agent("closing").await.unwrap(); + removed(closing_turn.await.unwrap(), "closing"); + assert!(!scratch.path().join("closing-wrote").exists()); + assert!(gate + .list_pending_for_agent(Some("closing")) + .unwrap() + .is_empty()); + + // Reusing the public id installs a fresh barrier: the new request + // remains answerable through the same context-free gate entry point. + let (replacement, _replacement_provider) = + parking_agent(&runtime, "closing", scratch.path()).await; + let replacement_turn = { + let replacement = replacement.clone(); + tokio::spawn(async move { replacement.turn("write the marker").send().await }) + }; + let replacement_request = eventually("replacement to park", || { + replacement.approvals().pending().ok()?.into_iter().next() + }) + .await; + assert_ne!(replacement_request.request_id, closing_request.request_id); + assert!(gate + .decide( + &replacement_request.request_id, + openhuman_embed::ApprovalDecision::ApproveOnce, + ) + .unwrap() + .is_some()); + assert!(replacement_turn.await.unwrap().is_ok()); + assert!(scratch.path().join("closing-wrote").exists()); + runtime.remove_agent("closing").await.unwrap(); + // ── an unknown id is refused ── let unknown = runtime .remove_agent("nope") From d4bd6c5fd3340e83a10316c3110cc0f1b2d2a1a2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 11 Oct 2026 00:06:10 +0300 Subject: [PATCH 12/18] docs(embed): refresh compiled capabilities and removal coverage Co-authored-by: Medulla --- docs/TEST-COVERAGE-MATRIX.md | 4 ++-- docs/gitbooks/en/developing/embed/api-index.json | 2 +- docs/gitbooks/en/developing/embed/api-index.md | 2 +- gitbooks/developing/embed/api-index.json | 2 +- gitbooks/developing/embed/api-index.md | 2 +- llms-full.txt | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/TEST-COVERAGE-MATRIX.md b/docs/TEST-COVERAGE-MATRIX.md index a7d1ab78575..c0e04119f47 100644 --- a/docs/TEST-COVERAGE-MATRIX.md +++ b/docs/TEST-COVERAGE-MATRIX.md @@ -659,7 +659,7 @@ The thread JSONL store moved to `tinyagents_session::threads` (`vendor/tinyagent | 16.1.8 | MCP servers declared programmatically | RU | `crates/openhuman-embed/src/harness/mcp_tests.rs` | ✅ | stdio/HTTP transports mutually exclusive, enabled by default, env accumulation, allow/deny filters, auth carried through | | 16.1.9 | Skill bundles installed into the workspace | RU | `crates/openhuman-embed/src/harness/skills_tests.rs` | ✅ | Copied not linked (discovery rejects symlinked bundles and manifests by design); symlinks skipped both as bundles and inside them; `Inherit` refuses the copy | | 16.1.10 | One runtime per process | RU+RI | `crates/openhuman-embed/src/runtime/builder_tests.rs`, `crates/openhuman-embed/tests/harness_embed.rs` | ✅ | A second `Runtime` (or `Harness`) returns `AlreadyRunning` rather than sharing process-global keyring/event-bus/subscribers; a failed build releases the slot so a retry is possible | -| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs`, `crates/openhuman-embed/tests/agent_lifecycle.rs`, `crates/openhuman-core/src/security/approval/registration_scope_tests.rs`, `crates/openhuman-embed/src/agent/lifecycle_tests.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused; removal settles pending approvals with `agent_removed` before cancelling turns; a claimed removal immediately refuses new admission; instance-owned approval registration closes before the denial snapshot and rejects late registrations without persisting rows; polling and callback approval decisions serialize with the removal claim | +| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs`, `crates/openhuman-embed/tests/agent_lifecycle.rs`, `crates/openhuman-core/src/security/approval/registration_scope_tests.rs`, `crates/openhuman-embed/src/agent/lifecycle_tests.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused; removal settles pending approvals with `agent_removed` before cancelling turns; a claimed removal immediately refuses new admission; instance-owned approval registration closes before the denial snapshot and rejects late registrations without persisting rows; polling and callback approval decisions serialize with the removal claim; the shared gate also refuses direct, agent-scoped and RPC decisions during the denial-snapshot gap | | 16.1.12 | API-key credential for library mode | RU+RI | `crates/openhuman-core/src/security/credentials/api_key_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs` | ✅ | Key stored as an `api-key` auth profile; wins over an expired session; managed inference sends it as a bearer with no `x-api-key`; auth state reports `credential = "api-key"` | | 16.1.14 | Host-owned tool policy extension | RI | `crates/openhuman-embed/tests/host_only_tools.rs` | ✅ | Host executors declare read-only requirements through the public `ToolPolicy` facade; HostOnly advertises the exact host catalog and refuses built-in shell/write attempts. Application-specific query validation and data handling belong to the host. | | 16.1.15 | Strict structured output and bounded repair | RI | `crates/openhuman-embed/tests/structured_validation.rs`, `crates/openhuman-embed/tests/structured_turns.rs`, `crates/openhuman-embed/tests/unit/turn_usage.rs` | ✅ | Wrong types and numeric combinators refused, invalid/external schemas make no call, repair bounded with usage, terminal length rejected even for complete JSON; successful charged usage reaches outcome/meter, invalid buyer charges stay unknown, root overlays retain child spend and session context. | @@ -667,7 +667,7 @@ The thread JSONL store moved to `tinyagents_session::threads` (`vendor/tinyagent | 16.1.19 | Pinned standalone Embed source consumer | Script+RI | `scripts/__tests__/embed-consumer-bootstrap.test.mjs` | ✅ | Exact SHA verification, inherited workspace edition, generated active patches, stable locked offline builds with optional Embed enabled/disabled, default dependency tree without HTTP, no untracked secrets or Git metadata, refused malformed/escaping patches and existing destinations | | 16.1.17 | Enforced shared turn/run budgets | RU+RI | `crates/openhuman-embed/tests/budget_fanout.rs`, `vendor/tinyagents/vendor/tinyinference/crates/tinyinference-llm/src/model/budget_tests.rs` | ✅ | Atomic parent/child reservations, physical provider admission, conservative unknown spend, cancellation and output cap | | 16.1.18 | Ordered bounded fanout | RI | `crates/openhuman-embed/tests/budget_fanout.rs` | ✅ | Input-order results, per-branch and child error isolation, branch ceilings, shared-budget concurrent refusal and empty fanout | -| 16.1.13 | Awaited per-turn cancellation | RU+RI | `crates/openhuman-embed/tests/turn_cancellation.rs`, `crates/openhuman-embed/tests/process_cancellation.rs`, `crates/openhuman-embed/src/turn_control_tests.rs` | ✅ | Before send, during inference and during a builtin shell command; agent reuse, concurrent cancellation handles, failure/drop acknowledgement, bounded and unbounded descendant termination plus direct-child reaping on Linux; deadline errors stay typed when the native cancellation relay interrupts dispatch; later cancellation/deadlines do not replace completed outcomes | +| 16.1.13 | Awaited per-turn cancellation | RU+RI | `crates/openhuman-embed/tests/turn_cancellation.rs`, `crates/openhuman-embed/tests/process_cancellation.rs`, `crates/openhuman-embed/src/turn_control_tests.rs` | ✅ | Before send, during inference and during a builtin shell command; agent reuse, concurrent cancellation handles, failure/drop acknowledgement, bounded and unbounded descendant termination plus direct-child reaping on Linux; cancellation acknowledgement waits for descendants even when they close output pipes; deadline errors stay typed when the native cancellation relay interrupts dispatch; later cancellation/deadlines do not replace completed outcomes | | 16.1.21 | Agent and turn hook isolation | RI | `crates/openhuman-embed/tests/scoped_hooks.rs` | ✅ | Additive same-named callbacks, overlapping worker turns, one-turn callback lifetime, resumed-session snapshots, core-scheduled turn callbacks and reuse of a removed agent id without registry leakage | | 16.1.22 | Agent-turn route attribution headers | RU+RI | `crates/openhuman-embed/tests/route_headers.rs`, `crates/openhuman-embed/src/turn_tests.rs` | ✅ | Per-route headers reach only the named endpoint and turn, remain absent on other agents and backend requests, and are redacted in Debug; wire fields pinned against the registered controller schema | | 16.1.23 | Tool-hook turn identity and cwd | RU+RI | `crates/openhuman-embed/tests/tool_hook_context.rs`, `crates/openhuman-core/src/hooks/bridge_tests.rs` | ✅ | Before and after hooks report the worker/session and working root used by builtin shell; a later turn returns to its agent default; configured hooks use resolved cwd even when tool arguments omit it | diff --git a/docs/gitbooks/en/developing/embed/api-index.json b/docs/gitbooks/en/developing/embed/api-index.json index 456344da074..e3ec1b6e36b 100644 --- a/docs/gitbooks/en/developing/embed/api-index.json +++ b/docs/gitbooks/en/developing/embed/api-index.json @@ -444,7 +444,7 @@ "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/docs/gitbooks/en/developing/embed/api-index.md b/docs/gitbooks/en/developing/embed/api-index.md index ddf5ad9bfe1..9b495063a64 100644 --- a/docs/gitbooks/en/developing/embed/api-index.md +++ b/docs/gitbooks/en/developing/embed/api-index.md @@ -196,7 +196,7 @@ The compiled capability report describes this build: "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/gitbooks/developing/embed/api-index.json b/gitbooks/developing/embed/api-index.json index 456344da074..e3ec1b6e36b 100644 --- a/gitbooks/developing/embed/api-index.json +++ b/gitbooks/developing/embed/api-index.json @@ -444,7 +444,7 @@ "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/gitbooks/developing/embed/api-index.md b/gitbooks/developing/embed/api-index.md index ddf5ad9bfe1..9b495063a64 100644 --- a/gitbooks/developing/embed/api-index.md +++ b/gitbooks/developing/embed/api-index.md @@ -196,7 +196,7 @@ The compiled capability report describes this build: "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, diff --git a/llms-full.txt b/llms-full.txt index baf1935eacb..f1a2d9f0664 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -2750,7 +2750,7 @@ This matrix comes from the default-feature compiled capability-report example. R | `skills` | Yes | | `storage-file` | No | | `storage-mongodb` | No | -| `storage-sqlite` | No | +| `storage-sqlite` | Yes | | `tinymemes` | Yes | | `voice` | No | | `web3` | No | @@ -3005,7 +3005,7 @@ The compiled capability report describes this build: "skills": true, "storage-file": false, "storage-mongodb": false, - "storage-sqlite": false, + "storage-sqlite": true, "tinymemes": true, "voice": false, "web3": false, From dfddbad81809d5b4e0085f6c42d531e3b60548fd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 11 Oct 2026 00:38:36 +0300 Subject: [PATCH 13/18] fix: close agent admission before approval barriers and surface flows atomically Co-authored-by: Medulla --- .../src/security/approval/flow_surface.rs | 22 ++++ .../security/approval/flow_surface_tests.rs | 83 ++++++++++++++ .../src/security/approval/gate_intercept.rs | 108 ++++++++---------- .../src/security/approval/mod.rs | 1 + .../security/approval/registration_scope.rs | 24 ++-- .../approval/registration_scope_tests.rs | 36 ++++++ crates/openhuman-embed/src/agent/lifecycle.rs | 12 +- .../src/agent/lifecycle_tests.rs | 48 ++++++++ docs/TEST-COVERAGE-MATRIX.md | 2 +- 9 files changed, 257 insertions(+), 79 deletions(-) create mode 100644 crates/openhuman-core/src/security/approval/flow_surface.rs create mode 100644 crates/openhuman-core/src/security/approval/flow_surface_tests.rs diff --git a/crates/openhuman-core/src/security/approval/flow_surface.rs b/crates/openhuman-core/src/security/approval/flow_surface.rs new file mode 100644 index 00000000000..e15eaa4b753 --- /dev/null +++ b/crates/openhuman-core/src/security/approval/flow_surface.rs @@ -0,0 +1,22 @@ +//! Orders request registration and its approval surfaces after workspace resolution. + +use super::ApprovalScope; + +/// Resolve the workspace before registering and publishing any approval surface. +/// Removal during resolution refuses registration; accepted registration and +/// publication finish together before removal. No barrier is held across the await. +pub(super) async fn register_after_workspace( + scope: Option<&ApprovalScope>, + workspace: impl std::future::Future, + register: impl FnOnce(W) -> T, +) -> Result { + let workspace = workspace.await; + match scope { + Some(scope) => scope.with_open(|| register(workspace)), + None => Ok(register(workspace)), + } +} + +#[cfg(test)] +#[path = "flow_surface_tests.rs"] +mod tests; diff --git a/crates/openhuman-core/src/security/approval/flow_surface_tests.rs b/crates/openhuman-core/src/security/approval/flow_surface_tests.rs new file mode 100644 index 00000000000..9f6233bfd67 --- /dev/null +++ b/crates/openhuman-core/src/security/approval/flow_surface_tests.rs @@ -0,0 +1,83 @@ +use super::*; +use std::cell::Cell; +use std::future::Future; +use std::sync::Arc; +use std::task::{Context, Poll, Wake, Waker}; + +struct Noop; +impl Wake for Noop { + fn wake(self: Arc) {} +} + +#[test] +fn a_closed_instance_registers_nothing_and_publishes_neither_flow_surface() { + let scope = ApprovalScope::default(); + scope.close("agent_removed"); + let publications = Cell::new(0); + let mut publication = std::pin::pin!(register_after_workspace( + Some(&scope), + std::future::ready(17), + |_| publications.set(publications.get() + 1), + )); + let waker = Waker::from(Arc::new(Noop)); + assert_eq!( + publication.as_mut().poll(&mut Context::from_waker(&waker)), + Poll::Ready(Err("agent_removed".to_owned())) + ); + assert_eq!(publications.get(), 0); +} + +#[test] +fn removal_during_workspace_lookup_suppresses_registration_and_all_surfaces() { + let scope = ApprovalScope::default(); + let ready = Cell::new(false); + let workspace = std::future::poll_fn(|_| { + if ready.get() { + Poll::Ready(17) + } else { + Poll::Pending + } + }); + let publications = Cell::new(0); + let mut publication = std::pin::pin!(register_after_workspace(Some(&scope), workspace, |_| { + publications.set(publications.get() + 1) + },)); + let waker = Waker::from(Arc::new(Noop)); + let mut cx = Context::from_waker(&waker); + assert_eq!(publication.as_mut().poll(&mut cx), Poll::Pending); + assert_eq!( + publications.get(), + 0, + "registration preceded workspace resolution" + ); + scope.close("agent_removed"); + ready.set(true); + assert_eq!( + publication.as_mut().poll(&mut cx), + Poll::Ready(Err("agent_removed".to_owned())) + ); + assert_eq!(publications.get(), 0); +} + +#[test] +fn a_live_or_unscoped_flow_registers_once_with_its_resolved_workspace() { + let scope = ApprovalScope::default(); + for scope in [Some(&scope), None] { + let publications = Cell::new(0); + let mut publication = std::pin::pin!(register_after_workspace( + scope, + std::future::ready(Some(("ws_test", 17))), + |workspace| { + assert_eq!(workspace, Some(("ws_test", 17))); + publications.set(publications.get() + 1); + 42 + }, + )); + let waker = Waker::from(Arc::new(Noop)); + assert_eq!( + publication.as_mut().poll(&mut Context::from_waker(&waker)), + Poll::Ready(Ok(42)) + ); + assert_eq!(publications.get(), 1); + } +} diff --git a/crates/openhuman-core/src/security/approval/gate_intercept.rs b/crates/openhuman-core/src/security/approval/gate_intercept.rs index d4d95a3ec9a..75eb097e748 100644 --- a/crates/openhuman-core/src/security/approval/gate_intercept.rs +++ b/crates/openhuman-core/src/security/approval/gate_intercept.rs @@ -503,7 +503,7 @@ impl ApprovalGate { let (tx, rx) = oneshot::channel::(); let scope = crate::core::runtime::CoreContext::current_host_overrides() .and_then(|overrides| overrides.approval_scope()); - let register = || -> anyhow::Result<()> { + let register = |workspace| -> anyhow::Result<()> { { let mut waiters = self.waiters.lock(); waiters.insert(request_id.clone(), tx); @@ -554,12 +554,55 @@ impl ApprovalGate { expires_at: expires_at.map(|t| t.to_rfc3339()), agent_id: agent_id.clone(), }); + // Publish every surface inside the same registration barrier. + // Workspace lookup completed before creating a row, so neither + // removal nor a fast human decision can precede these surfaces. + if let Some(ApprovalSourceContext::Flow { + flow_id, run_id, .. + }) = &source_context + { + tracing::info!(request_id = %request_id, flow_id = %flow_id, + run_id = %run_id, tool = tool_name, + "[approval::gate] flow-origin park — surfacing flow_approval_request + notification"); + BUS.publish(DomainEvent::FlowApprovalRequested { + request_id: request_id.clone(), + flow_id: flow_id.clone(), + run_id: run_id.clone(), + tool_name: tool_name.to_string(), + summary: action_summary.to_string(), + agent_id: agent_id.clone(), + }); + publish_flow_gate_notification( + &request_id, + flow_id, + run_id, + tool_name, + action_summary, + workspace, + ); + } Ok(()) }; - let registered = match scope.as_ref() { - Some(scope) => scope.with_open(register), - None => Ok(register()), - }; + let registered = super::flow_surface::register_after_workspace( + scope.as_deref(), + async { + if !matches!(&source_context, Some(ApprovalSourceContext::Flow { .. })) { + return None; + } + // Resolve before registration; never hold the barrier across + // asynchronous workspace I/O. An unbound live notification + // still surfaces the request when workspace lookup fails. + match crate::config::active_workspace_snapshot().await { + Ok((dir, revision)) => Some((crate::config::workspace_handle(&dir), revision)), + Err(error) => { + tracing::warn!(request_id = %request_id, + "[approval::gate] could not resolve the active workspace for the flow approval notification ({error}); publishing it unbound"); + None + } + } + }, + register, + ).await; match registered { Err(resolution) => { return ( @@ -584,61 +627,6 @@ impl ApprovalGate { Ok(Ok(())) => {} } - // Flow-origin surface bridge (flow-approval-surface, PR3): a flow run - // has no chat thread/client to route the generic `ApprovalRequested` - // through (both are `None` above, so the web-channel bridge silently - // drops it — see `web_chat::event_bus`'s - // `ApprovalSurfaceSubscriber`), which is exactly the silent-deadlock - // bug this correlation fixes. Broadcast a dedicated - // `flow_approval_request` socket event (no thread/client required, - // unlike the chat path) plus a `CoreNotification` with the three - // flow-scoped decision actions, so the Workflows UI can surface and - // resolve the park without polling. - if let Some(ApprovalSourceContext::Flow { - flow_id, run_id, .. - }) = &source_context - { - tracing::info!( - request_id = %request_id, - flow_id = %flow_id, - run_id = %run_id, - tool = tool_name, - "[approval::gate] flow-origin park — surfacing flow_approval_request + notification" - ); - BUS.publish(DomainEvent::FlowApprovalRequested { - request_id: request_id.clone(), - flow_id: flow_id.clone(), - run_id: run_id.clone(), - tool_name: tool_name.to_string(), - summary: action_summary.to_string(), - agent_id: agent_id.clone(), - }); - // The workspace the flow parked in, so the approval banner is - // dropped by a client that has since switched away rather than - // approving this workspace's call from another one. Fails open on - // a resolve failure: an unbound notification still reaches the - // user, whereas not publishing recreates the silent deadlock this - // bridge exists to fix. - let workspace = match crate::config::active_workspace_snapshot().await { - Ok((dir, revision)) => Some((crate::config::workspace_handle(&dir), revision)), - Err(error) => { - tracing::warn!( - request_id = %request_id, - "[approval::gate] could not resolve the active workspace for the flow approval notification ({error}); publishing it unbound" - ); - None - } - }; - publish_flow_gate_notification( - &request_id, - flow_id, - run_id, - tool_name, - action_summary, - workspace, - ); - } - tracing::info!( request_id = %request_id, tool = tool_name, diff --git a/crates/openhuman-core/src/security/approval/mod.rs b/crates/openhuman-core/src/security/approval/mod.rs index 030be04e7a4..b3cb6840eda 100644 --- a/crates/openhuman-core/src/security/approval/mod.rs +++ b/crates/openhuman-core/src/security/approval/mod.rs @@ -13,6 +13,7 @@ //! list-based `ApprovalManager` that consumed it was removed once the gate //! became the sole control. +mod flow_surface; pub mod gate; pub mod redact; mod registration_scope; diff --git a/crates/openhuman-core/src/security/approval/registration_scope.rs b/crates/openhuman-core/src/security/approval/registration_scope.rs index 3017d2f70d9..a171e42729a 100644 --- a/crates/openhuman-core/src/security/approval/registration_scope.rs +++ b/crates/openhuman-core/src/security/approval/registration_scope.rs @@ -1,41 +1,39 @@ //! Serializes an agent instance's approval registration and decisions with removal. -use std::sync::Mutex; +use std::sync::{Mutex, OnceLock}; /// An instance-owned barrier; reusing an agent ID requires a fresh scope. #[derive(Debug, Default)] pub struct ApprovalScope { - closed: Mutex>, + closed: Mutex<()>, + reason: OnceLock, } impl ApprovalScope { /// Stop registration and decisions; wait for accepted work to finish. /// The first teardown reason remains authoritative. pub fn close(&self, reason: &str) { - let mut closed = self + // Publish closure before waiting for accepted work. Turn admission and + // callers waiting to acquire the barrier must already see removal. + self.reason.get_or_init(|| reason.to_owned()); + let _closed = self .closed .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - if closed.is_none() { - *closed = Some(reason.to_owned()); - } } - /// Whether removal has closed this instance's registration and decision barrier. + /// Whether closing has begun, without waiting for accepted work to finish. pub fn is_closed(&self) -> bool { - self.closed - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .is_some() + self.reason.get().is_some() } /// Run synchronous registration or a decision while holding the removal barrier. pub(crate) fn with_open(&self, register: impl FnOnce() -> T) -> Result { - let closed = self + let _closed = self .closed .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - if let Some(reason) = closed.as_ref() { + if let Some(reason) = self.reason.get() { return Err(reason.clone()); } // Keep the barrier through persistence and approval event/waiter publication. diff --git a/crates/openhuman-core/src/security/approval/registration_scope_tests.rs b/crates/openhuman-core/src/security/approval/registration_scope_tests.rs index 7436514b9a0..c358c94ceac 100644 --- a/crates/openhuman-core/src/security/approval/registration_scope_tests.rs +++ b/crates/openhuman-core/src/security/approval/registration_scope_tests.rs @@ -1,6 +1,42 @@ use super::ApprovalScope; use std::sync::{mpsc, Arc}; +#[test] +fn closing_is_observable_before_an_accepted_registration_finishes() { + let scope = Arc::new(ApprovalScope::default()); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let registering = scope.clone(); + let registration = std::thread::spawn(move || { + registering.with_open(|| { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + }) + }); + entered_rx.recv().unwrap(); + let removing = scope.clone(); + let removal = std::thread::spawn(move || removing.close("agent_removed")); + let (observed_tx, observed_rx) = mpsc::channel(); + let observing = scope.clone(); + let observation = std::thread::spawn(move || { + while !observing.is_closed() { + std::thread::yield_now(); + } + observed_tx.send(()).unwrap(); + }); + // The registration stays suspended throughout this observation. A blocking + // is_closed cannot report removal until the registration is released. + let visible = observed_rx.recv_timeout(std::time::Duration::from_secs(2)); + release_tx.send(()).unwrap(); + assert_eq!(registration.join().unwrap(), Ok(())); + removal.join().unwrap(); + observation.join().unwrap(); + assert!( + visible.is_ok(), + "removal stayed invisible behind registration" + ); +} + #[test] fn removal_refuses_late_registration_without_creating_a_pending_row() { let scope = ApprovalScope::default(); diff --git a/crates/openhuman-embed/src/agent/lifecycle.rs b/crates/openhuman-embed/src/agent/lifecycle.rs index d112e963608..28d1d3a5bb2 100644 --- a/crates/openhuman-embed/src/agent/lifecycle.rs +++ b/crates/openhuman-embed/src/agent/lifecycle.rs @@ -32,7 +32,7 @@ impl ApprovalState { .decisions .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - if self.claimed.load(Ordering::SeqCst) { + if self.scope.is_closed() || self.claimed.load(Ordering::SeqCst) { None } else { Some(decide()) @@ -40,13 +40,15 @@ impl ApprovalState { } fn claim_removal(&self, reason: &str) -> bool { + // Publish closing before either mutex can block. Admission and all + // approval surfaces share that irreversible, nonblocking observation. + // Accepted core work finishes before the denial snapshot; accepted + // facade work finishes before this caller can claim that snapshot. + self.scope.close(reason); let _decision = self .decisions .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - // The shared gate closes before this claim becomes observable, so - // RPC/chat decisions cannot approve during the later denial snapshot. - self.scope.close(reason); !self.claimed.swap(true, Ordering::SeqCst) } } @@ -109,7 +111,7 @@ impl Lifecycle { let mut watcher = self.removed.subscribe(); // Claiming removal closes admission immediately; waking existing turns // waits until their approvals have been settled with the removal reason. - if self.approvals.claimed.load(Ordering::SeqCst) || *watcher.borrow_and_update() { + if self.approvals.scope.is_closed() || *watcher.borrow_and_update() { log::debug!("[embed][agent] turn refused: agent removed id={agent_id}"); return Err(removed()); } diff --git a/crates/openhuman-embed/src/agent/lifecycle_tests.rs b/crates/openhuman-embed/src/agent/lifecycle_tests.rs index 85ce46721a0..cc3e3390a23 100644 --- a/crates/openhuman-embed/src/agent/lifecycle_tests.rs +++ b/crates/openhuman-embed/src/agent/lifecycle_tests.rs @@ -5,6 +5,54 @@ fn removed(outcome: Result<(), CoreError>) -> bool { matches!(outcome, Err(CoreError::AgentRemoved { ref agent_id, .. }) if agent_id == "a") } +#[tokio::test] +async fn removal_refuses_turns_while_an_accepted_facade_decision_is_still_running() { + let lifecycle = Arc::new(Lifecycle::new()); + let approvals = lifecycle.approval_state(); + let deciding = approvals.clone(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let decision = std::thread::spawn(move || { + deciding.with_live(|| { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + 7 + }) + }); + entered_rx.recv().unwrap(); + let removing = lifecycle.clone(); + let removal = std::thread::spawn(move || removing.mark_removed("agent_removed")); + // Closing is observable while the facade decision still owns its mutex; + // the watch notification must wait for that accepted decision to finish. + let closing = tokio::time::timeout(Duration::from_secs(2), async { + while !lifecycle.approval_scope().is_closed() { + tokio::task::yield_now().await; + } + }) + .await; + let ran = AtomicBool::new(false); + let outcome = lifecycle + .admit("a", "test", async { + ran.store(true, Ordering::SeqCst); + Ok(()) + }) + .await; + let notified = *lifecycle.removed().borrow(); + release_tx.send(()).unwrap(); + assert_eq!(decision.join().unwrap(), Some(7)); + assert!(removal.join().unwrap()); + assert!( + closing.is_ok(), + "closing waited for the facade decision mutex" + ); + assert!( + !notified, + "accepted decision was not allowed to finish before notification" + ); + assert!(removed(outcome)); + assert!(!ran.load(Ordering::SeqCst)); +} + #[tokio::test] async fn a_live_agent_runs_its_turns() { let lifecycle = Lifecycle::new(); diff --git a/docs/TEST-COVERAGE-MATRIX.md b/docs/TEST-COVERAGE-MATRIX.md index c0e04119f47..823ade21c33 100644 --- a/docs/TEST-COVERAGE-MATRIX.md +++ b/docs/TEST-COVERAGE-MATRIX.md @@ -659,7 +659,7 @@ The thread JSONL store moved to `tinyagents_session::threads` (`vendor/tinyagent | 16.1.8 | MCP servers declared programmatically | RU | `crates/openhuman-embed/src/harness/mcp_tests.rs` | ✅ | stdio/HTTP transports mutually exclusive, enabled by default, env accumulation, allow/deny filters, auth carried through | | 16.1.9 | Skill bundles installed into the workspace | RU | `crates/openhuman-embed/src/harness/skills_tests.rs` | ✅ | Copied not linked (discovery rejects symlinked bundles and manifests by design); symlinks skipped both as bundles and inside them; `Inherit` refuses the copy | | 16.1.10 | One runtime per process | RU+RI | `crates/openhuman-embed/src/runtime/builder_tests.rs`, `crates/openhuman-embed/tests/harness_embed.rs` | ✅ | A second `Runtime` (or `Harness`) returns `AlreadyRunning` rather than sharing process-global keyring/event-bus/subscribers; a failed build releases the slot so a retry is possible | -| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs`, `crates/openhuman-embed/tests/agent_lifecycle.rs`, `crates/openhuman-core/src/security/approval/registration_scope_tests.rs`, `crates/openhuman-embed/src/agent/lifecycle_tests.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused; removal settles pending approvals with `agent_removed` before cancelling turns; a claimed removal immediately refuses new admission; instance-owned approval registration closes before the denial snapshot and rejects late registrations without persisting rows; polling and callback approval decisions serialize with the removal claim; the shared gate also refuses direct, agent-scoped and RPC decisions during the denial-snapshot gap | +| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs`, `crates/openhuman-embed/tests/agent_lifecycle.rs`, `crates/openhuman-core/src/security/approval/registration_scope_tests.rs`, `crates/openhuman-core/src/security/approval/flow_surface_tests.rs`, `crates/openhuman-embed/src/agent/lifecycle_tests.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused; removal settles pending approvals with `agent_removed` before cancelling turns; closing immediately refuses new admission even while an accepted registration or facade decision holds its barrier; instance-owned approval registration closes before the denial snapshot and rejects late registrations without persisting rows; polling and callback approval decisions serialize with the removal claim; the shared gate also refuses direct, agent-scoped and RPC decisions during the denial-snapshot gap; flow workspace resolution precedes registration, and registration publishes generic, flow and desktop approval surfaces together under the instance barrier | | 16.1.12 | API-key credential for library mode | RU+RI | `crates/openhuman-core/src/security/credentials/api_key_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs` | ✅ | Key stored as an `api-key` auth profile; wins over an expired session; managed inference sends it as a bearer with no `x-api-key`; auth state reports `credential = "api-key"` | | 16.1.14 | Host-owned tool policy extension | RI | `crates/openhuman-embed/tests/host_only_tools.rs` | ✅ | Host executors declare read-only requirements through the public `ToolPolicy` facade; HostOnly advertises the exact host catalog and refuses built-in shell/write attempts. Application-specific query validation and data handling belong to the host. | | 16.1.15 | Strict structured output and bounded repair | RI | `crates/openhuman-embed/tests/structured_validation.rs`, `crates/openhuman-embed/tests/structured_turns.rs`, `crates/openhuman-embed/tests/unit/turn_usage.rs` | ✅ | Wrong types and numeric combinators refused, invalid/external schemas make no call, repair bounded with usage, terminal length rejected even for complete JSON; successful charged usage reaches outcome/meter, invalid buyer charges stay unknown, root overlays retain child spend and session context. | From 299d67267f2ea0d2e1eb8fb08183c8f311a023ec Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 11 Oct 2026 00:42:07 +0300 Subject: [PATCH 14/18] fix(ci): honor fleet example build configuration Co-authored-by: Medulla --- scripts/__tests__/run-embed-examples.test.mjs | 20 +++++++++++++++++++ scripts/run-embed-examples.mjs | 20 ++++++++++++++++--- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/scripts/__tests__/run-embed-examples.test.mjs b/scripts/__tests__/run-embed-examples.test.mjs index ce3da711a49..8c896d40893 100644 --- a/scripts/__tests__/run-embed-examples.test.mjs +++ b/scripts/__tests__/run-embed-examples.test.mjs @@ -40,3 +40,23 @@ test("all example runs share the same required feature graph", () => { }); assert.deepEqual(featureArguments, ["mcp,skills", "mcp,skills"]); }); + + +test("fleet measurements retain their declared release and minimal-feature invocation", () => { + const examples = discoverExamples().filter(({name}) => name === "linux_fleet" || name === "mcp"); + const fleet = examples.find(({name}) => name === "linux_fleet"); + assert.equal(fleet.profile, "release"); + assert.equal(fleet.defaultFeatures, false); + const calls = new Map(); + runExamples({examples, environment: {}, run(_command, args) { + const name = args[args.indexOf("--example") + 1]; + calls.set(name, args); + return {status: 0, stdout: `EXAMPLE_OK ${name}\n`, stderr: ""}; + }}); + assert.ok(calls.get("linux_fleet").includes("--release")); + assert.ok(calls.get("linux_fleet").includes("--no-default-features")); + assert.equal(calls.get("linux_fleet").includes("--features"), false); + assert.equal(calls.get("mcp").includes("--release"), false); + assert.equal(calls.get("mcp").includes("--no-default-features"), false); + assert.ok(calls.get("mcp").includes("mcp")); +}); diff --git a/scripts/run-embed-examples.mjs b/scripts/run-embed-examples.mjs index 7f11b2b625d..4849c69e246 100644 --- a/scripts/run-embed-examples.mjs +++ b/scripts/run-embed-examples.mjs @@ -12,7 +12,9 @@ export function discoverExamples(directory = resolve(root, "crates/openhuman-emb if (!source.includes("//! Title:") || !source.includes("//! Run:") || !source.includes("// ANCHOR:")) { throw new Error(`${file}: missing example metadata or documentation anchor`); } - return { name: file.slice(0, -3), features: feature && feature !== "default" ? [feature] : [] }; + const profile = source.match(/^\/\/! Profile: (.+)$/m)?.[1]?.trim() ?? "dev"; + const defaultFeatures = source.match(/^\/\/! Default features: (.+)$/m)?.[1]?.trim() !== "disabled"; + return { name: file.slice(0, -3), features: feature && feature !== "default" ? [feature] : [], profile, defaultFeatures }; }); } export function offlineEnvironment(environment) { @@ -29,10 +31,22 @@ export function assertExampleOutput(name, result) { } } export function runExamples({ run = spawnSync, examples = discoverExamples(), environment = process.env } = {}) { - // Keep the feature graph identical for every run so Cargo reuses one build. - const features = [...new Set(examples.flatMap((example) => example.features))].sort(); + // Share one feature graph per declared build configuration. Measurements + // retain their release/minimal graph rather than inheriting other examples. + const configuration = (example) => `${example.profile ?? "dev"}:${example.defaultFeatures !== false}`; + const featureGroups = new Map(); + for (const example of examples) { + const key = configuration(example); + const features = featureGroups.get(key) ?? new Set(); + for (const feature of example.features) features.add(feature); + featureGroups.set(key, features); + } for (const example of examples) { const args = ["run", "--quiet", "-p", "openhuman-embed", "--example", example.name]; + if (example.profile === "release") args.push("--release"); + else if (example.profile && example.profile !== "dev") args.push("--profile", example.profile); + if (example.defaultFeatures === false) args.push("--no-default-features"); + const features = [...featureGroups.get(configuration(example))].sort(); if (features.length) args.push("--features", features.join(",")); const result = run(resolve(root, "scripts/ci-cancel-aware.sh"), ["cargo", ...args], { cwd: root, encoding: "utf8", env: offlineEnvironment(environment), timeout: 1800000, maxBuffer: 16 * 1024 * 1024, From 3392da4c6648ac546117d452dad5d03b4ef98b95 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 11 Oct 2026 00:53:15 +0300 Subject: [PATCH 15/18] docs(embed): fix strict rustdoc links and clarify approval surfaces Co-authored-by: Medulla --- .../src/security/approval/gate_intercept.rs | 4 ++-- crates/openhuman-embed/src/agent/spec.rs | 5 +++-- crates/openhuman-embed/src/turn_control.rs | 2 +- crates/openhuman-embed/src/turn_types.rs | 8 ++++---- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/crates/openhuman-core/src/security/approval/gate_intercept.rs b/crates/openhuman-core/src/security/approval/gate_intercept.rs index 75eb097e748..ec6e6c11579 100644 --- a/crates/openhuman-core/src/security/approval/gate_intercept.rs +++ b/crates/openhuman-core/src/security/approval/gate_intercept.rs @@ -555,8 +555,8 @@ impl ApprovalGate { agent_id: agent_id.clone(), }); // Publish every surface inside the same registration barrier. - // Workspace lookup completed before creating a row, so neither - // removal nor a fast human decision can precede these surfaces. + // Workspace lookup cannot delay a flow surface after the generic + // request; scoped removal and decisions wait for every surface. if let Some(ApprovalSourceContext::Flow { flow_id, run_id, .. }) = &source_context diff --git a/crates/openhuman-embed/src/agent/spec.rs b/crates/openhuman-embed/src/agent/spec.rs index 6ef9ad28d25..0a45d619a81 100644 --- a/crates/openhuman-embed/src/agent/spec.rs +++ b/crates/openhuman-embed/src/agent/spec.rs @@ -364,8 +364,9 @@ impl AgentSpec { /// The agent's own in-process tools, built fresh for every turn. /// /// Until this existed, an embedder's tools could only reach an agent over - /// [`mcp`](Self::mcp): a spec is data, and the session behind it is rebuilt - /// from that data on every turn, so a `Box` had nowhere to live + /// the `mcp` builder (with the `mcp` feature enabled): a spec is data, and + /// the session behind it is rebuilt from that data on every turn, so a + /// `Box` had nowhere to live /// in between. The cost was paid by the model — a discovery call to learn /// what the server offers, an `mcp_call_tool` envelope whose inner /// `arguments` object no provider can validate or constrain decoding diff --git a/crates/openhuman-embed/src/turn_control.rs b/crates/openhuman-embed/src/turn_control.rs index bcfa0aca633..1549866b976 100644 --- a/crates/openhuman-embed/src/turn_control.rs +++ b/crates/openhuman-embed/src/turn_control.rs @@ -13,7 +13,7 @@ impl Turn { /// Run the turn. /// /// Establishes the origin and progress scopes described in the module docs, - /// then dispatches through [`call`](crate::call::call) so the + /// then dispatches through the shared call handler so the /// `{result, logs}` envelope, [`DomainSet`](openhuman_core::core::runtime::DomainSet) /// gating and error classification are handled the same way as every other /// facade method. diff --git a/crates/openhuman-embed/src/turn_types.rs b/crates/openhuman-embed/src/turn_types.rs index 6ffe8dbee52..816e45ab75e 100644 --- a/crates/openhuman-embed/src/turn_types.rs +++ b/crates/openhuman-embed/src/turn_types.rs @@ -166,7 +166,7 @@ pub struct TurnRequest { #[serde(skip_serializing_if = "Option::is_none")] pub temperature: Option, /// Conversation this turn belongs to. The core does **not** mint one, so - /// [`Turn::send`] does; see [`TurnOutcome::session_id`]. + /// [`Turn::send`](crate::Turn::send) does; see [`TurnOutcome::session_id`]. #[serde(skip_serializing_if = "Option::is_none")] pub thread_id: Option, /// Per-turn working directory for the agent's filesystem and shell tools. @@ -215,14 +215,14 @@ pub struct TurnOutcome { pub reply: String, /// The conversation this turn ran in — the caller's `session_id` when one /// was supplied, otherwise the one minted for it. Pass it to the next - /// [`Turn::session`] to continue the conversation. + /// [`Turn::session`](crate::Turn::session) to continue the conversation. pub session_id: String, /// What the turn spent: tokens, cost, context window, and any synchronous /// children it ran. /// /// Present only when the turn returned. A turn that **failed** also spent /// what it spent, and there is no outcome to carry it on -- use - /// [`Turn::meter`] for that, which fires either way. + /// [`Turn::meter`](crate::Turn::meter) for that, which fires either way. /// /// `None` when the turn ran against a caller-built runtime's orchestrator /// rather than a runtime-owned [`Agent`](crate::Agent): that path answers @@ -230,7 +230,7 @@ pub struct TurnOutcome { /// report from. `None` also when the session reported nothing at all. pub usage: Option, /// [`reply`](Self::reply) parsed as JSON, when the turn asked for a JSON - /// [`response_format`](Turn::response_format) and the reply parses. + /// [`response_format`](crate::Turn::response_format) and the reply parses. /// `None` otherwise -- including a reply the model did not shape, which /// the host should treat as a failed structured answer. pub structured: Option, From 49fc2565723afefc61db2078ace938b76dd0ea27 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 11 Oct 2026 00:55:37 +0300 Subject: [PATCH 16/18] test(embed): exercise callback approvals through the scoped gate Co-authored-by: Medulla --- .../tests/approval_callbacks.rs | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 crates/openhuman-embed/tests/approval_callbacks.rs diff --git a/crates/openhuman-embed/tests/approval_callbacks.rs b/crates/openhuman-embed/tests/approval_callbacks.rs new file mode 100644 index 00000000000..7205df1caeb --- /dev/null +++ b/crates/openhuman-embed/tests/approval_callbacks.rs @@ -0,0 +1,105 @@ +//! Callback approvals use the same instance-owned gate as polling approvals. + +mod common; + +use std::sync::Arc; +use std::time::Duration; + +use common::{ + offline_config, route, runtime, scripted_provider, stub_backend, tool_call_completion, +}; +use openhuman_core::security::AutonomyLevel; +use openhuman_embed::{ + Access, AgentDefinitionSpec, AgentSpec, AgentTurnOrigin, ApprovalDecision, ApprovalHandler, + PendingApproval, Runtime, ToolScopeSpec, TrustedAccess, Workspace, +}; + +struct ApproveAndRecord(tokio::sync::mpsc::UnboundedSender); + +#[async_trait::async_trait] +impl ApprovalHandler for ApproveAndRecord { + async fn decide(&self, request: &PendingApproval) -> ApprovalDecision { + self.0 + .send(request.clone()) + .expect("approval receiver remains live"); + ApprovalDecision::ApproveOnce + } +} + +#[test] +fn callback_receives_the_owned_request_and_releases_its_parked_tool() { + runtime().block_on(async { tokio::spawn(scenario()).await.unwrap() }); +} + +async fn scenario() { + let backend = stub_backend().await; + let scratch = tempfile::tempdir().unwrap(); + let marker = scratch.path().join("approved-tool"); + let provider = scripted_provider( + vec![tool_call_completion( + "shell", + &serde_json::json!({"command": format!("touch {}", marker.display())}).to_string(), + )], + "approved reply", + ) + .await; + let mut config = offline_config(); + config.autonomy.enabled = true; + config.autonomy.level = AutonomyLevel::Full; + config.autonomy.auto_approve_all = true; + let runtime = Runtime::builder() + .config(config) + .workspace(Workspace::Ephemeral) + .backend_url(backend.uri()) + .access(Access::full()) + .build() + .await + .unwrap(); + let (requests, mut received) = tokio::sync::mpsc::unbounded_channel(); + let agent = runtime + .agent( + AgentSpec::new("callback-worker") + .provider(route(&provider, "callback-model")) + .access( + Access::supervised() + .origin(AgentTurnOrigin::WebChat { + thread_id: "callback-thread".into(), + client_id: "callback-client".into(), + request_id: None, + }) + .auto_approve(Vec::::new()) + .auto_approve_all(false) + .trust( + scratch.path().display().to_string(), + TrustedAccess::ReadWrite, + ), + ) + .definition( + AgentDefinitionSpec::new().tools(ToolScopeSpec::Named(vec!["shell".into()])), + ) + .approval_handler(Arc::new(ApproveAndRecord(requests))), + ) + .unwrap(); + let turn = tokio::spawn(agent.turn("Run the approved shell operation").send()); + let request = tokio::time::timeout(Duration::from_secs(10), received.recv()) + .await + .expect("the callback receives the parked request") + .expect("approval channel stays live"); + assert_eq!(request.agent_id.as_deref(), Some("callback-worker")); + assert_eq!(request.tool_name, "shell"); + assert!(!request.request_id.is_empty()); + let outcome = tokio::time::timeout(Duration::from_secs(10), turn) + .await + .expect("callback decision releases the turn") + .unwrap() + .unwrap(); + assert_eq!(outcome.reply, "approved reply"); + assert!(marker.exists(), "the approved tool actually executed"); + assert!(agent.approvals().pending().unwrap().is_empty()); + runtime.remove_agent("callback-worker").await.unwrap(); + assert!(agent.approvals().pending().unwrap().is_empty()); + assert!(matches!( + agent.turn("after removal").send().await, + Err(openhuman_embed::CoreError::AgentRemoved { .. }) + )); +} From 22e2ab292e7b04d68cd0227139f048ab0745bcd0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 11 Oct 2026 00:58:08 +0300 Subject: [PATCH 17/18] test(embed): cover flow approval surfaces and removed instances Co-authored-by: Medulla --- .../openhuman-embed/tests/flow_approvals.rs | 343 ++++++++++++++++++ 1 file changed, 343 insertions(+) create mode 100644 crates/openhuman-embed/tests/flow_approvals.rs diff --git a/crates/openhuman-embed/tests/flow_approvals.rs b/crates/openhuman-embed/tests/flow_approvals.rs new file mode 100644 index 00000000000..a2ce8e691a2 --- /dev/null +++ b/crates/openhuman-embed/tests/flow_approvals.rs @@ -0,0 +1,343 @@ +//! Flow approval surfaces use the embedded instance's workspace and removal barrier. + +mod common; + +use std::time::Duration; + +use openhuman_core::agent::turn_origin::{self, TrustedAutomationSource}; +use openhuman_core::core::events::DomainEvent; +use openhuman_core::core::runtime::{AgentContextRegistry, CoreContext}; +use openhuman_core::security::approval::gate::{FlowRunContext, APPROVAL_FLOW_RUN_CONTEXT}; +use openhuman_core::security::approval::{ApprovalGate, ApprovalSourceContext, GateOutcome}; +use openhuman_embed::{ + Access, AgentDefinitionSpec, AgentSpec, AgentTurnOrigin, ApprovalDecision, Runtime, + ToolScopeSpec, TrustedAccess, Workspace, +}; + +fn origin(flow_id: &str) -> AgentTurnOrigin { + AgentTurnOrigin::TrustedAutomation { + job_id: flow_id.to_owned(), + source: TrustedAutomationSource::Workflow { + require_approval: true, + }, + } +} + +fn run_context(flow_id: &str, run_id: &str) -> FlowRunContext { + FlowRunContext { + flow_id: flow_id.to_owned(), + run_id: run_id.to_owned(), + } +} + +#[test] +fn flow_surfaces_correlate_decisions_and_closed_instances_never_publish() { + common::runtime().block_on(async { + tokio::spawn(async { + let backend = common::stub_backend().await; + let scratch = tempfile::tempdir().unwrap(); + let marker = scratch.path().join("denied-flow-tool"); + let provider = common::scripted_provider( + vec![common::tool_call_completion( + "shell", + &serde_json::json!({"command": format!("touch '{}'", marker.display())}) + .to_string(), + )], + "flow-denied", + ) + .await; + let runtime = Runtime::builder() + .config(common::offline_config()) + .workspace(Workspace::Ephemeral) + .backend_url(backend.uri()) + .access(Access::supervised().auto_approve_all(false)) + .build() + .await + .unwrap(); + let spec = |id: &str, flow_id: &str| { + AgentSpec::new(id) + .provider(common::route(&provider, "flow-fixture")) + .action_dir(scratch.path()) + .access( + Access::supervised() + .origin(origin(flow_id)) + .auto_approve_all(false) + .auto_approve(Vec::::new()) + .trust( + scratch.path().display().to_string(), + TrustedAccess::ReadWrite, + ), + ) + .definition( + AgentDefinitionSpec::new() + .tools(ToolScopeSpec::Named(vec!["shell".to_owned()])), + ) + }; + let agent = runtime.agent(spec("flow-agent", "flow-live")).unwrap(); + let context = AgentContextRegistry::get("flow-agent").unwrap(); + let scope = context.host_overrides().unwrap().approval_scope().unwrap(); + assert!(!scope.is_closed()); + let (workspace, _) = CoreContext::scope( + context.clone(), + openhuman_core::config::active_workspace_snapshot(), + ) + .await + .unwrap(); + let workspace_handle = openhuman_core::config::workspace_handle(&workspace); + let gate = ApprovalGate::try_global().unwrap(); + let mut events = openhuman_core::core::bus::BUS.get().unwrap().receiver(); + let mut notifications = + openhuman_core::desktop::notifications::bus::subscribe_core_notifications(); + let parking_gate = gate.clone(); + let park = tokio::spawn(CoreContext::scope( + context, + turn_origin::with_origin(origin("flow-live"), async move { + APPROVAL_FLOW_RUN_CONTEXT + .scope(run_context("flow-live", "run-direct"), async move { + parking_gate + .intercept_audited_for_call( + "shell", + "flow action", + serde_json::json!({"command": "redacted"}), + Some("flow-call"), + ) + .await + }) + .await + }), + )); + let request_id = tokio::time::timeout(Duration::from_secs(5), async { + let mut generic_request = None; + loop { + match events.recv().await.unwrap() { + DomainEvent::ApprovalRequested { + request_id, + tool_name, + action_summary, + args_redacted, + thread_id, + client_id, + tool_call_id, + expires_at, + agent_id, + } if agent_id.as_deref() == Some("flow-agent") => { + assert_eq!(tool_name, "shell"); + assert_eq!(action_summary, "flow action"); + assert_eq!(args_redacted["command"], "redacted"); + assert_eq!((thread_id, client_id), (None, None)); + assert_eq!(tool_call_id.as_deref(), Some("flow-call")); + assert!(expires_at.is_some()); + generic_request = Some(request_id); + } + DomainEvent::FlowApprovalRequested { + request_id, + flow_id, + run_id, + tool_name, + summary, + agent_id, + } if flow_id == "flow-live" => { + assert_eq!(generic_request.as_deref(), Some(request_id.as_str())); + assert_eq!(run_id, "run-direct"); + assert_eq!(tool_name, "shell"); + assert_eq!(summary, "flow action"); + assert_eq!(agent_id.as_deref(), Some("flow-agent")); + break request_id; + } + _ => {} + } + } + }) + .await + .expect("flow and generic approval surfaces arrive together"); + let notification = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let notification = notifications.recv().await.unwrap(); + if notification.id == format!("flow-gate-approval:{request_id}") { + break notification; + } + } + }) + .await + .unwrap(); + assert_eq!( + notification.workspace.as_deref(), + Some(workspace_handle.as_str()) + ); + assert!(notification.workspace_revision.is_some()); + let actions = notification.actions.unwrap(); + assert_eq!( + actions + .iter() + .map(|action| action.action_id.as_str()) + .collect::>(), + ["approve_once", "approve_always_for_flow", "deny"] + ); + for action in actions { + let payload = action.payload.unwrap(); + assert_eq!(payload["request_id"], request_id); + assert_eq!(payload["flow_id"], "flow-live"); + assert_eq!(payload["run_id"], "run-direct"); + assert_eq!(payload["tool_name"], "shell"); + assert_eq!(payload["summary"], "flow action"); + assert_eq!(payload["decision"], action.action_id); + } + let rows = gate.list_pending_for_agent(Some("flow-agent")).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].request_id, request_id); + assert!(matches!(&rows[0].source_context, + Some(ApprovalSourceContext::Flow { flow_id, run_id, node_id }) + if flow_id == "flow-live" && run_id == "run-direct" && node_id.is_none())); + assert!(gate + .decide(&request_id, ApprovalDecision::Deny) + .unwrap() + .is_some()); + let (outcome, audited_request) = park.await.unwrap(); + assert!(matches!(outcome, GateOutcome::Deny { .. })); + // Only an allowed call returns an execution-audit id; a refusal + // correlates through the durable row and ApprovalDecided event. + assert!(audited_request.is_none()); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let DomainEvent::ApprovalDecided { + request_id: decided_request, + tool_name, + decision, + tool_call_id, + agent_id, + .. + } = events.recv().await.unwrap() + { + if decided_request == request_id { + assert_eq!(tool_name, "shell"); + assert_eq!(decision, "deny"); + assert_eq!(tool_call_id.as_deref(), Some("flow-call")); + assert_eq!(agent_id.as_deref(), Some("flow-agent")); + break; + } + } + } + }) + .await + .unwrap(); + assert!(gate + .decide(&request_id, ApprovalDecision::ApproveOnce) + .unwrap() + .is_none()); + + // Exercise the actual tool dispatcher too: a refused flow approval + // must produce a denied tool result and never create the marker. + let turn_agent = agent.clone(); + let turn = tokio::spawn(async move { + APPROVAL_FLOW_RUN_CONTEXT + .scope( + run_context("flow-live", "run-tool"), + turn_agent.turn("write the marker").send(), + ) + .await + }); + let request = common::eventually("flow tool to park", || { + agent.approvals().pending().ok()?.into_iter().next() + }) + .await; + agent + .approvals() + .decide(&request.request_id, ApprovalDecision::Deny) + .unwrap(); + let outcome = turn.await.unwrap().unwrap(); + assert_eq!(outcome.reply, "flow-denied"); + assert!(!marker.exists()); + let requests = common::chat_requests(&provider).await; + // Human approval refusals deliberately omit the policy-failure + // marker so the model can explain that the action was not done. + assert!(requests.iter().any(|request| common::tool_results(request) + .contains("This action was refused and must not be performed this turn"))); + + // A retained context belongs to its old instance after removal. + // Its late flow registration must create neither rows nor surfaces. + let _removed = runtime.agent(spec("flow-removed", "flow-closed")).unwrap(); + let old_context = AgentContextRegistry::get("flow-removed").unwrap(); + let old_scope = old_context + .host_overrides() + .unwrap() + .approval_scope() + .unwrap(); + runtime.remove_agent("flow-removed").await.unwrap(); + assert!(old_scope.is_closed()); + let mut late_events = openhuman_core::core::bus::BUS.get().unwrap().receiver(); + let mut late_notifications = + openhuman_core::desktop::notifications::bus::subscribe_core_notifications(); + let (outcome, request) = tokio::time::timeout( + Duration::from_secs(5), + CoreContext::scope( + old_context, + turn_origin::with_origin( + origin("flow-closed"), + APPROVAL_FLOW_RUN_CONTEXT.scope( + run_context("flow-closed", "run-closed"), + gate.intercept_audited( + "shell", + "late flow action", + serde_json::json!({}), + ), + ), + ), + ), + ) + .await + .unwrap(); + assert!( + matches!(outcome, GateOutcome::Deny { reason } if reason.contains("agent_removed")) + ); + assert!(request.is_none()); + assert!(gate + .list_pending_for_agent(Some("flow-removed")) + .unwrap() + .is_empty()); + assert!( + tokio::time::timeout(Duration::from_millis(100), async { + loop { + match late_events.recv().await.unwrap() { + DomainEvent::ApprovalRequested { agent_id, .. } + if agent_id.as_deref() == Some("flow-removed") => + { + break + } + DomainEvent::FlowApprovalRequested { flow_id, .. } + if flow_id == "flow-closed" => + { + break + } + _ => {} + } + } + }) + .await + .is_err(), + "removed instance published an approval surface" + ); + assert!( + tokio::time::timeout(Duration::from_millis(100), async { + loop { + let notification = late_notifications.recv().await.unwrap(); + if notification.actions.as_ref().is_some_and(|actions| { + actions.iter().any(|action| { + action + .payload + .as_ref() + .is_some_and(|payload| payload["flow_id"] == "flow-closed") + }) + }) { + break; + } + } + }) + .await + .is_err(), + "removed instance published an actionable notification" + ); + }) + .await + .unwrap(); + }); +} From c82fa023f52c23006ffebb9498bb7ca9c3c1d63a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 11 Oct 2026 01:02:35 +0300 Subject: [PATCH 18/18] test(embed): map callback and flow approval regressions Co-authored-by: Medulla --- docs/TEST-COVERAGE-MATRIX.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/TEST-COVERAGE-MATRIX.md b/docs/TEST-COVERAGE-MATRIX.md index 823ade21c33..73756897153 100644 --- a/docs/TEST-COVERAGE-MATRIX.md +++ b/docs/TEST-COVERAGE-MATRIX.md @@ -659,7 +659,7 @@ The thread JSONL store moved to `tinyagents_session::threads` (`vendor/tinyagent | 16.1.8 | MCP servers declared programmatically | RU | `crates/openhuman-embed/src/harness/mcp_tests.rs` | ✅ | stdio/HTTP transports mutually exclusive, enabled by default, env accumulation, allow/deny filters, auth carried through | | 16.1.9 | Skill bundles installed into the workspace | RU | `crates/openhuman-embed/src/harness/skills_tests.rs` | ✅ | Copied not linked (discovery rejects symlinked bundles and manifests by design); symlinks skipped both as bundles and inside them; `Inherit` refuses the copy | | 16.1.10 | One runtime per process | RU+RI | `crates/openhuman-embed/src/runtime/builder_tests.rs`, `crates/openhuman-embed/tests/harness_embed.rs` | ✅ | A second `Runtime` (or `Harness`) returns `AlreadyRunning` rather than sharing process-global keyring/event-bus/subscribers; a failed build releases the slot so a retry is possible | -| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs`, `crates/openhuman-embed/tests/agent_lifecycle.rs`, `crates/openhuman-core/src/security/approval/registration_scope_tests.rs`, `crates/openhuman-core/src/security/approval/flow_surface_tests.rs`, `crates/openhuman-embed/src/agent/lifecycle_tests.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused; removal settles pending approvals with `agent_removed` before cancelling turns; closing immediately refuses new admission even while an accepted registration or facade decision holds its barrier; instance-owned approval registration closes before the denial snapshot and rejects late registrations without persisting rows; polling and callback approval decisions serialize with the removal claim; the shared gate also refuses direct, agent-scoped and RPC decisions during the denial-snapshot gap; flow workspace resolution precedes registration, and registration publishes generic, flow and desktop approval surfaces together under the instance barrier | +| 16.1.11 | Many agents on one runtime | RU+RI | `crates/openhuman-embed/src/agent/spec_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs`, `crates/openhuman-embed/tests/agent_lifecycle.rs`, `crates/openhuman-embed/tests/approval_callbacks.rs`, `crates/openhuman-embed/tests/flow_approvals.rs`, `crates/openhuman-core/src/security/approval/registration_scope_tests.rs`, `crates/openhuman-core/src/security/approval/flow_surface_tests.rs`, `crates/openhuman-embed/src/agent/lifecycle_tests.rs` | ✅ | Three agents with different providers, access tiers, skills, MCP servers and action dirs; turns land on their own provider; per-agent MCP visibility; thread-scoped resume; duplicate/invalid ids and widening are refused; removal settles pending approvals with `agent_removed` before cancelling turns; closing immediately refuses new admission even while an accepted registration or facade decision holds its barrier; instance-owned approval registration closes before the denial snapshot and rejects late registrations without persisting rows; polling and callback approval decisions serialize with the removal claim; the shared gate also refuses direct, agent-scoped and RPC decisions during the denial-snapshot gap; flow workspace resolution precedes registration, and registration publishes generic, flow and desktop approval surfaces together under the instance barrier | | 16.1.12 | API-key credential for library mode | RU+RI | `crates/openhuman-core/src/security/credentials/api_key_tests.rs`, `crates/openhuman-embed/tests/runtime_agents.rs` | ✅ | Key stored as an `api-key` auth profile; wins over an expired session; managed inference sends it as a bearer with no `x-api-key`; auth state reports `credential = "api-key"` | | 16.1.14 | Host-owned tool policy extension | RI | `crates/openhuman-embed/tests/host_only_tools.rs` | ✅ | Host executors declare read-only requirements through the public `ToolPolicy` facade; HostOnly advertises the exact host catalog and refuses built-in shell/write attempts. Application-specific query validation and data handling belong to the host. | | 16.1.15 | Strict structured output and bounded repair | RI | `crates/openhuman-embed/tests/structured_validation.rs`, `crates/openhuman-embed/tests/structured_turns.rs`, `crates/openhuman-embed/tests/unit/turn_usage.rs` | ✅ | Wrong types and numeric combinators refused, invalid/external schemas make no call, repair bounded with usage, terminal length rejected even for complete JSON; successful charged usage reaches outcome/meter, invalid buyer charges stay unknown, root overlays retain child spend and session context. |