diff --git a/AGENTS.md b/AGENTS.md index 7496a64a1c4..26add9095ee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ Architecture: [overview](gitbooks/developing/architecture.md), | `crates/openhuman-core/src/core/` | CLI, controller contract (`Outcome`, schemas) and in-process dispatch, controller registry, event bus, runtime composition; no business logic and no JSON-RPC server | | `crates/openhuman-cli/` | The `openhuman-core` binary (`src/main.rs`), the developer/benchmark bins (`src/bin/`), and every root `tests/*.rs` / `examples/*.rs` target; depends on `openhuman-tinyhumans` for the backend transport the core does not carry | | `crates/openhuman-embed/` | Typed library facade for embedding the core in another product | -| `crates/openhuman-rpc/` | JSON-RPC 2.0 over the core: envelopes, HTTP client, and the server (router, Socket.IO, listener, `run_server*`) used by app, CLI and TUI; plus `session_store` (`session-store` feature), the on-disk session store (`session_raw/`, `session_db/`, `tinyagents_store/`, turn states) behind TinyAgents' session store port, which the app, CLI and TUI install. Core and embed reach session state through the port (`agent::session_store`); a few legacy paths still fall back to workspace files when no store is installed | +| `crates/openhuman-rpc/` | JSON-RPC 2.0 over the core: envelopes, HTTP client, and the server (router, Socket.IO, listener, `run_server*`) used by app, CLI and TUI; plus `session_store` (`session-store` feature), the on-disk session store (`session_raw/`, `session_db/`, `tinyagents_store/`, turn states) behind TinyAgents' session store port, which the app, CLI and TUI install. Core and embed reach session state through the port (`agent::session_store`); a few legacy paths still fall back to workspace files when no store is installed. With a storage URL (`OPENHUMAN_STORAGE_URL` / `[storage] url`), `install_for_host` installs TinyAgents' `DriverSessionStores` over that backend instead (core `storage` domain) | | `crates/openhuman-tinyhumans/` | The TinyHumans layer above embed: SDK-backed backend transport, a `RuntimeBuilder` that boots connected, and the host-side login/session owner (login-token exchange, `/auth/me`, current-user cache, credential handoff) used by app and TUI | | `crates/openhuman-tui/` | Standalone terminal frontend | | `tests/` | Rust integration and JSON-RPC tests | diff --git a/Cargo.lock b/Cargo.lock index 773ac3331fa..12944261a20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,6 +72,19 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -629,6 +642,29 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "bson" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969a9ba84b0ff843813e7249eed1678d9b6607ce5a3b8f0a47af3fcf7978e6e" +dependencies = [ + "ahash", + "base64 0.22.1", + "bitvec", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "indexmap", + "js-sys", + "once_cell", + "rand 0.9.5", + "serde", + "serde_bytes", + "serde_json", + "time", + "uuid", +] + [[package]] name = "bstr" version = "1.13.1" @@ -1038,7 +1074,7 @@ dependencies = [ "getrandom 0.2.17", "getrandom 0.3.4", "hmac 0.12.1", - "pbkdf2", + "pbkdf2 0.12.2", "rand 0.9.5", "sha2 0.10.9", "thiserror 1.0.69", @@ -1156,6 +1192,26 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "convert_case" version = "0.10.0" @@ -1394,6 +1450,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.23" @@ -1677,6 +1742,28 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +[[package]] +name = "derive-syn-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive-where" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e2b94854e8576378ccda7c8de8a66ed8b4e8acbd2c50ec3418ea6c8aaf4b567" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -1708,6 +1795,7 @@ dependencies = [ "quote", "rustc_version", "syn 2.0.119", + "unicode-xid", ] [[package]] @@ -2627,6 +2715,76 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-net" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c480823ed7c2c5d0f09c41020cb6b7c28029ce60ec42dc942158dcf22f8e0a4d" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.2", + "thiserror 2.0.20", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12b92608f679a6fa515dd1d15c1ff89443026e391200a2c840c7afcba482893d" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.2", + "ring", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3da5255c95d5a716857d54b5b8f4e8d67c3484d3beaaaae2ce25063b3ba981" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot", + "rand 0.10.2", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror 2.0.20", + "tokio", + "tracing", +] + [[package]] name = "hifijson" version = "0.5.0" @@ -3042,11 +3200,27 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result 0.4.1", + "windows-sys 0.61.2", +] + [[package]] name = "ipnet" version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +dependencies = [ + "serde", +] [[package]] name = "is_terminal_polyfill" @@ -3361,24 +3535,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "libdeflate-sys" -version = "1.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7870e5fbd2766179a937c725fb11f4ca0ef025d982beb61bd3ce755425bd19c" -dependencies = [ - "cc", -] - -[[package]] -name = "libdeflater" -version = "1.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cbb8285d2e2bd4b8d4155840ae81929354b4ba0d7b3c272c84761ffa2141e1f" -dependencies = [ - "libdeflate-sys", -] - [[package]] name = "libm" version = "0.2.16" @@ -3473,7 +3629,7 @@ dependencies = [ "indexmap", "itoa", "log", - "md-5", + "md-5 0.10.6", "nom 8.0.0", "rand 0.10.2", "rangemap", @@ -3521,6 +3677,54 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0d28e293f2b8c9d2b2d1c0193bd3c1cbdc0d2cf396888dc01162f7cf9d6c3f3" +[[package]] +name = "macro_magic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" +dependencies = [ + "const-random", + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" +dependencies = [ + "macro_magic_core", + "quote", + "syn 2.0.119", +] + [[package]] name = "mail-parser" version = "0.11.9" @@ -3570,6 +3774,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "md5" version = "0.8.1" @@ -3666,6 +3880,99 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "mongocrypt" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8426a875ded61430d4a811dbfda7633b6b8af0225c547fc6c28b8b0aa7d79a13" +dependencies = [ + "bson", + "mongocrypt-sys", + "once_cell", + "serde", +] + +[[package]] +name = "mongocrypt-sys" +version = "0.1.6+1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851fac73f7fe22f6a3ab87f720ce509cae7c9fd08e7dd27866cc232dee07ccf4" + +[[package]] +name = "mongodb" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af84443ae9878f59a6818686f339064479f42cdf293c4c637d8fc9c0a8c5eed6" +dependencies = [ + "base64 0.22.1", + "bitflags 2.13.2", + "bson", + "derive-where", + "derive_more", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hickory-net", + "hickory-proto", + "hickory-resolver", + "hmac 0.13.0", + "macro_magic", + "md-5 0.11.0", + "mongocrypt", + "mongodb-internal-macros", + "pbkdf2 0.13.0", + "percent-encoding", + "rand 0.9.5", + "rustc_version_runtime", + "rustls", + "serde", + "serde_bytes", + "serde_with", + "sha1 0.11.0", + "sha2 0.11.0", + "socket2", + "stringprep", + "strsim", + "take_mut", + "thiserror 2.0.20", + "tokio", + "tokio-rustls", + "tokio-util", + "typed-builder", + "uuid", + "webpki-roots 1.0.9", +] + +[[package]] +name = "mongodb-internal-macros" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504024bbb83ab1bf1512007f7288b54d3a6343043f3f619c26a153346773598d" +dependencies = [ + "macro_magic", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "moxcms" version = "0.8.1" @@ -4228,6 +4535,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -4334,6 +4645,7 @@ dependencies = [ "tinyruntime-pyserver", "tinysearch-bus", "tinyskills", + "tinystoragedrivers", "tinytools", "tinytools-agent", "tinytools-std", @@ -4615,20 +4927,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "oxipng" -version = "9.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c613f0f566526a647c7473f6a8556dbce22c91b13485ee4b4ec7ab648e4973" -dependencies = [ - "bitvec", - "indexmap", - "libdeflater", - "log", - "rgb", - "rustc-hash 2.1.3", -] - [[package]] name = "palette" version = "0.7.7" @@ -4712,6 +5010,15 @@ dependencies = [ "hmac 0.12.1", ] +[[package]] +name = "pbkdf2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "pdf-extract" version = "0.12.1" @@ -5006,6 +5313,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -5551,6 +5869,12 @@ dependencies = [ "webpki-roots 1.0.9", ] +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "rfc6979" version = "0.4.0" @@ -5561,15 +5885,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "rgb" -version = "0.8.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" -dependencies = [ - "bytemuck", -] - [[package]] name = "ring" version = "0.17.14" @@ -5654,6 +5969,16 @@ dependencies = [ "semver", ] +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + [[package]] name = "rustix" version = "1.1.5" @@ -6010,6 +6335,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -6047,6 +6382,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -6086,6 +6422,28 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" +dependencies = [ + "serde_core", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -6530,6 +6888,39 @@ dependencies = [ "windows 0.54.0", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.2", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys 0.8.7", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + [[package]] name = "tap" version = "1.0.1" @@ -6737,6 +7128,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinyagents-definition" version = "2.1.3" @@ -6756,6 +7156,7 @@ dependencies = [ "serde", "serde_json", "tinyagents-harness", + "tinyagents-tasks", "tinyinference-llm", "tinytools", "tokio", @@ -6774,7 +7175,6 @@ dependencies = [ "dirs 6.0.0", "flate2", "futures", - "oxipng", "regex", "reqwest", "rusqlite", @@ -6791,6 +7191,7 @@ dependencies = [ "tinyinference-image", "tinyinference-llm", "tinyinference-video", + "tinystoragedrivers-core", "tinytools", "tinytools-agent", "tokio", @@ -6826,6 +7227,7 @@ dependencies = [ "tinyagents-harness", "tinyagents-runtime", "tinyagents-session", + "tinyagents-tasks", "tinyinference-llm", "tinytools", "tokio", @@ -6883,12 +7285,30 @@ dependencies = [ "tempfile", "thiserror 2.0.20", "tinyagents-harness", + "tinyagents-tasks", + "tinystoragedrivers-core", + "tinystoragedrivers-sqlite", "tinytools-agent", "tokio", "tracing", "uuid", ] +[[package]] +name = "tinyagents-tasks" +version = "2.1.3" +dependencies = [ + "anyhow", + "async-trait", + "serde", + "serde_json", + "tinyagents-harness", + "tinyinference-llm", + "tinytools", + "tokio", + "tracing", +] + [[package]] name = "tinybox-core" version = "0.1.15" @@ -7161,6 +7581,7 @@ dependencies = [ "tinyflows", "tinyflows-catalog", "tinyflows-schedule", + "tinystoragedrivers-sqlite", "tokio", "tracing", "uuid", @@ -7520,6 +7941,60 @@ dependencies = [ "url", ] +[[package]] +name = "tinystoragedrivers" +version = "0.4.0" +dependencies = [ + "tinystoragedrivers-core", + "tinystoragedrivers-file", + "tinystoragedrivers-mongodb", + "tinystoragedrivers-sqlite", +] + +[[package]] +name = "tinystoragedrivers-core" +version = "0.4.0" +dependencies = [ + "async-trait", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokio", +] + +[[package]] +name = "tinystoragedrivers-file" +version = "0.4.0" +dependencies = [ + "async-trait", + "libc", + "serde", + "serde_json", + "tinystoragedrivers-core", + "tokio", +] + +[[package]] +name = "tinystoragedrivers-mongodb" +version = "0.4.0" +dependencies = [ + "async-trait", + "futures-util", + "mongodb", + "serde_json", + "tinystoragedrivers-core", +] + +[[package]] +name = "tinystoragedrivers-sqlite" +version = "0.4.0" +dependencies = [ + "rusqlite", + "serde_json", + "tinystoragedrivers-core", + "tokio", +] + [[package]] name = "tinystr" version = "0.8.4" @@ -7575,7 +8050,7 @@ dependencies = [ "reqwest", "rustix", "serde_json", - "sha2 0.10.9", + "sha2 0.11.0", "tinytools", "tokio", "tracing", @@ -7809,6 +8284,7 @@ checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "futures-util", "libc", @@ -8089,6 +8565,26 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +[[package]] +name = "typed-builder" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "typed-path" version = "0.12.3" @@ -8178,6 +8674,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "universal-hash" version = "0.5.1" @@ -8807,6 +9309,12 @@ dependencies = [ "wacore", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -8986,6 +9494,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + [[package]] name = "windows-result" version = "0.1.2" diff --git a/Cargo.toml b/Cargo.toml index f9eb04a23f0..23d063e4490 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,14 @@ openhuman-tinyhumans = { path = "crates/openhuman-tinyhumans", default-features tinytools = { path = "vendor/tinyagents/vendor/tinytools/crates/tinytools" } tinytools-agent = { path = "vendor/tinyagents/vendor/tinytools/crates/tinytools-agent" } +[patch."https://github.com/tinyhumansai/tinystoragedrivers"] +# tinyflows names the storage crates by git tag so it can build on its own. +# Redirect them to the copy tinyagents vendors — the one the session store and +# the core `storage` domain link — so there is exactly one `DocumentStore` +# trait and one `libsqlite3-sys` (`links = "sqlite3"`) in the graph. +tinystoragedrivers-core = { path = "vendor/tinyagents/vendor/tinystoragedrivers/crates/tinystoragedrivers-core" } +tinystoragedrivers-sqlite = { path = "vendor/tinyagents/vendor/tinystoragedrivers/crates/tinystoragedrivers-sqlite" } + [patch."https://github.com/tinyhumansai/tinyinference"] tinyinference-core = { path = "vendor/tinyagents/vendor/tinyinference/crates/tinyinference-core" } tinyinference-embeddings = { path = "vendor/tinyagents/vendor/tinyinference/crates/tinyinference-embeddings" } diff --git a/crates/openhuman-app/Cargo.lock b/crates/openhuman-app/Cargo.lock index 71370624785..2b413f0fe2a 100644 --- a/crates/openhuman-app/Cargo.lock +++ b/crates/openhuman-app/Cargo.lock @@ -537,18 +537,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "bitvec" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - [[package]] name = "blake2" version = "0.10.6" @@ -2155,12 +2143,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - [[package]] name = "futures" version = "0.3.34" @@ -3492,24 +3474,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "libdeflate-sys" -version = "1.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7870e5fbd2766179a937c725fb11f4ca0ef025d982beb61bd3ce755425bd19c" -dependencies = [ - "cc", -] - -[[package]] -name = "libdeflater" -version = "1.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cbb8285d2e2bd4b8d4155840ae81929354b4ba0d7b3c272c84761ffa2141e1f" -dependencies = [ - "libdeflate-sys", -] - [[package]] name = "libloading" version = "0.7.4" @@ -4579,6 +4543,7 @@ dependencies = [ "tinyruntime-pyserver", "tinysearch-bus", "tinyskills", + "tinystoragedrivers", "tinytools", "tinytools-agent", "tinytools-std", @@ -4831,20 +4796,6 @@ dependencies = [ "thiserror 2.0.20", ] -[[package]] -name = "oxipng" -version = "9.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c613f0f566526a647c7473f6a8556dbce22c91b13485ee4b4ec7ab648e4973" -dependencies = [ - "bitvec", - "indexmap 2.14.2", - "libdeflater", - "log", - "rgb", - "rustc-hash", -] - [[package]] name = "pango" version = "0.18.3" @@ -5427,12 +5378,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - [[package]] name = "rand" version = "0.8.8" @@ -5742,15 +5687,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "rgb" -version = "0.8.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" -dependencies = [ - "bytemuck", -] - [[package]] name = "ring" version = "0.17.14" @@ -6911,12 +6847,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - [[package]] name = "tar" version = "0.4.46" @@ -7443,6 +7373,7 @@ dependencies = [ "serde", "serde_json", "tinyagents-harness", + "tinyagents-tasks", "tinyinference-llm", "tinytools", "tokio", @@ -7461,7 +7392,6 @@ dependencies = [ "dirs 6.0.0", "flate2", "futures", - "oxipng", "regex", "reqwest 0.12.28", "rusqlite", @@ -7478,6 +7408,7 @@ dependencies = [ "tinyinference-image", "tinyinference-llm", "tinyinference-video", + "tinystoragedrivers-core", "tinytools", "tinytools-agent", "tokio", @@ -7513,6 +7444,7 @@ dependencies = [ "tinyagents-harness", "tinyagents-runtime", "tinyagents-session", + "tinyagents-tasks", "tinyinference-llm", "tinytools", "tokio", @@ -7570,12 +7502,30 @@ dependencies = [ "tempfile", "thiserror 2.0.20", "tinyagents-harness", + "tinyagents-tasks", + "tinystoragedrivers-core", + "tinystoragedrivers-sqlite", "tinytools-agent", "tokio", "tracing", "uuid", ] +[[package]] +name = "tinyagents-tasks" +version = "2.1.3" +dependencies = [ + "anyhow", + "async-trait", + "serde", + "serde_json", + "tinyagents-harness", + "tinyinference-llm", + "tinytools", + "tokio", + "tracing", +] + [[package]] name = "tinybox-core" version = "0.1.15" @@ -7862,6 +7812,7 @@ dependencies = [ "tinyflows", "tinyflows-catalog", "tinyflows-schedule", + "tinystoragedrivers-sqlite", "tokio", "tracing", "uuid", @@ -8221,6 +8172,34 @@ dependencies = [ "url", ] +[[package]] +name = "tinystoragedrivers" +version = "0.4.0" +dependencies = [ + "tinystoragedrivers-core", +] + +[[package]] +name = "tinystoragedrivers-core" +version = "0.4.0" +dependencies = [ + "async-trait", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokio", +] + +[[package]] +name = "tinystoragedrivers-sqlite" +version = "0.4.0" +dependencies = [ + "rusqlite", + "serde_json", + "tinystoragedrivers-core", + "tokio", +] + [[package]] name = "tinystr" version = "0.8.4" @@ -8276,7 +8255,7 @@ dependencies = [ "reqwest 0.12.28", "rustix", "serde_json", - "sha2 0.10.9", + "sha2 0.11.0", "tinytools", "tokio", "tracing", @@ -10245,15 +10224,6 @@ dependencies = [ "windows-version", ] -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] - [[package]] name = "x11" version = "2.21.0" diff --git a/crates/openhuman-app/Cargo.toml b/crates/openhuman-app/Cargo.toml index abc70646cdc..323e2458de1 100644 --- a/crates/openhuman-app/Cargo.toml +++ b/crates/openhuman-app/Cargo.toml @@ -324,6 +324,13 @@ tinyinference-llm = { path = "../../vendor/tinyagents/vendor/tinyinference/crate tinyinference-local = { path = "../../vendor/tinyagents/vendor/tinyinference/crates/tinyinference-local" } tinyinference-voice = { path = "../../vendor/tinyagents/vendor/tinyinference/crates/tinyinference-voice" } +[patch."https://github.com/tinyhumansai/tinystoragedrivers"] +# tinyflows names the storage crates by git tag; redirect them to the copy +# tinyagents vendors so this world links one `DocumentStore` trait and one +# `libsqlite3-sys`. Mirrors the root Cargo.toml. +tinystoragedrivers-core = { path = "../../vendor/tinyagents/vendor/tinystoragedrivers/crates/tinystoragedrivers-core" } +tinystoragedrivers-sqlite = { path = "../../vendor/tinyagents/vendor/tinystoragedrivers/crates/tinystoragedrivers-sqlite" } + [patch.crates-io] # Keep reqwest defaults disabled in this independent Cargo world too. Without # this patch, the registry release reintroduces native-tls/OpenSSL on Linux. diff --git a/crates/openhuman-cli/Cargo.toml b/crates/openhuman-cli/Cargo.toml index dafaf89b1df..e4be4568c93 100644 --- a/crates/openhuman-cli/Cargo.toml +++ b/crates/openhuman-cli/Cargo.toml @@ -341,6 +341,10 @@ hosting = ["openhuman-core/hosting", "openhuman-tinyhumans/hosting"] modules = ["openhuman-core/modules", "openhuman-tinyhumans/modules"] voice = ["openhuman-core/voice", "openhuman-tinyhumans/voice"] web3 = ["openhuman-core/web3", "openhuman-tinyhumans/web3"] +# Storage drivers for `[storage] url` (see the core `storage` domain). +storage-sqlite = ["openhuman-core/storage-sqlite", "openhuman-tinyhumans/storage-sqlite"] +storage-mongodb = ["openhuman-core/storage-mongodb", "openhuman-tinyhumans/storage-mongodb"] +storage-file = ["openhuman-core/storage-file", "openhuman-tinyhumans/storage-file"] runtime-node = ["openhuman-core/runtime-node", "openhuman-tinyhumans/runtime-node"] media = ["openhuman-core/media", "openhuman-tinyhumans/media"] flows = ["openhuman-core/flows", "openhuman-tinyhumans/flows"] diff --git a/crates/openhuman-core/Cargo.toml b/crates/openhuman-core/Cargo.toml index bef4275f5e7..daedb2d542a 100644 --- a/crates/openhuman-core/Cargo.toml +++ b/crates/openhuman-core/Cargo.toml @@ -149,7 +149,10 @@ tinyagents-graph = { path = "../../vendor/tinyagents/crates/tinyagents-graph", f "sqlite", ] } tinyagents-registry = { path = "../../vendor/tinyagents/crates/tinyagents-registry" } -tinyagents-session = { path = "../../vendor/tinyagents/crates/tinyagents-session" } +tinyagents-session = { path = "../../vendor/tinyagents/crates/tinyagents-session", features = ["storage-drivers"] } +# The storage ports and the URL-to-backend facade (`storage` domain). No driver +# by default: the `storage-*` features below add them. +tinystoragedrivers = { path = "../../vendor/tinyagents/vendor/tinystoragedrivers/crates/tinystoragedrivers", default-features = false } tinyagents-runtime = { path = "../../vendor/tinyagents/crates/tinyagents-runtime" } tinyagents-orchestration = { path = "../../vendor/tinyagents/crates/tinyagents-orchestration" } tinyinference-core = { path = "../../vendor/tinyagents/vendor/tinyinference/crates/tinyinference-core" } @@ -665,6 +668,12 @@ tokio = { version = "1", features = ["test-util"] } proptest = "1" [features] +# Storage drivers for `[storage] url` / `OPENHUMAN_STORAGE_URL` (see the +# `storage` domain). Memory is always available. Off in the product until a +# desktop domain moves onto the ports; a cloud build turns on `storage-mongodb`. +storage-sqlite = ["tinystoragedrivers/sqlite"] +storage-mongodb = ["tinystoragedrivers/mongodb"] +storage-file = ["tinystoragedrivers/file"] # THE CONTRIBUTOR SET — not the product set. # # This list is what a bare `cargo check`, `cargo test` and rust-analyzer diff --git a/crates/openhuman-core/src/agent/tinyagents/reaper.rs b/crates/openhuman-core/src/agent/tinyagents/reaper.rs index 82b91d91f56..266cf6c8c50 100644 --- a/crates/openhuman-core/src/agent/tinyagents/reaper.rs +++ b/crates/openhuman-core/src/agent/tinyagents/reaper.rs @@ -13,7 +13,19 @@ use tinyagents_session::transcript::import::ops::open_session_stores; /// Reap every run left non-terminal by a previous process, returning the number /// of runs moved to `Cancelled`. Best-effort; never blocks boot. +/// +/// Skipped on a storage backend other processes may share (MongoDB): there a +/// non-terminal status can belong to a run another replica is still driving, +/// and cancelling it would hide that run from active and late-attach status. pub(crate) async fn reap_orphaned_runs(workspace: &Path) -> usize { + reap_unless_shared(workspace, crate::storage::installed_is_shared()).await +} + +async fn reap_unless_shared(workspace: &Path, shared: bool) -> usize { + if shared { + log::info!("[agent] startup run sweep skipped: the storage backend is shared"); + return 0; + } log::debug!( "[agent] startup run sweep workspace={}", workspace.display() diff --git a/crates/openhuman-core/src/agent/tinyagents/reaper_tests.rs b/crates/openhuman-core/src/agent/tinyagents/reaper_tests.rs index f483b3b1dd9..bf150571a93 100644 --- a/crates/openhuman-core/src/agent/tinyagents/reaper_tests.rs +++ b/crates/openhuman-core/src/agent/tinyagents/reaper_tests.rs @@ -35,6 +35,19 @@ async fn workspace_adapter_reaps_and_empty_workspace_is_a_noop() { let _ = std::fs::remove_dir_all(&tmp); } +/// A shared backend never reaps: the run may belong to another replica. +#[tokio::test] +async fn a_shared_backend_leaves_running_work_alone() { + let tmp = std::env::temp_dir().join(format!("oh-reaper-{}", uuid::Uuid::new_v4())); + let store = FileStatusStore::new(open_session_stores(&tmp).kv); + let run = seed_status(&store, ExecutionStatus::Running).await; + assert_eq!(reap_unless_shared(&tmp, true).await, 0); + let status = store.get_status(&run).await.unwrap().expect("present"); + assert_eq!(status.status, ExecutionStatus::Running); + assert_eq!(reap_unless_shared(&tmp, false).await, 1); + let _ = std::fs::remove_dir_all(&tmp); +} + /// The sweep has to happen on the path a **build-only** embedder takes. /// /// `CoreRuntime::invoke` dispatches `openhuman.agent_runs_active` the moment diff --git a/crates/openhuman-core/src/config/ops/loader/snapshot.rs b/crates/openhuman-core/src/config/ops/loader/snapshot.rs index b128191f98c..b52837e8c84 100644 --- a/crates/openhuman-core/src/config/ops/loader/snapshot.rs +++ b/crates/openhuman-core/src/config/ops/loader/snapshot.rs @@ -15,6 +15,11 @@ pub fn snapshot_config_json(config: &Config) -> Result Result<()> { let store = crate::security::keyring::SecretStore::new(parent_dir, true); encrypt_optional_secret(&store, &mut config.api_key, "api_key")?; + // A MongoDB URL carries the database password. + encrypt_optional_secret(&store, &mut config.storage.url, "storage.url")?; encrypt_optional_secret(&store, &mut config.seltz.api_key, "seltz.api_key")?; encrypt_optional_secret( diff --git a/crates/openhuman-core/src/config/schema/load/secrets_tests.rs b/crates/openhuman-core/src/config/schema/load/secrets_tests.rs index 8fd4b9f6007..41e959dc946 100644 --- a/crates/openhuman-core/src/config/schema/load/secrets_tests.rs +++ b/crates/openhuman-core/src/config/schema/load/secrets_tests.rs @@ -18,3 +18,27 @@ fn gemini_key_round_trips_with_encryption_enabled() { Some("gemini-sentinel") ); } + +#[test] +fn storage_url_round_trips_and_fails_closed_without_the_key() { + let dir = tempfile::tempdir().unwrap(); + let mut config = Config::default(); + config.config_path = dir.path().join("config.toml"); + config.secrets.encrypt = true; + config.storage.url = Some("mongodb://app:hunter2@db.internal/openhuman".into()); + + encrypt_config_secrets(&mut config).unwrap(); + let sealed = config.storage.url.clone().unwrap(); + assert!(sealed.starts_with("enc2:") && !sealed.contains("hunter2")); + decrypt_config_secrets(&mut config, dir.path()).unwrap(); + assert_eq!( + config.storage.url.as_deref(), + Some("mongodb://app:hunter2@db.internal/openhuman") + ); + + // A key that cannot open the URL must not clear it into "no URL". + let other = tempfile::tempdir().unwrap(); + config.storage.url = Some(sealed.clone()); + decrypt_config_secrets(&mut config, other.path()).unwrap(); + assert_eq!(config.storage.url.as_deref(), Some(sealed.as_str())); +} diff --git a/crates/openhuman-core/src/config/schema/mod.rs b/crates/openhuman-core/src/config/schema/mod.rs index 21a75995400..3d1f5611564 100644 --- a/crates/openhuman-core/src/config/schema/mod.rs +++ b/crates/openhuman-core/src/config/schema/mod.rs @@ -29,6 +29,8 @@ mod hooks; pub use hooks::HooksConfig; pub mod hosting; pub use hosting::HostingConfig; +pub mod storage; +pub use storage::StorageConfig; mod identity_cost; mod load; pub use load::{ diff --git a/crates/openhuman-core/src/config/schema/storage.rs b/crates/openhuman-core/src/config/schema/storage.rs new file mode 100644 index 00000000000..a91e4fd51d4 --- /dev/null +++ b/crates/openhuman-core/src/config/schema/storage.rs @@ -0,0 +1,61 @@ +//! Storage backend configuration (`[storage]`). +//! +//! Where the core keeps durable state that has moved onto the +//! `tinystoragedrivers` ports. Empty (the default) keeps the classic on-disk +//! layout under the workspace, which is what the desktop app uses. A URL +//! points the process at one backend instead: +//! +//! ```toml +//! [storage] +//! url = "mongodb://app:secret@db.internal/openhuman" # cloud, multi-tenant +//! # url = "sqlite:/var/lib/openhuman/storage" # one SQLite dir +//! # url = "memory" # tests, keeps nothing +//! ``` +//! +//! `OPENHUMAN_STORAGE_URL` overrides it (see [`crate::storage`]). This +//! section is bootstrap configuration: it is always read from the file or the +//! environment, never from storage itself. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(default)] +pub struct StorageConfig { + /// The storage URL: `memory`, `sqlite:`, `mongodb://…/`, + /// `mongodb+srv://…/` or `file:`. Empty or absent keeps the + /// classic on-disk layout. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, +} + +impl std::fmt::Debug for StorageConfig { + /// Redacts credentials: a MongoDB URL carries the database password. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StorageConfig") + .field("url", &self.url.as_deref().map(redact_url)) + .finish() + } +} + +/// `url` with any `user:password@` userinfo replaced by `***@`, and the +/// query string and fragment dropped (they can carry tokens, such as MongoDB's +/// `authMechanismProperties`). +pub fn redact_url(url: &str) -> String { + let end = url.find(['?', '#']).unwrap_or(url.len()); + let (base, trimmed) = (&url[..end], end < url.len()); + let mut out = match (base.find("://"), base.rfind('@')) { + (Some(scheme), Some(at)) if at > scheme => { + format!("{}://***@{}", &base[..scheme], &base[at + 1..]) + } + _ => base.to_string(), + }; + if trimmed { + out.push_str("?***"); + } + out +} + +#[cfg(test)] +#[path = "storage_tests.rs"] +mod tests; diff --git a/crates/openhuman-core/src/config/schema/storage_tests.rs b/crates/openhuman-core/src/config/schema/storage_tests.rs new file mode 100644 index 00000000000..c29661752a6 --- /dev/null +++ b/crates/openhuman-core/src/config/schema/storage_tests.rs @@ -0,0 +1,43 @@ +use super::*; + +#[test] +fn defaults_to_the_classic_layout() { + assert_eq!(StorageConfig::default().url, None); + let parsed: StorageConfig = toml::from_str("").unwrap(); + assert_eq!(parsed, StorageConfig::default()); +} + +#[test] +fn reads_a_url_from_toml() { + let parsed: StorageConfig = toml::from_str("url = \"memory\"").unwrap(); + assert_eq!(parsed.url.as_deref(), Some("memory")); +} + +#[test] +fn debug_never_prints_credentials() { + let config = StorageConfig { + url: Some("mongodb://app:hunter2@db.internal/openhuman".into()), + }; + let shown = format!("{config:?}"); + assert!(!shown.contains("hunter2"), "{shown}"); + assert!( + shown.contains("mongodb://***@db.internal/openhuman"), + "{shown}" + ); + assert_eq!(redact_url("sqlite:/tmp/x"), "sqlite:/tmp/x"); + assert_eq!(redact_url("mongodb://h/db"), "mongodb://h/db"); +} + +#[test] +fn redact_drops_query_and_fragment_tokens() { + assert_eq!( + redact_url( + "mongodb://db.internal/openhuman?authMechanismProperties=AWS_SESSION_TOKEN%3Asecret" + ), + "mongodb://db.internal/openhuman?***" + ); + assert_eq!( + redact_url("mongodb://app:pw@db/x?token=secret#frag"), + "mongodb://***@db/x?***" + ); +} diff --git a/crates/openhuman-core/src/config/schema/types/config.rs b/crates/openhuman-core/src/config/schema/types/config.rs index c38ef51ce99..1e1d277a21e 100644 --- a/crates/openhuman-core/src/config/schema/types/config.rs +++ b/crates/openhuman-core/src/config/schema/types/config.rs @@ -421,6 +421,11 @@ pub struct Config { #[serde(default)] pub hosting: HostingConfig, + /// Storage backend for state on the `tinystoragedrivers` ports + /// (`[storage]`). Empty keeps the classic on-disk layout. + #[serde(default)] + pub storage: StorageConfig, + #[serde(default)] pub voice_server: VoiceServerConfig, diff --git a/crates/openhuman-core/src/config/schema/types/config_clone.rs b/crates/openhuman-core/src/config/schema/types/config_clone.rs index d5fd546e587..ffcf84d1e39 100644 --- a/crates/openhuman-core/src/config/schema/types/config_clone.rs +++ b/crates/openhuman-core/src/config/schema/types/config_clone.rs @@ -91,6 +91,7 @@ impl Clone for Config { runtime_pool: self.runtime_pool.clone(), tokenjuice: self.tokenjuice.clone(), hosting: self.hosting.clone(), + storage: self.storage.clone(), voice_server: self.voice_server.clone(), voice_providers: self.voice_providers.clone(), voice_live: self.voice_live.clone(), diff --git a/crates/openhuman-core/src/config/schema/types/defaults.rs b/crates/openhuman-core/src/config/schema/types/defaults.rs index f48399118f4..2737b331569 100644 --- a/crates/openhuman-core/src/config/schema/types/defaults.rs +++ b/crates/openhuman-core/src/config/schema/types/defaults.rs @@ -104,6 +104,7 @@ impl Default for Config { runtime_pool: RuntimePoolConfig::default(), tokenjuice: TokenjuiceConfig::default(), hosting: HostingConfig::default(), + storage: StorageConfig::default(), voice_server: VoiceServerConfig::default(), voice_providers: Vec::new(), voice_live: crate::config::schema::voice_live::LiveVoiceConfig::default(), diff --git a/crates/openhuman-core/src/lib.rs b/crates/openhuman-core/src/lib.rs index 5320a5acbd4..2570dd4289b 100644 --- a/crates/openhuman-core/src/lib.rs +++ b/crates/openhuman-core/src/lib.rs @@ -79,6 +79,7 @@ pub mod sandbox; pub mod search; pub mod security; pub mod skills; +pub mod storage; #[cfg(feature = "e2e-test-support")] pub mod test_support; pub mod threads; diff --git a/crates/openhuman-core/src/storage/README.md b/crates/openhuman-core/src/storage/README.md new file mode 100644 index 00000000000..5a98b3884af --- /dev/null +++ b/crates/openhuman-core/src/storage/README.md @@ -0,0 +1,48 @@ +# storage + +The process's storage backend on the +[tinystoragedrivers](https://github.com/tinyhumansai/tinystoragedrivers) +ports, vendored through `vendor/tinyagents/vendor/tinystoragedrivers`. + +One URL picks it: `OPENHUMAN_STORAGE_URL`, else `[storage] url` in +`config.toml` (`config::StorageConfig`). With neither set, the desktop +default, nothing is opened and every domain keeps the classic on-disk layout +under the workspace. + +| URL | Driver | Cargo feature | +| --- | --- | --- | +| `memory` | in-process, keeps nothing | always | +| `sqlite:` (a `.db` file, or a directory with one file per database) | SQLite | `storage-sqlite` | +| `mongodb://…/`, `mongodb+srv://…/` | MongoDB, one database shared by every scope | `storage-mongodb` | +| `file:` | JSON and JSONL files | `storage-file` | + +A URL for a driver the build lacks fails at `open`, naming the feature. None +of the features are in the shipped desktop product yet; a cloud build turns +on `storage-mongodb`. + +## Entry points + +- `configured_url(&Config)` / `url_from(env, &Config)`: the URL in effect. +- `open(url)`: parse and open a backend (credentials are redacted in logs). +- `install(backend)` / `installed()` / `clear()`: the process slot domains + read the backend from. +- `scope_for_agent(agent_id)`: the storage scope an agent's records live + under, the same mapping TinyAgents' `DriverSessionStores` uses, so every + domain agrees. +- `driver_is_shared(driver)` / `installed_is_shared()`: whether other + processes may write the same backend (MongoDB). Boot-time recovery, such + as the orphaned-run sweep, is skipped on a shared backend. + +## Consumers + +- The session store: `openhuman_rpc::session_store::install_for_host` opens + the configured backend before boot and installs `DriverSessionStores` + over it. See that module's README. + +## Boundaries + +The ports, drivers, scopes and conformance suites are tinystoragedrivers'; +the session store over them is `tinyagents-session`. This domain only +resolves the URL, opens the backend, and holds it for the process. The +`[storage]` section is bootstrap configuration and is never read from +storage itself. diff --git a/crates/openhuman-core/src/storage/mod.rs b/crates/openhuman-core/src/storage/mod.rs new file mode 100644 index 00000000000..0e8f26f4eae --- /dev/null +++ b/crates/openhuman-core/src/storage/mod.rs @@ -0,0 +1,118 @@ +//! The process's storage backend on the `tinystoragedrivers` ports. +//! +//! One URL picks where durable state that has moved onto the storage ports +//! lives: `OPENHUMAN_STORAGE_URL`, else `[storage] url` in `config.toml` +//! ([`crate::config::StorageConfig`]). With neither set — the desktop default — +//! nothing here is opened and every domain keeps the classic on-disk layout +//! under the workspace. +//! +//! When a URL is set, the host opens it once at startup ([`open`]) and +//! installs it ([`install`]); domains reach it through [`installed`] and bind +//! their records to an agent with [`scope_for_agent`]. The first consumer is +//! the session store: the host installs TinyAgents' `DriverSessionStores` +//! over this backend, so transcripts, turn states, records and journals live +//! in it, one scope per agent. +//! +//! Drivers are Cargo features of this crate: `storage-sqlite`, +//! `storage-mongodb`, `storage-file` (memory is always available). A URL for a +//! driver the build does not carry fails at [`open`] naming the feature, so a +//! misconfigured deployment stops at boot instead of at its first write. + +use std::sync::{Arc, LazyLock, RwLock}; + +pub use tinystoragedrivers::{ + Scope, ScopedStorage, StorageBackend, StorageConfig as StorageUrl, StorageError, +}; + +use crate::config::schema::storage::redact_url; +use crate::config::Config; + +/// The environment variable that overrides `[storage] url`. +pub const STORAGE_URL_VAR: &str = "OPENHUMAN_STORAGE_URL"; + +static BACKEND: LazyLock>>> = + LazyLock::new(|| RwLock::new(None)); + +/// The storage URL in effect: [`STORAGE_URL_VAR`], else `config`'s +/// `[storage] url`. Blank values count as unset. `None` keeps the classic +/// on-disk layout. +pub fn configured_url(config: &Config) -> Option { + url_from(std::env::var(STORAGE_URL_VAR).ok(), config) +} + +/// [`configured_url`] with the environment read made explicit, so the rule +/// is testable without mutating process-wide state. +pub fn url_from(env: Option, config: &Config) -> Option { + env.into_iter() + .chain(config.storage.url.clone()) + .map(|url| url.trim().to_string()) + .find(|url| !url.is_empty()) +} + +/// Parses and opens the backend `url` names. +/// +/// # Errors +/// +/// An unparseable URL, a driver this build was compiled without (the error +/// names the Cargo feature), or a backend that cannot be reached. +pub async fn open(url: &str) -> Result, StorageError> { + let parsed = StorageUrl::parse(url)?; + tracing::info!( + target: "openhuman::storage", + driver = parsed.driver(), + url = %redact_url(url), + "[storage] opening the configured backend" + ); + tinystoragedrivers::open(&parsed).await +} + +/// Makes `backend` the process's storage backend; returns the previous one. +pub fn install(backend: Arc) -> Option> { + let mut slot = BACKEND + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + slot.replace(backend) +} + +/// The installed backend, when the host configured one. +pub fn installed() -> Option> { + BACKEND + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() +} + +/// Removes the installed backend; returns whether there was one. +pub fn clear() -> bool { + BACKEND + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + .is_some() +} + +/// Whether a backend with this driver name may be shared by several +/// processes at once. A MongoDB database can be; SQLite files, the memory +/// driver and plain files belong to the one process that opened them. +/// +/// Boot-time recovery (interrupting in-flight turns, reaping orphaned runs) +/// is only sound on a backend no other process can be writing to. +pub fn driver_is_shared(driver: &str) -> bool { + driver == "mongodb" +} + +/// Whether the installed backend may be shared with other processes; `false` +/// when none is installed. See [`driver_is_shared`]. +pub fn installed_is_shared() -> bool { + installed().is_some_and(|backend| driver_is_shared(backend.driver())) +} + +/// The storage scope agent `agent_id`'s records live under — the same +/// mapping the session store uses, so every domain agrees on it. +pub fn scope_for_agent(agent_id: &str) -> Scope { + tinyagents_session::DriverSessionStores::scope_for(agent_id) +} + +#[cfg(test)] +#[path = "mod_tests.rs"] +mod tests; diff --git a/crates/openhuman-core/src/storage/mod_tests.rs b/crates/openhuman-core/src/storage/mod_tests.rs new file mode 100644 index 00000000000..c9e2b8e1c1a --- /dev/null +++ b/crates/openhuman-core/src/storage/mod_tests.rs @@ -0,0 +1,67 @@ +use super::*; + +fn config_with(url: Option<&str>) -> Config { + let mut config = Config::default(); + config.storage.url = url.map(str::to_string); + config +} + +#[test] +fn the_environment_wins_over_the_config_file() { + let config = config_with(Some("sqlite:/from/config")); + assert_eq!( + url_from(Some("memory".into()), &config).as_deref(), + Some("memory") + ); + assert_eq!( + url_from(None, &config).as_deref(), + Some("sqlite:/from/config") + ); +} + +#[test] +fn blank_values_keep_the_classic_layout() { + assert_eq!(url_from(None, &config_with(None)), None); + assert_eq!(url_from(Some(" ".into()), &config_with(Some(""))), None); + assert_eq!( + url_from(Some(" ".into()), &config_with(Some(" memory "))).as_deref(), + Some("memory"), + "a blank override falls through to the file" + ); +} + +#[tokio::test] +async fn opens_memory_and_refuses_garbage() { + let backend = open("memory").await.unwrap(); + assert_eq!(backend.driver(), "memory"); + assert!(open("ftp://nope").await.is_err()); +} + +#[test] +fn install_installed_and_clear() { + // One test owns the process slot, so it cannot race another. + clear(); + assert!(installed().is_none()); + let backend: Arc = Arc::new(tinystoragedrivers::MemoryStorage::new()); + assert!(install(Arc::clone(&backend)).is_none()); + assert!(installed().is_some_and(|got| Arc::ptr_eq(&got, &backend))); + assert!(install(Arc::new(tinystoragedrivers::MemoryStorage::new())).is_some()); + assert!(clear()); + assert!(!clear()); +} + +#[test] +fn agent_scopes_match_the_session_store() { + assert_eq!(scope_for_agent("agent-7").as_str(), "agent-7"); + assert!(scope_for_agent("has a space") + .as_str() + .starts_with("sha256:")); +} + +#[test] +fn only_mongodb_is_shared_between_processes() { + assert!(driver_is_shared("mongodb")); + for driver in ["sqlite", "memory", "file"] { + assert!(!driver_is_shared(driver), "{driver}"); + } +} diff --git a/crates/openhuman-core/src/tools/impl/network/host_tests.rs b/crates/openhuman-core/src/tools/impl/network/host_tests.rs index 09c62de3910..fbf4fa80d18 100644 --- a/crates/openhuman-core/src/tools/impl/network/host_tests.rs +++ b/crates/openhuman-core/src/tools/impl/network/host_tests.rs @@ -47,7 +47,7 @@ fn web_fetch_advertises_its_pre_move_contract() { "schema": { "properties": { "max_bytes": { - "description": "Truncate body at this many bytes (default 1_000_000).", + "description": "Cap the returned text at this many bytes (default 1_000_000). Bounds the OUTPUT — the extracted markdown, or the raw body with raw:true — never the markup the extractor reads, so lowering it cannot cost you content the page actually had.", "minimum": 1, "type": "integer" }, diff --git a/crates/openhuman-embed/Cargo.toml b/crates/openhuman-embed/Cargo.toml index 8240910c0b8..c5317d2edb5 100644 --- a/crates/openhuman-embed/Cargo.toml +++ b/crates/openhuman-embed/Cargo.toml @@ -17,6 +17,10 @@ hosting = ["openhuman-core/hosting"] modules = ["openhuman-core/modules"] voice = ["openhuman-core/voice"] web3 = ["openhuman-core/web3"] +# Storage drivers for `[storage] url` (see the core `storage` domain). +storage-sqlite = ["openhuman-core/storage-sqlite"] +storage-mongodb = ["openhuman-core/storage-mongodb"] +storage-file = ["openhuman-core/storage-file"] runtime-node = ["openhuman-core/runtime-node"] media = ["openhuman-core/media"] flows = ["openhuman-core/flows"] diff --git a/crates/openhuman-rpc/Cargo.toml b/crates/openhuman-rpc/Cargo.toml index 0571b806096..c7bba86139d 100644 --- a/crates/openhuman-rpc/Cargo.toml +++ b/crates/openhuman-rpc/Cargo.toml @@ -38,7 +38,7 @@ server = [ # `session_raw/` / `session_db/` / `tinyagents_store/` layout behind # TinyAgents' session store port. On with `server`; the TUI, which runs the # core without the server, turns it on by itself. -session-store = ["dep:tinyagents-session", "dep:tinyagents-harness", "dep:chrono", "dep:anyhow"] +session-store = ["dep:tinyagents-session", "dep:tinyagents-harness", "dep:chrono", "dep:anyhow", "tinyagents-session/storage-drivers"] # Forwarded so the server's Sentry-routing tests run against a core that # reports to Sentry. crash-reporting = ["openhuman-core/crash-reporting"] diff --git a/crates/openhuman-rpc/src/server/shims.rs b/crates/openhuman-rpc/src/server/shims.rs index 230e5e5b899..776a8e8d961 100644 --- a/crates/openhuman-rpc/src/server/shims.rs +++ b/crates/openhuman-rpc/src/server/shims.rs @@ -168,9 +168,9 @@ async fn run_server_with_services( } // The desktop app and the CLI keep conversations in the classic on-disk - // layout; the core itself carries no storage. Installed before boot so - // its recovery sweep runs. - crate::session_store::install(); + // layout unless a storage URL is configured; the core itself carries no + // storage. Installed before boot so its recovery sweep runs. + crate::session_store::install_for_host().await?; let runtime = builder.build().await?; super::serve::serve(&runtime, ready_tx, shutdown_token).await } diff --git a/crates/openhuman-rpc/src/session_store/README.md b/crates/openhuman-rpc/src/session_store/README.md index dcd873d1515..041ba090e28 100644 --- a/crates/openhuman-rpc/src/session_store/README.md +++ b/crates/openhuman-rpc/src/session_store/README.md @@ -62,11 +62,37 @@ tinyagents_store/{kv,journal}/ run status, goals, todos, turn journal ``` +## A storage-backed store instead + +`install_for_host()` is what the server shims and the TUI call. With no +storage URL configured (`OPENHUMAN_STORAGE_URL`, else `[storage] url` in +`config.toml`) it is `install()` above, unchanged. With one, it opens that +backend (`openhuman_core::storage::open`), makes it the process's storage +backend, and installs TinyAgents' `DriverSessionStores` over it instead: + +```text + install_for_host() + storage URL? no -> install() (the layout above) + yes -> storage::open(url) -> storage::install(backend) + DriverSessionStores::new(backend) + .recover_on_open(driver != "mongodb") + -> agent::session_store::install(provider) +``` + +Every agent then gets its own storage scope (its id, or `sha256:` of an id +that is not a valid scope), so agents sharing one MongoDB database never see +each other's transcripts, turn states, records or journals. A single-process +backend (SQLite, memory, files) interrupts an agent's in-flight turns the +first time it is opened; MongoDB does not, because another process may own +them. A URL that cannot be parsed or opened fails the boot rather than +falling back to local files. `install_for_url` is the same with the URL +already resolved, for tests and hosts that read it themselves. + ## Layout | File | What it does | | --- | --- | -| [`mod.rs`](mod.rs) | `SqliteSessionStores` (`at`, `resolving`), its `SessionStoreProvider` impl (`for_agent`, `recover`, `destination_key`, `workspace_dir`), and `install()`. | +| [`mod.rs`](mod.rs) | `SqliteSessionStores` (`at`, `resolving`), its `SessionStoreProvider` impl (`for_agent`, `recover`, `destination_key`, `workspace_dir`), `install()`, and `install_for_host()` / `install_for_url()`, which install `DriverSessionStores` when a storage URL is configured. | ## Key types and entry points @@ -113,6 +139,16 @@ runs as a doctest. cargo test -p openhuman-rpc session_store ``` +## Scope of the storage-backed layout + +A configured storage URL moves what the installed `SessionStoreProvider` +serves: the agent loop's transcripts, turn states, records and journal. Host +readers that still resolve `workspace/session_raw` or the workspace turn-state +and journal files directly (thread history paging, turn-state RPCs, run replay, +cron origin delivery, graph subagent transcripts) have not moved onto the +provider yet and are tracked as follow-ups; until they do, treat a storage URL +as opt-in and not yet a drop-in for the desktop layout. + ## Further reading - [`gitbooks/developing/architecture/agent-harness.md`](../../../../gitbooks/developing/architecture/agent-harness.md): the agent harness. diff --git a/crates/openhuman-rpc/src/session_store/mod.rs b/crates/openhuman-rpc/src/session_store/mod.rs index 9434cab6618..a65ef0bb3ed 100644 --- a/crates/openhuman-rpc/src/session_store/mod.rs +++ b/crates/openhuman-rpc/src/session_store/mod.rs @@ -139,6 +139,79 @@ pub fn install() { ))); } +/// Installs the session store the host's configuration asks for, before the +/// core boots. +/// +/// With no storage URL (`OPENHUMAN_STORAGE_URL`, else `[storage] url`) this is +/// [`install`]: the classic on-disk layout, unchanged. With one, the backend is +/// opened, made the process's storage backend +/// ([`openhuman_core::storage::install`]), and TinyAgents' +/// `DriverSessionStores` is installed over it: every agent's transcripts, turn +/// states, records and journal in that backend, one storage scope per agent. +/// +/// A single-process backend (SQLite, memory, files) also interrupts an +/// agent's in-flight turns the first time it is opened, since only an earlier +/// process can have left them. MongoDB does not: several processes may share +/// that database, and another one may own those turns. +/// +/// # Errors +/// +/// When a URL is configured but cannot be parsed or opened. A deployment that +/// asked for a backend must not quietly fall back to local files. +pub async fn install_for_host() -> anyhow::Result<()> { + let url = match std::env::var(openhuman_core::storage::STORAGE_URL_VAR) { + Ok(url) if !url.trim().is_empty() => Some(url.trim().to_string()), + _ => match openhuman_core::config::rpc::load_config_with_timeout().await { + Ok(config) => openhuman_core::storage::configured_url(&config), + // An unreadable config keeps the desktop booting on the classic + // layout, as it always has. Remote deployments pin the backend + // with `OPENHUMAN_STORAGE_URL`, which never reads the config. + Err(error) => { + tracing::warn!( + "[session_store] config unavailable ({error}); keeping the on-disk layout" + ); + None + } + }, + }; + install_for_url(url).await +} + +/// [`install_for_host`] with the URL already resolved: `None` installs the +/// classic on-disk store, a URL opens that backend and installs +/// `DriverSessionStores` over it. +/// +/// # Errors +/// +/// When `url` cannot be parsed or opened. +pub async fn install_for_url(url: Option) -> anyhow::Result<()> { + use anyhow::Context as _; + + let Some(url) = url else { + // Drop a backend an earlier call installed, so storage operations do + // not keep writing to it while the classic layout is in force. + openhuman_core::storage::clear(); + install(); + return Ok(()); + }; + let backend = openhuman_core::storage::open(&url) + .await + .context("opening the configured storage backend")?; + let single_process = !openhuman_core::storage::driver_is_shared(backend.driver()); + let provider = tinyagents_session::DriverSessionStores::new(Arc::clone(&backend)) + .context("starting the session store bridge")? + .recover_on_open(single_process); + // Only a fully working bridge makes the backend the process's storage. + openhuman_core::storage::install(backend); + tracing::info!( + target: "openhuman_rpc::session_store", + recover_on_open = single_process, + "[session_store] installed the storage-backed session store" + ); + openhuman_core::agent::session_store::install(Arc::new(provider)); + Ok(()) +} + #[cfg(test)] #[path = "mod_tests.rs"] mod tests; diff --git a/crates/openhuman-rpc/src/session_store/mod_tests.rs b/crates/openhuman-rpc/src/session_store/mod_tests.rs index 009c79c2041..f1829d18d34 100644 --- a/crates/openhuman-rpc/src/session_store/mod_tests.rs +++ b/crates/openhuman-rpc/src/session_store/mod_tests.rs @@ -48,3 +48,131 @@ fn recovery_interrupts_turns_left_in_flight() { Some(TurnLifecycle::Interrupted) ); } + +/// The provider and storage slots are process-wide; these tests take turns. +static SLOTS: Mutex<()> = Mutex::new(()); + +/// Puts the process-global storage backend back as a test found it. +fn restore_backend(previous: Option>) { + match previous { + Some(backend) => { + openhuman_core::storage::install(backend); + } + None => { + openhuman_core::storage::clear(); + } + } +} + +#[tokio::test] +async fn a_storage_url_installs_the_driver_backed_store() { + let _turn = SLOTS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let previous = openhuman_core::agent::session_store::installed(); + let previous_backend = openhuman_core::storage::installed(); + install_for_url(Some("memory".into())).await.unwrap(); + + let provider = openhuman_core::agent::session_store::installed().unwrap(); + assert!( + provider + .destination_key() + .is_some_and(|key| key.starts_with("memory://")), + "{:?}", + provider.destination_key() + ); + assert_eq!( + openhuman_core::storage::installed().map(|b| b.driver()), + Some("memory") + ); + // Agents are kept apart in the shared backend. + let alice = provider.for_agent("alice"); + alice + .turn_states + .put(&TurnState::started("t", "r", 8, "2026-01-01T00:00:00Z")) + .unwrap(); + assert!(provider + .for_agent("bob") + .turn_states + .get("t") + .unwrap() + .is_none()); + + restore_backend(previous_backend); + openhuman_core::agent::session_store::restore(previous); +} + +#[tokio::test] +async fn no_url_keeps_the_classic_layout() { + let _turn = SLOTS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let previous = openhuman_core::agent::session_store::installed(); + let previous_backend = openhuman_core::storage::installed(); + install_for_url(None).await.unwrap(); + let provider = openhuman_core::agent::session_store::installed().unwrap(); + assert!( + provider.workspace_dir().is_some(), + "the file layout is installed" + ); + assert!(openhuman_core::storage::installed().is_none()); + restore_backend(previous_backend); + openhuman_core::agent::session_store::restore(previous); +} + +#[tokio::test] +async fn an_unusable_url_fails_the_boot() { + let _turn = SLOTS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let previous = openhuman_core::agent::session_store::installed(); + let error = install_for_url(Some("ftp://nowhere".into())) + .await + .unwrap_err(); + assert!(format!("{error:#}").contains("storage"), "{error:#}"); + openhuman_core::agent::session_store::restore(previous); +} + +#[tokio::test] +async fn restoring_the_classic_layout_clears_a_previous_backend() { + let _turn = SLOTS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let previous = openhuman_core::agent::session_store::installed(); + let previous_backend = openhuman_core::storage::installed(); + install_for_url(Some("memory".into())).await.unwrap(); + assert!(openhuman_core::storage::installed().is_some()); + install_for_url(None).await.unwrap(); + assert!(openhuman_core::storage::installed().is_none()); + restore_backend(previous_backend); + openhuman_core::agent::session_store::restore(previous); +} + +#[tokio::test] +async fn the_host_reads_the_storage_url_from_the_environment() { + let _turn = SLOTS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let previous = openhuman_core::agent::session_store::installed(); + let previous_backend = openhuman_core::storage::installed(); + let var = openhuman_core::storage::STORAGE_URL_VAR; + let old = std::env::var_os(var); + + std::env::set_var(var, " memory "); + let booted = install_for_host().await; + let driver = openhuman_core::storage::installed().map(|b| b.driver()); + + std::env::set_var(var, "ftp://nowhere"); + let refused = install_for_host().await; + + match old { + Some(value) => std::env::set_var(var, value), + None => std::env::remove_var(var), + } + restore_backend(previous_backend); + openhuman_core::agent::session_store::restore(previous); + + booted.unwrap(); + assert_eq!(driver, Some("memory")); + assert!(refused.is_err(), "an unusable env URL fails the boot"); +} diff --git a/crates/openhuman-tinyhumans/Cargo.toml b/crates/openhuman-tinyhumans/Cargo.toml index 64b7e63f335..0d9609c1e03 100644 --- a/crates/openhuman-tinyhumans/Cargo.toml +++ b/crates/openhuman-tinyhumans/Cargo.toml @@ -25,6 +25,10 @@ hosting = ["openhuman-embed/hosting"] modules = ["openhuman-embed/modules"] voice = ["openhuman-embed/voice"] web3 = ["openhuman-embed/web3"] +# Storage drivers for `[storage] url` (see the core `storage` domain). +storage-sqlite = ["openhuman-embed/storage-sqlite"] +storage-mongodb = ["openhuman-embed/storage-mongodb"] +storage-file = ["openhuman-embed/storage-file"] runtime-node = ["openhuman-embed/runtime-node"] media = ["openhuman-embed/media"] flows = ["openhuman-embed/flows"] diff --git a/crates/openhuman-tui/src/runner.rs b/crates/openhuman-tui/src/runner.rs index a95d526224b..8bfe6664a9c 100644 --- a/crates/openhuman-tui/src/runner.rs +++ b/crates/openhuman-tui/src/runner.rs @@ -176,8 +176,9 @@ async fn async_main( // In-process core: full domains (channel.web_chat needs DomainGroup::Channels, // so harness() is not enough), no RPC transport, no background services. - // Conversations in the classic on-disk layout, as the desktop keeps them. - openhuman_rpc::session_store::install(); + // Conversations in the classic on-disk layout, as the desktop keeps them, + // unless a storage URL (`OPENHUMAN_STORAGE_URL` / `[storage] url`) is set. + openhuman_rpc::session_store::install_for_host().await?; let runtime = Arc::new( CoreBuilder::new(HostKind::detect_standalone()) .domains(DomainSet::full()) diff --git a/scripts/ci/agent-runtime-boundary-baseline.json b/scripts/ci/agent-runtime-boundary-baseline.json index d6036653b13..a6f86a6ca3d 100644 --- a/scripts/ci/agent-runtime-boundary-baseline.json +++ b/scripts/ci/agent-runtime-boundary-baseline.json @@ -23,7 +23,7 @@ { "rule": "openhuman-runtime-bridge", "path": "crates/openhuman-core/src/agent/subagent_host/ops/graph/dispatch.rs", - "line": 197, + "line": 198, "text": "let run_result = Box::pin(run_turn_via_tinyagents_shared(", "occurrence": 1 }, @@ -44,7 +44,7 @@ { "rule": "openhuman-runtime-bridge", "path": "crates/openhuman-core/src/agent/tinyagents/mod.rs", - "line": 92, + "line": 93, "text": "pub(crate) use turn_runner::{run_root_turn_via_hosted_agent, run_turn_via_tinyagents_shared};", "occurrence": 1 }, @@ -58,14 +58,14 @@ { "rule": "openhuman-runtime-bridge", "path": "crates/openhuman-core/src/agent/tinyagents/turn_runner.rs", - "line": 88, + "line": 89, "text": "pub(crate) async fn run_turn_via_tinyagents_shared(", "occurrence": 1 }, { "rule": "openhuman-runtime-bridge", "path": "crates/openhuman-core/src/agent/tinyagents/turn_runner.rs", - "line": 268, + "line": 269, "text": "} = assemble_turn_harness(", "occurrence": 1 }, @@ -422,7 +422,7 @@ { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/agent/session_host/builder/host_tools.rs", - "line": 388, + "line": 367, "text": "let root = crate::agent::turn_workspace::current()?;", "occurrence": 1 }, @@ -499,56 +499,56 @@ { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/agent/turn_origin.rs", - "line": 175, + "line": 179, "text": "pub fn current_is_user_authored() -> bool {", "occurrence": 1 }, { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/agent/turn_origin.rs", - "line": 179, + "line": 183, "text": "tokio::task_local! {", "occurrence": 1 }, { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/agent/turn_origin.rs", - "line": 184, + "line": 188, "text": "pub static AGENT_TURN_ORIGIN: AgentTurnOrigin;", "occurrence": 1 }, { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/agent/turn_origin.rs", - "line": 212, + "line": 216, "text": "AGENT_TURN_ORIGIN.scope(origin, Box::pin(fut)).await", "occurrence": 1 }, { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/agent/turn_origin.rs", - "line": 219, + "line": 223, "text": "AGENT_TURN_ORIGIN.try_with(|o| o.clone()).ok()", "occurrence": 1 }, { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/agent/turn_origin.rs", - "line": 274, + "line": 278, "text": "Some(origin) => with_origin(origin, fut).await,", "occurrence": 1 }, { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/agent/turn_origin.rs", - "line": 310, + "line": 314, "text": "Some(origin) => with_origin(origin, fut).await,", "occurrence": 2 }, { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/agent/turn_origin.rs", - "line": 373, + "line": 377, "text": "pub fn current_request_id() -> Option {", "occurrence": 1 }, @@ -604,7 +604,7 @@ { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/cron/origin.rs", - "line": 165, + "line": 166, "text": "let turn = turn_origin::current();", "occurrence": 1 }, @@ -681,7 +681,7 @@ { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/flows/ops/execution.rs", - "line": 272, + "line": 306, "text": "with_origin(", "occurrence": 1 }, @@ -786,7 +786,7 @@ { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/inference/provider/openhuman_backend_model.rs", - "line": 497, + "line": 480, "text": "fn with_thread_id(request: ModelRequest, thread_id: Option<&str>) -> ModelRequest {", "occurrence": 1 }, @@ -863,7 +863,7 @@ { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/mcp/server/tools/dispatch.rs", - "line": 374, + "line": 375, "text": "crate::agent::turn_origin::with_origin(origin, agent.run_single(&prompt)),", "occurrence": 1 }, @@ -884,7 +884,7 @@ { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/security/approval/gate_intercept.rs", - "line": 24, + "line": 53, "text": "let origin = turn_origin::current().unwrap_or(AgentTurnOrigin::Unknown);", "occurrence": 1 }, @@ -961,7 +961,7 @@ { "rule": "openhuman-task-local", "path": "crates/openhuman-core/src/voice/live/session.rs", - "line": 201, + "line": 202, "text": "Box::pin(with_origin(", "occurrence": 1 }, @@ -1504,6 +1504,13 @@ "text": "pub use tinyagents_harness::error::{Result, TinyAgentsError};", "occurrence": 1 }, + { + "rule": "tinyagents-upstream-reexport", + "path": "vendor/tinyagents/crates/tinyagents-graph/src/orchestration.rs", + "line": 7, + "text": "pub use tinyagents_tasks::*;", + "occurrence": 1 + }, { "rule": "tinyagents-upstream-reexport", "path": "vendor/tinyagents/crates/tinyagents-harness/src/tool/discover/mod.rs", @@ -1511,6 +1518,13 @@ "text": "pub use tinytools::rank::{Bm25Index, tokenize};", "occurrence": 1 }, + { + "rule": "tinyagents-upstream-reexport", + "path": "vendor/tinyagents/crates/tinyagents-orchestration/src/status/mod.rs", + "line": 20, + "text": "pub use tinyagents_tasks::NoEquivalentStatus;", + "occurrence": 1 + }, { "rule": "tinyagents-upstream-reexport", "path": "vendor/tinyagents/crates/tinyagents-orchestration/src/subagent/policy.rs", @@ -1542,7 +1556,7 @@ { "rule": "tinyagents-upstream-reexport", "path": "vendor/tinyagents/crates/tinyagents-session/src/port/mod.rs", - "line": 49, + "line": 58, "text": "pub use tinyagents_harness::store::{AppendStore, Store};", "occurrence": 1 } diff --git a/scripts/ci/check-dep-sim-calibration.sh b/scripts/ci/check-dep-sim-calibration.sh index b0170ba2081..60a5011c5a5 100755 --- a/scripts/ci/check-dep-sim-calibration.sh +++ b/scripts/ci/check-dep-sim-calibration.sh @@ -99,8 +99,10 @@ cd "$(dirname "$0")/../.." # five names from the Linux flows graph; native builds remain at three. # 311 -> 320 on 2026-10-07: tinyskills v0.2.8 (#7054) brings cap-std and its # dependencies into the flows graph. -# This matches the current `flows:342:320:3` entry in +# 320 -> 313 on 2026-10-09: the tinyagents and tinyflows mains shed seven +# names from the flows graph. +# This matches the current `flows:335:313:2` entry in # scripts/kernel-floor.limits; its preceding entries are historical. -EXPECTED_NAMES=320 +EXPECTED_NAMES=313 exec python3 scripts/dep-sim.py --cut-nothing --expect-names "${EXPECTED_NAMES}" diff --git a/scripts/ci/checkout-submodules.sh b/scripts/ci/checkout-submodules.sh index 611f92aa5c6..6dc81c86360 100755 --- a/scripts/ci/checkout-submodules.sh +++ b/scripts/ci/checkout-submodules.sh @@ -19,6 +19,9 @@ NESTED=( # tinyagents-live (the live voice agent runner) builds the vendored # tinyliveagents library by path. "vendor/tinyagents vendor/tinyliveagents" + # The session store and the core `storage` domain link the storage drivers + # tinyagents vendors; the root [patch] points tinyflows' git dependency here. + "vendor/tinyagents vendor/tinystoragedrivers" # The native TinyConnectors test module is built from its own manifest, so # the root workspace's tinybus patch does not apply to this dependency. "vendor/tinyconnectors vendor/tinybus" diff --git a/scripts/kernel-floor.limits b/scripts/kernel-floor.limits index d36e03add29..790c700dca6 100644 --- a/scripts/kernel-floor.limits +++ b/scripts/kernel-floor.limits @@ -24,6 +24,11 @@ # fs-set-times, maybe-owned and rustix-linux-procfs into # the always-on flows graph. Measured on CI (Linux): # 342 packages / 320 names / 3 native. +# 335/313/2 2026-10-09 The tinyagents and tinyflows mains (storage drivers +# on tinystoragedrivers v0.4.0) shed seven packages, seven +# names and one native build from the flows graph. +# Measured on CI (Linux): 335 packages / 313 names / 2 +# native. # 339/318/3 2026-10-04 Workspace attachment intake adds lossless PNG # optimization through oxipng (11 packages/names, including # libdeflate-sys C compilation). The merged TinyBox jail @@ -658,4 +663,4 @@ # not add a native build dependency. # 332/311/3 2026-10-04 Memory lifecycle refresh removed five packages/names # from the Linux flows graph; native builds remain at 3. -flows:342:320:3 +flows:335:313:2 diff --git a/vendor/tinyagents b/vendor/tinyagents index 1df6ad019ca..d54a85368a6 160000 --- a/vendor/tinyagents +++ b/vendor/tinyagents @@ -1 +1 @@ -Subproject commit 1df6ad019ca02681cbfb26276d734b1723dbc708 +Subproject commit d54a85368a6e6fbc3f21d2e8613b31b31ec74629 diff --git a/vendor/tinyflows b/vendor/tinyflows index a48f7c57b53..f9df351f2cd 160000 --- a/vendor/tinyflows +++ b/vendor/tinyflows @@ -1 +1 @@ -Subproject commit a48f7c57b535cdd2af3951594cb5a2ca7a91eb6c +Subproject commit f9df351f2cd73db99dd9ea19560001a4c35a12bc