diff --git a/crates/openhuman-core/src/inference/context_window.rs b/crates/openhuman-core/src/inference/context_window.rs index fce1f3da662..c3732a02755 100644 --- a/crates/openhuman-core/src/inference/context_window.rs +++ b/crates/openhuman-core/src/inference/context_window.rs @@ -10,6 +10,11 @@ //! through `tinyinference_llm::model::discover` (bounded, cached per //! endpoint and model), lowered by any window the provider stated in a //! context-overflow error on that endpoint (learned by the chat adapters). +//! Ollama's OpenAI-compatible listing carries no window, so for the local +//! Ollama provider and any endpoint that looks like an Ollama server +//! (including a custom OpenAI-compatible provider at `:11434/v1`) the same +//! discovery asks the native `POST /api/show`; its result, or its failure, +//! is cached and the whole lookup is time-bounded. //! 3. **Local runtime profile** for local providers (Ollama, LM Studio, ...). //! 4. **Static guess**: the tier aliases, the cost catalog, and the generic //! id-pattern table ([`super::model_context::static_context_window_for_model`]), @@ -23,7 +28,8 @@ use std::collections::{HashMap, HashSet}; use std::sync::{Mutex, OnceLock}; use tinyinference_llm::model::discover::{ - discover_model_limits_with, model_limits_cache, ModelLimitsCache, ModelListingFetcher, + discover_model_limits_with, model_limits_cache, DiscoveryRequest, ModelLimitsCache, + ModelListingFetcher, }; use crate::config::Config; @@ -132,6 +138,27 @@ fn warn_static_guess(model: &str, window: Option, provider: &str) { } } +/// Discovery request for the built-in local Ollama provider: its OpenAI +/// listing carries no window, so the native `/api/show` probe is forced on. +/// A custom OpenAI-compatible provider pointed at an Ollama server gets the +/// same probe from the endpoint auto-detection in `tinyinference-llm`. +fn ollama_limits_request(model: &str, config: &Config) -> Option { + let root = tinyinference_local::ollama::ollama_base_url_from_override( + config.local_ai.base_url.as_deref(), + ); + let model = model + .split_once('@') + .map_or(model, |(model, _)| model) + .trim(); + if model.is_empty() { + return None; + } + Some( + DiscoveryRequest::new(format!("{}/v1", root.trim_end_matches('/')), model) + .with_ollama_native(true), + ) +} + /// The fetcher discovery uses. Unit tests never reach the network. fn default_fetcher() -> Box { #[cfg(test)] @@ -234,10 +261,17 @@ async fn resolve_inner( } let local_kind = tinyinference_local::profile::kind_from_provider_string(provider); - if local_kind.is_none() { - if let Some(request) = + let request = match local_kind { + None => { crate::inference::provider::factory::model_limits_request(role, provider, model, config) - { + } + Some(tinyinference_local::profile::LocalProviderKind::Ollama) => { + ollama_limits_request(model, config) + } + Some(_) => None, + }; + { + if let Some(request) = request { let limits = discover_model_limits_with(fetcher, cache, &request).await; if let Some((window, limits)) = limits .as_ref() diff --git a/crates/openhuman-core/src/inference/context_window_tests.rs b/crates/openhuman-core/src/inference/context_window_tests.rs index fbe7ec1fb85..eadc235c050 100644 --- a/crates/openhuman-core/src/inference/context_window_tests.rs +++ b/crates/openhuman-core/src/inference/context_window_tests.rs @@ -26,6 +26,8 @@ const V41_FLASH: &str = "deepseek/deepseek-v4.1-flash"; #[derive(Default)] struct FakeFetcher { routes: Vec<(String, Value)>, + posts: Vec<(String, Value)>, + post_calls: StdMutex>, calls: StdMutex>, } @@ -35,6 +37,15 @@ impl FakeFetcher { self } + fn with_post(mut self, url: &str, body: Value) -> Self { + self.posts.push((url.to_string(), body)); + self + } + + fn post_count(&self) -> usize { + self.post_calls.lock().unwrap().len() + } + fn call_count(&self) -> usize { self.calls.lock().unwrap().len() } @@ -54,6 +65,20 @@ impl ModelListingFetcher for FakeFetcher { .map(|(_, body)| body.clone()) .ok_or_else(|| tinyinference_llm::Error::Catalog(format!("GET {url} returned 404"))) } + + async fn post_json( + &self, + url: &str, + _headers: &[(String, String)], + _body: &Value, + ) -> tinyinference_llm::Result { + self.post_calls.lock().unwrap().push(url.to_string()); + self.posts + .iter() + .find(|(route, _)| route == url) + .map(|(_, body)| body.clone()) + .ok_or_else(|| tinyinference_llm::Error::Catalog(format!("POST {url} returned 404"))) + } } fn openrouter_listing() -> Value { @@ -230,7 +255,7 @@ async fn unknown_model_without_provider_data_is_unknown() { } #[tokio::test] -async fn local_routes_use_the_local_profile_without_discovery() { +async fn local_ollama_without_a_reachable_server_falls_back_to_the_local_profile() { let tmp = TempDir::new().unwrap(); let config = openrouter_config(&tmp); let fetcher = FakeFetcher::default(); @@ -245,7 +270,93 @@ async fn local_routes_use_the_local_profile_without_discovery() { .await; assert!(resolved.window.is_some()); assert_eq!(resolved.source, WindowSource::LocalProfile); - assert_eq!(fetcher.call_count(), 0); +} + +fn ollama_show() -> Value { + json!({ + "model_info": { "general.architecture": "qwen3", "qwen3.context_length": 40960 }, + "capabilities": ["completion", "tools"] + }) +} + +#[tokio::test] +async fn builtin_local_ollama_reads_the_window_from_api_show() { + let tmp = TempDir::new().unwrap(); + let mut config = openrouter_config(&tmp); + config.local_ai.base_url = Some("http://127.0.0.1:11434".to_string()); + let fetcher = + FakeFetcher::default().with_post("http://127.0.0.1:11434/api/show", ollama_show()); + let resolved = resolve_context_window_with( + &fetcher, + &ModelLimitsCache::default(), + "chat", + "ollama:qwen3:14b", + "qwen3:14b", + &config, + ) + .await; + assert_eq!(resolved.window, Some(40_960)); + assert_eq!(resolved.source, WindowSource::ProviderReported); +} + +#[tokio::test] +async fn custom_openai_provider_at_ollama_v1_reads_the_window_from_api_show() { + let tmp = TempDir::new().unwrap(); + let mut config = openrouter_config(&tmp); + config.cloud_providers.push(CloudProviderCreds { + id: "p_ollama".to_string(), + slug: "myollama".to_string(), + label: "Ollama".to_string(), + endpoint: "http://127.0.0.1:11434/v1".to_string(), + auth_style: AuthStyle::None, + ..Default::default() + }); + // `/v1/models` lists the model without any window, as Ollama does. + let fetcher = FakeFetcher::default() + .with( + "http://127.0.0.1:11434/v1/models", + json!({"data": [{"id": "qwen3:14b", "object": "model", "owned_by": "library"}]}), + ) + .with_post("http://127.0.0.1:11434/api/show", ollama_show()); + let cache = ModelLimitsCache::default(); + let resolve = || { + resolve_context_window_with( + &fetcher, + &cache, + "chat", + "myollama:qwen3:14b", + "qwen3:14b", + &config, + ) + }; + let resolved = resolve().await; + assert_eq!(resolved.window, Some(40_960)); + assert_eq!(resolved.source, WindowSource::ProviderReported); + // Cached: the second turn makes no further requests. + resolve().await; + assert_eq!(fetcher.post_count(), 1); +} + +#[tokio::test] +async fn failed_ollama_probe_is_remembered_and_not_retried() { + let tmp = TempDir::new().unwrap(); + let mut config = openrouter_config(&tmp); + config.local_ai.base_url = Some("http://127.0.0.1:11434".to_string()); + let fetcher = FakeFetcher::default(); + let cache = ModelLimitsCache::default(); + for _ in 0..3 { + let resolved = resolve_context_window_with( + &fetcher, + &cache, + "chat", + "ollama:qwen3:14b", + "qwen3:14b", + &config, + ) + .await; + assert_eq!(resolved.source, WindowSource::LocalProfile); + } + assert_eq!(fetcher.post_count(), 1); } #[test] diff --git a/vendor/tinyagents b/vendor/tinyagents index 42fcd8e7ff6..0699f94cbb6 160000 --- a/vendor/tinyagents +++ b/vendor/tinyagents @@ -1 +1 @@ -Subproject commit 42fcd8e7ff67d47e842cf09bafbfa04407065902 +Subproject commit 0699f94cbb6db21c72807cb8a0f123754976c59e