Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions crates/aster-cli/src/jev.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,15 @@ pub(crate) enum Advice {
Stop,
}

#[cfg_attr(not(feature = "jev"), allow(dead_code))]
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum RouteAdvice {
Unavailable,
Picked { entry: String, probability: f64 },
Unsure { entry: String, probability: f64 },
Failed(String),
}

pub(crate) struct Advisor {
#[cfg(feature = "jev")]
client: Option<aster_jev::JevClient>,
Expand Down Expand Up @@ -93,6 +102,33 @@ impl Advisor {
Advice::None
}
}

pub(crate) async fn route(&self, message: &str, entries: &[(String, String)]) -> RouteAdvice {
#[cfg(feature = "jev")]
{
let Some(client) = &self.client else {
return RouteAdvice::Unavailable;
};
match client.route(message, entries).await {
Ok(pick) if pick.probability >= aster_jev::ROUTE_CONFIDENCE_MIN => {
RouteAdvice::Picked {
entry: pick.entry,
probability: pick.probability,
}
}
Ok(pick) => RouteAdvice::Unsure {
entry: pick.entry,
probability: pick.probability,
},
Err(e) => RouteAdvice::Failed(format!("{e:#}")),
}
}
#[cfg(not(feature = "jev"))]
{
let _ = (message, entries);
RouteAdvice::Unavailable
}
}
}

#[cfg(feature = "jev")]
Expand Down
83 changes: 82 additions & 1 deletion crates/aster-cli/src/jev_tests.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
#[cfg(feature = "jev")]
use super::*;

#[cfg(feature = "jev")]
Expand All @@ -14,3 +13,85 @@ fn snapshot_keeps_the_last_entries_bounded() {
let wire = vec![serde_json::json!({ "role": "user", "content": long })];
assert!(snapshot(&wire).chars().count() < 600);
}

#[cfg(feature = "jev")]
async fn advisor_answering(
status: u16,
answers: serde_json::Value,
) -> (Advisor, wiremock::MockServer) {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/systemone"))
.respond_with(ResponseTemplate::new(status).set_body_json(answers))
.mount(&server)
.await;
let advisor = Advisor {
client: Some(aster_jev::JevClient::new("key").base_url(server.uri())),
};
(advisor, server)
}

#[cfg(feature = "jev")]
fn route_answer(entry: &str, probability: f64) -> serde_json::Value {
serde_json::json!({ "answers": { "entry": {
"type": "choice",
"choice": entry,
"probabilities": { entry: probability },
"confidence": probability,
} } })
}

#[cfg(feature = "jev")]
fn entries() -> Vec<(String, String)> {
vec![
("everyday".to_string(), "small edits".to_string()),
("deep".to_string(), "hard debugging".to_string()),
]
}

#[cfg(feature = "jev")]
#[tokio::test]
async fn route_takes_a_confident_pick() {
let (advisor, _server) = advisor_answering(200, route_answer("deep", 0.9)).await;
assert_eq!(
advisor.route("why does this deadlock", &entries()).await,
RouteAdvice::Picked {
entry: "deep".to_string(),
probability: 0.9,
}
);
}

#[cfg(feature = "jev")]
#[tokio::test]
async fn route_below_the_bar_is_unsure() {
let (advisor, _server) = advisor_answering(200, route_answer("deep", 0.4)).await;
assert_eq!(
advisor.route("rename x", &entries()).await,
RouteAdvice::Unsure {
entry: "deep".to_string(),
probability: 0.4,
}
);
}

#[cfg(feature = "jev")]
#[tokio::test]
async fn route_failure_is_reported() {
let (advisor, _server) = advisor_answering(500, serde_json::json!({})).await;
assert!(matches!(
advisor.route("rename x", &entries()).await,
RouteAdvice::Failed(_)
));
}

#[tokio::test]
async fn route_without_a_client_is_unavailable() {
let advisor = Advisor::resolve(&Experimental::default());
assert_eq!(
advisor.route("rename x", &[]).await,
RouteAdvice::Unavailable
);
}
26 changes: 26 additions & 0 deletions crates/aster-cli/src/mom.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ use anyhow::Result;
use aster_mom::{Catalog, Engine, ModelEntry, Power, Resolver, Selection, Signals, Thinking};
use clap::{Args, Subcommand};

use crate::jev::RouteAdvice;

const JEV_ROUTER: &str = "jev";

/// Everything needed to reach one resolved model from a turn task.
#[derive(Debug, Clone)]
pub struct RouterTarget {
Expand Down Expand Up @@ -461,6 +465,25 @@ pub async fn consult_router(
plan: &RouterPlan,
message: &str,
) -> Option<String> {
match crate::jev::current().route(message, &plan.entries).await {
RouteAdvice::Picked { entry, probability } => {
log_router(JEV_ROUTER, &format!("picked {entry} (p={probability:.2})"));
return Some(entry);
}
RouteAdvice::Unsure { entry, probability } => {
log_router(
JEV_ROUTER,
&format!("unsure: {entry} at p={probability:.2}, keeping start-with"),
);
return None;
}
RouteAdvice::Failed(err) => log_router(
JEV_ROUTER,
&format!("call failed: {err}; asking the router model"),
),
RouteAdvice::Unavailable => {}
}

let mut client = client.clone();
if client.base_url().trim_end_matches('/') != plan.router.base_url.trim_end_matches('/') {
client.set_endpoint(&plan.router.base_url, plan.router.key.clone());
Expand Down Expand Up @@ -590,6 +613,9 @@ async fn route(message: &str) -> Result<()> {
anyhow::bail!("give me a message to route, e.g. aster mom route fix this typo");
}
let repo_root = std::env::current_dir().unwrap_or_default();
if let Ok(settings) = crate::settings::Settings::load(Some(&repo_root)) {
crate::jev::init(&settings.experimental);
}
let Some(session) = MomSession::load(&repo_root) else {
println!("no mom.yaml found (looked in the project root, .agents/, and ~/.aster)");
return Ok(());
Expand Down
76 changes: 76 additions & 0 deletions crates/aster-jev/src/jev_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,82 @@ async fn parses_typed_answers() {
);
}

fn entries() -> Vec<(String, String)> {
vec![
(
"everyday".to_string(),
"small edits and questions".to_string(),
),
(
"deep".to_string(),
"multi-step design and debugging".to_string(),
),
]
}

#[tokio::test]
async fn route_sends_entries_as_choice_criteria() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/systemone"))
.and(body_partial_json(serde_json::json!({
"state": "User message:\nwhy does the retry loop deadlock?",
"questions": { "entry": {
"type": "choice",
"criteria": {
"everyday": "small edits and questions",
"deep": "multi-step design and debugging",
},
} },
})))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"answers": { "entry": choice_answer("deep", 0.91) }
})))
.mount(&server)
.await;

let client = JevClient::new("key").base_url(server.uri());
let pick = client
.route("why does the retry loop deadlock?", &entries())
.await
.unwrap();
assert_eq!(
pick,
RoutePick {
entry: "deep".to_string(),
probability: 0.91,
}
);
}

#[tokio::test]
async fn route_rejects_an_undeclared_entry() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/systemone"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"answers": { "entry": choice_answer("scout", 0.99) }
})))
.mount(&server)
.await;

let client = JevClient::new("key").base_url(server.uri());
assert!(client.route("rename x", &entries()).await.is_err());
}

#[tokio::test]
async fn route_reports_a_failed_call() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/systemone"))
.respond_with(ResponseTemplate::new(500).set_body_json(serde_json::json!({})))
.mount(&server)
.await;

let client = JevClient::new("key").base_url(server.uri());
assert!(client.route("rename x", &entries()).await.is_err());
}

#[tokio::test]
async fn confident_stop_is_advised() {
let server = MockServer::start().await;
Expand Down
46 changes: 46 additions & 0 deletions crates/aster-jev/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ const ADVISE_TIMEOUT_SECS: u64 = 2;
/// A suggestion below this probability of its chosen option is ignored.
pub const ADVISE_CONFIDENCE_MIN: f64 = 0.8;

pub const ROUTE_CONFIDENCE_MIN: f64 = 0.6;

const DEFAULT_BASE_URL: &str = "https://api.typesafe.ai";
const DEFAULT_MODEL: &str = "jev-latest";

Expand Down Expand Up @@ -264,6 +266,50 @@ impl JevClient {
_ => None,
}
}

pub async fn route(&self, message: &str, entries: &[(String, String)]) -> Result<RoutePick> {
let mut questions = BTreeMap::new();
questions.insert(
"entry".to_string(),
Question::Choice {
instructions: "Which model entry is best suited to answer this user message? \
Multi-step design, debugging, or refactoring work goes to a \
stronger entry; small mechanical asks go to a cheaper one."
.to_string(),
criteria: entries.to_vec(),
},
);
let state = format!("User message:\n{}", truncate(message, MAX_STATE_CHARS));
let answers = self.evaluate(&state, &questions).await?;
match answers.get("entry") {
Some(Answer::Choice {
choice,
probabilities,
confidence,
}) => {
anyhow::ensure!(
entries.iter().any(|(name, _)| name == choice),
"jev picked {choice:?}, which is not a declared entry"
);
let probability = probabilities
.get(choice)
.copied()
.or(*confidence)
.unwrap_or(0.0);
Ok(RoutePick {
entry: choice.clone(),
probability,
})
}
other => anyhow::bail!("jev returned no entry choice: {other:?}"),
}
}
}

#[derive(Debug, Clone, PartialEq)]
pub struct RoutePick {
pub entry: String,
pub probability: f64,
}

/// What the agent loop should do at a round boundary.
Expand Down
7 changes: 5 additions & 2 deletions docs/CONFIG.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,11 @@ change or disappear in any release.
[Jev](https://typesafe.ai) classifier at each tool round whether the agent
loop should continue, retry, ask you, or stop. Advisory only: a
low-confidence answer is ignored, the round cap still binds, and a failed
check changes nothing. Needs a build with the `jev` cargo feature and
`ASTER_JEV_API_KEY` set.
check changes nothing. With a mom router enabled, Jev also answers the
router first: it picks an entry as a typed choice, a pick under 0.6
probability keeps `start-with`, and a failed call falls through to the
router model. Every consultation lands in `~/.aster/logs/mom-router.jsonl`.
Needs a build with the `jev` cargo feature and `ASTER_JEV_API_KEY` set.

## Where the file lives

Expand Down
Loading