Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
85d8f19
chore(vendor): add tinytools dependency
senamakel Oct 10, 2026
7ccae17
test(harness): add tool argument recovery tests
senamakel Oct 10, 2026
9f417f6
feat(agent_loop): add tool execution to the agent loop
senamakel Oct 10, 2026
eaadacd
refactor(harness): extract tool call wrapper into its own module
senamakel Oct 10, 2026
bfba206
test(harness): add tool call wrapper tests
senamakel Oct 10, 2026
8f05a61
feat(agent_loop): unwrap tool calls nested in pseudo wrapper tools
senamakel Oct 10, 2026
6c9fbc0
feat(discover): add bridge for tool discovery
senamakel Oct 10, 2026
2e2f5ed
refactor(todos): move todo tests into a dedicated module
senamakel Oct 10, 2026
be756f3
test(todos): cover todo tool edge cases
senamakel Oct 10, 2026
6675624
test(harness): cover truncated tool call handling in agent loop
senamakel Oct 10, 2026
b8c406f
refactor(agent_loop): extract tool call handling into helper
senamakel Oct 10, 2026
96f95cd
fix(agent_loop): stop retrying tool calls repeatedly cut off by outpu…
senamakel Oct 10, 2026
be75e87
test(agent_loop): populate truncation fields in turn recovery fixture
senamakel Oct 10, 2026
4842f83
refactor(harness): extract response recovery into its own module
senamakel Oct 10, 2026
6d67c47
docs(harness): clarify invalid-args error carries a compact signature
senamakel Oct 10, 2026
06f083b
docs(harness): document wrapped tool call retargeting and per-propert…
senamakel Oct 10, 2026
bae3ac9
fix(observability): correct OTLP trace export handling
senamakel Oct 10, 2026
0973d61
style(observability): reformat otlp attribute calls
senamakel Oct 10, 2026
7d8afa8
chore: files changed crates/tinyagents-harness/src/observability/trac…
senamakel Oct 10, 2026
42fcd8e
fix(agent_loop): keep skill argument on tools named by name
senamakel Oct 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions crates/tinyagents-graph/src/todos/mod_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,58 @@ mod tool_tests {
assert!(res.output().contains("pass `todos`"), "{}", res.output());
}

/// Production captures: models name an item's text `title`, `item`,
/// `text`, `task` or `description` instead of `content`. Each is the
/// content; `content` wins when several are present, and an unrelated
/// `id` is ignored rather than rejected.
#[tokio::test]
async fn content_aliases_are_accepted() {
let tool = TodoTool::new(store());
let res = run(
&tool,
Some("t"),
json!({ "todos": [
{ "id": 1, "title": "A", "status": "in_progress" },
{ "id": "2", "item": "B" },
{ "text": "C", "status": "pending" },
{ "task": "D" },
{ "description": "E" },
{ "content": "F", "title": "ignored", "description": "also ignored" },
{ "title": " ", "task": "G" }
] }),
)
.await;
assert!(!res.is_error, "{res:?}");
let contents: Vec<&str> = raw(&res)["todos"]
.as_array()
.unwrap()
.iter()
.map(|item| item["content"].as_str().unwrap())
.collect();
assert_eq!(contents, ["A", "B", "C", "D", "E", "F", "G"]);
}

/// The harness validates arguments against the schema before the tool
/// runs, so an alias the tool accepts must not be a schema error.
#[test]
fn schema_admits_content_aliases_and_a_missing_status() {
let tool = TodoTool::new(store());
let schema = tool.parameters_schema();
for item in [
json!({ "title": "A", "status": "pending" }),
json!({ "id": 1, "task": "B" }),
json!({ "content": "C" }),
] {
let args = json!({ "todos": [item] });
tinyagents_harness::tool::validate_against_schema(&schema, &args)
.unwrap_or_else(|err| panic!("{args}: {err}"));
}
assert_eq!(
schema["properties"]["todos"]["items"]["properties"]["content"]["type"], "string",
"`content` stays the canonical, advertised field"
);
}

#[tokio::test]
async fn invariant_violation_is_a_soft_error() {
let tool = TodoTool::new(store());
Expand Down
48 changes: 44 additions & 4 deletions crates/tinyagents-graph/src/todos/tool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,49 @@ pub struct TodoTool {
/// One item as the model writes it. `status` accepts the Claude-style
/// `pending` / `in_progress` / `completed` plus the aliases [`parse_status`]
/// knows (`todo`, `done`, ...).
///
/// `content` is canonical, but models also name the item's text `title`,
/// `item`, `text`, `task` or `description` (all seen in production). Each is
/// its own field rather than a serde alias so an item carrying two of them is
/// not a "duplicate field" error: the first non-blank one in
/// [`TodoArg::text`]'s order wins. Unknown keys (`id`, `priority`) are
/// ignored.
#[derive(Deserialize)]
struct TodoArg {
content: String,
#[serde(default)]
content: Option<String>,
#[serde(default)]
title: Option<String>,
#[serde(default)]
item: Option<String>,
#[serde(default)]
text: Option<String>,
#[serde(default)]
task: Option<String>,
#[serde(default)]
description: Option<String>,
#[serde(default)]
status: Option<String>,
}

impl TodoArg {
/// The item's text: `content`, else the first non-blank alias.
fn text(&self) -> &str {
[
&self.content,
&self.title,
&self.item,
&self.text,
&self.task,
&self.description,
]
.into_iter()
.filter_map(|field| field.as_deref().map(str::trim))
.find(|text| !text.is_empty())
.unwrap_or_default()
}
}

impl TodoTool {
/// Creates the `todo` tool backed by `store`.
pub fn new(store: Arc<dyn Store>) -> Self {
Expand Down Expand Up @@ -109,7 +145,7 @@ fn parse_items(raw: &Value) -> std::result::Result<Vec<TodoItem>, String> {
serde_json::from_value(raw.clone()).map_err(|e| format!("invalid `todos`: {e}"))?;
let mut items = Vec::with_capacity(args.len());
for arg in args {
let content = arg.content.trim();
let content = arg.text();
if content.is_empty() {
return Err("every todo needs non-empty `content`".to_string());
}
Expand All @@ -131,6 +167,11 @@ fn parameters_schema() -> Value {
"description": "The full list, in order. Pass null to read the current list.",
"items": {
"type": "object",
// `content` is the advertised field. It is not
// `required`: the tool also reads the `title` / `task` /
// ... spellings models drift to (see `TodoArg`), and a
// schema rejection here would stop the call before the
// tool could. A blank item is still refused, by the tool.
"properties": {
Comment thread
senamakel marked this conversation as resolved.
"content": { "type": "string" },
"status": {
Expand All @@ -141,8 +182,7 @@ fn parameters_schema() -> Value {
"completed", "complete", "done", "finished"
]
}
},
"required": ["content", "status"]
}
}
}
},
Expand Down
7 changes: 7 additions & 0 deletions crates/tinyagents-harness/src/agent_loop/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ mod response_recovery;
mod run_loop;
pub(crate) mod stream;
mod structured_plan;
mod tool_call_wrapper;
Comment thread
senamakel marked this conversation as resolved.
mod tool_changes;
mod tool_surface;
mod tool_timing;
Expand Down Expand Up @@ -182,6 +183,12 @@ mod test;
#[path = "tool_rules_tests.rs"]
mod tool_rules_test;
#[cfg(test)]
#[path = "tools_args_recovery_tests.rs"]
Comment thread
senamakel marked this conversation as resolved.
mod tools_args_recovery_test;
#[cfg(test)]
#[path = "truncated_tool_call_tests.rs"]
Comment thread
senamakel marked this conversation as resolved.
mod truncated_tool_call_test;
#[cfg(test)]
#[path = "unknown_tool_tests.rs"]
mod unknown_tool_test;

Expand Down
49 changes: 49 additions & 0 deletions crates/tinyagents-harness/src/agent_loop/response_recovery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,41 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
..
} = *turn;
ctx.truncated_call_positions.clear();
ctx.truncated_repeat_positions.clear();
if self.policy.reject_truncated_tool_calls
&& !tool_calls.is_empty()
&& crate::finish_reason::is_length_stop(response.finish_reason.as_deref())
{
let truncated_positions = truncated_call_positions(tool_calls);
if !truncated_positions.is_empty() {
let truncated_names: std::collections::BTreeSet<String> = truncated_positions
Comment thread
senamakel marked this conversation as resolved.
.iter()
.map(|&index| tool_calls[index].name.clone())
.collect();
// A tool already warned that its calls are too large, cut off
// once more: the model is re-sending the same oversized shape
// and the output cap is not going to move. Stop here instead
// of spending the rest of the retry budget on it.
if let Some(name) = truncated_names
.iter()
.find(|name| turn_recovery.truncated_repeat_names.contains(*name))
{
tracing::warn!(
target: "tinyagents::agent_loop",
run_id = %ctx.run_id(),
call_id = %call_id,
tool = %name,
"[agent_loop] the same tool keeps being cut off by the output token limit; stopping"
);
messages.pop();
ctx.retract_transcript(messages.len());
return Err(TinyAgentsError::LimitExceeded(format!(
"run `{}` stopped: tool `{name}` was cut off by the output token limit \
on consecutive turns after being told to split the call into smaller \
ones (RunPolicy::reject_truncated_tool_calls)",
ctx.run_id(),
)));
}
if turn_recovery.truncated_tool_call_retries_used
>= self.policy.truncated_tool_call_retries
{
Expand All @@ -78,8 +107,25 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
)));
}
turn_recovery.truncated_tool_call_retries_used += 1;
let repeated: std::collections::BTreeSet<String> = truncated_names
Comment thread
senamakel marked this conversation as resolved.
.intersection(&turn_recovery.truncated_tool_names)
.cloned()
.collect();
// Give the retry room to finish the call.
turn_recovery.boost_max_tokens(attempt_max_tokens);
if !repeated.is_empty() {
tracing::info!(
target: "tinyagents::agent_loop",
run_id = %ctx.run_id(),
call_id = %call_id,
tools = ?repeated,
"[agent_loop] same tool cut off by the output limit twice; answering with the stop-repeating corrective"
);
}
turn_recovery

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

priority high tests uncertain

Track repeated truncations by invocation, not tool name

The repeat detection still keys on the tool name: truncated_tool_names and truncated_repeat_names are BTreeSet<String> of names, and the matching in tools.rs answers a cut-off call with the stop-retrying corrective — or ends the run — whenever any call to that same tool name is cut off again. A legitimate, differently-shaped invocation of the same tool (e.g. a small save_workflow call after a huge one was cut) inherits the previous call's strike and can terminate the run on its first truncation. Track the strike per call shape or per invocation (for example a hash of the tool name plus argument length, or a counter of consecutive truncated turns that re-sent the same call), so a new, smaller call is judged on its own.

[RULE] wrong-key-tracking ·

.truncated_repeat_names
.extend(repeated.iter().cloned());
turn_recovery.truncated_tool_names = truncated_names;
tracing::info!(
target: "tinyagents::agent_loop",
run_id = %ctx.run_id(),
Expand Down Expand Up @@ -138,6 +184,9 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
}
if truncated_positions.contains(&index) {
ctx.truncated_call_positions.insert(batch_index);
if repeated.contains(&call.name) {
ctx.truncated_repeat_positions.insert(batch_index);
}
}
batch_index += 1;
}
Expand Down
Loading
Loading