Repository navigation
fix(harness): recover common tool-call argument mistakes and stop repeating cut-off calls #365
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
85d8f19
7ccae17
9f417f6
eaadacd
bfba206
8f05a61
6c9fbc0
2e2f5ed
be756f3
6675624
b8c406f
96f95cd
be75e87
4842f83
6d67c47
06f083b
bae3ac9
0973d61
7d8afa8
42fcd8e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -51,12 +51,41 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> { | |
| .. | ||
| } = *turn; | ||
| ctx.truncated_call_positions.clear(); | ||
| ctx.truncated_repeat_positions.clear(); | ||
| if self.policy.reject_truncated_tool_calls | ||
| && !tool_calls.is_empty() | ||
| && crate::finish_reason::is_length_stop(response.finish_reason.as_deref()) | ||
| { | ||
| let truncated_positions = truncated_call_positions(tool_calls); | ||
| if !truncated_positions.is_empty() { | ||
| let truncated_names: std::collections::BTreeSet<String> = truncated_positions | ||
|
senamakel marked this conversation as resolved.
|
||
| .iter() | ||
| .map(|&index| tool_calls[index].name.clone()) | ||
| .collect(); | ||
| // A tool already warned that its calls are too large, cut off | ||
| // once more: the model is re-sending the same oversized shape | ||
| // and the output cap is not going to move. Stop here instead | ||
| // of spending the rest of the retry budget on it. | ||
| if let Some(name) = truncated_names | ||
| .iter() | ||
| .find(|name| turn_recovery.truncated_repeat_names.contains(*name)) | ||
| { | ||
| tracing::warn!( | ||
| target: "tinyagents::agent_loop", | ||
| run_id = %ctx.run_id(), | ||
| call_id = %call_id, | ||
| tool = %name, | ||
| "[agent_loop] the same tool keeps being cut off by the output token limit; stopping" | ||
| ); | ||
| messages.pop(); | ||
| ctx.retract_transcript(messages.len()); | ||
| return Err(TinyAgentsError::LimitExceeded(format!( | ||
| "run `{}` stopped: tool `{name}` was cut off by the output token limit \ | ||
| on consecutive turns after being told to split the call into smaller \ | ||
| ones (RunPolicy::reject_truncated_tool_calls)", | ||
| ctx.run_id(), | ||
| ))); | ||
| } | ||
| if turn_recovery.truncated_tool_call_retries_used | ||
| >= self.policy.truncated_tool_call_retries | ||
| { | ||
|
|
@@ -78,8 +107,25 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> { | |
| ))); | ||
| } | ||
| turn_recovery.truncated_tool_call_retries_used += 1; | ||
| let repeated: std::collections::BTreeSet<String> = truncated_names | ||
|
senamakel marked this conversation as resolved.
|
||
| .intersection(&turn_recovery.truncated_tool_names) | ||
| .cloned() | ||
| .collect(); | ||
| // Give the retry room to finish the call. | ||
| turn_recovery.boost_max_tokens(attempt_max_tokens); | ||
| if !repeated.is_empty() { | ||
| tracing::info!( | ||
| target: "tinyagents::agent_loop", | ||
| run_id = %ctx.run_id(), | ||
| call_id = %call_id, | ||
| tools = ?repeated, | ||
| "[agent_loop] same tool cut off by the output limit twice; answering with the stop-repeating corrective" | ||
| ); | ||
| } | ||
| turn_recovery | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Track repeated truncations by invocation, not tool name The repeat detection still keys on the tool name: [RULE] wrong-key-tracking · |
||
| .truncated_repeat_names | ||
| .extend(repeated.iter().cloned()); | ||
| turn_recovery.truncated_tool_names = truncated_names; | ||
| tracing::info!( | ||
| target: "tinyagents::agent_loop", | ||
| run_id = %ctx.run_id(), | ||
|
|
@@ -138,6 +184,9 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> { | |
| } | ||
| if truncated_positions.contains(&index) { | ||
| ctx.truncated_call_positions.insert(batch_index); | ||
| if repeated.contains(&call.name) { | ||
| ctx.truncated_repeat_positions.insert(batch_index); | ||
| } | ||
| } | ||
| batch_index += 1; | ||
| } | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.