Skip to content

Commit 823d260

Browse files
authored
Merge pull request #37 from tinyhumansai/issue/204
Recover mixed DeepSeek DSML tool calls
2 parents 15ad9df + 0e1f94c commit 823d260

2 files changed

Lines changed: 128 additions & 9 deletions

File tree

‎crates/tinytools-agent/src/parse/grammar/tagged.rs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ use crate::parse::call_object::{AliasPolicy, read_calls};
3737
use crate::parse::json_values::{
3838
extract_first_json_value_with_end, extract_json_values, find_json_end,
3939
};
40+
use crate::repair::args::from_call_object;
4041
use crate::repair::json::{recover_object, strip_code_fence};
4142
use crate::types::{CallSource, ParseOptions, ParsedToolCall};
4243

@@ -94,6 +95,19 @@ static NAMED_INVOKE_OPEN_RE: LazyLock<Option<Regex>> = LazyLock::new(|| {
9495
.ok()
9596
});
9697

98+
/// The malformed name-parameter spelling emitted after JSON arguments by
99+
/// `DeepSeek` when its DSML stream leaks into a text `<tool_call>` block.
100+
static DSML_NAME_PARAMETER_RE: LazyLock<Option<Regex>> = LazyLock::new(|| {
101+
Regex::new(
102+
r#"(?is)^\s*(?:</[||]{1,2}\s*DSML\s*[||]{1,2}\s*parameter\s*>\s*)*<[||]{1,2}\s*DSML\s*[||]{1,2}\s*parameter\s+name\s*=\s*"name"\s*:\s*"([^"]+)"[^>]*(?:>|})"#,
103+
)
104+
.ok()
105+
});
106+
107+
/// `DeepSeek`'s group terminator, used to leave following assistant prose intact.
108+
static DSML_CALLS_CLOSE_RE: LazyLock<Option<Regex>> =
109+
LazyLock::new(|| Regex::new(r"(?is)</[||]{1,2}\s*DSML\s*[||]{1,2}\s*calls\s*>").ok());
110+
97111
/// First match of `re` in `haystack`, as `(start, end)`.
98112
fn find_re(re: &LazyLock<Option<Regex>>, haystack: &str) -> Option<(usize, usize)> {
99113
re.as_ref()
@@ -340,6 +354,14 @@ impl Tagged {
340354
})
341355
.or_else(|| extract_first_json_value_with_end(after));
342356
if let Some((value, consumed)) = recovered {
357+
let (dsml_calls, dsml_end) = recover_dsml_calls(after);
358+
if !dsml_calls.is_empty() {
359+
return Probe::Found(Block {
360+
start: opener.start,
361+
end: body_start + dsml_end,
362+
decoded: Decoded::Calls(dsml_calls),
363+
});
364+
}
343365
let calls = read_calls(
344366
&value,
345367
AliasPolicy::Marked,
@@ -382,6 +404,55 @@ impl Tagged {
382404
}
383405
}
384406

407+
/// Recovers JSON argument objects followed by `DeepSeek`'s name parameter.
408+
/// JSON boundaries are found before scanning markup, so tag-like text inside
409+
/// string arguments cannot be mistaken for DSML framing.
410+
fn recover_dsml_calls(after: &str) -> (Vec<ParsedToolCall>, usize) {
411+
let Some(re) = DSML_NAME_PARAMETER_RE.as_ref() else {
412+
return (Vec::new(), 0);
413+
};
414+
let mut calls = Vec::new();
415+
let mut cursor = 0;
416+
while cursor < after.len() {
417+
let Some((arguments, json_end)) = extract_first_json_value_with_end(&after[cursor..])
418+
else {
419+
break;
420+
};
421+
let json_end = cursor + json_end;
422+
let metadata = &after[json_end..];
423+
let inline_name = arguments.get("name").and_then(serde_json::Value::as_str);
424+
let name_match = re.captures(metadata);
425+
let name = inline_name.or_else(|| {
426+
name_match
427+
.as_ref()
428+
.and_then(|m| m.get(1).map(|v| v.as_str()))
429+
});
430+
let Some(name) = name.map(str::trim) else {
431+
break;
432+
};
433+
if name.is_empty() {
434+
break;
435+
}
436+
let name = name.to_owned();
437+
let args = from_call_object(&arguments);
438+
calls.push(ParsedToolCall::new(name, args, CallSource::TaggedJson));
439+
// An inline name identifies this JSON value, but does not frame a
440+
// later value in trailing text. Continue only when DSML explicitly
441+
// connects the values with a name parameter.
442+
let Some(name_match) = name_match else {
443+
cursor = json_end;
444+
break;
445+
};
446+
cursor = json_end + name_match.get(0).map_or(0, |m| m.end());
447+
}
448+
let end = if calls.is_empty() {
449+
0
450+
} else {
451+
find_re(&DSML_CALLS_CLOSE_RE, &after[cursor..]).map_or(cursor, |(_, end)| cursor + end)
452+
};
453+
(calls, end)
454+
}
455+
385456
/// Whether a tag-family marker is a closer (`</tool_call>`, `<|/tool_call|>`).
386457
/// Skips the same whitespace class `TAG_RE`'s `\s` does (not just space and
387458
/// tab), so a marker like `<\n/tool_call>` — which the regex matches as one

‎crates/tinytools-agent/src/parse/test/tagged.rs‎

Lines changed: 57 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,33 @@ fn unclosed_tag_with_balanced_json_still_recovers() {
8080
assert_eq!(calls.len(), 1);
8181
}
8282

83+
#[test]
84+
fn unclosed_tag_recovers_dsml_parameter_calls() {
85+
let response = concat!(
86+
"<tool_call>\n",
87+
r#"{"arguments":{"path":"work/extract.py","content":"t = re.sub(r'<script.*?</script>', ' ', t, flags=re.S)"}}"#,
88+
"</|DSML| parameter>\n",
89+
"<|DSML| parameter name=\"name\":\"file_write\"}</|DSML| parameter>\n",
90+
"</|DSML| invoke>\n",
91+
"<|DSML| invoke>\n",
92+
r#"{"arguments":{"command":"ls work"}}"#,
93+
"</|DSML| parameter>\n",
94+
"<|DSML| parameter name=\"name\":\"shell\"}</|DSML| parameter>\n",
95+
"</|DSML| invoke>\n",
96+
"</|DSML| calls>Done.",
97+
);
98+
let (text, calls) = parse(response);
99+
assert_eq!(text, "Done.");
100+
assert_eq!(calls.len(), 2);
101+
assert_eq!(calls[0].name, "file_write");
102+
assert_eq!(
103+
calls[0].arguments["content"],
104+
"t = re.sub(r'<script.*?</script>', ' ', t, flags=re.S)"
105+
);
106+
assert_eq!(calls[1].name, "shell");
107+
assert_eq!(calls[1].arguments["command"], "ls work");
108+
}
109+
83110
#[test]
84111
fn unclosed_tag_recovery_preserves_unrelated_markup_after_the_json() {
85112
// No tag-family marker exists anywhere after this opener, so a `</div>`
@@ -632,12 +659,8 @@ fn a_bare_dsml_invoke_carries_its_name_in_the_body() {
632659
/// A block that decodes to nothing must not bury the calls after it.
633660
///
634661
/// Verbatim from a `deepseek` turn: an unterminated `<tool_call>` whose body is
635-
/// `{"arguments":{…}}` with no name — genuinely unrecoverable, since the name
636-
/// survived only in a corrupted `<|DSML| parameter name="name":"file_write"}`
637-
/// line and inventing one is never right — followed by a complete
638-
/// `<|DSML| invoke>`. The failed block used to run to end-of-text and take
639-
/// the good call with it, so the whole response parsed as prose and both calls
640-
/// were lost.
662+
/// `{"arguments":{…}}` with a name in a corrupted DSML parameter line,
663+
/// followed by a complete `<|DSML| invoke>`. Both calls should survive.
641664
#[test]
642665
fn an_undecodable_block_does_not_swallow_the_call_after_it() {
643666
let raw = concat!(
@@ -656,11 +679,12 @@ fn an_undecodable_block_does_not_swallow_the_call_after_it() {
656679
let outcome = super::parse_known(raw, &["file_write", "shell"]);
657680
assert_eq!(
658681
outcome.calls.len(),
659-
1,
660-
"the well-formed call survives its malformed neighbour: {:?}",
682+
2,
683+
"both calls survive their mixed framing: {:?}",
661684
outcome.calls
662685
);
663-
assert_eq!(outcome.calls[0].name, "shell");
686+
assert_eq!(outcome.calls[0].name, "file_write");
687+
assert_eq!(outcome.calls[1].name, "shell");
664688

665689
// Reduced to the essential shape, so a future change that reintroduces the
666690
// swallow fails here with less noise.
@@ -717,6 +741,30 @@ fn recovery_leaves_a_named_invoke_after_a_complete_malformed_body() {
717741
assert_eq!(outcome.calls[0].name, "shell");
718742
}
719743

744+
#[test]
745+
fn unclosed_inline_name_does_not_scan_later_json_from_narrative() {
746+
let raw = concat!(
747+
"<tool_call>{\"name\":\"first\",\"arguments\":{}} prose ",
748+
"{\"name\":\"shell\",\"arguments\":{\"command\":\"ls\"}}"
749+
);
750+
let (text, calls) = parse(raw);
751+
assert_eq!(calls.len(), 1, "{calls:?}");
752+
assert_eq!(calls[0].name, "first");
753+
assert!(text.contains("prose"));
754+
}
755+
756+
#[test]
757+
fn unclosed_inline_name_decodes_stringified_and_aliased_arguments() {
758+
for raw in [
759+
r#"<tool_call>{"name":"echo","arguments":"{\"value\":\"x\"}"}"#,
760+
r#"<tool_call>{"name":"echo","args":{"value":"x"}}"#,
761+
] {
762+
let (_, calls) = parse(raw);
763+
assert_eq!(calls.len(), 1, "{calls:?}");
764+
assert_eq!(calls[0].arguments, serde_json::json!({"value": "x"}));
765+
}
766+
}
767+
720768
#[test]
721769
fn a_named_invoke_precedes_a_later_bare_invoke_closer() {
722770
let raw = concat!(

0 commit comments

Comments
 (0)