Skip to content

Commit dfe4c73

Browse files
tarastaras
authored
Merge pull request #988 from taras/agent/testagent-model-options-984-delivery
Support model and effort configuration in TestAgent scenarios
2 parents 33ab20f + dd308df commit dfe4c73

31 files changed

Lines changed: 1312 additions & 27 deletions

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -662,6 +662,10 @@ jobs:
662662
with:
663663
bun-version: 1.4.0
664664

665+
- uses: denoland/setup-deno@e95548e56dfa95d4e1a28d6f422fafe75c4c26fb # v2.0.3
666+
with:
667+
deno-version: v2.9.5
668+
665669
- run: bun install
666670

667671
- name: Test

‎packages/test-agent/README.md‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,42 @@ the unnamed session, not every named session. Each test and each working
9797
directory receive independent scenario state, even when they use the same
9898
mapping. A test may finish with later scenario stages unused.
9999

100+
## Exercise model and effort selection
101+
102+
Declare fictional choices on `TestAgent.Scenario` and keep the workflow's
103+
ordinary Session configuration:
104+
105+
```md
106+
<TestAgent agent="reviewer">
107+
<TestAgent.Scenario session="review" src="./agents/review.md"
108+
models={{selected:"small",options:[
109+
{id:"small",efforts:{selected:"low",options:["low","high"]}},
110+
{id:"review",efforts:{selected:"brief",options:["brief","deep"]}}
111+
]}} />
112+
<Test name="uses review configuration">
113+
<Agent name="reviewer"><Session name="review" model="review" effort="deep">
114+
<Prompt>Review this change.</Prompt>
115+
</Session></Agent>
116+
</Test>
117+
</TestAgent>
118+
```
119+
120+
In `agents/review.md`, observe what the prompt actually consumed:
121+
122+
```md
123+
<WhenPrompt template="Review this change." model="review" effort="deep" />
124+
Reviewed with the requested configuration.
125+
```
126+
127+
The IDs are exact strings supplied by the test. Each nonempty catalog has a
128+
selected member and unique options. A model remembers its own effort selection
129+
when the session switches away and back. Omitted or null catalogs represent
130+
unavailable choices; ordinary unconfigured Sessions still run. A missing
131+
WhenPrompt constraint skips that check, while null asserts unavailable choices.
132+
A mismatch leaves the stage active and fails the test. The same catalog syntax
133+
works in nested Execution declarations; separate Tests and children start from
134+
their own defaults. No real provider or credentials are needed.
135+
100136
## Use a failure to fix the workflow
101137

102138
If the document sends this instead:

‎packages/test-agent/mod.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,3 +58,5 @@ export { runTestAgentWorker } from "./src/worker/run.ts";
5858
export { installChildTestAgent, testAgentChildDeclaration } from "./src/child-configuration.ts";
5959
export type { PlanProviderAssembly, PlanProviderPolicy } from "./src/child-configuration.ts";
6060
export type { ChildTestAgentInstallation } from "./src/child-configuration.ts";
61+
62+
export type { ScenarioModels, ScenarioModel, ScenarioEfforts } from "@executablemd/testing";

‎packages/test-agent/src/SessionConfiguration.defaults.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
<WhenPrompt template="defaults" model="fixture-small" effort="low" />
2+
declared defaults
3+
4+
<WhenPrompt template="changed" model="fixture-small" effort="high" />
5+
changed defaults

‎packages/test-agent/src/SessionConfiguration.refusal-behavior.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
<WhenPrompt template="restored" model="A" effort="low" />
2+
restored
3+
4+
<WhenPrompt template="still restored" model="A" effort="low" />
5+
still restored

‎packages/test-agent/src/SessionConfiguration.refusals.md‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<TestAgent agent="reviewer">
2+
<TestAgent.Scenario session="review" src="./SessionConfiguration.refusal-behavior.md" models={{selected:"A",options:[{id:"A",efforts:{selected:"low",options:["low","high"]}},{id:"B",efforts:{selected:"brief",options:["brief","deep"]}},{id:"none"}]}} />
3+
<TestAgent.Scenario session="unavailable" src="./SessionConfiguration.unavailable.md" />
4+
<Test name="invalid configuration starts no prompt and restores the original session" timeout="60s">
5+
<Agent name="reviewer">
6+
<ExpectRefusal message="Unknown model">
7+
<Session name="review" model="unknown"><Prompt>not started</Prompt></Session>
8+
</ExpectRefusal>
9+
<ExpectRefusal message="Invalid effort level">
10+
<Session name="review" model="B" effort="high"><Prompt>not started</Prompt></Session>
11+
</ExpectRefusal>
12+
<Session name="review"><Prompt>restored</Prompt></Session>
13+
<ExpectRefusal message="Effort choices are unavailable">
14+
<Session name="review" model="none" effort="high"><Prompt>not started</Prompt></Session>
15+
</ExpectRefusal>
16+
<Session name="review"><Prompt>still restored</Prompt></Session>
17+
</Agent>
18+
</Test>
19+
<Test name="unavailable configuration starts no prompt">
20+
<Agent name="reviewer">
21+
<ExpectRefusal message="Model choices are unavailable">
22+
<Session name="unavailable" model="A"><Prompt>not started</Prompt></Session>
23+
</ExpectRefusal>
24+
<Session name="unavailable"><Prompt>unavailable</Prompt></Session>
25+
</Agent>
26+
</Test>
27+
</TestAgent>

‎packages/test-agent/src/SessionConfiguration.review.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<WhenPrompt template="restored" model="fixture-small" effort="low" />
2+
small low
3+
<WhenPrompt template="configured" model="fixture-review" effort="deep" />
4+
review deep
5+
<WhenPrompt template="remember high" model="fixture-small" effort="high" />
6+
small high
7+
<WhenPrompt template="other model" model="fixture-review" effort="brief" />
8+
review brief
9+
<WhenPrompt template="stored effort" model="fixture-small" effort="high" />
10+
stored high
11+
<WhenPrompt template="same model" model="fixture-small" effort="high" />
12+
same high
13+
<WhenPrompt template="restored high" model="fixture-small" effort="high" />
14+
restored high

‎packages/test-agent/src/SessionConfiguration.test.md‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Scenario model and effort selection
2+
3+
<Let as="catalog" value={{selected:"fixture-small",options:[{id:"fixture-small",efforts:{selected:"low",options:["low","high"]}},{id:"fixture-review",efforts:{selected:"brief",options:["brief","deep"]}},{id:"no-effort"}]}} />
4+
<TestAgent agent="reviewer">
5+
<TestAgent.Scenario session="review" src="./SessionConfiguration.review.md" models={catalog} />
6+
<TestAgent.Scenario session="defaults" src="./SessionConfiguration.defaults.md" models={catalog} />
7+
<TestAgent.Scenario session="unavailable" src="./SessionConfiguration.unavailable.md" models={null} />
8+
9+
<Test name="configuration and dependent effort memory share one session" timeout="60s">
10+
<Agent name="reviewer">
11+
<Session name="review"><Prompt as="restored">restored</Prompt></Session>
12+
<AssertStringIncludes actual={restored} expected="small low" />
13+
<Session name="review" model="fixture-review" effort="deep"><Prompt as="configured">configured</Prompt></Session>
14+
<AssertStringIncludes actual={configured} expected="review deep" />
15+
<Session name="review" model="fixture-small" effort="high"><Prompt>remember high</Prompt></Session>
16+
<Session name="review" model="fixture-review" effort="brief"><Prompt>other model</Prompt></Session>
17+
<Session name="review" model="fixture-small"><Prompt>stored effort</Prompt></Session>
18+
<Session name="review" model="fixture-small"><Prompt>same model</Prompt></Session>
19+
<Session name="review"><Prompt>restored high</Prompt></Session>
20+
</Agent>
21+
</Test>
22+
23+
<Test name="catalog defaults are used without Session settings">
24+
<Agent name="reviewer">
25+
<Session name="defaults"><Prompt>defaults</Prompt></Session>
26+
<Session name="defaults" model="fixture-small" effort="high"><Prompt>changed</Prompt></Session>
27+
</Agent>
28+
</Test>
29+
<Test name="an independent Test starts at declared defaults">
30+
<Agent name="reviewer"><Session name="defaults"><Prompt>defaults</Prompt></Session></Agent>
31+
</Test>
32+
<Test name="unavailable choices allow an unconfigured Session">
33+
<Agent name="reviewer">
34+
<Session name="unavailable"><Prompt>unavailable</Prompt></Session>
35+
</Agent>
36+
</Test>
37+
</TestAgent>
38+
39+
<Test name="sibling nested Executions have independent catalogs">
40+
<Execution host="run" source={'<Agent name="reviewer"><Session name="defaults"><Prompt>defaults</Prompt></Session><Session name="defaults" model="fixture-small" effort="high"><Prompt>changed</Prompt></Session></Agent>'} as="first">
41+
<TestAgent agent="reviewer"><TestAgent.Scenario session="defaults" src="./SessionConfiguration.defaults.md" models={catalog} /></TestAgent>
42+
<AssertEquals actual={first.result.ok} expected={true} />
43+
</Execution>
44+
<Execution host="run" source={'<Agent name="reviewer"><Session name="defaults"><Prompt>defaults</Prompt></Session></Agent>'} as="second">
45+
<TestAgent agent="reviewer"><TestAgent.Scenario session="defaults" src="./SessionConfiguration.defaults.md" models={catalog} /></TestAgent>
46+
<AssertEquals actual={second.result.ok} expected={true} />
47+
</Execution>
48+
</Test>

‎packages/test-agent/src/SessionConfiguration.unavailable.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
<WhenPrompt template="unavailable" model={null} effort={null} />
2+
choices unavailable

‎packages/test-agent/src/child-configuration.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ import {
5656
import type { ScenarioDeclaration } from "./components.ts";
5757
import { TEST_AGENT_PROVIDER } from "./provider.ts";
5858

59+
import { normalizeModels } from "./configuration.ts";
60+
5961
const TEST_AGENT = "TestAgent";
6062
const SCENARIO = "TestAgent.Scenario";
6163

@@ -107,6 +109,13 @@ function open(collect: {
107109
refuse(`<${SCENARIO}> requires a "src" prop.`);
108110
return "";
109111
}
112+
let models;
113+
try {
114+
models = normalizeModels(props.models);
115+
} catch (error) {
116+
refuse(`<${SCENARIO}> ${error instanceof Error ? error.message : String(error)}`);
117+
return "";
118+
}
110119
const read = yield* readScenarioSource(src);
111120
if (!read.ok) {
112121
refuse(`<${SCENARIO}> cannot read the behavior document "${src}": ${read.error.message}`);
@@ -128,6 +137,7 @@ function open(collect: {
128137
session,
129138
rootDir: read.value.rootDir,
130139
document: read.value.document,
140+
...(models === undefined ? {} : { models }),
131141
});
132142
return "";
133143
};
@@ -266,6 +276,7 @@ export function* installChildTestAgent(
266276
rootDir: scenario.rootDir,
267277
document: { path: scenario.document.path, source: scenario.document.source },
268278
duplicate: false,
279+
models: normalizeModels(scenario.models),
269280
});
270281
}
271282
const partition = yield* provisionPartition({

0 commit comments

Comments
 (0)