Skip to content

Commit ec7688e

Browse files
committed
fix(workflows): a settle over un-reverified edits never claims passed
verify_settled ended 'passed' on verify_EVER_passed: a green verify followed by edits nothing re-verified could still settle green. The settle end now grounds on the same tree probe as finish_run (_tree_is_verify_green), so the two clean ends cannot disagree; the stale-green case downgrades to reason='settled' with its own summary ('edits after the last green verify were never re-verified').
1 parent ecf696f commit ec7688e

2 files changed

Lines changed: 91 additions & 11 deletions

File tree

‎src/agent6/workflows/loop.py‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2082,7 +2082,11 @@ def _turn_stop_checks( # noqa: PLR0911 - a flat precedence ladder of terminal c
20822082
f"LOOP: verify_settled at iter {turn.iteration} (idle {state.verify_settled_idle})"
20832083
)
20842084
self._final_checkpoint(turn.iteration)
2085-
if state.verify_ever_passed:
2085+
# Ground on the TREE, not on verify_ever_passed: a green verify
2086+
# followed by un-reverified edits must not settle as "passed"
2087+
# (finish_run grounds on the same probe, so the two clean ends
2088+
# cannot disagree).
2089+
if state.verify_ever_passed and self._tree_is_verify_green(state) is not False:
20862090
self._emit_run_end_passed(reason="verify_settled", iterations=turn.iteration)
20872091
return RunResult(
20882092
completed=True,
@@ -2091,23 +2095,31 @@ def _turn_stop_checks( # noqa: PLR0911 - a flat precedence ladder of terminal c
20912095
iterations=turn.iteration,
20922096
tool_calls=state.tool_calls,
20932097
)
2094-
# Gateless seed: the work is committed and the worker went quiet,
2095-
# but NOTHING verified it; ending "passed" here put a lie on every
2098+
# The work is committed and the worker went quiet, but nothing
2099+
# verified the FINAL tree; ending "passed" here put a lie on every
20962100
# surface (observed: an empty repo inferred no verify, the run
20972101
# built a whole project, and the banner read "verify passed").
20982102
self._pass_pending_root_tasks()
20992103
self._emit("run.end", reason="settled", iterations=turn.iteration, all_passed=False)
2104+
if state.verify_ever_passed:
2105+
summary = (
2106+
"the worker settled, but edits after the last green verify were"
2107+
" never re-verified"
2108+
)
2109+
elif self.config.workflow.verify_command:
2110+
# A command can exist here only via mid-run adoption (an
2111+
# operator-set one is never gateless).
2112+
summary = (
2113+
"the worker settled after committing work; the adopted verify never passed"
2114+
)
2115+
else:
2116+
summary = (
2117+
"the worker settled after committing work; no verify command existed to gate it"
2118+
)
21002119
return RunResult(
21012120
completed=True,
21022121
reason="settled",
2103-
# A command can exist here only via mid-run adoption (an
2104-
# operator-set one is never gateless); say which truth applies.
2105-
summary=(
2106-
"the worker settled after committing work; the adopted verify never passed"
2107-
if self.config.workflow.verify_command
2108-
else "the worker settled after committing work; no verify command existed"
2109-
" to gate it"
2110-
),
2122+
summary=summary,
21112123
iterations=turn.iteration,
21122124
tool_calls=state.tool_calls,
21132125
)

‎tests/unit/test_workflow.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1606,6 +1606,74 @@ def dispatch(self, name: str, raw_input: dict[str, Any]) -> ToolResult:
16061606
assert result.completed is True
16071607

16081608

1609+
def test_drive_loop_settle_after_unreverified_edits_is_not_passed(tmp_path: Path) -> None:
1610+
"""A green verify followed by edits that never re-verify must not settle as
1611+
'verify passed': the settle end grounds on the same tree probe as
1612+
finish_run, so it downgrades to reason='settled' with the stale-green
1613+
summary (all_passed=False)."""
1614+
1615+
class ProviderStub:
1616+
def __init__(self) -> None:
1617+
self.calls = 0
1618+
1619+
def call(self, **kwargs: Any) -> ProviderResponse:
1620+
self.calls += 1
1621+
if self.calls == 1:
1622+
return _tool_resp("run_verify_command", tool_id="v1") # green
1623+
if self.calls == 2: # then an edit nothing re-verifies
1624+
return _tool_resp(
1625+
"apply_edit",
1626+
{"path": "a.py", "edits": [{"kind": "create", "new_string": "x = 2\n"}]},
1627+
tool_id="e1",
1628+
)
1629+
return _tool_resp("run_command", {"cmd": f"ls {self.calls}"}, tool_id=f"c{self.calls}")
1630+
1631+
class DispatcherStub:
1632+
def dispatch(self, name: str, raw_input: dict[str, Any]) -> ToolResult:
1633+
return ExecResult(
1634+
returncode=0, stdout="ok", stderr="", duration_s=0.1, exec_failed=False
1635+
)
1636+
1637+
config = SimpleNamespace(
1638+
workflow=SimpleNamespace(
1639+
require_verify_to_finish=False,
1640+
spec_recheck_on_finish=False,
1641+
verify_command=("true",),
1642+
metric=SimpleNamespace(goal=None),
1643+
)
1644+
)
1645+
events: list[dict[str, Any]] = []
1646+
1647+
class _Events:
1648+
def emit(self, event_type: str, /, **fields: Any) -> None:
1649+
events.append({"type": event_type, **fields})
1650+
1651+
wf = _wf(
1652+
root=tmp_path,
1653+
config=config,
1654+
mode="run",
1655+
provider=ProviderStub(),
1656+
dispatcher=DispatcherStub(),
1657+
max_iterations=30,
1658+
events=_Events(),
1659+
)
1660+
messages = [{"role": "user", "content": [{"type": "text", "text": "TASK:\ndo it"}]}]
1661+
with patch("agent6.workflows.loop.commit_all", return_value="sha1"):
1662+
result = wf._drive_loop( # pyright: ignore[reportPrivateUsage]
1663+
system="s",
1664+
conversation=Conversation.from_wire(messages),
1665+
tools=[],
1666+
tool_calls=0,
1667+
start_iteration=1,
1668+
root_task_id=None,
1669+
original_task="t",
1670+
)
1671+
assert result.reason == "settled"
1672+
assert "never re-verified" in result.summary
1673+
ends = [e for e in events if e["type"] == "run.end"]
1674+
assert ends and ends[-1]["all_passed"] is False
1675+
1676+
16091677
def test_drive_loop_verify_settled_does_not_fire_before_first_verify(tmp_path: Path) -> None:
16101678
"""The settled detector must stay dormant until verify has passed at least
16111679
once — a worker still reading toward its first green build must not be

0 commit comments

Comments
 (0)