diff --git a/scripts/ci/variation_prev_guard.py b/scripts/ci/variation_prev_guard.py index f6b5ea1e10..eb6262c208 100644 --- a/scripts/ci/variation_prev_guard.py +++ b/scripts/ci/variation_prev_guard.py @@ -500,6 +500,21 @@ def resolve_prev_targets( return out +def unresolved_prev_targets( + cited: "set[int] | list[int]", prev_targets: dict[str, dict] +) -> list[int]: + """Cited ``prev:`` numbers the gate could NOT resolve (#14550, 2nd defect). + + After the resolution phase, a cited number absent from ``prev_targets`` + is an ABSTENTION, not an attestation: the guard did not evaluate + invariants 2/3 for it. Left silent, a lookup failure and a measured pass + reach the merge-gate under the same green (#14515 was CLEAN and + mergeable with a `prev:` at an OPEN PR, purely because the `gh` + resolution happened to fail during its run). + """ + return sorted(n for n in cited if str(n) not in prev_targets) + + def _json_field(raw: "str | None", field: str) -> "str | None": """Return ``field`` from a JSON object payload, or None if unreadable.""" try: @@ -546,6 +561,7 @@ def main(argv: list[str] | None = None) -> int: prev_targets = (_read_prev_targets_file(args.prev_targets_file) if args.prev_targets_file else {}) + resolution_failed: list[int] = [] if args.resolve_targets: cited = set(find_prev_target_pr_numbers(body)) for msg in commits: @@ -561,10 +577,21 @@ def main(argv: list[str] | None = None) -> int: # an accusation: that is the defect this flag repairs. print(f"prev-target resolution failed ({e}); " "abstaining on invariants 2/3", file=sys.stderr) + # An abstention must be READABLE as one (#14550): the gate stays + # green (FN-safety unchanged) but the verdict names what it could + # not measure, and the ::warning:: becomes a run annotation through + # the workflow's existing `cat /tmp/verdict.err`. + resolution_failed = unresolved_prev_targets(cited, prev_targets) verdict = check(body, commits, current_pr=args.current_pr, prev_targets=prev_targets) + if resolution_failed: + verdict["resolution_failed"] = resolution_failed + targets = ", ".join(f"#{n}" for n in resolution_failed) + print(f"::warning::prev_guard abstention -- unresolvable target(s) " + f"{targets} (FN-safety): this green is an abstention, not an " + f"attestation (#14550)", file=sys.stderr) print(json.dumps(verdict, ensure_ascii=False)) return 0 if verdict["guard_pass"] else 1 diff --git a/scripts/tests/test_variation_prev_guard.py b/scripts/tests/test_variation_prev_guard.py index f6bb11f992..265f93267d 100644 --- a/scripts/tests/test_variation_prev_guard.py +++ b/scripts/tests/test_variation_prev_guard.py @@ -523,3 +523,60 @@ def test_fully_backticked_tag_is_still_evaluated(): v = vpg.check(body, current_pr=99999, prev_targets={"14548": {"kind": "pr", "merged": False}}) assert v["guard_pass"] is False + + +# --- #14550, second defect: a silent fail-open is an unearned attestation ---- +# ai-01 measured it on #14515: CLEAN, PR gate green, mergeable -- with a +# `prev:` at an OPEN PR, because the `gh` resolution happened to fail during +# THAT run. Same violation as four red PRs the same minute; only the +# abstention differed, and nothing in the verdict said so. + +def test_unresolved_prev_targets_pure_selector(): + # The selector names exactly what the gate could not measure: cited but + # absent from the resolved dict -- whether the lookup raised (network, + # gh absent) or the target itself did not resolve. + assert vpg.unresolved_prev_targets({14483}, {}) == [14483] + assert vpg.unresolved_prev_targets({14483, 7}, {"14483": {"kind": "pr", "merged": True}}) == [7] + assert vpg.unresolved_prev_targets(set(), {}) == [] + + +def test_resolution_failure_stays_green_but_flags_abstention( + tmp_path, capsys, monkeypatch): + # ACCEPTANCE 4 (#14550): resolution fails -> guard_pass: True AND + # resolution_failed non-empty. The FN-safety contract is unchanged + # (never accuse on a lookup failure); what changes is that the green + # stops being indistinguishable from a measured one. + body = tmp_path / "body.txt" + body.write_text( + "Grain: MED/tooling -- lane myia-po-2025:CoursIA-2 -- prev: MED/guard #14459", + encoding="utf-8") + + def _boom(missing, runner=None): + raise RuntimeError("gh absent (simulated #14515 condition)") + + monkeypatch.setattr(vpg, "resolve_prev_targets", _boom) + rc = vpg.main(["--body-file", str(body), "--current-pr", "14560", + "--resolve-targets"]) + captured = capsys.readouterr() + verdict = json.loads(captured.out) + assert rc == 0 + assert verdict["guard_pass"] is True + assert verdict["resolution_failed"] == [14459] + # The warning line is a GitHub Actions workflow command: the workflow + # `cat`s verdict.err inside the step, so this becomes a run annotation. + assert "::warning::" in captured.err + assert "#14459" in captured.err + + +def test_14515_real_body_with_resolved_open_target_fails(): + # ACCEPTANCE 5 (#14550) -- positive control of the fail-open: with the + # resolution SUCCEEDING, the real #14515 tag (`prev: MED/notebook-python + # #14483`, target OPEN) must go red. The abstention flag repairs + # visibility, never the predicate. + body = ("Grain: MED/notebook-python -- lane myia-po-2023:CoursIA -- " + "prev: MED/notebook-python #14483") + v = vpg.check(body, prev_targets={"14483": {"kind": "pr", "merged": False}}) + assert v["guard_pass"] is False + kinds = {h["kind"] for h in v["hits"]["prev_invalid"]} + assert "prev-not-merged" in kinds + assert "resolution_failed" not in v # check() has nothing to abstain on