From 6bbefb403326383a051b7cf97662743e1459db67 Mon Sep 17 00:00:00 2001 From: jsboige Date: Fri, 25 Sep 2026 00:16:02 +0200 Subject: [PATCH 1/2] fix(ci,#17668): extract parked-run release into shared script, port settle-polling to qc-research-monitor Both catalog-cron.yml and qc-research-monitor.yml carried an inline "Release the parked runs" step; the qc-research twin lacked the bounded settle-polling fix from #17634, proving two copies diverge. Single implementation now lives in scripts/ci/release_parked_runs.py, called by both workflows with the same env contract (RELEASE_SETTLE_POLLS added to qc-research-monitor). Behavior verified equivalent by 4 pytest controls. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/catalog-cron.yml | 57 +------- .github/workflows/qc-research-monitor.yml | 50 ++----- scripts/ci/release_parked_runs.py | 163 ++++++++++++++++++++++ tests/test_release_parked_runs.py | 153 ++++++++++++++++++++ 4 files changed, 328 insertions(+), 95 deletions(-) create mode 100644 scripts/ci/release_parked_runs.py create mode 100644 tests/test_release_parked_runs.py diff --git a/.github/workflows/catalog-cron.yml b/.github/workflows/catalog-cron.yml index 005ac485bb..3db36564fd 100644 --- a/.github/workflows/catalog-cron.yml +++ b/.github/workflows/catalog-cron.yml @@ -364,57 +364,6 @@ jobs: RELEASE_CEILING: "40" # Fenetre de sonde (x 15 s) : laisser GitHub creer les runs du head. RELEASE_SETTLE_POLLS: "8" - run: | - set -u - head_sha="$(git rev-parse HEAD)" - echo "::notice title=Catalog::releasing parked runs at $head_sha" - # Les runs `pull_request` du head n'existent pas encore a l'instant du - # push : GitHub les cree quelques secondes plus tard. Mesure du - # 2026-09-24 : push 08:47:03Z, requete 08:47:07-13Z -> "no parked - # run", runs crees 08:47:12Z puis gares -> PR de livraison BLOCKED. - # On sonde jusqu'a un compte non nul ET stable sur deux sondes, dans - # une fenetre bornee (RELEASE_SETTLE_POLLS x 15 s). - parked=""; prev=-1 - for _poll in $(seq 1 "$RELEASE_SETTLE_POLLS"); do - sleep 15 - parked="$(gh run list --repo "$GITHUB_REPOSITORY" --branch "$BRANCH" \ - --limit 100 \ - --json databaseId,headSha,conclusion,workflowName \ - --jq ".[] | select(.conclusion==\"action_required\") \ - | select(.headSha==\"$head_sha\") \ - | \"\(.databaseId)\t\(.workflowName)\"" || true)" - n="$(printf '%s\n' "$parked" | grep -c . || true)" - if [ "$n" -gt 0 ] && [ "$n" -eq "$prev" ]; then break; fi - prev="$n" - done - if [ -z "$parked" ]; then - echo "::notice title=Catalog::no parked run at this head -- nothing to release." - exit 0 - fi - total="$(printf '%s\n' "$parked" | grep -c . || true)" - echo "::notice title=Catalog::$total run(s) parked at this head, staggering ${RELEASE_STAGGER_SECONDS}s apart" - released=0; refused=0; deferred=0; seen=0 - while IFS="$(printf '\t')" read -r rid name; do - [ -z "$rid" ] && continue - seen=$((seen + 1)) - if [ "$seen" -gt "$RELEASE_CEILING" ]; then - deferred=$((deferred + 1)) - continue - fi - [ "$released" -gt 0 ] && sleep "$RELEASE_STAGGER_SECONDS" - if gh api -X POST "repos/$GITHUB_REPOSITORY/actions/runs/$rid/approve" >/dev/null 2>&1; then - released=$((released + 1)) - else - refused=$((refused + 1)) - echo "::warning title=Catalog::could not approve run $rid ($name)" - fi - done </dev/null 2>&1; then - released=$((released + 1)) - else - refused=$((refused + 1)) - echo "::warning title=QC-research::could not approve run $rid ($name)" - fi - done < "no parked run", runs + # crees 08:47:12Z puis gares -> PR de livraison BLOCKED. + RELEASE_SETTLE_POLLS: "8" + run: python3 scripts/ci/release_parked_runs.py --tag QC-research diff --git a/scripts/ci/release_parked_runs.py b/scripts/ci/release_parked_runs.py new file mode 100644 index 0000000000..97d517266d --- /dev/null +++ b/scripts/ci/release_parked_runs.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""release_parked_runs.py -- libere les runs `pull_request` gares par un push bot. + +## Pourquoi ce script existe + +Un commit pousse par github-actions[bot] sur une PR same-repo cree les runs +`pull_request` en `action_required` -- en attente d'une approbation manuelle +qui, mesuree sur 8 jours consecutifs, n'est jamais venue. Un run gare ne +s'execute jamais, donc le check requis qu'il porte est ABSENT a la tete (pas +rouge : absent), et la PR de livraison reste BLOCKED avec tous les checks +visibles verts (#17532 : #17177 resta BLOCKED 2 jours ; meme classe que +#10136 -- le catalogue de main est reste 9 jours stale pendant que le +workflow rapportait `success` chaque matin). + +Deuxieme piege, corrige par #17634 : les runs du head n'existent pas encore a +l'instant du push (GitHub les cree quelques secondes plus tard). Une requete +`gh run list` unique part avant leur creation et rend "no parked run", et les +runs restent gares. On sonde donc jusqu'a un compte non nul ET stable sur +deux sondes, dans une fenetre bornee. + +Ce script est l'implementation unique appelee par les deux workflows qui +livrent par PR bot (`catalog-cron.yml` et `qc-research-monitor.yml`) : deux +copies du meme correctif finissent par diverger -- c'est exactement ce qui a +produit #17668 (le jumeau qc-research n'avait pas le sondage borne de #17634). + +## Contrat + +- Variables d'env (meme contrat que l'ancien step inline) : + GH_TOKEN -- jeton GitHub (lu par le CLI `gh`) + GITHUB_REPOSITORY -- fixe par le runner + BRANCH -- branche de livraison garante + RELEASE_STAGGER_SECONDS -- delai entre deux approbations (defaut 15 : + 16 runs approuves en rafale sur un pool sature + ont tenu les checks `queued` 19 min, pendant + lesquelles "PR gate" a epuise le quota du jeton + d'installation, HTTP 403, et echoue fail-closed + -- mesure du 2026-09-21) + RELEASE_CEILING -- soupape, pas un plafond de debit (defaut 40) : + au-dela, c'est que les runs s'empilent sans + jamais s'executer ; le depassement est ECRIT + RELEASE_SETTLE_POLLS -- fenetre de sonde (defaut 8 x 15 s) : laisser + GitHub creer les runs du head + RELEASE_SETTLE_SECONDS -- espacement entre deux sondes (defaut 15 ; + overridable pour les tests) +- Sortie : exit 0 dans tous les cas geres (le step porte continue-on-error ; + la panne d'approbation doit rester VISIBLE via ::warning, pas rouge). +""" + +import os +import subprocess +import sys +import time + + +def notice(tag: str, message: str) -> None: + print(f"::notice title={tag}::{message}") + + +def warning(tag: str, message: str) -> None: + print(f"::warning title={tag}::{message}") + + +def run_git_head() -> str: + out = subprocess.run( + ["git", "rev-parse", "HEAD"], capture_output=True, text=True, check=True, + encoding="utf-8", errors="replace", + ) + return out.stdout.strip() + + +def list_parked(repo: str, branch: str, head_sha: str) -> list[tuple[str, str]]: + out = subprocess.run( + ["gh", "run", "list", "--repo", repo, "--branch", branch, "--limit", "100", + "--json", "databaseId,headSha,conclusion,workflowName"], + capture_output=True, text=True, encoding="utf-8", errors="replace", + ) + if out.returncode != 0: + return [] + import json + rows = json.loads(out.stdout or "[]") + return [ + (str(r["databaseId"]), r["workflowName"]) + for r in rows + if r.get("conclusion") == "action_required" and r.get("headSha") == head_sha + ] + + +def approve(repo: str, run_id: str) -> bool: + out = subprocess.run( + ["gh", "api", "-X", "POST", f"repos/{repo}/actions/runs/{run_id}/approve"], + capture_output=True, text=True, encoding="utf-8", errors="replace", + ) + return out.returncode == 0 + + +def main() -> int: + tag = "Catalog" + args = sys.argv[1:] + it = iter(args) + for arg in it: + if arg == "--tag": + tag = next(it, "Catalog") + else: + print(f"argument inconnu : {arg}", file=sys.stderr) + return 2 + + repo = os.environ["GITHUB_REPOSITORY"] + branch = os.environ["BRANCH"] + stagger = int(os.environ.get("RELEASE_STAGGER_SECONDS", "15")) + ceiling = int(os.environ.get("RELEASE_CEILING", "40")) + polls = int(os.environ.get("RELEASE_SETTLE_POLLS", "8")) + settle = int(os.environ.get("RELEASE_SETTLE_SECONDS", "15")) + + head_sha = run_git_head() + notice(tag, f"releasing parked runs at {head_sha}") + + parked: list[tuple[str, str]] = [] + prev = -1 + for _ in range(polls): + time.sleep(settle) + parked = list_parked(repo, branch, head_sha) + n = len(parked) + if n > 0 and n == prev: + break + prev = n + + if not parked: + notice(tag, "no parked run at this head -- nothing to release.") + return 0 + + notice(tag, f"{len(parked)} run(s) parked at this head, staggering {stagger}s apart") + released = refused = deferred = 0 + for seen, (run_id, name) in enumerate(parked, start=1): + if seen > ceiling: + deferred += 1 + continue + if released > 0: + time.sleep(stagger) + if approve(repo, run_id): + released += 1 + else: + refused += 1 + warning(tag, f"could not approve run {run_id} ({name})") + + notice(tag, f"released {released} run(s), refused {refused}, deferred {deferred}") + if deferred > 0: + warning( + tag, + f"{deferred} run(s) left parked by the ceiling of {ceiling} -- this is " + "NOT normal throughput. Runs are piling up at one head; investigate " + "before raising the ceiling.", + ) + if refused > 0: + warning( + tag, + f"{refused} run(s) still parked -- the delivery PR will stay BLOCKED. " + "Approve them by hand, or grant a token that may approve workflow runs.", + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_release_parked_runs.py b/tests/test_release_parked_runs.py new file mode 100644 index 0000000000..6c41f5366e --- /dev/null +++ b/tests/test_release_parked_runs.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""test_release_parked_runs.py — contrôles positifs du déblocage des runs garés. + +Couvre scripts/ci/release_parked_runs.py (issue #17668, port du sondage borné +de #17634 partagé entre catalog-cron.yml et qc-research-monitor.yml) : + - la course au release : les runs du head n'existent pas à la 1re sonde -> + la boucle re-sonde et casse au compte stable non nul (2e/3e sonde) + - filtrage : seuls les runs `action_required` du head courant comptent + - staggering + plafond RELEASE_CEILING : le run au-delà est différé SANS + tentative d'approbation + - approbation refusée -> warning, comptée, jamais de rouge (exit 0) + - aucune approbation quand la fenêtre de sonde reste vide + +Usage : + pytest tests/test_release_parked_runs.py -v +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +import types +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "ci" / "release_parked_runs.py" +HEAD = "abc123def" + + +def _load(monkeypatch, rows_by_probe, approve_ok=None): + spec = importlib.util.spec_from_file_location("rpr", SCRIPT) + rpr = importlib.util.module_from_spec(spec) + spec.loader.exec_module(rpr) + + state = {"list": 0, "approves": [], "settle_sleeps": 0, "stagger_sleeps": 0} + + def fake_run(cmd, capture_output=True, text=True, **kw): + if cmd[0] == "git": + assert cmd[:2] == ["git", "rev-parse"] + return types.SimpleNamespace(returncode=0, stdout=HEAD + "\n", stderr="") + if cmd[:2] == ["gh", "run"]: + state["list"] += 1 + rows = rows_by_probe(state["list"]) + return types.SimpleNamespace(returncode=0, stdout=json.dumps(rows), stderr="") + if cmd[:2] == ["gh", "api"]: + assert cmd[2:4] == ["-X", "POST"] + state["approves"].append(cmd[-1]) + ok = approve_ok(cmd[-1]) if approve_ok else True + return types.SimpleNamespace(returncode=0 if ok else 1, stdout="{}", stderr="") + raise AssertionError(f"commande inattendue : {cmd}") + + rpr.subprocess.run = fake_run + + def fake_sleep(seconds): + # les sleeps de sonde sont >= 1 s, le stagger entre approbations vaut + # la valeur configuree (0 dans ces tests) + if seconds >= 1: + state["settle_sleeps"] += 1 + else: + state["stagger_sleeps"] += 1 + + rpr.time.sleep = fake_sleep + + monkeypatch.setenv("GITHUB_REPOSITORY", "jsboige/CoursIA") + monkeypatch.setenv("BRANCH", "chore/test-pending") + monkeypatch.setenv("RELEASE_SETTLE_POLLS", "8") + monkeypatch.setenv("RELEASE_SETTLE_SECONDS", "1") + monkeypatch.setenv("RELEASE_STAGGER_SECONDS", "0") + monkeypatch.setenv("RELEASE_CEILING", "40") + monkeypatch.setenv("GH_TOKEN", "fake") + + return rpr, state + + +def _run_main(rpr, capsys, monkeypatch): + monkeypatch.setattr(sys, "argv", ["release_parked_runs.py", "--tag", "T"]) + rc = rpr.main() + return rc, capsys.readouterr().out + + +def _rows(*ids): + return [ + {"databaseId": i, "headSha": HEAD, "conclusion": "action_required", + "workflowName": f"w{i}"} + for i in ids + ] + + +def test_course_au_release_sondage_borne(monkeypatch, capsys): + """Runs apparaissant à la 2e sonde : casse au compte stable, puis release.""" + rows_by_probe = lambda n: [] if n == 1 else _rows(101, 102) + [ + {"databaseId": 999, "headSha": "autre-tete", "conclusion": "action_required", + "workflowName": "bruit-autre-branche"}, + {"databaseId": 101, "headSha": HEAD, "conclusion": "success", + "workflowName": "deja-execute"}, + ] + rpr, state = _load(monkeypatch, rows_by_probe) + rc, out = _run_main(rpr, capsys, monkeypatch) + + assert rc == 0 + # 1re sonde vide, 2e sonde 2 runs, 3e sonde stable -> break + assert state["list"] == 3 + # seuls les 2 runs action_required du head sont approuves + assert state["approves"] == [ + "repos/jsboige/CoursIA/actions/runs/101/approve", + "repos/jsboige/CoursIA/actions/runs/102/approve", + ] + assert "released 2 run(s), refused 0, deferred 0" in out + assert "::warning" not in out + + +def test_plafond_differe_sans_tenter_lapprobation(monkeypatch, capsys): + """RELEASE_CEILING=2 sur 3 runs : le 3e est différé, pas d'appel API pour lui.""" + rows_by_probe = lambda n: [] if n == 1 else _rows(201, 202, 203) + rpr, state = _load(monkeypatch, rows_by_probe) + monkeypatch.setenv("RELEASE_CEILING", "2") + rc, out = _run_main(rpr, capsys, monkeypatch) + + assert rc == 0 + assert state["approves"] == [ + "repos/jsboige/CoursIA/actions/runs/201/approve", + "repos/jsboige/CoursIA/actions/runs/202/approve", + ] + assert "released 2 run(s), refused 0, deferred 1" in out + assert "left parked by the ceiling of 2" in out + + +def test_approbation_refusee_warning_pas_rouge(monkeypatch, capsys): + """Un approve refusé est compté + warning, le script reste exit 0.""" + rows_by_probe = lambda n: [] if n == 1 else _rows(301) + rpr, state = _load( + monkeypatch, rows_by_probe, + approve_ok=lambda url: url.endswith("/runs/301/approve") is False, + ) + rc, out = _run_main(rpr, capsys, monkeypatch) + + assert rc == 0 + assert state["approves"], "l'approbation doit être tentée" + assert "released 0 run(s), refused 1, deferred 0" in out + assert "could not approve run 301" in out + assert "delivery PR will stay BLOCKED" in out + + +def test_fenetre_vide_aucune_approbation(monkeypatch, capsys): + """Aucun run garé après toutes les sondes : notice + exit 0, zéro appel.""" + rpr, state = _load(monkeypatch, lambda n: []) + monkeypatch.setenv("RELEASE_SETTLE_POLLS", "3") + rc, out = _run_main(rpr, capsys, monkeypatch) + + assert rc == 0 + assert state["list"] == 3 + assert state["approves"] == [] + assert "no parked run at this head" in out From 822ec53e1666ca834062098cbe200c38a50b43ba Mon Sep 17 00:00:00 2001 From: jsboige Date: Fri, 25 Sep 2026 00:34:22 +0200 Subject: [PATCH 2/2] fix(tests,#17668): stop leaking fake subprocess.run/time.sleep into the pytest process The controls patched rpr.subprocess.run and rpr.time.sleep on the global singleton modules, so the fakes intercepted git calls of 84 unrelated tests in the same CI job. Inject SimpleNamespace stubs on the loaded module instance only. Co-Authored-By: Claude Sonnet 5 --- tests/test_release_parked_runs.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_release_parked_runs.py b/tests/test_release_parked_runs.py index 6c41f5366e..c4b8528671 100644 --- a/tests/test_release_parked_runs.py +++ b/tests/test_release_parked_runs.py @@ -49,7 +49,11 @@ def fake_run(cmd, capture_output=True, text=True, **kw): return types.SimpleNamespace(returncode=0 if ok else 1, stdout="{}", stderr="") raise AssertionError(f"commande inattendue : {cmd}") - rpr.subprocess.run = fake_run + # stubs injectes sur l'instance du module UNIQUEMENT : `rpr.subprocess` et + # `rpr.time` sont les singletons globaux -- les muter directement pollue + # tout le processus pytest (84 tests voisins touches au 1er passage, les + # fakes interceptant leurs appels git). + rpr.subprocess = types.SimpleNamespace(run=fake_run) def fake_sleep(seconds): # les sleeps de sonde sont >= 1 s, le stagger entre approbations vaut @@ -59,7 +63,7 @@ def fake_sleep(seconds): else: state["stagger_sleeps"] += 1 - rpr.time.sleep = fake_sleep + rpr.time = types.SimpleNamespace(sleep=fake_sleep) monkeypatch.setenv("GITHUB_REPOSITORY", "jsboige/CoursIA") monkeypatch.setenv("BRANCH", "chore/test-pending")