feat(compat): draft issue text when a compat column fails

.github/workflows/compat.yml

@@ -9,6 +9,7 @@ on:
 
 permissions:
   contents: read
+  actions: read # the draft step fetches the job's last green set
 
 concurrency:
   group: ${{ github.workflow }}
@@ -82,6 +83,18 @@ jobs:
             [ ! -e "$OUT/suites.txt" ] || { echo '```'; cat "$OUT/suites.txt"; echo '```'; }
           } >>"$GITHUB_STEP_SUMMARY"
 
+      # A red cell only shows in the site table, so draft the upstream issue text: what failed and
+      # which pins moved since the job's last green set. Posting it stays with a person.
+      - name: Issue draft
+        if: ${{ failure() && matrix.column != 'latest' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GREEN: compat-green-${{ matrix.column }}-${{ matrix.python }}-${{ matrix.os }}
+        run: |
+          mkdir -p "$OUT"
+          python3 scripts/compat_draft.py "$OUT" "$RUNNER_TEMP/prev-requirements.txt"
+          cat "$OUT/issue-draft.md" >>"$GITHUB_STEP_SUMMARY"
+
       - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         if: ${{ !cancelled() }}
         with:

CHANGELOG.md

@@ -9,6 +9,9 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Added
 
+- When a `floors`, `pre`, `conda-forge` or `suites` compat job fails, its summary and artifact now
+  hold a draft of an upstream issue: the failed test IDs (or the resolve error), the versions, the
+  pins that changed since the job's last green set, and the run URL. Posting it is left to a person.
 - `BWBENCH_SIZES=useeio` runs the bw2calc cases on USEEIO 1.1, a real database (US EPA,
   public domain, 388 commodities), fetched once by a pinned SHA-256 and checked against a
   reference solve. Only timings are kept ([method](docs/method.md#the-real-database)).

scripts/compat_draft.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Draft the text of an upstream issue for a compat job that failed.
+
+Usage: compat_draft.py OUT PREV_REQS.
+
+OUT is the job's output folder (requirements.txt, suites.txt). Writes OUT/issue-draft.md: the
+package versions, what failed (the FAILED node IDs from suites.txt, or the resolve error when
+there is no requirements.txt), the pins that changed since the job's last green set, and the run
+URL. PREV_REQS is that set's requirements.txt. When the environment variable GREEN names the
+job's artifact (`compat-green-COLUMN-PYTHON-OS`), it is fetched there first through `gh`, and
+GITHUB_TOKEN is needed with `actions: read`. Green artifacts expire after 90 days, so a job red
+for longer has no set to compare with, and the draft says so. Posting the draft stays with a person.
+"""
+
+import io
+import json
+import os
+import sys
+import zipfile
+from pathlib import Path
+
+from compat_site import GREEN, REPO, gh, versions
+
+
+def fetch_green(name: str, dest: Path) -> None:
+    """Save the requirements.txt of the newest unexpired NAME artifact from main, if there is one."""
+    lines = gh("--paginate", f"repos/{REPO}/actions/artifacts?name={name}&per_page=100", "--jq", ".artifacts[] | @json")
+    for a in map(json.loads, lines.splitlines()):
+        if a["workflow_run"]["head_branch"] == "main" and not a["expired"]:
+            with zipfile.ZipFile(io.BytesIO(gh(f"repos/{REPO}/actions/artifacts/{a['id']}/zip"))) as z:
+                dest.write_bytes(z.read("requirements.txt"))
+            return
+
+
+def pins(requirements: str) -> dict[str, str]:
+    """Every name==version in a requirements.txt."""
+    return {k: v.split()[0] for k, v in (line.split("==", 1) for line in requirements.splitlines() if "==" in line)}
+
+
+def draft(out: Path, prev: Path, run_url: str) -> str:
+    """The issue text for the job whose files are in OUT."""
+    reqs = out / "requirements.txt"
+    text = [f"Compat job `{out.name}` failed: {run_url}", ""]
+    if reqs.exists():
+        now = reqs.read_text()
+        text += ["Versions in the set:", *(f"- {k} {v}" for k, v in versions(now).items()), ""]
+    else:
+        text += ["The resolve failed: there is no requirements.txt (the job log has uv's or pixi's error).", ""]
+    suites = out / "suites.txt"
+    if suites.exists():
+        bad = [ln for ln in suites.read_text().splitlines() if ln.startswith(" ") or not ln.endswith(" passed")]
+        text += ["Failed:", "```", *bad, "```", ""]
+    if not prev.exists():
+        text += ["No green set to compare with: the last one expired (they are kept 90 days) or there is none.", ""]
+    elif reqs.exists():
+        old, new = pins(prev.read_text()), pins(now)
+        changed = [
+            f"- {k}: {old.get(k, '(new)')} -> {new.get(k, '(gone)')}"
+            for k in sorted(old | new)
+            if old.get(k) != new.get(k)
+        ]
+        text += ["Pins changed since the last green set:", *(changed or ["- none"]), ""]
+    return "\n".join(text)
+
+
+def main(out: Path, prev: Path) -> None:
+    """Write OUT/issue-draft.md."""
+    if green := os.environ.get("GREEN"):
+        if not green.startswith(GREEN):
+            msg = f"{green}: not a {GREEN}* artifact name"
+            raise SystemExit(msg)
+        fetch_green(green, prev)
+    server = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
+    run_url = f"{server}/{REPO}/actions/runs/{os.environ.get('GITHUB_RUN_ID', '')}"
+    (out / "issue-draft.md").write_text(draft(out, prev, run_url))
+
+
+if __name__ == "__main__":
+    main(Path(sys.argv[1]), Path(sys.argv[2]))

tests/test_compat_draft.py

@@ -0,0 +1,77 @@
+"""scripts/compat_draft.py on fixture files, and with `gh` stubbed out for the green fetch."""
+
+import io
+import json
+import os
+import subprocess
+import sys
+import zipfile
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[1]
+SCRIPT = REPO / "scripts" / "compat_draft.py"
+
+GH = """#!/bin/sh
+case "$*" in
+*/zip) cat "$FAKE/green.zip" ;;
+*/artifacts*) cat "$FAKE/artifacts.jsonl" ;;
+*) exit 1 ;;
+esac
+"""
+
+PREV = "brightway25==1.1.0\nbw2calc==3.0.0\nnumpy==2.5.0\nscipy==1.16.0\n"
+NOW = "brightway25==1.1.0\nbw2calc==3.1.0\nnumpy==2.5.0\nnewdep==1.0\n"
+
+
+def run(
+    tmp_path: Path, *, now: str | None, suites: str | None = None, prev: str | None = PREV, green: bool = False
+) -> str:
+    out = tmp_path / "3.12-suites"
+    out.mkdir()
+    if now is not None:
+        (out / "requirements.txt").write_text(now)
+    if suites is not None:
+        (out / "suites.txt").write_text(suites)
+    prev_path = tmp_path / "prev.txt"
+    env = {
+        **{k: v for k, v in os.environ.items() if k.startswith("COVERAGE_")},  # for `patch = ["subprocess"]`
+        "PATH": f"{tmp_path}:/usr/bin:/bin",
+        "GITHUB_RUN_ID": "42",
+    }
+    if prev is not None and not green:
+        prev_path.write_text(prev)
+    if green:
+        (tmp_path / "gh").write_text(GH)
+        (tmp_path / "gh").chmod(0o755)
+        buf = io.BytesIO()
+        with zipfile.ZipFile(buf, "w") as z:
+            z.writestr("requirements.txt", PREV)
+        (tmp_path / "green.zip").write_bytes(buf.getvalue())
+        a = {"id": 5, "expired": False, "workflow_run": {"head_branch": "main"}}
+        (tmp_path / "artifacts.jsonl").write_text(json.dumps(a) + "\n")
+        env |= {"FAKE": str(tmp_path), "GREEN": "compat-green-suites-3.12-ubuntu-latest"}
+    subprocess.run([sys.executable, SCRIPT, out, prev_path], env=env, check=True)
+    return (out / "issue-draft.md").read_text()
+
+
+def test_failed_suite_names_nodes_and_changed_pins(tmp_path: Path) -> None:
+    suites = "bw2data 4.8 passed\nbw2calc 3.1.0 failed\n    FAILED tests/test_a.py::test_x\n"
+    text = run(tmp_path, now=NOW, suites=suites)
+    assert "actions/runs/42" in text
+    assert "FAILED tests/test_a.py::test_x" in text
+    assert "bw2data 4.8 passed" not in text
+    assert "- bw2calc: 3.0.0 -> 3.1.0" in text
+    assert "- newdep: (new) -> 1.0" in text
+    assert "- scipy: 1.16.0 -> (gone)" in text
+    assert "numpy: " not in text.split("Pins changed")[1]
+
+
+def test_resolve_failure_and_no_green_set(tmp_path: Path) -> None:
+    text = run(tmp_path, now=None, prev=None)
+    assert "resolve failed" in text
+    assert "No green set to compare with" in text
+
+
+def test_fetches_the_last_green_set(tmp_path: Path) -> None:
+    text = run(tmp_path, now=NOW, green=True)
+    assert "- bw2calc: 3.0.0 -> 3.1.0" in text