diff --git a/.github/workflows/notebook-link-render-check.yml b/.github/workflows/notebook-link-render-check.yml new file mode 100644 index 0000000000..6ab3ca465b --- /dev/null +++ b/.github/workflows/notebook-link-render-check.yml @@ -0,0 +1,154 @@ +name: notebook-link-render-check + +# EPIC #10921 — surface l'etat des liens .ipynb dans les READMEs de series. +# +# Mode advisory : exit 0 en TOUTE circonstance, le signal est porte par un +# label (delta signe sur les READMEs touches par la PR). Bloquer le merge sur +# ce label releve de la politique de la lane, pas du CI -- la correction en +# volume (2096 liens) est deliberement hors scope (cf. PR body, G.4). +# +# Pourquoi advisory : +# * Le constat mesurable au depot est 100 % BRUT (2096/2096, mesure au cycle +# c.891 / HEAD 2e28c2a05). Cause racine = `_quarto.yml` `notebook-preview: +# false` (Quarto ne genere AUCUN `.html` sibling). Une garde bloquante +# fermerait systematiquement toute PR touchant un README de serie jusqu'a +# la decision strategique (bascule `notebook-preview: true` vs URLs GitHub +# blob viewer) -- cette decision est **exterieure** a cette PR. +# * Ce workflow est un **instrument** : il signale l'evolution par-PR (delta +# sur les READMEs modifies), pas la jauge depot-entier. La jauge vit dans +# le commentaire de mesure cycle c.891 sur l'EPIC #10921. +# +# Format du label : `notebook-link-render-delta: +/-N (M readmes)` -- delta +# signe du nombre de liens BRUT introduits (+) ou draines (-) par les +# READMEs modifies par la PR vs main (merge-base). 0 readmes = PR sans +# README de serie ; delta 0 = PR neutre. + +on: + pull_request: + paths: + - 'MyIA.AI.Notebooks/**/README.md' + - 'scripts/notebook_tools/check_notebook_link_render.py' + - '.github/workflows/notebook-link-render-check.yml' + workflow_dispatch: {} + +concurrency: + group: notebook-link-render-check-${{ github.ref }} + cancel-in-progress: true + +jobs: + check-link-render: + name: check_notebook_link_render.py (advisory) + # Routage #14283 tranche 4 : balayage Python pur, stdlib-only. + # Declencheur pull_request : on garde la garde same-repo (un PR de fork + # n'atteint jamais le runner ephemere auto-heberge ; aucun code de fork + # n'est execute cote workflow). + # Garde universelle parentheisee (forme acceptee par + # `test_universal_pull_request_guard_is_accepted`, #13874) -- la + # disjonction explicite `null || == repo` couvre pull_request et + # pull_request_target en un seul predicat, et les forks PR se font + # skipper proprement par le checker avant d'atteindre le runner. + if: ${{ github.event.pull_request.head.repo.full_name == null || github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: [self-hosted, coursia-ephemeral, coursia-linux] + permissions: + pull-requests: write + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + filter: blob:none + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Compute delta on PR READMEs vs main + id: scan + run: | + set +e + MERGE_BASE=$(git merge-base origin/main HEAD) + # Liste des READMEs de serie modifies par la PR (filtre paths du trigger). + mapfile -t PR_READMES < <(git diff --name-only --diff-filter=d "$MERGE_BASE" HEAD -- 'MyIA.AI.Notebooks/**/README.md' | sort -u) + n_readmes=${#PR_READMES[@]} + pr_brut=0; main_brut=0 + for readme in "${PR_READMES[@]}"; do + [ -z "$readme" ] && continue + # Le parent du README dans le repo (le checker a besoin du VRAI chemin + # pour resoudre les liens .ipynb relatifs -- le dump dans /tmp casse + # la resolution par defaut, d'ou --link-root). + readme_dir="$(dirname "$readme")" + # Version PR (HEAD). + if git show "HEAD:$readme" > /tmp/_pr_readme.md 2>/dev/null; then + p=$(cd /tmp && python "$GITHUB_WORKSPACE/scripts/notebook_tools/check_notebook_link_render.py" --tracked-only --json /tmp/_pr_readme.md --link-root "$GITHUB_WORKSPACE/$readme_dir" 2>/dev/null \ + | python -c "import json,sys; print(json.load(sys.stdin)['summary']['totals']['BRUT'])" 2>/dev/null || echo 0) + pr_brut=$((pr_brut + p)) + fi + # Version main (merge-base). + if git show "$MERGE_BASE:$readme" > /tmp/_main_readme.md 2>/dev/null; then + m=$(cd /tmp && python "$GITHUB_WORKSPACE/scripts/notebook_tools/check_notebook_link_render.py" --tracked-only --json /tmp/_main_readme.md --link-root "$GITHUB_WORKSPACE/$readme_dir" 2>/dev/null \ + | python -c "import json,sys; print(json.load(sys.stdin)['summary']['totals']['BRUT'])" 2>/dev/null || echo 0) + main_brut=$((main_brut + m)) + fi + done + delta=$((pr_brut - main_brut)) + if [ "$delta" -gt 0 ]; then delta_sign="+"; else delta_sign=""; fi + echo "PR READMEs scanned: $n_readmes | PR BRUT: $pr_brut | main BRUT: $main_brut | delta: ${delta_sign}${delta}" + echo "delta_brut=$delta" >> "$GITHUB_OUTPUT" + echo "delta_sign=$delta_sign" >> "$GITHUB_OUTPUT" + echo "pr_readmes=$n_readmes" >> "$GITHUB_OUTPUT" + set -e + + - name: Apply signal label (advisory) + uses: actions/github-script@v7 + with: + script: | + // Label delta signe (jamais bloquant) sur la PR -- ce que la PR a + // reellement modifie (N readmes scannes, +/- BRUT). + const delta = '${{ steps.scan.outputs.delta_brut }}'; + const deltaSign = '${{ steps.scan.outputs.delta_sign }}'; + const prReadmes = '${{ steps.scan.outputs.pr_readmes }}'; + const deltaLabel = `notebook-link-render-delta: ${deltaSign}${delta} (${prReadmes} readmes)`; + + // Cleanup des anciens labels notebook-link-render-* de runs precedents. + let existing = []; + try { + const { data } = await github.rest.issues.listLabelsOnIssue({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + existing = data; + } catch (e) { + core.warning(`listLabelsOnIssue failed: ${e.message}`); + } + const toRemove = existing + .filter(l => l.name && l.name.startsWith('notebook-link-render-')) + .map(l => l.name); + for (const name of toRemove) { + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + name, + }); + } catch (e) { + core.info(`removeLabel ${name} ignored: ${e.status || e.message}`); + } + } + + // Poser le label delta. Si le label n'existe pas encore dans le + // repo (premiere utilisation), on ne fait pas crasher le job + // advisory -- warning, pas error. + try { + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [deltaLabel], + }); + } catch (e) { + core.warning(`addLabels "${deltaLabel}" failed (label may not exist yet): ${e.message}`); + } + core.info(`Advisory label set: ${deltaLabel}`); diff --git a/scripts/ci/check_self_hosted_runner_policy.py b/scripts/ci/check_self_hosted_runner_policy.py index 77b9e8bd33..2355cb7436 100644 --- a/scripts/ci/check_self_hosted_runner_policy.py +++ b/scripts/ci/check_self_hosted_runner_policy.py @@ -204,6 +204,16 @@ "translation-drift.yml", "translation-sync.yml", "variation-light-genre.yml", + # tranche 6 (c.903, owner myia-po-2026:CoursIA-2) : garde advisory pur-Python + # declenchee sur pull_request filtrant les `MyIA.AI.Notebooks/**/README.md`, + # scan delta-PR vs main (merge-base), pose un label signe (jamais exit != 0). + # Meme profil que machine-dep-timing-advisory / pr-path-collision-advisory / + # exercises-advisory / catalog-drift : stdlib-only, lecture seule du repo, + # aucun GITHUB_TOKEN cote job. Le job porte la garde universelle parenthe- + # seee (cf. test_universal_guard_with_combined_target_is_accepted, #13874) + # pour les forks PRs qui se font skipper proprement par pr_gate. + # Rollback = revert de cette PR (l'entree disparait de l'allowlist). + "notebook-link-render-check.yml", } GITHUB_HOSTED_LABELS = { "ubuntu-latest", diff --git a/scripts/notebook_tools/check_notebook_link_render.py b/scripts/notebook_tools/check_notebook_link_render.py new file mode 100644 index 0000000000..2da0f4b49d --- /dev/null +++ b/scripts/notebook_tools/check_notebook_link_render.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Check notebook link rendering in series READMEs. + +Scans ``MyIA.AI.Notebooks/**/README.md`` and classifies each markdown link to a +``.ipynb`` file as one of: + +* ``RENDU`` : a sibling ``.html`` exists locally (Quarto has rendered the notebook). +* ``BRUT`` : only the ``.ipynb`` exists locally (raw link, no local preview). +* ``MANQUE`` : neither the ``.ipynb`` nor any ``.html`` sibling exists (dangling). + +The instrument is **diagnostic, not corrective**. The current root cause of the +high BRUT rate is ``_quarto.yml`` ``notebook-preview: false`` (which prevents +Quarto from rendering ``.ipynb`` files at all). Bulk replacement of links is a +composite change (G.4) and is **deliberately out of scope** for this tool — +strategy decisions (``notebook-preview: true`` vs. URLs to GitHub blob viewer) +belong to a follow-up epic PR, not an audit sweep. + +Usage:: + + python scripts/notebook_tools/check_notebook_link_render.py [path] [options] + + path root to scan (default: ``MyIA.AI.Notebooks``) + --tracked-only restrict to ``git ls-files`` intersection + --json emit machine-readable JSON + --verbose per-link verdicts on stdout + --fail-on VERDICT non-zero exit if any link matches VERDICT + (one of: RENDU, BRUT, MANQUE, ANY) +""" +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import urllib.parse +from collections import Counter +from pathlib import Path + +EXCLUDE_TOKENS = (".ipynb_checkpoints", "_archives", "/.git/") + +LINK_PATTERN = re.compile( + r"(?:\[[^\]]*\]\()([^\s)]+\.ipynb)(?:\)|\#)", + re.IGNORECASE, +) + + +def tracked_files(root: Path) -> set[str] | None: + """Posix paths (relative to CWD) of tracked files under ``root``. + + Returns ``None`` on ``git ls-files`` failure — caller falls back to scanning + everything on disk. + """ + try: + out = subprocess.run( + ["git", "ls-files", "--", str(root)], + capture_output=True, text=True, encoding="utf-8", errors="replace", + check=True, + ).stdout + except (subprocess.CalledProcessError, OSError) as err: + print(f"[!] git ls-files failed ({err}); scanning everything.", + file=sys.stderr) + return None + return {Path(line).as_posix() for line in out.splitlines() if line} + + +def head_commit() -> str: + try: + return subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, text=True, encoding="utf-8", errors="replace", + check=True, + ).stdout.strip() + except (subprocess.CalledProcessError, OSError): + return "?" + + +def classify_link(target: Path, repo_root: Path) -> str: + """Classify a single resolved ``.ipynb`` link target. + + * ``RENDU`` if a sibling ``.html`` exists next to the ``.ipynb``. + * ``MANQUE`` if the ``.ipynb`` itself does not exist (dangling). + * ``BRUT`` otherwise — the ``.ipynb`` exists but no ``.html`` preview. + """ + nb = target if target.suffix == ".ipynb" else target.with_suffix(".ipynb") + if not nb.exists(): + return "MANQUE" + html = nb.with_suffix(".html") + if html.exists(): + return "RENDU" + return "BRUT" + + +def extract_links(readme: Path, repo_root: Path, link_root: Path | None = None) -> list[dict]: + """Extract ``.ipynb`` markdown links from a single README. + + Returns per-link records: ``link_text``, ``raw_target``, ``resolved`` (posix + relative to CWD if inside repo, else absolute), ``verdict``. + + ``link_root`` controls how relative link paths are resolved: by default + they are joined against the README's parent directory (markdown semantics). + Passing an explicit ``link_root`` makes the checker resolve links against + that directory instead — useful in CI when the README has been dumped to + ``/tmp/`` and the real notebook files live in the repo. + """ + try: + text = readme.read_text(encoding="utf-8", errors="replace") + except OSError as err: + return [{"error": f"read failed: {err}", "verdict": "PARSE_ERROR"}] + base = (link_root or readme.parent).resolve() + records = [] + for match in LINK_PATTERN.finditer(text): + raw = match.group(1) + # Strip any URL fragment or query, then percent-decode (markdown links to + # accented filenames often embed ``%C3%A9`` etc. — see SemanticKernel README). + clean = urllib.parse.unquote(raw.split("#", 1)[0].split("?", 1)[0]) + # Resolve against ``base`` (markdown parent by default; override via link_root). + candidate = (base / clean).resolve() + try: + relative = candidate.relative_to(repo_root.resolve()).as_posix() + except ValueError: + relative = candidate.as_posix() # outside repo — surface absolute + verdict = classify_link(candidate, repo_root) + records.append({ + "link_text": raw, + "resolved": relative, + "verdict": verdict, + }) + return records + + +def scan(target: Path, tracked_only: bool, repo_root: Path, + link_root: Path | None = None) -> list[dict]: + """Scan ``target`` recursively for README.md files and classify their links. + + ``link_root``, when provided, overrides the markdown-parent directory for + resolving relative ``.ipynb`` link paths. See ``extract_links``. + """ + if not target.exists(): + print(f"[!] target does not exist: {target}", file=sys.stderr) + return [] + tracked = tracked_files(target) if tracked_only else None + + readmes = sorted(target.rglob("README.md")) if target.is_dir() else [target] + out = [] + for readme in readmes: + posix = readme.as_posix() + if any(tok in posix for tok in EXCLUDE_TOKENS): + continue + if tracked is not None and posix not in tracked: + continue + links = extract_links(readme, repo_root, link_root) + out.append({"readme": posix, "links": links}) + return out + + +def summarize(records: list[dict]) -> dict: + """Global + per-README summary buckets. Verdicts: RENDU / BRUT / MANQUE.""" + per_readme = [] + global_counter: Counter[str] = Counter() + readmes_with_links = 0 + readmes_100pct_brut = 0 + for entry in records: + verdicts = [l.get("verdict") for l in entry["links"]] + c = Counter(v for v in verdicts if v in {"RENDU", "BRUT", "MANQUE"}) + n_links = sum(c.values()) + if n_links: + readmes_with_links += 1 + if c["RENDU"] == 0 and c["MANQUE"] == 0: + readmes_100pct_brut += 1 + per_readme.append({ + "readme": entry["readme"], + "n_links": n_links, + "RENDU": c["RENDU"], + "BRUT": c["BRUT"], + "MANQUE": c["MANQUE"], + }) + global_counter.update(c) + return { + "n_readmes_scanned": len(records), + "n_readmes_with_links": readmes_with_links, + "n_readmes_100pct_brut": readmes_100pct_brut, + "totals": { + "RENDU": global_counter["RENDU"], + "BRUT": global_counter["BRUT"], + "MANQUE": global_counter["MANQUE"], + }, + "per_readme": per_readme, + } + + +def emit_human(summary: dict, records: list[dict], verbose: bool, head: str) -> None: + totals = summary["totals"] + n_links = sum(totals.values()) + print(f"check_notebook_link_render @ {head}") + print(f" readmes scanned : {summary['n_readmes_scanned']}") + print(f" readmes w/ links: {summary['n_readmes_with_links']}") + print(f" 100% BRUT : {summary['n_readmes_100pct_brut']}") + print(f" totals : {n_links} links") + pct = lambda v: f"{(100.0 * v / n_links):5.1f}%" if n_links else " 0.0%" + print(f" RENDU : {totals['RENDU']:5d} {pct(totals['RENDU'])}") + print(f" BRUT : {totals['BRUT']:5d} {pct(totals['BRUT'])}") + print(f" MANQUE : {totals['MANQUE']:5d} {pct(totals['MANQUE'])}") + if not verbose: + return + for entry in records: + if not entry["links"]: + continue + print(f"\n {entry['readme']}") + for link in entry["links"]: + print(f" [{link['verdict']:6s}] {link.get('resolved', link.get('link_text', '?'))}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Classify notebook links in series READMEs as RENDU/BRUT/MANQUE.", + ) + parser.add_argument( + "path", nargs="?", default="MyIA.AI.Notebooks", + help="root to scan (default: MyIA.AI.Notebooks)", + ) + parser.add_argument( + "--tracked-only", action="store_true", + help="restrict to git ls-files intersection", + ) + parser.add_argument( + "--json", action="store_true", + help="emit JSON instead of human-readable summary", + ) + parser.add_argument( + "--verbose", action="store_true", + help="per-link verdicts on stdout (ignored with --json)", + ) + parser.add_argument( + "--fail-on", choices=("RENDU", "BRUT", "MANQUE", "ANY"), default=None, + help="non-zero exit if any link matches this verdict", + ) + parser.add_argument( + "--link-root", default=None, + help="resolve relative .ipynb links against this directory " + "(default: the README's own parent). Useful in CI when the README " + "has been dumped to /tmp/ and notebooks live in the real repo.", + ) + args = parser.parse_args() + + repo_root = Path.cwd() + target = Path(args.path) + link_root = Path(args.link_root) if args.link_root else None + records = scan(target, args.tracked_only, repo_root, link_root) + summary = summarize(records) + + if args.json: + print(json.dumps({ + "head": head_commit(), + "target": target.as_posix(), + "tracked_only": args.tracked_only, + "link_root": link_root.as_posix() if link_root else None, + "summary": summary, + "records": records, + }, indent=2, ensure_ascii=False)) + else: + emit_human(summary, records, args.verbose, head_commit()) + + if args.fail_on: + totals = summary["totals"] + threshold = sum(totals.values()) if args.fail_on == "ANY" else totals[args.fail_on] + if threshold: + print(f"[fail] {threshold} link(s) matched --fail-on {args.fail_on}", + file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())