fix(catalog,#14831): build_git_metadata leve au lieu de publier un catalogue degrade

.github/workflows/catalog-drift.yml

@@ -78,14 +78,36 @@ jobs:
 
       # Canonical regeneration (local to the runner only — nothing is written
       # back to the branch). git-tracked notebooks only, for CI determinism.
+      # generate_catalog.py sort en 2 quand `git log` n'a pas pu rendre
+      # l'historique des notebooks (#14831). Ce detecteur read-only, qui se
+      # declare toujours vert, ABSORBE cet abandon au lieu de rougir -- et ce
+      # n'est pas de la complaisance : sans metadonnees git, le catalogue
+      # regenere classe TOUT en UNREVIEWED, si bien que le diff de derive
+      # calcule juste apres mesurerait la panne du runner, pas la derive de la
+      # PR. Sans historique il n'y a pas de derive mesurable : on l'annonce en
+      # annotation et on s'arrete la. Tout autre code de sortie reste un echec.
       - name: Regenerate catalog + README markers
+        id: regen
         run: |
           git fetch origin main --depth=1 || true
+          set +e
           python scripts/notebook_tools/generate_catalog.py --json-only --git-tracked-only
+          rc=$?
+          set -e
+          if [ "$rc" -eq 2 ]; then
+            echo "git_unavailable=true" >> "$GITHUB_OUTPUT"
+            echo "::notice title=Catalog drift check skipped::Les metadonnees git etaient indisponibles sur ce runner (diagnostic complet dans le journal de ce step). La derive n'est pas mesurable sans elles, et ce job read-only ne bloque pas la PR. Le catalogue est regenere quotidiennement sur main par catalog-cron.yml."
+            exit 0
+          fi
+          if [ "$rc" -ne 0 ]; then
+            echo "generate_catalog.py a rendu rc=$rc" >&2
+            exit "$rc"
+          fi
           python scripts/notebook_tools/expand_catalog_markers.py
 
       - name: Detect drift
         id: diff
+        if: steps.regen.outputs.git_unavailable != 'true'
         run: |
           # Only the catalog JSON and the README marker files are in scope.
           git add COURSE_CATALOG.generated.json MyIA.AI.Notebooks/README.md 'MyIA.AI.Notebooks/*/README.md' 2>/dev/null || true

scripts/notebook_tools/generate_catalog.py

@@ -28,6 +28,7 @@
 import re
 import subprocess
 import sys
+import time
 import unicodedata
 from collections import Counter
 from pathlib import Path
@@ -93,29 +94,92 @@ def estimate_duration(cells_code: int, kernel: str, requirements: dict) -> str:
     return "15min"
 
 
+# Plafond mesure, pas devine. Sur le pool `[self-hosted, coursia-ephemeral,
+# coursia-linux]` qui publie le catalogue (checkout `fetch-depth: 0`), sept runs
+# du 2026-09-13 rendent ce step entre 10 s et 64 s. Les trois runs sous 25 s
+# impriment `Preserved curated fields ... for 1 entries` (sain) ; les trois a
+# 40 s et plus impriment `... for 1094 entries` (degrade) -- ils ont brule
+# 30 s dans le delai, obtenu {} en silence, et publie un catalogue ou
+# `scientific_review` vaut UNREVIEWED partout. C'est #14831.
+#
+# `git log` ayant ete TUE a 30 s, ce qu'il lui fallait reellement n'a jamais ete
+# observe : le plafond est donc releve genereusement plutot qu'ajuste au plus
+# juste. L'echec bruyant ci-dessous reste la vraie garantie -- si meme ce plafond
+# se revelait insuffisant, la generation s'arrete en le disant au lieu de
+# publier un catalogue faux en concluant `success`.
+GIT_LOG_TIMEOUT_SECONDS = 180
+
+# Restreindre l'historique aux notebooks : c'est le seul sous-arbre que le
+# parser ci-dessous retient, et 40 % de la sortie de `git log` n'en releve pas.
+# Equivalence verifiee en passant les deux sorties par ce meme parser
+# (1335 notebooks dates de part et d'autre, aucune cle et aucun champ divergents,
+# 1,0 Mo -> 0,6 Mo). Allegement sur -- gain de temps NON demontre : en local les
+# deux formes mesurent 0,45 s contre 0,48 s, soit rien. Le remede de #14831 est
+# le plafond ci-dessus, pas cette ligne.
+GIT_LOG_PATHSPEC = "MyIA.AI.Notebooks"
+
+
+class GitMetadataUnavailable(RuntimeError):
+    """Raised when ``git log`` could not produce notebook history.
+
+    Returning ``{}`` instead is indistinguishable from "no notebook has any
+    history". That silence is the defect of #14831: every ``last_validator``
+    reads falsy, ``classify_scientific_review`` can open no gate and falls
+    through to UNREVIEWED, and ``_merge_curated_fields`` then restores
+    ``last_validation``/``last_validator`` from ``origin/main`` -- the two
+    fields that would have exposed the failure -- while ``scientific_review``,
+    absent from ``CURATED_GIT_FIELDS``, passes through untouched. The run
+    concludes ``success`` and publishes a catalog that is green and wrong.
+    """
+
+
 def build_git_metadata() -> dict[str, dict]:
     """Build last-commit metadata for all notebooks via git log.
 
     Returns dict keyed by relative path (forward slashes) with:
         last_validation: ISO date of last commit touching the file
         last_validator: email of last committer
         issues_prs: list of '#NNN' references from commit messages
+
+    Raises:
+        GitMetadataUnavailable: git timed out, is missing, or exited non-zero.
+            Whether that aborts the run or degrades it is the caller's
+            decision (``--allow-degraded-git``) -- never this function's, and
+            never silent.
     """
+    started = time.monotonic()
     try:
         result = subprocess.run(
-            ["git", "log", "--name-only", "--format=COMMIT:%ai|%ae|%s"],
-            capture_output=True, text=True, encoding="utf-8", errors="replace", cwd=str(REPO_ROOT), timeout=30,
+            [
+                "git", "log", "--name-only", "--format=COMMIT:%ai|%ae|%s",
+                "--", GIT_LOG_PATHSPEC,
+            ],
+            capture_output=True, text=True, encoding="utf-8", errors="replace",
+            cwd=str(REPO_ROOT), timeout=GIT_LOG_TIMEOUT_SECONDS,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise GitMetadataUnavailable(
+            f"'git log' a depasse le delai de {GIT_LOG_TIMEOUT_SECONDS}s "
+            f"(cwd={REPO_ROOT})"
+        ) from exc
+    except FileNotFoundError as exc:
+        raise GitMetadataUnavailable(
+            f"executable 'git' introuvable (cwd={REPO_ROOT})"
+        ) from exc
+
+    elapsed = time.monotonic() - started
+    if result.returncode != 0:
+        stderr = (result.stderr or "").strip().replace("\n", " | ")[:500]
+        raise GitMetadataUnavailable(
+            f"'git log' a rendu rc={result.returncode} en {elapsed:.1f}s "
+            f"(cwd={REPO_ROOT}) -- stderr: {stderr or '(vide)'}"
         )
-        if result.returncode != 0:
-            return {}
-    except (subprocess.TimeoutExpired, FileNotFoundError):
-        return {}
 
     metadata: dict[str, dict] = {}
     current_date = ""
     current_email = ""
     current_subject = ""
-    prefix = "MyIA.AI.Notebooks/"
+    prefix = f"{GIT_LOG_PATHSPEC}/"
 
     for line in result.stdout.split("\n"):
         if line.startswith("COMMIT:"):
@@ -139,6 +203,10 @@ def build_git_metadata() -> dict[str, dict]:
                     "issues_prs": [f"#{n}" for n in issues[:5]],
                 }
 
+    # Emitted on every run so a healthy pass is legible too: a low
+    # "Preserved curated fields" count only means something next to the number
+    # of notebooks git actually dated here.
+    print(f"Git metadata: {len(metadata)} notebooks dates en {elapsed:.1f}s")
     return metadata
 
 
@@ -1560,11 +1628,41 @@ def main():
         "--git-tracked-only", action="store_true",
         help="Only include notebooks tracked by git (for CI consistency)",
     )
+    parser.add_argument(
+        "--allow-degraded-git", action="store_true",
+        help=(
+            "Generate even when 'git log' fails, leaving git metadata empty. "
+            "Without it a git failure aborts, rather than publishing a catalog "
+            "whose scientific_review silently reads UNREVIEWED (#14831)."
+        ),
+    )
     args = parser.parse_args()
 
     scan_exclusions: Counter[str] = Counter()
     pedagogical = not args.all
-    git_meta = build_git_metadata()
+    try:
+        git_meta = build_git_metadata()
+    except GitMetadataUnavailable as exc:
+        print(f"ERREUR: metadonnees git indisponibles -- {exc}", file=sys.stderr)
+        print(
+            "  Sans elles, chaque notebook perd last_validation/last_validator, "
+            "scientific_review retombe a UNREVIEWED, et _merge_curated_fields "
+            "restaure justement les deux champs qui auraient revele la panne "
+            "(#14831). Le catalogue produit serait vert et faux.",
+            file=sys.stderr,
+        )
+        if not args.allow_degraded_git:
+            print(
+                "  Abandon sans ecrire de catalogue. Utiliser --allow-degraded-git "
+                "pour generer malgre tout (hors depot git, par exemple).",
+                file=sys.stderr,
+            )
+            sys.exit(2)
+        print(
+            "  --allow-degraded-git: poursuite avec des metadonnees git vides.",
+            file=sys.stderr,
+        )
+        git_meta = {}
     forensic_meta = build_forensic_metadata()
     head_sha = get_head_sha()
     # Merge forensic metadata into git_meta (forensic wins on overlap for forensic-only keys)

scripts/notebook_tools/tests/test_generate_catalog.py

@@ -6,10 +6,13 @@
 """
 
 import json
+import subprocess
 import sys
 import tempfile
 from collections import Counter
 from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
 
 import pytest
 
@@ -2039,5 +2042,110 @@ def test_sans_compteur_comportement_inchange(self, tmp_path, monkeypatch):
         assert [e["path"] for e in entries] == ["SerieA/keep.ipynb"]
 
 
+# --- build_git_metadata : l'echec est bruyant, jamais un dict vide (#14831) ---
+
+
+class TestBuildGitMetadataLoud:
+    """Un `git log` en echec doit lever, pas rendre {}.
+
+    Le dict vide etait indiscernable de « aucun notebook n'a d'historique » :
+    chaque `last_validator` devenait falsy, `classify_scientific_review`
+    retombait sur UNREVIEWED, puis `_merge_curated_fields` restaurait
+    `last_validation`/`last_validator` depuis origin/main -- les deux champs
+    qui auraient trahi la panne -- pendant que `scientific_review`, absent de
+    CURATED_GIT_FIELDS, passait degrade jusqu'au catalogue publie. Le run se
+    concluait `success`. Ces tests verifient que chacune des trois sorties
+    d'echec nomme sa cause, et que le chemin nominal continue de rendre ses
+    entrees.
+    """
+
+    def test_returncode_non_nul_leve_en_nommant_rc_et_stderr(self):
+        import generate_catalog as gc
+
+        with patch("generate_catalog.subprocess.run") as mock_run:
+            mock_run.return_value = SimpleNamespace(
+                returncode=128, stdout="",
+                stderr="fatal: detected dubious ownership in repository",
+            )
+            with pytest.raises(gc.GitMetadataUnavailable) as exc:
+                gc.build_git_metadata()
+
+        msg = str(exc.value)
+        assert "rc=128" in msg
+        assert "dubious ownership" in msg
+
+    def test_stderr_vide_reste_lisible(self):
+        """Un rc non nul sans stderr ne doit pas rendre un message tronque :
+        c'est le cas ou l'operateur n'a que le rc pour diagnostiquer."""
+        import generate_catalog as gc
+
+        with patch("generate_catalog.subprocess.run") as mock_run:
+            mock_run.return_value = SimpleNamespace(
+                returncode=129, stdout="", stderr="",
+            )
+            with pytest.raises(gc.GitMetadataUnavailable) as exc:
+                gc.build_git_metadata()
+
+        msg = str(exc.value)
+        assert "rc=129" in msg
+        assert "(vide)" in msg
+
+    def test_timeout_leve_en_nommant_le_delai(self):
+        import generate_catalog as gc
+
+        with patch("generate_catalog.subprocess.run") as mock_run:
+            mock_run.side_effect = subprocess.TimeoutExpired(
+                cmd="git log", timeout=gc.GIT_LOG_TIMEOUT_SECONDS,
+            )
+            with pytest.raises(gc.GitMetadataUnavailable) as exc:
+                gc.build_git_metadata()
+
+        assert str(gc.GIT_LOG_TIMEOUT_SECONDS) in str(exc.value)
+
+    def test_git_absent_leve(self):
+        import generate_catalog as gc
+
+        with patch("generate_catalog.subprocess.run") as mock_run:
+            mock_run.side_effect = FileNotFoundError(
+                2, "No such file or directory", "git",
+            )
+            with pytest.raises(gc.GitMetadataUnavailable) as exc:
+                gc.build_git_metadata()
+
+        assert "git" in str(exc.value)
+
+    def test_controle_positif_le_chemin_nominal_rend_ses_entrees(self):
+        """Contre-controle : un detecteur se valide par ses faux negatifs.
+
+        Verifie du meme coup que le commit le plus recent gagne (git log est
+        antichronologique) et que le filtre de prefixe tient.
+        """
+        import generate_catalog as gc
+
+        stdout = "\n".join([
+            "COMMIT:2026-09-13 10:00:00 +0200|dev@example.org|feat: nb (#123) et (#124)",
+            "",
+            "MyIA.AI.Notebooks/Serie/n.ipynb",
+            "scripts/hors_perimetre.py",
+            "COMMIT:2026-09-01 08:00:00 +0200|autre@example.org|ancien (#99)",
+            "",
+            "MyIA.AI.Notebooks/Serie/n.ipynb",
+            "MyIA.AI.Notebooks/Serie/m.ipynb",
+        ])
+
+        with patch("generate_catalog.subprocess.run") as mock_run:
+            mock_run.return_value = SimpleNamespace(
+                returncode=0, stdout=stdout, stderr="",
+            )
+            meta = gc.build_git_metadata()
+
+        assert set(meta) == {"Serie/n.ipynb", "Serie/m.ipynb"}
+        # Le commit le plus recent gagne sur n.ipynb (premier vu = garde).
+        assert meta["Serie/n.ipynb"]["last_validation"] == "2026-09-13"
+        assert meta["Serie/n.ipynb"]["last_validator"] == "dev@example.org"
+        assert meta["Serie/n.ipynb"]["issues_prs"] == ["#123", "#124"]
+        assert meta["Serie/m.ipynb"]["last_validation"] == "2026-09-01"
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])