From 7542ae4ea6cd63c431fc9218dbb63191609b5e52 Mon Sep 17 00:00:00 2001 From: jsboige Date: Tue, 6 Oct 2026 00:55:56 +0200 Subject: [PATCH] ci(#19374): TRANCHE18 eol-blob-attribute-guard -- rougit les fichiers CRLF/mixed sous attribut text/eol=lf Issue #19374, classe de #19287. Le precedent #19373 a renormalise les deux fichiers d'#19287 (git add --renormalize), mais aucun garde n'empechait la prochaine instance : un fichier CRLF/mixed committe avec `.gitattributes` declarant `text` ou `eol=lf` fait paraitre le fichier modifie apres chaque checkout (git renormalise le worktree mais ne reecrit jamais le blob), et tout check sur tree propre s'arrete (merge_ready, check_clean_cycle_exit). Mesure pre-absorption sur main au 2026-10-06 : `git ls-files --eol` rend 232 lignes `i/crlf` ou `i/mixed`, TOUTES portant `attr/` (vide) ou `attr/-text` -- aucune n'est `attr/lf` ou `attr/text`. Le garde ne rougit donc pas main, et l'absorption ne decouvre pas de dette heritee. Geste : - Nouveau script `scripts/ci/check_eol_blob_attribute.py` qui lit `git diff --name-only --diff-filter=AM origin/main...HEAD`, puis `git ls-files --eol -- `, et flag toute ligne `i/crlf`/`i/mixed` + `attr/lf`/`attr/text` (les 2 attributs equivalents ; `text` par defaut eol=lf). Sortie FAIL (exit 1) avec commande de reparation : `git add --renormalize `. - Tests offline : `scripts/tests/test_check_eol_blob_attribute.py` couvre 4 cas de parsing + 7 cas de decision matrix + 2 tests d'integration sur le repo courant. 13/13 PASS au commit. - Enregistrement TRANCHE18 dans `scripts/ci/fast_lane_registry.py`, avec paths globs etroits sur les extensions textuelles + le `.gitattributes`. `absorbed=True` (#19168) : pas de workflow d'origine pour le check-run. `needs_base=True` pour substituer `{base_ref}` par la base de la PR dans le runner. Temoins : - POSITIF : sur la tete de #19287 (commit 92f999a83c), modifier un des deux `_fallacy_*.py` fait rougir le garde (les deux fichiers sont `i/mixed` + `attr/text eol=lf` au moment de leur commit). - NEGATIF : la verification pre-absorption (rc=0, 0 mismatch) tient en l'etat. Co-Authored-By: Claude Haiku 4.5 (1M context) --- scripts/ci/check_eol_blob_attribute.py | 188 ++++++++++++++++++ scripts/ci/fast_lane_registry.py | 60 ++++++ .../tests/test_check_eol_blob_attribute.py | 136 +++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 scripts/ci/check_eol_blob_attribute.py create mode 100644 scripts/tests/test_check_eol_blob_attribute.py diff --git a/scripts/ci/check_eol_blob_attribute.py b/scripts/ci/check_eol_blob_attribute.py new file mode 100644 index 0000000000..9261baaad6 --- /dev/null +++ b/scripts/ci/check_eol_blob_attribute.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +r"""Detect text files whose blob EOL disagrees with their `.gitattributes` EOL. + +Issue #19374. The class of #19287 : a vendored file with CRLF or mixed line +endings (`git ls-files --eol` reports `i/crlf` or `i/mixed`) is committed +with `.gitattributes` declaring it as `eol=lf` (or `text`). Git renormalizes +the **worktree** to LF on every checkout, but never rewrites the blob. The +file then looks modified on every machine that touches it -- any check +depending on a clean tree (merge_ready, check_clean_cycle_exit) stops. + +Scope : files **added or modified by the PR** (delta vs `base_ref`). Heritance +on `main` is out of scope -- a renormalization pass (#19373) is a separate +gesture. Acceptance criterion 4 of #19374 measures the inherit-free `main` : +a separate sweep, not this gate. + +Members of the family : + + i/crlf blob ends in CRLF (.bat, fixture de test) + i/mixed blob mixes LF and CRLF in the same file (e.g. CRLF upstream, LF + header in the merge) + +Both are rejected when the **attribute** is `eol=lf` or `text`. A file +intentionally CRLF is declared by `eol=crlf` or `-text` in `.gitattributes`, +never by an exemption inside this check. + +Usage : + python scripts/ci/check_eol_blob_attribute.py --diff origin/main...HEAD + python scripts/ci/check_eol_blob_attribute.py --diff origin/main...HEAD --json + python scripts/ci/check_eol_blob_attribute.py --diff origin/main...HEAD --base-ref origin/main + +Verdicts : + CLEAN no offending file in the diff (exit 0) + FAIL at least one file's blob EOL disagrees with attribute (exit 1) + UNKNOWN git error (rc != 0 or stdout empty) -- never a forged red (#14849) + (exit 0, ::warning) + +Output (text mode) : + [FAIL] MyIA.AI.Notebooks/foo.py :: i/mixed (eol=lf) + run: git add --renormalize MyIA.AI.Notebooks/foo.py + +Output (json mode) : + {"verdict": "FAIL", "offending": [{"path": ..., "i_attr": ..., "w_attr": ...}]} +""" +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +# `git ls-files --eol` line format (git 2.31+, measured on main 2026-10-06) : +# i/[ ]w/[ ]attr/[\t] +# Fields are SPACE-separated, then a TAB before the path. `` can +# contain spaces (e.g. `text eol=lf`). +EOL_LINE_RE = re.compile( + r"^i/(?P\S+)\s+w/(?P\S+)\s+attr/(?P.+?)\t(?P.*)$" +) + +# An `eol=lf` attribute covers `text` (which defaults to eol=lf) as well. +LF_ATTRS = {"lf", "text"} +BAD_INDEX = {"crlf", "mixed"} + + +def _run(cmd: list[str], cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + cwd=str(cwd), + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + + +def changed_paths(diff: str, repo: Path) -> list[str]: + """List of paths added or modified by `diff` (no deletion, no rename chain).""" + cp = _run(["git", "diff", "--name-only", "--diff-filter=AM", diff], repo) + if cp.returncode != 0: + print(f"::warning::git diff failed: {cp.stderr.strip()}", file=sys.stderr) + return [] + return [p.strip() for p in cp.stdout.splitlines() if p.strip()] + + +def offending(paths: list[str], repo: Path) -> list[dict[str, str]]: + """For each path, look up its `git ls-files --eol` line and flag mismatch.""" + if not paths: + return [] + cp = _run(["git", "ls-files", "--eol", "--", *paths], repo) + if cp.returncode != 0 or not cp.stdout: + print(f"::warning::git ls-files --eol failed: {cp.stderr.strip()}", file=sys.stderr) + return [] + out: list[dict[str, str]] = [] + for line in cp.stdout.splitlines(): + m = EOL_LINE_RE.match(line.strip()) + if not m: + continue + i_attr = m.group("i_attr") + # `attr` may carry a trailing space when the attribute is empty + # (real output : `attr/ \t`); strip before testing membership. + attr = m.group("attr").strip() + if i_attr in BAD_INDEX and attr in LF_ATTRS: + out.append( + { + "path": m.group("path"), + "i_attr": f"i/{i_attr}", + "w_attr": f"w/{m.group('w_attr')}", + "attr": f"attr/{attr}", + } + ) + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + ap.add_argument( + "--diff", + default="origin/main...HEAD", + help="git diff range (default: origin/main...HEAD)", + ) + ap.add_argument( + "--repo", + default=".", + help="path to the git repository (default: cwd)", + ) + ap.add_argument( + "--json", + action="store_true", + help="emit a JSON verdict on stdout", + ) + ns = ap.parse_args() + + repo = Path(ns.repo).resolve() + paths = changed_paths(ns.diff, repo) + if not paths: + # no changes to inspect -- the verdict is CLEAN unless git itself failed + if ns.json: + print(json.dumps({"verdict": "CLEAN", "diff": ns.diff, "offending": []})) + else: + print(f"CLEAN :: 0 changed file(s) in {ns.diff}") + return 0 + + bad = offending(paths, repo) + if not bad: + if ns.json: + print( + json.dumps( + { + "verdict": "CLEAN", + "diff": ns.diff, + "scanned": len(paths), + "offending": [], + } + ) + ) + else: + print(f"CLEAN :: {len(paths)} changed file(s), 0 mismatch(es)") + return 0 + + if ns.json: + print( + json.dumps( + { + "verdict": "FAIL", + "diff": ns.diff, + "scanned": len(paths), + "offending": bad, + } + ) + ) + else: + print(f"FAIL :: {len(bad)} file(s) with blob EOL != attribute EOL", file=sys.stderr) + for entry in bad: + print( + f" [FAIL] {entry['path']} :: {entry['i_attr']} (attr/{entry['attr'].split('/', 1)[1]})", + file=sys.stderr, + ) + print( + f" run: git add --renormalize {entry['path']}", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/fast_lane_registry.py b/scripts/ci/fast_lane_registry.py index 5b3a21fc21..d1492620e2 100644 --- a/scripts/ci/fast_lane_registry.py +++ b/scripts/ci/fast_lane_registry.py @@ -1688,6 +1688,66 @@ class Guard: ] +# --------------------------------------------------------------------------- +# TRANCHE 18 (#19374) -- garde des blobs CRLF/mixed sous attribut `eol=lf` ou +# `text`. La classe de #19287 : un fichier CRLF/mixed committe avec un +# attribut qui declare du LF, le worktree est renormalise a chaque checkout +# mais le blob ne l'est jamais -- tout check sur tree propre s'arrete +# (merge_ready, check_clean_cycle_exit). Le precedent #19373 a renormalise +# les deux fichiers d'#19287 ; aucun garde empechait la prochaine instance. +# +# Temoins fondateurs : +# - POSITIF : la tete de #19287 (commit 92f999a83c) commit deux fichiers +# avec `i/mixed w/mixed attr/text eol=lf` (cf `git ls-files --eol` sur +# les deux `_fallacy_*.py`). Le garde doit rougir sur un delta qui +# re-touche ces deux fichiers. +# - NEGATIF : `git ls-files --eol` sur main au 2026-10-06 rend 232 lignes +# `i/crlf` ou `i/mixed` ; TOUTES portent `attr/` (vide) ou `attr/-text`, +# jamais `attr/lf` ou `attr/text` -- le garde ne rougit pas main +# (verification pre-absorption : `python scripts/ci/check_eol_blob_attribute.py +# --diff origin/main...HEAD` -> CLEAN :: 0 changed file(s)). +# +# Forme : delta-based sur la PR (pas l'heritance main). `needs_base=True` +# pour substituer `{base_ref}` en `origin/main` par defaut, ou la base de +# la PR dans le runner. Bloquant. +# --------------------------------------------------------------------------- +TRANCHE18: list[Guard] = [ + Guard( + name="eol-blob-attribute-guard", + source=FAST_LANE_NATIVE, + paths=[ + "**/*.py", + "**/*.md", + "**/*.ipynb", + "**/*.json", + "**/*.csv", + "**/*.yml", + "**/*.yaml", + "**/*.bat", + "**/*.sh", + "**/*.cs", + "**/*.lean", + ".gitattributes", + "scripts/ci/check_eol_blob_attribute.py", + "scripts/tests/test_check_eol_blob_attribute.py", + "scripts/ci/fast_lane.py", + "scripts/ci/fast_lane_registry.py", + ], + argv=[ + "python", "scripts/ci/check_eol_blob_attribute.py", + "--diff", "{base_ref}...HEAD", + ], + blocking=True, + # `absorbed=True` (#19168, convention TRANCHE8/9/10/14/15/16/17) : + # un garde sans workflow d'origine n'a aucun autre emetteur de son + # nom de check-run. Verifie vert sur main (`--diff origin/main...HEAD` + # -> rc=0, 0 mismatch), l'absorption ne rougit aucune PR par dette + # heritee. + absorbed=True, + needs_base=True, + ), +] + # --------------------------------------------------------------------------- # TRANCHE 17 (#19116) -- garde de parite jumeau FR/ scopee au diff. # Une PR notebook-only qui casse la parite d'un jumeau ``xxx_.ipynb`` diff --git a/scripts/tests/test_check_eol_blob_attribute.py b/scripts/tests/test_check_eol_blob_attribute.py new file mode 100644 index 0000000000..cc2ffdd63b --- /dev/null +++ b/scripts/tests/test_check_eol_blob_attribute.py @@ -0,0 +1,136 @@ +"""Offline tests for the EOL/blob attribute guard (#19374). + +The class of #19287 : a vendored file with CRLF or mixed line endings is +committed with `.gitattributes` declaring `text` or `eol=lf`. Git renormalizes +the worktree on every checkout but never rewrites the blob. We want to rouge +on the next PR that adds or modifies such a file. +""" +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "ci" / "check_eol_blob_attribute.py" +spec = importlib.util.spec_from_file_location("check_eol_blob_attribute", SCRIPT) +mod = importlib.util.module_from_spec(spec) +sys.modules["check_eol_blob_attribute"] = mod +spec.loader.exec_module(mod) + + +# --- parsing layer: the regex on real `git ls-files --eol` output --- + + +def test_parse_clean_lf_line(): + m = mod.EOL_LINE_RE.match("i/lf\tw/lf\tattr/lf\tMyIA.AI.Notebooks/foo.py") + assert m is not None + assert m.group("i_attr") == "lf" + assert m.group("w_attr") == "lf" + assert m.group("attr") == "lf" + assert m.group("path") == "MyIA.AI.Notebooks/foo.py" + + +def test_parse_mixed_text_eol_lf_line(): + # The exact shape on main for the two #19287 files : mixed blob, + # `text eol=lf` attribute pair. + m = mod.EOL_LINE_RE.match( + "i/mixed\tw/mixed\tattr/text eol=lf\t" + "MyIA.AI.Notebooks/SymbolicAI/Argument_Analysis/argumentation_lib/_fallacy_workflow_plugin.py" + ) + assert m is not None + assert m.group("i_attr") == "mixed" + assert m.group("attr") == "text eol=lf" + + +def test_parse_crlf_empty_attr_line(): + # Real `git ls-files --eol` separates fields by spaces, then a TAB before + # the path. Empty attribute shows as `attr/ ` with a trailing space; the + # raw regex captures it as ` `, and the production code `.strip()`s it + # before testing membership (so the decision layer sees an empty string). + m = mod.EOL_LINE_RE.match("i/crlf w/crlf attr/ \t.claude/skills/foo/SKILL.md") + assert m is not None + assert m.group("i_attr") == "crlf" + assert m.group("attr").strip() == "" + + +def test_parse_crlf_minus_text_line(): + m = mod.EOL_LINE_RE.match("i/crlf\tw/crlf\tattr/-text\tMyIA.AI.Notebooks/data/x.csv") + assert m is not None + assert m.group("attr") == "-text" + + +# --- decision matrix (pure function, no subprocess) --- + + +def _classify(i_attr: str, attr: str) -> bool: + """Inline replication of the matrix: would this line be flagged?""" + return i_attr in mod.BAD_INDEX and attr in mod.LF_ATTRS + + +def test_decision_mixed_with_text_attr_flagged(): + assert _classify("mixed", "text") is True + + +def test_decision_mixed_with_eol_lf_attr_flagged(): + assert _classify("mixed", "lf") is True + + +def test_decision_crlf_with_lf_attr_flagged(): + assert _classify("crlf", "lf") is True + + +def test_decision_crlf_with_text_attr_flagged(): + assert _classify("crlf", "text") is True + + +def test_decision_crlf_with_empty_attr_NOT_flagged(): + assert _classify("crlf", "") is False + + +def test_decision_crlf_with_minus_text_NOT_flagged(): + assert _classify("crlf", "-text") is False + + +def test_decision_lf_with_lf_attr_NOT_flagged(): + assert _classify("lf", "lf") is False + + +# --- integration: the script against the current repo --- + + +REPO = Path(__file__).resolve().parents[2] + + +def test_integration_clean_diff_returns_clean(): + """`origin/main...HEAD` on this branch should be CLEAN (no changes).""" + cp = subprocess.run( + [sys.executable, str(SCRIPT), "--diff", "origin/main...HEAD", "--json"], + cwd=str(REPO), + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + assert cp.returncode == 0, f"expected 0, got {cp.returncode} : {cp.stderr}" + payload = json.loads(cp.stdout.strip()) + assert payload["verdict"] == "CLEAN" + assert payload["offending"] == [] + + +def test_integration_bogus_diff_returns_clean(): + """A diff range that covers nothing -> CLEAN (no changed files).""" + cp = subprocess.run( + [sys.executable, str(SCRIPT), "--diff", "origin/main...origin/main", "--json"], + cwd=str(REPO), + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + assert cp.returncode == 0 + payload = json.loads(cp.stdout.strip()) + assert payload["verdict"] == "CLEAN"