fix(guard,#17623): deaccent advisory — repli lexical ACCENT_PAIRS (notebook entierement desaccentue)

.github/workflows/markdown-deaccent-advisory.yml

@@ -121,7 +121,7 @@ jobs:
           unset_label() { gh pr edit "$PR_NUMBER" --remove-label "$1" 2>/dev/null || true; }
 
           ensure_label "$LABEL_DEACCENT" \
-            "Un notebook modifie introduit une desaccentuation auto-flaggee (#14325, registre TRANCHE6)" "C4A64C"
+            "Un notebook modifie introduit une desaccentuation signalee — auto ou lexique (#14325/#17623, registre TRANCHE6)" "C4A64C"
           ensure_label "$LABEL_DEACCENT_BLOCKED" \
             "Un notebook modifie est a prose non francaise et n'est pas analyse (#14325)" "BFD4F2"
 
@@ -134,9 +134,10 @@ jobs:
             [ -z "$nb" ] && continue
             [ -f "$nb" ] || continue
             # Le detecteur rend rc=2 sur `--fail-on-findings` quand une forme
-            # auto est introduite ; on capture la sortie dans tous les cas pour
-            # distinguer les 4 cas (clean / blocked-language / candidates /
-            # error).
+            # auto OU lexique (#17623 : repli dictionnaire ACCENT_PAIRS, sans
+            # jumeau interne) est introduite ; on capture la sortie dans tous
+            # les cas pour distinguer les 4 cas (clean / blocked-language /
+            # candidates / error).
             out=$(python scripts/notebook_tools/detect_markdown_deaccent.py \
                     --json --fail-on-findings "$nb" 2>&1) && rc=0 || rc=$?
             if [ "$rc" -eq 0 ]; then
@@ -149,7 +150,7 @@ jobs:
               echo "$out" >> "$GITHUB_STEP_SUMMARY"
               echo '```' >> "$GITHUB_STEP_SUMMARY"
               echo "</details>" >> "$GITHUB_STEP_SUMMARY"
-              echo "::notice title=Markdown deaccent advisory::$nb introduit des candidats auto ; voir resume."
+              echo "::notice title=Markdown deaccent advisory::$nb introduit des candidats (auto/lexique) ; voir resume."
             elif [ "$rc" -eq 1 ]; then
               # rc=1 = vacuous scan / fichier non trouve. Traite comme
               # "non applicable" sans bloquer.
@@ -169,7 +170,7 @@ jobs:
           if [ "$ANY_AUTO" -gt 0 ]; then
             set_label "$LABEL_DEACCENT"
             echo "" >> "$GITHUB_STEP_SUMMARY"
-            echo "FIX: un ou plusieurs notebooks introduisent des candidats auto de desaccentuation (cf #14325, registre TRANCHE6). Voir le detail par notebook ci-dessus. La relecture peut justifier la forme nue (code, citation verbatim, libelle de figure) ou demander une cure locale." >> "$GITHUB_STEP_SUMMARY"
+            echo "FIX: un ou plusieurs notebooks introduisent des candidats (auto/lexique) de desaccentuation (cf #14325/#17623, registre TRANCHE6). Voir le detail par notebook ci-dessus. La relecture peut justifier la forme nue (code, citation verbatim, libelle de figure) ou demander une cure locale." >> "$GITHUB_STEP_SUMMARY"
           else
             unset_label "$LABEL_DEACCENT"
             echo "RESULT: aucune desaccentuation auto introduite par cette PR (advisory)." >> "$GITHUB_STEP_SUMMARY"

scripts/notebook_tools/detect_markdown_deaccent.py

@@ -3,10 +3,11 @@
 
 A "desaccented" word is an unaccented surface form whose accent-stripped twin
 exists (accented) elsewhere in the *same* notebook -- the internal positive
-control: without an accented twin, nothing is asserted. E.g. `theoreme` is
-flagged only if `theoreme`'s strip equals `theoreme` (no accents) and an
-accented `theoreme` (the `theoreme` spelling, stripped) was seen in that
-notebook.
+control. E.g. `theoreme` is flagged (bucket `auto`) only if an accented
+`théorème` was seen in that notebook. When no accented twin exists ANYWHERE --
+the entirely-desaccented notebook of #17623, invisible to the twin control --
+the word is still flagged (bucket `lexicon`) when its stripped form belongs to
+the conservative cure dictionary of `detect_accent_stripping.py`.
 
 This is the family seated by #14064: the user's coquille `individaux` was the
 visible tip of a systematic markdown desaccentuation. The historical,
@@ -18,8 +19,8 @@
 words, not typos, and together accounted for `6 084` (16%) of the ceiling.
 
 Hardening therefore does NOT try to raise precision by scoring context (that
-would overfit). It does two things, and validates by FALSE NEGATIVES (never by
-hits):
+would overfit). It does three things, and validates by FALSE NEGATIVES (never
+by hits):
 
 1. **Exclusion lists.** A closed set of unaccented forms that are either (a)
    legitimate homographs of a distinct French word (`des`/`dès`, `sur`/`sûr`,
@@ -31,6 +32,17 @@
    never as auto-flagged candidates.
 2. **FR/EN prose gate.** A notebook whose markdown is English-dominant is not a
    French-desaccentuation candidate at all; it is skipped with `language=en`.
+3. **Lexicon fallback (#17623, route 1).** The internal positive control is
+   blind to a notebook whose prose is ENTIRELY desaccented: no accented twin
+   exists anywhere, so the worst offender reported `0`. When a French notebook
+   has no twin for a form, the detector now consults the conservative cure
+   dictionary `ACCENT_PAIRS` of `detect_accent_stripping.py` (imported, never
+   copied -- organ-first) and reports the hit in a distinct `lexicon` bucket.
+   `ACCENT_PAIRS` only contains pairs whose stripped form is NOT a valid
+   standalone French word, so the homograph classes that plagued the 96%-FP
+   historical scan are excluded by construction; the local exclusion lists
+   (homographs like `tache`/`tâche`, which the dictionary wrongly admits, and
+   EN cognats) take precedence over the lexicon anyway.
 
 Acceptance is by the forms it MUST catch, written as tests, never by its hit
 count: `theoreme`, `etat`, `donnees`, `equilibre`, `entrainement` must be
@@ -41,10 +53,12 @@
                                    [--fail-on-findings]
 
 Output: one line per notebook `COUNT TOTAL (AUTO auto / HOM homograph / COG
-en-cognat), PATH`, then a machine-readable summary. A scan that matches no
-notebook is an error (exit 1) -- a vacuous `0/0` is never a clean scan (the
-scan_md_hierarchy.py lesson, #3968). `--fail-on-findings` exits 2 when any
-notebook has auto-flagged candidates (PR-gate use).
+en-cognat / LEX lexicon), PATH`, then a machine-readable summary. A scan that
+matches no notebook is an error (exit 1) -- a vacuous `0/0` is never a clean
+scan (the scan_md_hierarchy.py lesson, #3968). `--fail-on-findings` exits 2
+when any notebook has auto-flagged or lexicon candidates (advisory wiring
+aggregates this rc to success, #17623 -- hardening it into a blocking gate is
+the coordinator's call).
 """
 import argparse
 import json
@@ -53,6 +67,14 @@
 import unicodedata
 from pathlib import Path
 
+# Sibling import (organ-first, #17623 route 1): the lexicon is the conservative
+# cure dictionary of detect_accent_stripping.py -- imported, never copied. If
+# the two organs ever diverge on a pair, they diverge together.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from detect_accent_stripping import ACCENT_PAIRS  # noqa: E402
+
+LEXICON_DEACCENTED = frozenset(ACCENT_PAIRS)
+
 _WORD_RE = re.compile(r"[A-Za-zÀ-ÿ]{3,}")
 # Inline code, fenced code blocks, and inline math. Desaccented words inside
 # code/math are not prose defects; stripping them mirrors the issue's
@@ -132,11 +154,14 @@ def _markdown_texts(nb: dict) -> list[str]:
 
 
 def find_candidates(nb: dict) -> dict:
-    """Return {language, auto, homograph, en_cognat} candidate word->count maps.
+    """Return {language, auto, lexicon, homograph, en_cognat} word->count maps.
 
     Each bucket maps the *stripped lowercase key* to the number of unaccented
-    surface occurrences. `auto` only, is what a PR gate should fail on; the
-    other buckets are transparency, not defects.
+    surface occurrences. `auto` (accented twin present) and `lexicon` (no twin,
+    form in the conservative cure dictionary -- the entirely-desaccented
+    notebook of #17623) are findings; the other buckets are transparency, not
+    defects. The local exclusion lists take precedence over the lexicon: a
+    homograph the dictionary wrongly admits (`tache`) must stay silent.
     """
     texts = _markdown_texts(nb)
     # Internal positive control: strip(key) -> an accented surface form.
@@ -147,8 +172,8 @@ def find_candidates(nb: dict) -> dict:
             if word != _strip_accents(word):
                 accented.setdefault(_strip_accents(word).lower(), word)
 
-    result = {"language": _classify_language(texts), "auto": {}, "homograph": {},
-              "en_cognat": {}}
+    result = {"language": _classify_language(texts), "auto": {}, "lexicon": {},
+              "homograph": {}, "en_cognat": {}}
     if result["language"] != "fr":
         return result
 
@@ -158,14 +183,16 @@ def find_candidates(nb: dict) -> dict:
             if word != _strip_accents(word):
                 continue
             key = word.lower()
-            if key not in accented:
-                continue
-            bucket = (
-                "auto"
-                if key not in HOMOGRAPH_EXCLUSIONS and key not in EN_COGNAT_EXCLUSIONS
-                else ("homograph" if key in HOMOGRAPH_EXCLUSIONS else "en_cognat")
-            )
-            result[bucket][key] = result[bucket].get(key, 0) + 1
+            excluded = key in HOMOGRAPH_EXCLUSIONS or key in EN_COGNAT_EXCLUSIONS
+            if key in accented:
+                bucket = (
+                    "auto"
+                    if not excluded
+                    else ("homograph" if key in HOMOGRAPH_EXCLUSIONS else "en_cognat")
+                )
+                result[bucket][key] = result[bucket].get(key, 0) + 1
+            elif key in LEXICON_DEACCENTED and not excluded:
+                result["lexicon"][key] = result["lexicon"].get(key, 0) + 1
     return result
 
 
@@ -188,7 +215,7 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("paths", nargs="*", help="notebook(s) or directory(ies)")
     parser.add_argument("--json", action="store_true", help="emit JSON report")
     parser.add_argument("--fail-on-findings", action="store_true",
-                        help="exit 2 when any notebook has auto-flagged candidates")
+                        help="exit 2 when any notebook has auto or lexicon candidates")
     args = parser.parse_args(argv)
 
     if not args.paths:
@@ -221,13 +248,14 @@ def main(argv: list[str] | None = None) -> int:
         rel = path.as_posix()
         report[rel] = result
         n_auto = _total(result["auto"])
+        n_lex = _total(result["lexicon"])
         n_hom = _total(result["homograph"])
         n_cog = _total(result["en_cognat"])
-        n_tot = n_auto + n_hom + n_cog
+        n_tot = n_auto + n_hom + n_cog + n_lex
         if not args.json:
-            print(f"{n_tot} TOTAL ({n_auto} auto / {n_hom} hom / {n_cog} cog) "
-                  f"[{result['language']}], {rel}")
-        if n_auto and args.fail_on_findings:
+            print(f"{n_tot} TOTAL ({n_auto} auto / {n_hom} hom / {n_cog} cog / "
+                  f"{n_lex} lex) [{result['language']}], {rel}")
+        if (n_auto or n_lex) and args.fail_on_findings:
             exit_code = 2
 
     if args.json:

scripts/notebook_tools/tests/test_detect_markdown_deaccent.py

@@ -138,6 +138,64 @@ def test_language_stopwords_nonempty():
     assert len(EN_STOPWORDS) > 20
 
 
+# --- #17623 route 1: lexicon fallback (the entirely-desaccented notebook) ---
+
+def test_fully_desaccented_french_notebook_signaled_via_lexicon():
+    """The structural false negative of #17623: French prose with NO accented
+    twin anywhere (the worst offender reported 0 before the fallback) is now
+    signaled through the conservative cure dictionary (bucket `lexicon`)."""
+    prose = ("Le probleme est que l'entrainement remplacee de donnees utilise "
+             "un meme modele pour chaque serie.")
+    nb = _notebook([_md(prose)])
+    result = find_candidates(nb)
+    assert result["language"] == "fr", "this notebook must classify as French prose"
+    for form in ("probleme", "entrainement", "donnees", "meme", "modele", "remplacee"):
+        assert form in result["lexicon"], (
+            f"false negative: {form} must be lexicon-flagged, "
+            f"got lexicon={sorted(result['lexicon'])}"
+        )
+    assert result["auto"] == {}
+
+
+def test_lexicon_words_with_internal_twin_stay_auto():
+    """A word whose accented twin exists stays in `auto` (positive control
+    unchanged); the lexicon bucket never duplicates it."""
+    accented = "le problème de l'entraînement"
+    unaccented = "le probleme et l'entrainement"
+    nb = _notebook([_md(accented), _md(unaccented)])
+    result = find_candidates(nb)
+    assert "probleme" in result["auto"]
+    assert "entrainement" in result["auto"]
+    assert result["lexicon"] == {}
+
+
+def test_homographs_not_signaled_by_lexicon():
+    """Acceptance #17623: `des`/`sur`/`mesure`/`cote` are not in the dictionary
+    (stripped forms that ARE valid French words); `tache` IS in the dictionary
+    but the local HOMOGRAPH_EXCLUSIONS must take precedence -- none of the five
+    may be lexicon-signaled, twin or no twin."""
+    prose = "le des partitif, sur la table, une mesure exacte, le cote et la tache"
+    nb = _notebook([_md(prose)])
+    result = find_candidates(nb)
+    for form in ("des", "sur", "mesure", "cote", "tache"):
+        assert form not in result["lexicon"], (
+            f"homograph {form} must NOT be lexicon-signaled"
+        )
+        assert form not in result["auto"]
+
+
+def test_english_notebook_gate_skips_lexicon():
+    """English-dominant prose → language=en → no lexicon signal either: the
+    FR/EN gate protects the fallback exactly as it protects the twin control."""
+    en_prose = ("The probleme with this entrainement is that the donnees are "
+                "noisy across the board.")
+    nb = _notebook([_md(en_prose)])
+    result = find_candidates(nb)
+    assert result["language"] == "en"
+    assert result["lexicon"] == {}
+    assert result["auto"] == {}
+
+
 def test_french_stopword_shared_word_not_in_both_notably():
     """A word used as an FR stopword should not be double-qualifying as EN. This is
     a soft guard: `sur` is an FR preposition and happens to be `sur` as a verb; the