diff --git a/.github/workflows/evict-orphan-caches.yml b/.github/workflows/evict-orphan-caches.yml index 5308b30e61..6f51bc8150 100644 --- a/.github/workflows/evict-orphan-caches.yml +++ b/.github/workflows/evict-orphan-caches.yml @@ -23,8 +23,18 @@ name: Evict orphan Actions caches # # Defaut : tous les lundis 03:00 UTC (apres la nuit europeenne, avant # l'activite US). Ajustable via workflow_dispatch. +# +# Reouverture de #16088 (2026-09-28) : le sweep passe AUSSI a chaque push sur +# main. Chaque merge ecrit ses overlays CodeQL (un par langue) ; mesure du +# 28/09, 33 overlays en ~5 h (3,96 Go) avec un quota a 10,14 / 10 Go, et le +# cache `.lake` de knot_lean evince moins de 10 h apres sa sauvegarde. Un +# passage hebdomadaire ne suit pas cette cadence, et le cron de ce depot +# est servi avec des heures de retard. Le script garde les 3 overlays les +# plus recents par famille (`--keep-latest`, defaut 3) et evince le reste. on: + push: + branches: [main] schedule: - cron: "0 3 * * 1" workflow_dispatch: diff --git a/scripts/ci/evict_orphan_caches.py b/scripts/ci/evict_orphan_caches.py index ff4a56b03b..cdde254123 100644 --- a/scripts/ci/evict_orphan_caches.py +++ b/scripts/ci/evict_orphan_caches.py @@ -159,6 +159,15 @@ def _parse_args() -> argparse.Namespace: help="Evict caches whose last_accessed_at is older than this many hours " "(default: 24). A cache never hit has last_accessed_at == created_at.", ) + p.add_argument( + "--keep-latest", + type=int, + default=3, + help="Per overlay family (config hash + lang + toolchain), keep only the " + "N most recent caches still classified KEEP and evict the older ones " + "(default: 3; 0 disables). Reopening of #16088: every overlay of main " + "is an ancestor of main, so the two other criteria never catch them.", + ) p.add_argument( "--repo", default=os.environ.get("GITHUB_REPOSITORY", "jsboige/CoursIA"), @@ -300,6 +309,7 @@ def _classify_cache( sha = m.group("sha40") rec["sha40"] = sha rec["lang"] = m.group("lang") + rec["family"] = f"{m.group('random8')}-{m.group('lang')}-{m.group('toolchain')}" reasons: list[str] = [] ancestor_status = _is_ancestor(sha, remote, main_branch) @@ -324,6 +334,34 @@ def _classify_cache( return rec +def _mark_superseded(records: list[dict[str, Any]], keep_latest: int) -> None: + """Evict the overlays of a family beyond its `keep_latest` most recent ones. + + Only records still classified KEEP are counted and touched : an overlay + already evicted for another reason does not take one of the kept slots, + and a REFUSE (undecidable ancestry) is never turned into a deletion. + Order is `created_at` descending ; `keep_latest <= 0` disables the rule. + + Why (reopening of #16088, 2026-09-28) : each overlay of `main` carries a + SHA of `main`, hence an ancestor, and the quota fills in a few hours while + the age window is 7 days. Measured : 33 overlays kept, 3.96 GB, quota at + 10.14 / 10 GB, and the knot_lean `.lake` cache evicted within 10 h of its + save -- every Lean build of that lake then ran cold. + """ + if keep_latest <= 0: + return + families: dict[str, list[dict[str, Any]]] = {} + for r in records: + if r["verdict"] == "KEEP" and "family" in r: + families.setdefault(r["family"], []).append(r) + for members in families.values(): + members.sort(key=lambda r: _parse_iso(r["created_at"]), reverse=True) + for r in members[keep_latest:]: + r.update( + {"verdict": "EVICT", "reason": f"superseded_beyond_latest_{keep_latest}"} + ) + + def _print_human(records: list[dict[str, Any]], applied: bool) -> None: for r in records: verdict = r["verdict"] @@ -379,6 +417,7 @@ def main() -> int: ) for c in caches ] + _mark_superseded(records, args.keep_latest) applied = args.apply if applied: diff --git a/tests/test_evict_orphan_caches.py b/tests/test_evict_orphan_caches.py index aacd46a09f..58076194ee 100644 --- a/tests/test_evict_orphan_caches.py +++ b/tests/test_evict_orphan_caches.py @@ -206,6 +206,77 @@ def test_ancestor_check_unknown_refuses(self): self.assertEqual(rec["reason"], "ancestor_check_failed") +class TestMarkSuperseded(unittest.TestCase): + """Reouverture #16088 (2026-09-28) : garder les N overlays les plus + recents par famille, evincer les autres, meme ancetres de main.""" + + def _rec(self, family, hours_ago, verdict="KEEP", reason="recent_and_ancestor"): + created = _dt.datetime(2026, 9, 28, 4, 0, tzinfo=_dt.timezone.utc) - _dt.timedelta(hours=hours_ago) + return { + "key": f"{family}-{hours_ago}", + "family": family, + "created_at": _iso(created), + "verdict": verdict, + "reason": reason, + } + + def test_keeps_latest_n_per_family(self): + recs = [self._rec("py", h) for h in (5, 1, 3, 0, 4)] + eoc._mark_superseded(recs, 3) + kept = sorted(r["key"] for r in recs if r["verdict"] == "KEEP") + self.assertEqual(kept, ["py-0", "py-1", "py-3"]) + evicted = [r for r in recs if r["verdict"] == "EVICT"] + self.assertEqual(len(evicted), 2) + self.assertTrue(all(r["reason"] == "superseded_beyond_latest_3" for r in evicted)) + + def test_families_are_independent(self): + recs = [self._rec("py", h) for h in (0, 1)] + [self._rec("cs", h) for h in (0, 1, 2)] + eoc._mark_superseded(recs, 2) + evicted = [r["key"] for r in recs if r["verdict"] == "EVICT"] + self.assertEqual(evicted, ["cs-2"]) + + def test_already_evicted_does_not_take_a_slot(self): + # Un overlay deja evince (SHA orphelin) ne compte pas parmi les N gardes. + recs = [ + self._rec("py", 0, verdict="EVICT", reason="sha_not_ancestor_of_main"), + self._rec("py", 1), + self._rec("py", 2), + ] + eoc._mark_superseded(recs, 2) + self.assertEqual([r["verdict"] for r in recs], ["EVICT", "KEEP", "KEEP"]) + self.assertEqual(recs[0]["reason"], "sha_not_ancestor_of_main") + + def test_refuse_is_never_turned_into_eviction(self): + recs = [self._rec("py", h) for h in (0, 1)] + recs.append(self._rec("py", 2, verdict="REFUSE", reason="ancestor_check_failed")) + eoc._mark_superseded(recs, 1) + self.assertEqual(recs[2]["verdict"], "REFUSE") + self.assertEqual([r["verdict"] for r in recs[:2]], ["KEEP", "EVICT"]) + + def test_zero_disables(self): + recs = [self._rec("py", h) for h in range(6)] + eoc._mark_superseded(recs, 0) + self.assertTrue(all(r["verdict"] == "KEEP" for r in recs)) + + def test_records_without_family_are_untouched(self): + rec = {"key": "lake-x", "created_at": "2026-09-28T00:00:00Z", + "verdict": "REFUSE", "reason": "not_codeql_overlay"} + eoc._mark_superseded([rec], 1) + self.assertEqual(rec["verdict"], "REFUSE") + + def test_classify_sets_family(self): + now = _dt.datetime.now(_dt.timezone.utc) + cache = { + "id": 1, + "key": TestClassifyCache.KEY_ORPHAN, + "size_in_bytes": 1, + "created_at": _iso(now), + "last_accessed_at": _iso(now), + } + rec = eoc._classify_cache(cache, max_age_hours=24, remote="origin", main_branch="main") + self.assertEqual(rec["family"], "d953d79b74456ce0-python-2.27.0") + + class TestParseIso(unittest.TestCase): """Compat GitHub '...Z' suffix."""