Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/workflows/detect-dup-selftest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,13 @@ name: Detect-Duplicate Self-Test
# 3. Failure visibility: under the blind window the control MUST go
# missing and the CLI MUST exit 2 -- if blindness stops producing
# exit 2, the job goes red.
#
# #18039: the control pair #13050/#13051 is ~5000 numbers behind the head,
# so `--limit 600` alone can no longer reach it -- the workflow steps below
# keep their flags, and the CLI pours the control issues into the scanned
# lot itself (`gh issue view`, see _gh_issue_view). The detector still
# finds the burst by title+window; nothing is hardcoded. The unfetchable-
# control case (issue deleted) stays exit 2 by design.

on:
schedule:
Expand Down
53 changes: 53 additions & 0 deletions scripts/detect_duplicate_issues.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,15 @@
`--self-test` and the unit tests in
`scripts/tests/test_detect_duplicate_issues.py`.

Under `--self-test` the control issues are **poured into the scanned lot**
via `gh issue view` (#18039): a counted window (`--limit N`) slides with the
repository's head and the pair (~5000 numbers behind at #18039) had left it,
turning the daily self-test permanently red. The pour fetches the real
issues and adds them to the lot; the **detector** still has to find the pair
by title equality within the window -- nothing is hardcoded. If an issue of
a control pair cannot be fetched (deleted, transferred), it is absent from
the lot by definition: the control goes missing and the self-test exits 2.

What it does NOT do
-------------------
- It does NOT close duplicates. Closing is a human/coord decision because
Expand Down Expand Up @@ -161,6 +170,29 @@ def _gh_issue_list(limit: int, state: str = "all") -> list[dict]:
return json.loads(proc.stdout)


def _gh_issue_view(number: int) -> dict:
"""Fetch ONE issue by number via `gh issue view`.

Used only by the self-test pour (#18039): the positive-control pair
lives far behind the repository head, so a `--limit N` window can never
reach it. The row returned is the real GitHub record -- the detector
must still find the burst by itself.
"""
proc = subprocess.run(
[
"gh", "issue", "view", str(number),
"--json", "number,title,createdAt,state",
],
capture_output=True, text=True, encoding="utf-8", errors="replace",
timeout=60,
)
if proc.returncode != 0:
raise RuntimeError(
f"`gh issue view {number}` failed: {proc.stderr.strip()}"
)
return json.loads(proc.stdout)


def _parse_iso(ts: str) -> datetime:
"""Parse ISO 8601 returned by gh (Z-suffix) into aware datetime."""
return datetime.fromisoformat(ts.replace("Z", "+00:00"))
Expand Down Expand Up @@ -283,6 +315,27 @@ def _cli(argv: list[str] | None = None) -> int:
except RuntimeError as e:
print(str(e), file=sys.stderr)
return 1

if args.self_test:
# #18039: pour the positive-control issues into the lot. The pair
# is far behind the head and a counted window cannot see it; the
# pour adds the REAL GitHub records and the detector still has to
# find the burst. An unfetchable control issue stays out of the lot
# -> the control goes missing -> exit 2 (fail-closed).
seen = {int(d["number"]) for d in rows_raw}
for ctrl in KNOWN_POSITIVE_CONTROLS:
for n in ctrl:
if n in seen:
continue
try:
rows_raw.append(_gh_issue_view(n))
except RuntimeError as e:
print(
f"WARNING: control issue #{n} not fetchable -- "
f"counted as removed from the lot: {e}",
file=sys.stderr,
)

rows = [IssueRow.from_gh_dict(d) for d in rows_raw]
result = detect_burst_pairs(rows, window_seconds=args.window_seconds)

Expand Down
60 changes: 57 additions & 3 deletions scripts/tests/test_detect_duplicate_issues.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,22 +220,76 @@ def test_cli_exit_1_when_control_found(self):
(13051, "[Lean-2] exemple", "2026-08-26T01:39:14Z"),
)
from unittest import mock
with mock.patch.object(_mod, "_gh_issue_list", return_value=rows):
with mock.patch.object(_mod, "_gh_issue_list", return_value=rows), \
mock.patch.object(_mod, "_gh_issue_view") as view:
rc = _mod._cli(["--self-test", "--limit", "10"])
self.assertEqual(rc, 1)
# The pair is already in the lot -> the pour must NOT fetch again.
view.assert_not_called()

def test_cli_exit_2_when_control_missing(self):
"""Control missing -> exit 2, even though other pairs exist
(exit 2 takes precedence over exit 1)."""
(exit 2 takes precedence over exit 1). Both control issues are
unfetchable (deleted) -> removed from the lot by definition."""
rows = self._gh_rows(
(1, "foo", "2026-08-26T01:00:00Z"),
(2, "foo", "2026-08-26T01:00:01Z"),
)
from unittest import mock
with mock.patch.object(_mod, "_gh_issue_list", return_value=rows):
with mock.patch.object(_mod, "_gh_issue_list", return_value=rows), \
mock.patch.object(
_mod, "_gh_issue_view",
side_effect=RuntimeError("`gh issue view` failed: gone"),
):
rc = _mod._cli(["--self-test", "--limit", "10"])
self.assertEqual(rc, 2)

def test_self_test_pours_pair_outside_counted_window(self):
"""#18039 regression lock: the control pair lives BEHIND the
--limit window (repository grew past it). The pour fetches the two
real rows via `gh issue view` and adds them to the lot -- the
detector then finds the burst -> exit 1, NOT 2. This is the exact
shape that turned the daily self-test red since 2026-09-22."""
recent = self._gh_rows(
(18000, "modern topic", "2026-09-26T10:00:00Z"),
(18001, "another topic", "2026-09-26T11:00:00Z"),
)
control_rows = {
13050: {"number": 13050, "title": "[Lean-2] exemple",
"createdAt": "2026-08-26T01:39:13Z", "state": "OPEN"},
13051: {"number": 13051, "title": "[Lean-2] exemple",
"createdAt": "2026-08-26T01:39:14Z", "state": "OPEN"},
}
from unittest import mock
with mock.patch.object(_mod, "_gh_issue_list", return_value=recent), \
mock.patch.object(_mod, "_gh_issue_view",
side_effect=lambda n: control_rows[n]):
rc = _mod._cli(["--self-test", "--limit", "600",
"--window-seconds", "60"])
self.assertEqual(rc, 1)

def test_self_test_exit_2_when_one_control_issue_removed(self):
"""#18039 negative control: ONE issue of the pair is unfetchable
(removed) while the other is poured in. The lot holds a single
member -> no burst -> control missing -> exit 2."""
recent = self._gh_rows(
(18000, "modern topic", "2026-09-26T10:00:00Z"),
)
lone_row = {"number": 13050, "title": "[Lean-2] exemple",
"createdAt": "2026-08-26T01:39:13Z", "state": "OPEN"}

def fake_view(number):
if number == 13050:
return lone_row
raise RuntimeError("`gh issue view` failed: gone")

from unittest import mock
with mock.patch.object(_mod, "_gh_issue_list", return_value=recent), \
mock.patch.object(_mod, "_gh_issue_view", side_effect=fake_view):
rc = _mod._cli(["--self-test", "--limit", "600",
"--window-seconds", "60"])
self.assertEqual(rc, 2)

def test_blind_window_drops_control(self):
"""Window 0 (blind): the known pair's delta (1 s) falls outside the
window -> zero pairs, control reported MISSING. This is the exact
Expand Down
Loading