Skip to content

Commit 167473d

Browse files
zhanghuiclaude
andcommitted
test: make the slow/live selection collectable and survivable on Windows
Running `-m "slow or live_llm"` on the Windows box surfaced three problems in the suite itself, none in the product. 1. `tests/e2e/test_reflection_e2e.py` aborted collection of the whole run (`ModuleNotFoundError: No module named 'adapters'`), so zero tests executed. It imported `benchmarks.run` as a package, but since #425 the runner imports its siblings bare (`import adapters`) and only resolves with `benchmarks/` itself on sys.path -- the unit tests that use it already do that dance; this file now mirrors them. Verified against two detached worktrees: origin/main fails, e8612b9 (v1.3.0, before #425) collects. Not Windows-specific; CI never collects this file. The same refactor moved the prompts into `adapters/locomo.py`, added a `config` parameter to `_build_context`, and a `{current_date_line}` placeholder to ANSWER_PROMPT; the script body is adapted to those by signature. It has no `def test_` -- it is a manual driver with a `test_` filename -- and the adapted body has not been executed here; that needs a Tier 2 server and a LoCoMo run. 2. `test_real_sigint_during_phase_await_returns_130_with_resume_hint` sends itself `os.kill(os.getpid(), signal.SIGINT)`. On Windows that call is TerminateProcess with the signal number as exit code, so the test killed the pytest process mid-run: 27 of 29 had run, the summary and every traceback were lost, and `pytest_exit=2` was the literal SIGINT value, not pytest's "interrupted". Skipped on win32 with the mechanism in the reason. 3. `test_add_html_file_uri_parsed_into_buffer` built its uri as `f"file://{doc}"`, which on Windows is `file://C:\...` -- drive letter in the host slot. Same fix as the unit tests in f7a31c7: `Path.as_uri()`. Collection of the live selection is now clean (29 selected, 0 errors); the default `test_backfill_flags.py` selection still passes locally. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 parent 9c0fe18 commit 167473d

3 files changed

Lines changed: 35 additions & 18 deletions

File tree

‎tests/e2e/test_multimodal_add_e2e.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ async def test_add_html_file_uri_parsed_into_buffer(
129129
"sender_id": "alice",
130130
"role": "user",
131131
"timestamp": 1780304400000,
132-
"content": [{"type": "html", "uri": f"file://{doc}"}],
132+
"content": [{"type": "html", "uri": doc.as_uri()}],
133133
}
134134
],
135135
},

‎tests/e2e/test_reflection_e2e.py‎

Lines changed: 28 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,29 +10,38 @@
1010
from __future__ import annotations
1111

1212
import argparse
13+
import importlib
1314
import json
1415
import logging
1516
import sys
1617
import time
1718
from pathlib import Path
1819
from typing import Any
1920

20-
# benchmarks/run.py is the benchmark runner; add repo root to sys.path so
21-
# the benchmarks package is importable from any working directory.
22-
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
23-
24-
from benchmarks.run import (
25-
ANSWER_PROMPT,
26-
JUDGE_SYSTEM_PROMPT,
27-
JUDGE_USER_PROMPT,
28-
EverosClient,
29-
LLMClientPool,
30-
_build_context,
31-
_extract_final_answer,
32-
_extract_json,
33-
_parse_session_timestamp,
34-
print_section,
35-
)
21+
# benchmarks/run.py is a script, not a package module: it imports its
22+
# siblings bare (``import adapters``), which only resolves with
23+
# ``benchmarks/`` itself on sys.path. Importing it as ``benchmarks.run``
24+
# therefore fails at collection and, by pytest's default, aborts the whole
25+
# run before a single test executes. The unit tests that use the runner do
26+
# this same dance (``test_benchmark_cli_portability.py``); mirror it.
27+
_BENCH = Path(__file__).resolve().parents[2] / "benchmarks"
28+
if str(_BENCH) not in sys.path:
29+
sys.path.insert(0, str(_BENCH))
30+
_run = importlib.import_module("run")
31+
# The prompts moved from run.py into the per-benchmark adapters when the
32+
# runner was generalised (#425); this suite is LoCoMo conv_0, so LoCoMo's.
33+
_locomo = importlib.import_module("adapters.locomo")
34+
_LOCOMO_CONFIG = importlib.import_module("config").BenchmarkConfig.from_toml("locomo")
35+
ANSWER_PROMPT = _locomo.ANSWER_PROMPT
36+
JUDGE_SYSTEM_PROMPT = _locomo.JUDGE_SYSTEM_PROMPT
37+
JUDGE_USER_PROMPT = _locomo.JUDGE_USER_PROMPT
38+
EverosClient = _run.EverosClient
39+
LLMClientPool = _run.LLMClientPool
40+
_build_context = _run._build_context
41+
_extract_final_answer = _run._extract_final_answer
42+
_extract_json = _run._extract_json
43+
_parse_session_timestamp = _run._parse_session_timestamp
44+
print_section = _run.print_section
3645

3746
logger = logging.getLogger(__name__)
3847

@@ -292,8 +301,10 @@ def answer_and_judge(
292301
search_data.get("profiles", []),
293302
speaker_a,
294303
speaker_b,
304+
_LOCOMO_CONFIG,
295305
)
296-
prompt = ANSWER_PROMPT.format(context=context, question=query)
306+
# No session date is known here; the runner passes "" in that case too.
307+
prompt = ANSWER_PROMPT.format(context=context, current_date_line="", question=query)
297308
try:
298309
resp = llm_client.chat.completions.create(
299310
model=llm_model,

‎tests/integration/test_cli/test_backfill_flags.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import hashlib
3939
import os
4040
import signal
41+
import sys
4142
import threading
4243
import time
4344
from collections.abc import AsyncIterator
@@ -237,6 +238,11 @@ async def _raise_interrupt(*_args: object, **_kwargs: object) -> None:
237238

238239

239240
@pytest.mark.slow
241+
@pytest.mark.skipif(
242+
sys.platform == "win32",
243+
reason="os.kill(pid, SIGINT) is TerminateProcess on Windows: it kills the "
244+
"pytest process itself (exit code 2) instead of delivering a Ctrl-C",
245+
)
240246
def test_real_sigint_during_phase_await_returns_130_with_resume_hint(
241247
backfill_runtime: Path, monkeypatch: pytest.MonkeyPatch
242248
) -> None:

0 commit comments

Comments
 (0)