diff --git a/pdm.lock b/pdm.lock index ff71e6aa..64663945 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "drop", "hle", "ifbench", "ifeval", "iheval", "math", "multi-if", "ruler", "ruler-gen", "scicode", "t-eval", "test"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:0c005321b98f547f55c0042123575cd7d77c3663a2729ce6bfdc934fc9ce2299" +content_hash = "sha256:5c5c6e7508ade09cdd18e2dab299f92bf6ae94a045ee410bb07ee863a860e903" [[metadata.targets]] requires_python = ">=3.12,<3.15" @@ -15,7 +15,7 @@ name = "absl-py" version = "2.3.1" requires_python = ">=3.8" summary = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -groups = ["ifeval", "iheval"] +groups = ["ifeval", "iheval", "multi-if"] files = [ {file = "absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d"}, {file = "absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9"}, @@ -877,7 +877,7 @@ name = "immutabledict" version = "4.2.2" requires_python = "<4.0,>=3.8" summary = "Immutable wrapper around dictionaries (a fork of frozendict)" -groups = ["ifeval", "iheval"] +groups = ["ifeval", "iheval", "multi-if"] files = [ {file = "immutabledict-4.2.2-py3-none-any.whl", hash = "sha256:97c31d098a2c850e93a958badeef765e4736ed7942ec73e439facd764a3a7217"}, {file = "immutabledict-4.2.2.tar.gz", hash = "sha256:cb6ed3090df593148f94cb407d218ca526fd2639694afdb553dc4f50ce6feeca"}, diff --git a/pyproject.toml b/pyproject.toml index b8624ac7..75a761bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,13 +61,21 @@ ifeval = [ iheval = ["sieval[ifeval]", "rouge-score>=0.1.2"] multi-if = [ # Multi-IF vendors its own multilingual fork of the IFEval checkers, so this - # group is deliberately not `ifeval`'s: no absl/immutabledict (the fork uses - # stdlib logging and MappingProxyType), and emoji is required rather than - # optional because the CJK word counter behind every Chinese length - # constraint counts emoji as words. + # group is deliberately not `ifeval`'s: it restates what the fork reaches + # rather than reusing a list built for the other copy, and emoji is required + # rather than optional because the CJK word counter behind every Chinese + # length constraint counts emoji as words. "emoji>=2.15.0", "langdetect>=1.0.9", "nltk>=3.9.2", + # Not reached by the fork itself, but `multi_if_0shot_gen_fixed` grades + # through `sieval.community.instruction_following_eval_fixed`, whose repairs + # are shared with the ifeval family and so import google-research's + # `instructions` module -- these two are that module's. Needed at grading + # time, not import time, so omitting them fails inside `feedback()` with + # inference already paid for. + "absl-py>=2.3.1", + "immutabledict>=4.2.2", # Only reached when langdetect reports Thai for a model *response* (Thai is # not one of the dataset's eight languages). Declared anyway so that path # cannot die on ImportError mid-run; the import itself is deferred. diff --git a/sieval/community/instruction_following_eval/evaluation_lib.py b/sieval/community/instruction_following_eval/evaluation_lib.py index 4ad955cb..9125d2c9 100644 --- a/sieval/community/instruction_following_eval/evaluation_lib.py +++ b/sieval/community/instruction_following_eval/evaluation_lib.py @@ -19,6 +19,13 @@ """ # adapted from https://github.com/google-research/google-research/blob/f97f6adab57bd3065b24169bcfc559dc34d0db84/instruction_following_eval/evaluation_lib.py +# +# Local adaptation: the two graders take a keyword-only `instruction_dict`, +# defaulting to the vendored registry, so `ifeval_0shot_gen_fixed` can grade +# through the repaired checkers in +# `sieval.community.instruction_following_eval_fixed` without mutating a global +# that concurrently-graded samples share. Omitting the argument reproduces +# upstream's behaviour exactly, which is what the unqualified task does. import collections import dataclasses import json @@ -82,14 +89,18 @@ def write_outputs(output_jsonl_filename, outputs): def test_instruction_following_strict( inp, prompt_to_response, + *, + instruction_dict=None, ): """Tests response to see if instrutions are followed.""" + if instruction_dict is None: + instruction_dict = instructions_registry.INSTRUCTION_DICT response = prompt_to_response[inp.prompt] instruction_list = inp.instruction_id_list is_following_list = [] for index, instruction_id in enumerate(instruction_list): - instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction_cls = instruction_dict[instruction_id] instruction = instruction_cls(instruction_id) instruction.build_description(**inp.kwargs[index]) @@ -114,8 +125,12 @@ def test_instruction_following_strict( def test_instruction_following_loose( inp, prompt_to_response, + *, + instruction_dict=None, ): """Tests response for an upper bound for following instructions.""" + if instruction_dict is None: + instruction_dict = instructions_registry.INSTRUCTION_DICT response = prompt_to_response[inp.prompt] r = response.split("\n") response_remove_first = "\n".join(r[1:]).strip() @@ -139,7 +154,7 @@ def test_instruction_following_loose( is_following_list = [] for index, instruction_id in enumerate(instruction_list): - instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction_cls = instruction_dict[instruction_id] instruction = instruction_cls(instruction_id) instruction.build_description(**inp.kwargs[index]) diff --git a/sieval/community/instruction_following_eval_fixed.py b/sieval/community/instruction_following_eval_fixed.py new file mode 100644 index 00000000..676f0a96 --- /dev/null +++ b/sieval/community/instruction_following_eval_fixed.py @@ -0,0 +1,315 @@ +"""Repaired checkers for the IFEval instruction family. + +Three of the 25 IFEval checkers grade something other than what their own +instruction says. This module holds the repairs, as mixins that override one +method each, plus the registries that mount them. Nothing here changes what an +item *asks*: every repair is to the code that decides whether an answer complied. + +**Why this is original code in ``community/``, which the directory's CLAUDE.md +asks to be argued for.** The same three checkers are vendored twice — once under +``instruction_following_eval/`` (google-research) and once under ``multi_if/`` +(Meta's copy of the same file) — and their bodies are logic-identical, differing +only in one import path (``instructions_util.generate_keywords`` against a flat +``generate_keywords``) and one logging call, neither of which a repair touches. +So this is the ``_sympy_guards.py`` situation exactly: holding the fix *outside* +both vendored packages serves upstream alignment rather than working against it, +because the alternative is two copies of every repair inside files whose value is +being diffable against upstream — "a fix duplicated into two files is a fix that +will eventually exist in only one of them". + +**Nothing here is reachable from the unqualified tasks.** ``ifeval_0shot_gen`` +and ``multi_if_0shot_gen`` keep grading through the vendored registry, byte for +byte; the repairs are visible only to the ``_fixed`` siblings, which say so in +their names. A faithful port stays available for reproducing published numbers, +and a repaired one for measuring a model. + +The one change inside the vendored files is a keyword-only ``instruction_dict`` +parameter on their four grader functions, defaulting to the vendored registry. +It was preferred over re-implementing the grading loops here: those loops are +~25 lines each of upstream logic, and a copy would drift silently the next time +the vendored files are re-synced, whereas a parameter cannot. + +Every repair below is the narrowest one that fixes the defect, and each reduces +to upstream's own expression on the inputs upstream already handled — the tests +assert that reduction rather than asserting the repair in isolation. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import re +from collections.abc import Mapping + +import langdetect + +from sieval.community.instruction_following_eval import ( + instructions as _ifeval_upstream, +) +from sieval.community.instruction_following_eval import ( + instructions_registry as _ifeval_registry, +) +from sieval.community.multi_if import ifeval as _multi_if_upstream + +#: The three instruction ids whose checker this module replaces. Declared rather +#: than derived so a rename upstream fails loudly at registry-build time instead +#: of silently un-applying a fix. +NTH_PARAGRAPH_FIRST_WORD = "length_constraints:nth_paragraph_first_word" +LETTER_FREQUENCY = "keywords:letter_frequency" +ENGLISH_CAPITAL = "change_case:english_capital" + +FIXED_INSTRUCTION_IDS = frozenset( + {NTH_PARAGRAPH_FIRST_WORD, LETTER_FREQUENCY, ENGLISH_CAPITAL} +) + +#: Upstream's punctuation set for first-word extraction, lifted so the helper +#: below can run the same normalisation per token. +_PUNCTUATION = {".", ",", "?", "!", "'", '"'} + + +def _leading_word(token: str) -> str: + """Upstream's first-word normalisation, lifted verbatim to run per token. + + Character for character the loop upstream runs on ``paragraph.split()[0]``: + strip, drop leading quotes, then take letters up to the first punctuation + mark, lowercased. + """ + word = token.strip().lstrip("'").lstrip('"') + out = "" + for letter in word: + if letter in _PUNCTUATION: + break + out += letter.lower() + return out + + +def _is_ascii_letter(character: str) -> bool: + """Upstream's accepted range for a frequency letter, as the predicate it is.""" + return 97 <= ord(character.lower()) <= 122 + + +class _NthParagraphFirstWordFix: + """Index the paragraph list that was counted; compare a multi-token value. + + **Defect A — the list counted is not the list indexed.** Upstream splits on + ``\\n\\n``, then decrements a *count* for each blank chunk while indexing the + *unfiltered* list:: + + paragraphs = re.split(r"\\n\\n", value) + num_paragraphs = len(paragraphs) + for paragraph in paragraphs: + if not paragraph.strip(): + num_paragraphs -= 1 + ... + paragraph = paragraphs[self._nth_paragraph - 1] + + So a blank chunk at or before index ``nth - 1`` changes which paragraph is + checked — or hands the check a blank one — while the total it is compared + against is computed on the other reading. The common case is a response whose + *first* chunk is blank, which whole runs do uniformly depending on the chat + template. (Two newlines have to meet before ``re.split`` yields an empty + chunk, so a single leading ``\\n`` does not do it; a blank chunk *after* the + target index is harmless, both readings then agreeing.) The two are + reconciled the only way that keeps upstream's own ``num_paragraphs``: filter + once, and index what was counted. + + **Defect B — a ``first_word`` that is not one word.** Upstream compares + against ``paragraph.split()[0]``, a single whitespace-delimited token. Some + slots store a multi-token phrase there, and the prompts that carry them ask + for that same phrase, so the kwarg is faithful to the item and the comparison + is what is wrong: a single token can never equal a multi-token string, so + such a slot returns FAIL for every possible response. + + The repair reads how many tokens the *constraint's own value* spans and + compares that many tokens of the paragraph, each normalised by upstream's own + routine. For a single-token value — every well-formed item — ``n_tokens`` is + 1 and the expression reduces to upstream's, token for token. It does not + relax the comparison: the phrase must still open the paragraph, in order. + """ + + # Set by the upstream `build_description` this mixin inherits alongside. + _num_paragraphs: int + _nth_paragraph: int + _first_word: str + + def check_following(self, value: str) -> bool: + # One definition of "paragraph": a `\n\n`-separated chunk with content. + # Upstream computes the same number for `num_paragraphs`; it just kept + # the unfiltered list around to index into. + paragraphs = [p for p in re.split(r"\n\n", value) if p.strip()] + num_paragraphs = len(paragraphs) + + if not 1 <= self._nth_paragraph <= num_paragraphs: + return False + paragraph = paragraphs[self._nth_paragraph - 1].strip() + if not paragraph: # unreachable after the filter; kept as upstream's guard + return False + + # 1 for every well-formed item, which is upstream's path exactly. The + # `or 1` covers an empty constraint value, where `split()` yields nothing + # and slicing to 0 tokens would compare "" against "" and pass anything. + n_tokens = len(self._first_word.split()) or 1 + opening = " ".join(_leading_word(t) for t in paragraph.split()[:n_tokens]) + + return num_paragraphs == self._num_paragraphs and opening == self._first_word + + +class _LetterFrequencyFix: + """Keep the character the item names. + + Upstream's ``build_description`` accepts a letter only when it is a single + character in ``[a-z]``; anything else is silently replaced by + ``random.choice(string.ascii_letters)``, drawn *freshly on every call* — so + an item naming a non-alphabetic character is graded against a different + letter each time it is scored. + + Everything else is deferred to upstream — the frequency default, the relation + validation, the description pattern — so the only input whose handling + changes is a single character outside ``[a-z]``. Upstream's fallback survives + for values that are genuinely unusable (``None``, or more than one character + after stripping): there the item states nothing gradable, and inventing a + rule would be worse than upstream's. + """ + + _letter: str + _frequency: int + _comparison_relation: str + _description_pattern: str + + def build_description(self, *, letter=None, let_frequency=None, let_relation=None): + stripped = (letter or "").strip() + substituting = len(stripped) == 1 and not _is_ascii_letter(stripped) + # A placeholder upstream accepts, so its own validation still runs and no + # random draw is consumed -- consuming one would perturb the global RNG + # stream for every later checker, which is a side effect a grading fix + # has no business having. + description = super().build_description( # type: ignore[misc] + letter="a" if substituting else letter, + let_frequency=let_frequency, + let_relation=let_relation, + ) + if substituting: + self._letter = stripped.lower() + description = self._description_pattern.format( + letter=self._letter, + let_frequency=self._frequency, + let_relation=self._comparison_relation, + ) + return description + + +class _EnglishCapitalFix: + """Detect the language of a case-folded copy. + + Upstream is ``value.isupper() and langdetect.detect(value) == "en"``. + ``langdetect`` profiles are built from lowercase text, so ALL-CAPS input is + off-distribution for every profile and the verdict becomes unstable between + the Latin-script languages — the same English paragraph can detect as ``de``. + Because the checker's own precondition is that the response *is* all capitals, + every response it ever sees is in exactly the state the detector handles worst. + + Only the argument to ``detect`` changes in what is *graded*. ``isupper()`` is + still upstream's, still first, and still short-circuits, so the set of + responses reaching the detector is exactly the set upstream sends there, and + "all capital letters" is decided by code this mixin does not touch. The one + other difference is that overriding the whole method drops upstream's + log-on-exception (``logging.error`` in google-research's copy, ``logger.info`` + in Meta's -- the single line on which the two copies differ); it logged the + entire response, and the verdict it accompanied is unchanged. + + The sibling ``change_case:english_lowercase`` is deliberately *not* repaired: + it calls the detector on text that is already lowercase, which is the + condition the profiles were built for, so it has no defect to fix. + """ + + def check_following(self, value: str) -> bool: + assert isinstance(value, str) + try: + # `.lower()` only here: the response itself is never modified, and + # the capitals requirement is evaluated above on the original. + return value.isupper() and langdetect.detect(value.lower()) == "en" + except langdetect.LangDetectException: + # Upstream counts an undetectable text as following the instruction. + # Kept for parity, though `isupper()` above makes it near-unreachable + # -- an empty or uncased response fails before it. + return True + + +class IFEvalParagraphFirstWordCheckFixed( + _NthParagraphFirstWordFix, _ifeval_upstream.ParagraphFirstWordCheck +): + """The IFEval registry's copy, repaired.""" + + +class IFEvalLetterFrequencyCheckerFixed( + _LetterFrequencyFix, _ifeval_upstream.LetterFrequencyChecker +): + """The IFEval registry's copy, repaired.""" + + +class IFEvalCapitalLettersEnglishCheckerFixed( + _EnglishCapitalFix, _ifeval_upstream.CapitalLettersEnglishChecker +): + """The IFEval registry's copy, repaired.""" + + +class MultiIFParagraphFirstWordCheckFixed( + _NthParagraphFirstWordFix, _multi_if_upstream.ParagraphFirstWordCheck +): + """The Multi-IF registry's copy of the same checker, repaired identically.""" + + +class MultiIFLetterFrequencyCheckerFixed( + _LetterFrequencyFix, _multi_if_upstream.LetterFrequencyChecker +): + """The Multi-IF registry's copy of the same checker, repaired identically.""" + + +class MultiIFCapitalLettersEnglishCheckerFixed( + _EnglishCapitalFix, _multi_if_upstream.CapitalLettersEnglishChecker +): + """The Multi-IF registry's copy of the same checker, repaired identically.""" + + +def _build(base: Mapping[str, type], fixes: Mapping[str, type]) -> dict[str, type]: + """Overlay ``fixes`` on a copy of ``base``, refusing to add a new id. + + The refusal is the point: if an upstream re-sync renames one of the three + ids, the fix would otherwise be mounted under a key nothing looks up and the + ``_fixed`` task would quietly grade like the unqualified one. + """ + missing = sorted(set(fixes) - set(base)) + if missing: + raise KeyError( + f"instruction id(s) not in the upstream registry: {missing}. " + "The vendored checkers were renamed; update the constants in " + "sieval.community.instruction_following_eval_fixed." + ) + return {**base, **fixes} + + +def fixed_ifeval_registry() -> dict[str, type]: + """The IFEval registry with the three repaired checkers substituted in. + + A fresh dict each call, and the vendored ``INSTRUCTION_DICT`` is never + mutated: samples are graded concurrently, so a task that swapped the global + registry would change how *other* tasks grade mid-run. + """ + return _build( + _ifeval_registry.INSTRUCTION_DICT, + { + NTH_PARAGRAPH_FIRST_WORD: IFEvalParagraphFirstWordCheckFixed, + LETTER_FREQUENCY: IFEvalLetterFrequencyCheckerFixed, + ENGLISH_CAPITAL: IFEvalCapitalLettersEnglishCheckerFixed, + }, + ) + + +def fixed_multi_if_registry() -> dict[str, type]: + """The Multi-IF registry with the three repaired checkers substituted in.""" + return _build( + _multi_if_upstream.INSTRUCTION_DICT, + { + NTH_PARAGRAPH_FIRST_WORD: MultiIFParagraphFirstWordCheckFixed, + LETTER_FREQUENCY: MultiIFLetterFrequencyCheckerFixed, + ENGLISH_CAPITAL: MultiIFCapitalLettersEnglishCheckerFixed, + }, + ) diff --git a/sieval/community/multi_if/evaluation_lib.py b/sieval/community/multi_if/evaluation_lib.py index eb5c7ce9..36df4633 100644 --- a/sieval/community/multi_if/evaluation_lib.py +++ b/sieval/community/multi_if/evaluation_lib.py @@ -25,6 +25,12 @@ # 1. `import ifeval` -> `from . import ifeval` (upstream is a flat repo). # 2. The `Dict[str, float]` return annotations are corrected to `dict`: both # functions return lists, not floats, so upstream's annotation is wrong. +# 3. Both graders take a keyword-only `instruction_dict`, defaulting to the +# vendored registry, so `multi_if_0shot_gen_fixed` can grade through the +# repaired checkers in `sieval.community.instruction_following_eval_fixed` +# without mutating a global that concurrently-graded samples share. +# Omitting the argument reproduces upstream's behaviour exactly, which is +# what the unqualified task does. # Otherwise the bodies are byte-identical to upstream. from typing import Any @@ -32,13 +38,15 @@ from . import ifeval -def gen_acc_strict(x: dict[str, Any]) -> dict: +def gen_acc_strict(x: dict[str, Any], *, instruction_dict=None) -> dict: # reference: fbcode/gen_ai/github/fair_evals/evals/tasks/finetune/ifeval.py + if instruction_dict is None: + instruction_dict = ifeval.INSTRUCTION_DICT response = str(x["response"]) instruction_list = x["instruction_id_list"] is_following_list = [] for index, instruction_id in enumerate(instruction_list): - instruction_cls = ifeval.INSTRUCTION_DICT[instruction_id] + instruction_cls = instruction_dict[instruction_id] instruction = instruction_cls(instruction_id) instruction.build_description(**x["kwargs"][index]) @@ -54,7 +62,9 @@ def gen_acc_strict(x: dict[str, Any]) -> dict: } -def gen_acc_loose(x: dict[str, Any]) -> dict: +def gen_acc_loose(x: dict[str, Any], *, instruction_dict=None) -> dict: + if instruction_dict is None: + instruction_dict = ifeval.INSTRUCTION_DICT response = str(x["response"]) r = response.split("\n") response_remove_first = "\n".join(r[1:]).strip() @@ -77,7 +87,7 @@ def gen_acc_loose(x: dict[str, Any]) -> dict: instruction_list = x["instruction_id_list"] is_following_list = [] for index, instruction_id in enumerate(instruction_list): - instruction_cls = ifeval.INSTRUCTION_DICT[instruction_id] + instruction_cls = instruction_dict[instruction_id] instruction = instruction_cls(instruction_id) instruction.build_description(**x["kwargs"][index]) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 17ecf422..75e10eb3 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -1924,7 +1924,28 @@ "reference_impl": { "source": "google-research/instruction_following_eval", "url": "https://github.com/google-research/google-research/blob/f97f6adab57bd3065b24169bcfc559dc34d0db84/instruction_following_eval/evaluation_lib.py", - "notes": "evaluation_lib + instructions_registry vendored from google-research." + "notes": "evaluation_lib + instructions_registry vendored from google-research. Three of the 25 checkers grade something other than what their own instruction says (length_constraints:nth_paragraph_first_word, keywords:letter_frequency, change_case:english_capital). Kept as-is here, per the unqualified-name rule; ifeval_0shot_gen_fixed repairs them and carries the measured delta." + }, + "status": "stable", + "reference_kind": "value" + }, + { + "name": "ifeval_0shot_gen_fixed", + "display_name": "IFEval (0-shot, generative, corrected)", + "description": "IFEval with three repaired checkers; grading otherwise upstream's.", + "dataset": "ifeval", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "open-ended" + ], + "deps_group": "ifeval", + "model_type": "chat", + "reference_impl": { + "source": "google-research/instruction_following_eval", + "url": "https://github.com/google-research/google-research/blob/f97f6adab57bd3065b24169bcfc559dc34d0db84/instruction_following_eval/evaluation_lib.py", + "notes": "Same vendored evaluation_lib + instructions_registry as ifeval_0shot_gen, with three checkers replaced by the repaired subclasses in sieval.community.instruction_following_eval_fixed. DIVERGENCES (all three, exhaustive): (1) length_constraints:nth_paragraph_first_word — upstream decrements a paragraph count for each blank '\\n\\n' chunk but indexes the unfiltered list, so a blank chunk at or before the target index checks the wrong paragraph (or a blank one); the fix filters once and indexes what it counted. It also compares as many tokens as the constraint's own value spans, which is inert on IFEval (all 12 slots are single-token) and matters for Multi-IF. (2) keywords:letter_frequency — upstream replaces a letter outside [a-z] with a fresh random.choice per call; the fix keeps the character the item names (keys 1122 '#', 1129 '!'), and consumes no draw, so the global RNG stream other checkers share is unperturbed. (3) change_case:english_capital — upstream runs langdetect on ALL-CAPS text, which every profile is off-distribution for; the fix detects on a case-folded copy and leaves isupper() to decide the capitals requirement. Nothing else differs: the task overrides one method (_instruction_dict) of ifeval_0shot_gen and inherits prompting, grading, records and report. SCORE IMPACT, measured by running this task's own feedback/report and the unqualified task's over the same stored responses, langdetect seeded identically per arm — strict prompt-level 92.79→95.38 (+2.59, instruction-level 95.20→96.88) on a full-541 Intern-S2-Preview run, 94.82→95.19 (+0.37) on a second full-541 run, 65.00→67.00 (+2.00) on a 100-prompt Qwen3-30B-A3B thinking-on subset; loose prompt-level +0.55/+0.55/+2.00. Flips are bidirectional: the second run's loose reading moves english_capital 2 FAIL→PASS and 1 PASS→FAIL. Coverage 70 of 834 slots over 68 of 541 prompts (12.6%). It also removes nondeterminism: over 5 langdetect seeds the repaired task is seed-invariant on both full-541 sets (spread 0.00, no slot changing verdict) where upstream spans 0.18 and 0.37. The 100-prompt subset keeps a 2.00 spread at two slots that are not this task's to fix — change_case:english_lowercase (detects on already-lowercase text, deliberately unrepaired) and one english_capital whose response is a 94k-character repetition loop. REPRODUCTION NOTE: langdetect.detect is randomized and sieval never sets DetectorFactory.seed, so an unseeded A/B also flips checkers this task does not touch; pin the seed in both arms." }, "status": "stable", "reference_kind": "value" @@ -2188,11 +2209,33 @@ "reference_impl": { "source": "facebookresearch/Multi-IF", "url": "https://github.com/facebookresearch/Multi-IF/blob/1cdb53ed18499ad729e0766e5d3099dd5344406f/metrics.py", - "notes": "Multi-IF's own multilingual fork of the IFEval checkers is vendored (sieval.community.multi_if); the google-research IFEval sibling is NOT interchangeable with it. Upstream drives one pass per turn (--steps 1 2 3), one sample per turn; this task walks all three in one pass. Upstream reports fractions, this task percentages. Grading matches upstream's metrics_gen exactly, checked twice: offline on 535 conversations across all 8 languages (3,098 follow-lists), and on two full live runs by re-grading their own responses with upstream's graders — 128,258 per-constraint comparisons, 15 disagreements (0.012%), every one of them at a langdetect-routed checker (change_case:english_capital x10, length_constraints:number_sentences/number_words x1 each) or at the rejected-kwargs row (keywords:letter_frequency x3). Zero disagreements at any deterministic checker; re-derived per-language cells agree to 6e-02, `score` to 0.003. The two defects upstream cannot grade reproducibly itself: kwargs it rejects (letter='#' in 1122:18:en; empty keyword in 2616:4:zh — 6 of 13,447 turn-cells) send build_description to an unseeded random.choice, and langdetect is likewise unseeded and picks the counting algorithm behind every length constraint. Measured cost of both, re-grading identical responses 3x on the full set: 12 of 26,894 turn-cells flip and `score` spans 0.012 — two orders of magnitude under the +-0.4 sd that conversation sampling contributes. Tracked, not repaired, per the unqualified-name rule; fixing either needs a `_fixed` variant with a measured delta. PUBLISHED-NUMBER RESIDUAL (open). The only servable model carrying a first-party Multi-IF figure is Qwen3-32B: Qwen3 Technical Report (arXiv:2505.09388) Table 13 Thinking 73.0, Table 14 Non-thinking 70.7. Full 4,501-conversation runs at that report's own sampling knobs, 0 failures, denominators 4501/4501/4445, give `score` 78.73 (non-thinking) and 78.97 (thinking) — +8.03 and +5.97. No single reduction closes both: turn-3-only is nearest (71.38, +0.68; 71.23, -1.77) and is mechanically plausible because upstream emits one report per turn, so an integrator running --steps 3 would publish exactly that cell — but it contradicts the published ordering, since the two arms here are statistically indistinguishable (0.24 apart, bootstrap sd 0.4) where the report has Thinking +2.3. Upstream's own driver default max_new_tokens=1024 accounts for part of the level: a paired 600-conversation arm at that cap truncates 12.23% of turn-cells and loses 2.66 points, about a third of the gap. That the anchor is not the benchmark's protocol is visible in the report itself — its Table 11 gives OpenAI-o1 48.8 where the Multi-IF paper gives o1-preview 78.9 (three-turn average) and 70.7 (turn 3)." + "notes": "Multi-IF's own multilingual fork of the IFEval checkers is vendored (sieval.community.multi_if); the google-research IFEval sibling is NOT interchangeable with it. Upstream drives one pass per turn (--steps 1 2 3), one sample per turn; this task walks all three in one pass. Upstream reports fractions, this task percentages. Grading matches upstream's metrics_gen exactly, checked twice: offline on 535 conversations across all 8 languages (3,098 follow-lists), and on two full live runs by re-grading their own responses with upstream's graders — 128,258 per-constraint comparisons, 15 disagreements (0.012%), every one of them at a langdetect-routed checker (change_case:english_capital x10, length_constraints:number_sentences/number_words x1 each) or at the rejected-kwargs row (keywords:letter_frequency x3). Zero disagreements at any deterministic checker; re-derived per-language cells agree to 6e-02, `score` to 0.003. The two defects upstream cannot grade reproducibly itself: kwargs it rejects (letter='#' in 1122:18:en; empty keyword in 2616:4:zh — 6 of 13,447 turn-cells) send build_description to an unseeded random.choice, and langdetect is likewise unseeded and picks the counting algorithm behind every length constraint. Measured cost of both, re-grading identical responses 3x on the full set: 12 of 26,894 turn-cells flip and `score` spans 0.012 — two orders of magnitude under the +-0.4 sd that conversation sampling contributes. Tracked, not repaired, per the unqualified-name rule; fixing either needs a `_fixed` variant with a measured delta. That variant now exists as multi_if_0shot_gen_fixed, and it addresses the first only in part: it keeps the letter the 1122:18:en row names, and repairs two further checkers, but the empty-keyword row 2616:4:zh stays as upstream grades it and langdetect still routes the counting algorithm behind every length constraint in both tasks. PUBLISHED-NUMBER RESIDUAL (open). The only servable model carrying a first-party Multi-IF figure is Qwen3-32B: Qwen3 Technical Report (arXiv:2505.09388) Table 13 Thinking 73.0, Table 14 Non-thinking 70.7. Full 4,501-conversation runs at that report's own sampling knobs, 0 failures, denominators 4501/4501/4445, give `score` 78.73 (non-thinking) and 78.97 (thinking) — +8.03 and +5.97. No single reduction closes both: turn-3-only is nearest (71.38, +0.68; 71.23, -1.77) and is mechanically plausible because upstream emits one report per turn, so an integrator running --steps 3 would publish exactly that cell — but it contradicts the published ordering, since the two arms here are statistically indistinguishable (0.24 apart, bootstrap sd 0.4) where the report has Thinking +2.3. Upstream's own driver default max_new_tokens=1024 accounts for part of the level: a paired 600-conversation arm at that cap truncates 12.23% of turn-cells and loses 2.66 points, about a third of the gap. That the anchor is not the benchmark's protocol is visible in the report itself — its Table 11 gives OpenAI-o1 48.8 where the Multi-IF paper gives o1-preview 78.9 (three-turn average) and 70.7 (turn 3)." }, "status": "experimental", "reference_kind": "value" }, + { + "name": "multi_if_0shot_gen_fixed", + "display_name": "Multi-IF (0-shot, generative, corrected)", + "description": "Multi-IF with three repaired checkers; grading otherwise upstream's.", + "dataset": "multi_if", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "multilingual", + "multi-turn", + "open-ended" + ], + "deps_group": "multi-if", + "model_type": "chat", + "reference_impl": { + "source": "facebookresearch/Multi-IF", + "url": "https://github.com/facebookresearch/Multi-IF/blob/1cdb53ed18499ad729e0766e5d3099dd5344406f/metrics.py", + "notes": "Same vendored graders and multilingual checker fork as multi_if_0shot_gen, with three checkers replaced by the repaired subclasses in sieval.community.instruction_following_eval_fixed (shared with ifeval_0shot_gen_fixed; the two vendored copies of these three classes are logic-identical). DIVERGENCES (exhaustive): (1) length_constraints:nth_paragraph_first_word — paragraphs are counted and indexed on the same blank-filtered list, and the comparison spans as many tokens as the constraint's own value; 35 of 749 slots carry a first_word that is not one token (32 multi-token across 6 languages, 3 blank in conversation 2215:14:zh) and upstream FAILs every response on them unconditionally. The 3 blank slots stay ungradeable by design. (2) keywords:letter_frequency — the item's own character is kept instead of a fresh random.choice per call; 1 conversation, 1122:18:en, 3 turn-cells. (3) change_case:english_capital — langdetect runs on a case-folded copy, since ALL-CAPS text is off-distribution for every profile it ships. Nothing else differs: one method (_instruction_dict) is overridden and everything else inherited. SCORE IMPACT, measured by running this task's own feedback/report and the unqualified task's over 160 stored Qwen3-30B-A3B thinking-on conversations (1,218 slots), langdetect seeded identically per arm — score 69.07→70.52 (+1.45); per-turn overall 75.45→77.51 / 69.09→70.74 / 62.67→63.30; strict instruction-level 78.14→80.97 / 79.46→81.44 / 78.75→79.64. Flips 17 nth_paragraph_first_word + 3 english_capital (strict), 1 + 3 (loose), all FAIL→PASS. Loose moves less because its line-stripping retries already undo some instances of the paragraph defect. Coverage 1,121 slots over 515 of 4,501 conversations (11.4%). Unlike the IFEval sibling, which is seed-invariant after repair, score here still spans 0.18 over 5 langdetect seeds — the same 0.18 upstream spans, since that residue is language routing on genuinely multilingual text. REPRODUCTION NOTE: langdetect.detect is randomized and sieval never sets DetectorFactory.seed; pin it in both arms, since it also selects the counting algorithm behind every length constraint." + }, + "status": "stable", + "reference_kind": "value" + }, { "name": "openbookqa_kshot_gen", "display_name": "OpenBookQA (k-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 9da80d68..82d2ee3a 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -106,6 +106,9 @@ from .ifbench_0shot_gen import ( from .ifeval_0shot_gen import ( IFEvalZeroShotGenTask, ) +from .ifeval_0shot_gen_fixed import ( + IFEvalZeroShotGenFixedTask, +) from .iheval_0shot_gen import ( IHEvalZeroShotGenTask, ) @@ -142,6 +145,9 @@ from .mmmlu_kshot_clp import ( from .multi_if_0shot_gen import ( MultiIFZeroShotGenTask, ) +from .multi_if_0shot_gen_fixed import ( + MultiIFZeroShotGenFixedTask, +) from .openbookqa_kshot_gen import ( OpenBookQAFewShotGenTask, ) @@ -220,6 +226,7 @@ __all__ = [ "HumanEvalZeroShotBaseGenTask", "HumanEvalZeroShotGenTask", "IFBenchZeroShotGenTask", + "IFEvalZeroShotGenFixedTask", "IFEvalZeroShotGenTask", "IHEvalZeroShotGenTask", "IMOAnswerBenchZeroShotGenTask", @@ -232,6 +239,7 @@ __all__ = [ "MMLUProZeroShotGenTask", "MMLUZeroShotGenTask", "MMMLUKShotClpTask", + "MultiIFZeroShotGenFixedTask", "MultiIFZeroShotGenTask", "OpenBookQAFewShotGenTask", "PlatinumGSM8KZeroShotGenTask", diff --git a/sieval/tasks/ifeval_0shot_gen.py b/sieval/tasks/ifeval_0shot_gen.py index 81f02710..df0e1db9 100644 --- a/sieval/tasks/ifeval_0shot_gen.py +++ b/sieval/tasks/ifeval_0shot_gen.py @@ -43,7 +43,14 @@ reference_impl=ReferenceImpl( source="google-research/instruction_following_eval", url="https://github.com/google-research/google-research/blob/f97f6adab57bd3065b24169bcfc559dc34d0db84/instruction_following_eval/evaluation_lib.py", - notes="evaluation_lib + instructions_registry vendored from google-research.", + notes=( + "evaluation_lib + instructions_registry vendored from google-research. " + "Three of the 25 checkers grade something other than what their own " + "instruction says (length_constraints:nth_paragraph_first_word, " + "keywords:letter_frequency, change_case:english_capital). Kept as-is " + "here, per the unqualified-name rule; ifeval_0shot_gen_fixed repairs " + "them and carries the measured delta." + ), ), ) class IFEvalZeroShotGenTask( @@ -61,6 +68,15 @@ class IFEvalZeroShotGenTask( def __init__(self, dataset, model, name: str | None = None): super().__init__(dataset=dataset, model=model, name=name) + def _instruction_dict(self) -> dict[str, type] | None: + """Registry the checkers are looked up in; ``None`` is the vendored one. + + The single seam ``ifeval_0shot_gen_fixed`` needs. Everything else about + how a sample is prompted, graded and pooled is shared, so overriding this + cannot make the two tasks differ in any other way. + """ + return None + @override async def preprocess(self, raw, ctx): return build_prompt_record( @@ -120,8 +136,11 @@ async def feedback(self, post, ctx): # report() pools those raw counts rather than averaging the rates here. metrics: dict[str, bool | float] = {} detail = {} + instruction_dict = self._instruction_dict() for grade in _GRADES: - out = graders[grade](inp, {prompt: response}) + out = graders[grade]( + inp, {prompt: response}, instruction_dict=instruction_dict + ) followed = list(out.follow_instruction_list) metrics[f"{grade}_follow_all"] = out.follow_all_instructions metrics[f"{grade}_instruction_level"] = ( diff --git a/sieval/tasks/ifeval_0shot_gen_fixed.py b/sieval/tasks/ifeval_0shot_gen_fixed.py new file mode 100644 index 00000000..a9624820 --- /dev/null +++ b/sieval/tasks/ifeval_0shot_gen_fixed.py @@ -0,0 +1,133 @@ +"""IFEval 0-shot generative task, corrected — three repaired constraint checkers. + +``ifeval_0shot_gen`` keeps grading through the vendored google-research +registry, defects included, because that is what reproduces a published number. +This variant substitutes three repaired checkers and is therefore *not* a +reproduction: it measures whether a response obeyed the constraint its own +prompt states. + +**One method differs.** This class overrides +:meth:`~sieval.tasks.ifeval_0shot_gen.IFEvalZeroShotGenTask._instruction_dict` +and nothing else — the prompt, the strict/loose graders, the per-sample record +and the pooled report are all inherited, so the two tasks cannot diverge +anywhere except at the registry the checkers are looked up in. That is the +strongest available statement that the delta is the repair and not a second +change riding along with it. + +The three defects, and why each fix is the narrowest one, are documented on the +mixins in :mod:`sieval.community.instruction_following_eval_fixed`; the +divergences and the measured deltas are enumerated in ``notes`` below. Three +things those numbers do not say on their own: + +* **Coverage.** The three ids hold 70 of the pinned set's 834 constraint slots, + over 68 of 541 prompts. Every ``nth_paragraph_first_word`` slot carries a + single-token ``first_word``, so the multi-token half of that repair is inert + here and is exercised only by the Multi-IF sibling. +* **What fires the paragraph defect.** ``re.split(r"\\n\\n", ...)`` yields an + empty chunk only where two newlines meet, so the trigger is a response whose + first chunk is *blank*, not merely one that opens with a newline. Whole runs + do this uniformly and by different routes — 540 of 541 Intern-S2 responses + open with a space then a blank line, all 100 Qwen3 responses open with + ``\\n\\n`` outright, a second full-541 run does it in 0.2% — which is why the + per-set deltas span an order of magnitude rather than reading as noise. On a + run whose responses never open blank, this repair moves ``english_capital`` + and nothing else. +* **Seed ``langdetect`` before reproducing any of it.** ``langdetect.detect`` is + randomized and SiEval does not set ``DetectorFactory.seed``, so an unseeded + A/B flips *unrepaired* checkers too — ``change_case:english_lowercase`` + flipped between arms during this measurement and looked for a while like the + repair leaking through a shared RNG. That same unseededness is why + ``english_capital`` is worth repairing rather than merely worth noting. + +**Status.** ``stable``. The divergence is carried by the name, so ``status`` is +not gated on reproducing a published number — by construction this variant +cannot, since it grades differently on purpose. What a ``_fixed`` variant owes +instead is a quantified delta, measured here on three stored response sets, with +each repair reducing to upstream's own expression on the inputs upstream already +handled (asserted in the tests). ``experimental`` is for a faithful port whose +published anchor is not reachable, which is not a claim this task makes. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import override + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.tasks.ifeval_0shot_gen import IFEvalZeroShotGenTask + + +@sieval_task( + name="ifeval_0shot_gen_fixed", + display_name="IFEval (0-shot, generative, corrected)", + description="IFEval with three repaired checkers; grading otherwise upstream's.", + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "open-ended"), + deps_group="ifeval", + model_type="chat", + # The divergence is carried by the name, not by `status`: this task grades + # differently on purpose, so it claims no published number to be gated on. + # What a `_fixed` variant owes instead is the quantified delta in `notes`. + status="stable", + reference_kind="value", + reference_impl=ReferenceImpl( + source="google-research/instruction_following_eval", + url="https://github.com/google-research/google-research/blob/f97f6adab57bd3065b24169bcfc559dc34d0db84/instruction_following_eval/evaluation_lib.py", + notes=( + "Same vendored evaluation_lib + instructions_registry as " + "ifeval_0shot_gen, with three checkers replaced by the repaired " + "subclasses in sieval.community.instruction_following_eval_fixed. " + "DIVERGENCES (all three, exhaustive): (1) " + "length_constraints:nth_paragraph_first_word — upstream decrements a " + "paragraph count for each blank '\\n\\n' chunk but indexes the " + "unfiltered list, so a blank chunk at or before the target index " + "checks the wrong paragraph (or a blank one); the fix filters once " + "and indexes what it counted. It also " + "compares as many tokens as the constraint's own value spans, which is " + "inert on IFEval (all 12 slots are single-token) and matters for " + "Multi-IF. (2) keywords:letter_frequency — upstream replaces a letter " + "outside [a-z] with a fresh random.choice per call; the fix keeps the " + "character the item names (keys 1122 '#', 1129 '!'), and consumes no " + "draw, so the global RNG stream other checkers share is unperturbed. " + "(3) change_case:english_capital — upstream runs langdetect on ALL-CAPS " + "text, which every profile is off-distribution for; the fix detects on " + "a case-folded copy and leaves isupper() to decide the capitals " + "requirement. Nothing else differs: the task overrides one method " + "(_instruction_dict) of ifeval_0shot_gen and inherits prompting, " + "grading, records and report. " + "SCORE IMPACT, measured by running this task's own feedback/report " + "and the unqualified task's over the same stored responses, " + "langdetect seeded identically per arm — strict prompt-level " + "92.79→95.38 (+2.59, instruction-level 95.20→96.88) on a full-541 " + "Intern-S2-Preview run, 94.82→95.19 (+0.37) on a second full-541 run, " + "65.00→67.00 (+2.00) on a 100-prompt Qwen3-30B-A3B thinking-on subset; " + "loose prompt-level +0.55/+0.55/+2.00. Flips are bidirectional: the " + "second run's loose reading moves english_capital 2 FAIL→PASS and 1 " + "PASS→FAIL. Coverage 70 of 834 slots over 68 of 541 prompts (12.6%). " + "It also removes nondeterminism: over 5 langdetect seeds the repaired " + "task is seed-invariant on both full-541 sets (spread 0.00, no slot " + "changing verdict) where upstream spans 0.18 and 0.37. The 100-prompt " + "subset keeps a 2.00 spread at two slots that are not this task's to " + "fix — change_case:english_lowercase (detects on already-lowercase " + "text, deliberately unrepaired) and one english_capital whose response " + "is a 94k-character repetition loop. " + "REPRODUCTION NOTE: langdetect.detect is randomized and sieval never " + "sets DetectorFactory.seed, so an unseeded A/B also flips checkers " + "this task does not touch; pin the seed in both arms." + ), + ), +) +class IFEvalZeroShotGenFixedTask(IFEvalZeroShotGenTask): + @override + def _instruction_dict(self) -> dict[str, type] | None: + # Imported here, not at module scope, for the reason the base task + # lazy-imports `evaluation_lib`: registration imports every task module, + # and must not pay for the vendored checkers and langdetect. + from sieval.community.instruction_following_eval_fixed import ( + fixed_ifeval_registry, + ) + + # A fresh dict per call, never the vendored global: samples grade + # concurrently, and mutating the shared registry would change how the + # unqualified task grades in the same session. + return fixed_ifeval_registry() diff --git a/sieval/tasks/multi_if_0shot_gen.py b/sieval/tasks/multi_if_0shot_gen.py index 7e82acbf..59b76341 100644 --- a/sieval/tasks/multi_if_0shot_gen.py +++ b/sieval/tasks/multi_if_0shot_gen.py @@ -220,7 +220,12 @@ def _ensure_punkt_tab() -> None: "26,894 turn-cells flip and `score` spans 0.012 — two orders of " "magnitude under the +-0.4 sd that conversation sampling contributes. " "Tracked, not repaired, per the unqualified-name rule; fixing either " - "needs a `_fixed` variant with a measured delta. " + "needs a `_fixed` variant with a measured delta. That variant now " + "exists as multi_if_0shot_gen_fixed, and it addresses the first only " + "in part: it keeps the letter the 1122:18:en row names, and repairs " + "two further checkers, but the empty-keyword row 2616:4:zh stays as " + "upstream grades it and langdetect still routes the counting " + "algorithm behind every length constraint in both tasks. " "PUBLISHED-NUMBER RESIDUAL (open). The only servable model carrying a " "first-party Multi-IF figure is Qwen3-32B: Qwen3 Technical Report " "(arXiv:2505.09388) Table 13 Thinking 73.0, Table 14 Non-thinking " @@ -254,6 +259,15 @@ class MultiIFZeroShotGenTask( dict[str, float | str], ] ): + def _instruction_dict(self) -> dict[str, type] | None: + """Registry the checkers are looked up in; ``None`` is the vendored one. + + The single seam ``multi_if_0shot_gen_fixed`` needs. Everything else about + how a conversation is walked, graded and pooled is shared, so overriding + this cannot make the two tasks differ in any other way. + """ + return None + @override async def preprocess(self, raw, ctx): turns = raw["turns"] @@ -392,6 +406,7 @@ async def feedback(self, post, ctx): metrics: dict[str, bool | float] = {} detail: dict[str, dict] = {} + instruction_dict = self._instruction_dict() for index, turn in enumerate(graded, start=1): instruction_ids = list(turn["instruction_id_list"]) payload = { @@ -404,7 +419,8 @@ async def feedback(self, post, ctx): } detail[f"turn_{index}"] = {"instruction_id_list": instruction_ids} for grade in _GRADES: - followed = list(graders[grade](payload)["follow_instruction_list"]) + graded_turn = graders[grade](payload, instruction_dict=instruction_dict) + followed = list(graded_turn["follow_instruction_list"]) metrics[f"turn_{index}_{grade}_follow_all"] = all(followed) metrics[f"turn_{index}_{grade}_instruction_level"] = ( sum(followed) / len(followed) if followed else 0.0 diff --git a/sieval/tasks/multi_if_0shot_gen_fixed.py b/sieval/tasks/multi_if_0shot_gen_fixed.py new file mode 100644 index 00000000..47556926 --- /dev/null +++ b/sieval/tasks/multi_if_0shot_gen_fixed.py @@ -0,0 +1,139 @@ +"""Multi-IF 0-shot generative task, corrected — three repaired constraint checkers. + +``multi_if_0shot_gen`` keeps grading through Meta's vendored fork of the IFEval +checkers, defects included; its module docstring names two of them and says they +are tracked rather than repaired, "per the unqualified-name rule; fixing either +needs a ``_fixed`` variant with a measured delta". This is that variant. + +**One method differs.** This class overrides +:meth:`~sieval.tasks.multi_if_0shot_gen.MultiIFZeroShotGenTask._instruction_dict` +and nothing else. Conversation walking, per-turn grading, the cumulative +constraint lists, the per-language and per-turn pooling — all inherited, so the +two tasks cannot diverge anywhere except at the registry the checkers are looked +up in. + +The repairs are shared with ``ifeval_0shot_gen_fixed`` and documented on the +mixins in :mod:`sieval.community.instruction_following_eval_fixed`; Multi-IF's +vendored copies of these three checkers are logic-identical to google-research's +(the two copies differ only in one import path and one logging call), which is +why one mixin serves both registries. The divergences and the measured deltas +are enumerated in ``notes`` below. Three things those numbers do not say on +their own, two of them specific to this port: + +* ``length_constraints:nth_paragraph_first_word`` carries a ``first_word`` that + is *not a single whitespace-delimited token* in 35 of its 749 slots — 32 + multi-token (Hindi 12, Spanish 9, French 3, Portuguese 3, Russian 3, Italian + 2, English 0) and 3 blank, all three in one conversation (``2215:14:zh``). + Upstream compares against ``paragraph.split()[0]``, one token, so a + multi-token slot **returns FAIL for every possible response** — a check that + cannot pass measures nothing. The repair spans as many tokens as the + constraint's own value, each normalised by upstream's own routine, so it does + not relax the comparison and reduces to upstream's expression token for token + whenever the value is one token — every IFEval slot, and 714 of these. The 3 + blank slots stay ungradeable, since a constraint with no value states nothing + to check; this repair is not a backfill. +* ``keywords:letter_frequency`` with a non-``[a-z]`` letter appears in one + conversation, ``1122:18:en``, at 3 turn-cells (the constraint is cumulative, + so turn 1's slot recurs in turns 2 and 3). That is the row the unqualified + task's notes call out as one it "cannot grade reproducibly itself". +* The strict/loose asymmetry in the deltas is mechanical: the loose reading + already re-tries each response with its first and last lines stripped, which + happens to undo some instances of the paragraph-index defect. That also makes + loose the weaker evidence of the two — it was accidentally masking the defect, + not immune to it. + +**Caveat on reproducing the deltas: seed langdetect.** Identical to the IFEval +sibling's, and it binds harder here — ``langdetect`` also selects the word- and +sentence-counting algorithm behind every length constraint, in a set that is +multilingual by design. Unlike that sibling, which is *seed-invariant* after +repair on both full sets it was measured on, this task keeps a 0.18 ``score`` +spread over five seeds — the same 0.18 upstream has, because that residue is +language routing on genuine multilingual text, which no repair here touches and +none should. Pin ``langdetect.DetectorFactory.seed`` in both arms before +attributing a flip to a repair. + +**Status.** ``stable``. Its parent stays ``experimental`` because it is a +faithful port whose first-party published number is not reproducible under any +single reduction (see ``multi_if_0shot_gen``'s notes). That is a property of the +anchor, which this variant neither changes nor inherits: it claims no anchor at +all. Repairing three checkers moves further from that number, deliberately; what +it owes instead is a quantified delta, which ``notes`` carries. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import override + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.tasks.multi_if_0shot_gen import MultiIFZeroShotGenTask + + +@sieval_task( + name="multi_if_0shot_gen_fixed", + display_name="Multi-IF (0-shot, generative, corrected)", + description="Multi-IF with three repaired checkers; grading otherwise upstream's.", + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("multilingual", "multi-turn", "open-ended"), + deps_group="multi-if", + model_type="chat", + # The parent stays `experimental` because it is a faithful port whose + # published anchor is not reachable. That reason does not carry over: this + # variant grades differently on purpose and claims no anchor, so what it + # owes instead is the quantified delta in `notes`. + status="stable", + reference_kind="value", + reference_impl=ReferenceImpl( + source="facebookresearch/Multi-IF", + url="https://github.com/facebookresearch/Multi-IF/blob/1cdb53ed18499ad729e0766e5d3099dd5344406f/metrics.py", + notes=( + "Same vendored graders and multilingual checker fork as " + "multi_if_0shot_gen, with three checkers replaced by the repaired " + "subclasses in sieval.community.instruction_following_eval_fixed " + "(shared with ifeval_0shot_gen_fixed; the two vendored copies of these " + "three classes are logic-identical). DIVERGENCES (exhaustive): (1) " + "length_constraints:nth_paragraph_first_word — paragraphs are counted " + "and indexed on the same blank-filtered list, and the comparison spans " + "as many tokens as the constraint's own value; 35 of 749 slots carry a " + "first_word that is not one token (32 multi-token across 6 languages, " + "3 blank in conversation 2215:14:zh) and upstream FAILs every response " + "on them unconditionally. The 3 blank slots stay ungradeable by " + "design. (2) keywords:letter_frequency — the item's own character is " + "kept instead of a fresh random.choice per call; 1 conversation, " + "1122:18:en, 3 turn-cells. (3) change_case:english_capital — langdetect " + "runs on a case-folded copy, since ALL-CAPS text is off-distribution " + "for every profile it ships. Nothing else differs: one method " + "(_instruction_dict) is overridden and everything else inherited. " + "SCORE IMPACT, measured by running this task's own feedback/report " + "and the unqualified task's over 160 stored Qwen3-30B-A3B thinking-on " + "conversations (1,218 slots), langdetect seeded identically per arm — " + "score 69.07→70.52 (+1.45); per-turn overall 75.45→77.51 / " + "69.09→70.74 / 62.67→63.30; strict instruction-level 78.14→80.97 / " + "79.46→81.44 / 78.75→79.64. Flips 17 nth_paragraph_first_word + 3 " + "english_capital (strict), 1 + 3 (loose), all FAIL→PASS. Loose moves " + "less because its line-stripping retries already undo some instances " + "of the paragraph defect. Coverage 1,121 slots over 515 of 4,501 " + "conversations (11.4%). Unlike the IFEval sibling, which is " + "seed-invariant after repair, score here still spans 0.18 over 5 " + "langdetect seeds — the same 0.18 upstream spans, since that residue " + "is language routing on genuinely multilingual text. " + "REPRODUCTION NOTE: langdetect.detect is randomized and sieval never " + "sets DetectorFactory.seed; pin it in both arms, since it also selects " + "the counting algorithm behind every length constraint." + ), + ), +) +class MultiIFZeroShotGenFixedTask(MultiIFZeroShotGenTask): + @override + def _instruction_dict(self) -> dict[str, type] | None: + # Imported here, not at module scope, for the reason the base task + # lazy-imports `evaluation_lib`: registration imports every task module, + # and must not pay for the 3.5k-line checker fork and langdetect. + from sieval.community.instruction_following_eval_fixed import ( + fixed_multi_if_registry, + ) + + # A fresh dict per call, never the vendored global: samples grade + # concurrently, and mutating the shared registry would change how the + # unqualified task grades in the same session. + return fixed_multi_if_registry() diff --git a/tests/unit/community/test_instruction_following_eval_fixed.py b/tests/unit/community/test_instruction_following_eval_fixed.py new file mode 100644 index 00000000..ea234400 --- /dev/null +++ b/tests/unit/community/test_instruction_following_eval_fixed.py @@ -0,0 +1,421 @@ +"""Unit tests for the three repaired IFEval-family checkers. + +Every repair is asserted twice: once that it *reduces to upstream* on the inputs +upstream already handled — which is what makes the divergence exactly the defect +and not a rewrite riding along with it — and once that it changes the verdict on +the input that exposed the defect. + +`langdetect.detect` is randomized and SiEval never seeds it, so the +`change_case:english_capital` tests stub the detector rather than calling it. A +test that flips with the process RNG would be no evidence at all, and the point +of that repair is precisely that upstream calls the detector on input it cannot +read. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import random + +import langdetect +import pytest +from langdetect.lang_detect_exception import ErrorCode, LangDetectException + +from sieval.community.instruction_following_eval import ( + instructions as ifeval_upstream, +) +from sieval.community.instruction_following_eval import ( + instructions_registry as ifeval_registry, +) +from sieval.community.instruction_following_eval_fixed import ( + ENGLISH_CAPITAL, + FIXED_INSTRUCTION_IDS, + LETTER_FREQUENCY, + NTH_PARAGRAPH_FIRST_WORD, + IFEvalCapitalLettersEnglishCheckerFixed, + IFEvalLetterFrequencyCheckerFixed, + IFEvalParagraphFirstWordCheckFixed, + MultiIFCapitalLettersEnglishCheckerFixed, + MultiIFLetterFrequencyCheckerFixed, + MultiIFParagraphFirstWordCheckFixed, + _build, + fixed_ifeval_registry, + fixed_multi_if_registry, +) +from sieval.community.multi_if import ifeval as multi_if_upstream + +# --------------------------------------------------------------------------- +# registries +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("build", "upstream"), + [ + (fixed_ifeval_registry, ifeval_registry.INSTRUCTION_DICT), + (fixed_multi_if_registry, multi_if_upstream.INSTRUCTION_DICT), + ], + ids=["ifeval", "multi_if"], +) +def test_registry_replaces_exactly_the_three_declared_checkers(build, upstream): + fixed = build() + # Same ids: a repair may not add or drop a constraint type, or the two tasks + # would differ in *what* they grade rather than in how. + assert set(fixed) == set(upstream) + differing = {key for key, cls in fixed.items() if cls is not upstream[key]} + assert differing == set(FIXED_INSTRUCTION_IDS) + + +@pytest.mark.parametrize( + ("build", "upstream"), + [ + (fixed_ifeval_registry, ifeval_registry.INSTRUCTION_DICT), + (fixed_multi_if_registry, multi_if_upstream.INSTRUCTION_DICT), + ], + ids=["ifeval", "multi_if"], +) +def test_registry_is_a_fresh_dict_and_never_the_vendored_global(build, upstream): + # Samples grade concurrently. A registry that aliased or mutated the global + # would change how the *unqualified* task grades in the same session, which + # is the one thing the `_fixed` split exists to prevent. + before = dict(upstream) + first, second = build(), build() + assert first is not upstream + assert first is not second + first["punctuation:no_comma"] = object + assert upstream == before + assert upstream[NTH_PARAGRAPH_FIRST_WORD] is before[NTH_PARAGRAPH_FIRST_WORD] + + +def test_build_refuses_to_mount_a_fix_under_an_unknown_id(): + # The failure this guards against is silent: after an upstream rename the + # overlay would land on a key nothing looks up, and `_fixed` would grade + # identically to the unqualified task while still claiming a delta. + with pytest.raises(KeyError, match="renamed"): + _build({"a": int}, {"b": str}) + + +def test_build_names_the_offending_id(): + with pytest.raises(KeyError) as excinfo: + _build({"a": int}, {"length_constraints:nth_paragraph_first_word": str}) + assert "length_constraints:nth_paragraph_first_word" in str(excinfo.value) + + +@pytest.mark.parametrize( + ("fixed_cls", "upstream_cls"), + [ + (IFEvalParagraphFirstWordCheckFixed, ifeval_upstream.ParagraphFirstWordCheck), + (IFEvalLetterFrequencyCheckerFixed, ifeval_upstream.LetterFrequencyChecker), + ( + IFEvalCapitalLettersEnglishCheckerFixed, + ifeval_upstream.CapitalLettersEnglishChecker, + ), + ( + MultiIFParagraphFirstWordCheckFixed, + multi_if_upstream.ParagraphFirstWordCheck, + ), + (MultiIFLetterFrequencyCheckerFixed, multi_if_upstream.LetterFrequencyChecker), + ( + MultiIFCapitalLettersEnglishCheckerFixed, + multi_if_upstream.CapitalLettersEnglishChecker, + ), + ], +) +def test_each_fixed_checker_subclasses_the_one_it_replaces(fixed_cls, upstream_cls): + # Everything not named by a mixin -- `build_description`'s defaults and + # validation, `get_instruction_args`, the description pattern -- must stay + # upstream's, and subclassing is what makes that true by construction rather + # than by review. + assert issubclass(fixed_cls, upstream_cls) + + +# --------------------------------------------------------------------------- +# length_constraints:nth_paragraph_first_word +# --------------------------------------------------------------------------- + + +def _paragraph_pair(num_paragraphs: int, nth: int, first_word: str): + fixed = IFEvalParagraphFirstWordCheckFixed(NTH_PARAGRAPH_FIRST_WORD) + upstream = ifeval_upstream.ParagraphFirstWordCheck(NTH_PARAGRAPH_FIRST_WORD) + for checker in (fixed, upstream): + checker.build_description( + num_paragraphs=num_paragraphs, nth_paragraph=nth, first_word=first_word + ) + return fixed, upstream + + +# Responses with no blank `\n\n` chunk and a single-token target: exactly the +# inputs upstream already handles, where the repair must be invisible. +_UNAFFECTED_RESPONSES = [ + "alpha one\n\nbeta two\n\ngamma three", + "alpha one\n\nWRONG two\n\ngamma three", + 'alpha one\n\n"beta" two\n\ngamma three', + "alpha one\n\nbeta. two\n\ngamma three", + "alpha one\n\nbeta two", + "alpha one\n\nbeta two\n\ngamma three\n\ndelta four", + " alpha one\n\n beta two\n\n gamma three", + "alpha", +] + + +@pytest.mark.parametrize("response", _UNAFFECTED_RESPONSES) +@pytest.mark.parametrize("nth", [1, 2, 3]) +def test_paragraph_first_word_reduces_to_upstream_when_no_chunk_is_blank(response, nth): + fixed, upstream = _paragraph_pair(num_paragraphs=3, nth=nth, first_word="beta") + assert fixed.check_following(response) == upstream.check_following(response) + + +def test_paragraph_first_word_indexes_the_list_it_counted(): + # The defect in one line: an empty leading chunk is excluded from the count + # but still occupies index 0, so paragraph 2 is read out of slot 2 of the + # *unfiltered* list -- which is paragraph 1. + response = "\n\nalpha one\n\nbeta two\n\ngamma three" + fixed, upstream = _paragraph_pair(num_paragraphs=3, nth=2, first_word="beta") + assert upstream.check_following(response) is False + assert fixed.check_following(response) is True + + +@pytest.mark.parametrize( + "response", + [ + # A blank line before the nth paragraph, wherever it comes from: + "alpha one\n\n\n\nbeta two\n\ngamma three", # a doubled break mid-response + "alpha one\n\n \n\nbeta two\n\ngamma three", # a whitespace-only chunk + "\n\nalpha one\n\nbeta two\n\ngamma three", # a leading blank line + ], +) +def test_paragraph_first_word_is_shifted_by_any_blank_chunk_not_just_a_leading_one( + response, +): + # Worth pinning separately: the defect is easy to describe as an artifact of + # responses that open with a blank line, and it is not -- any blank chunk + # *before* the nth paragraph does the same thing. (A blank chunk after it is + # harmless, since both readings then agree on the index and on the count.) + fixed, upstream = _paragraph_pair(num_paragraphs=3, nth=2, first_word="beta") + assert upstream.check_following(response) is False + assert fixed.check_following(response) is True + + +def test_paragraph_first_word_is_unaffected_by_a_trailing_blank_chunk(): + # The counting loop already discounts it and no index moves, so upstream and + # the repair agree -- pinned so the repair's scope is bounded from both + # sides rather than only demonstrated where it bites. + response = "alpha one\n\nbeta two\n\ngamma three\n\n" + fixed, upstream = _paragraph_pair(num_paragraphs=3, nth=2, first_word="beta") + assert upstream.check_following(response) is True + assert fixed.check_following(response) is True + + +def test_paragraph_first_word_still_fails_the_wrong_word(): + # The repair is not a blanket pass: it moves *which* paragraph is read, and + # that paragraph must still start with what was asked for. + response = "\nalpha one\n\nWRONG two\n\ngamma three" + fixed, _upstream = _paragraph_pair(num_paragraphs=3, nth=2, first_word="beta") + assert fixed.check_following(response) is False + + +def test_paragraph_first_word_still_fails_the_wrong_paragraph_count(): + # `num_paragraphs` is counted the same way it always was; only the indexing + # was reconciled with it. + response = "\nalpha one\n\nbeta two" + fixed, _upstream = _paragraph_pair(num_paragraphs=3, nth=2, first_word="beta") + assert fixed.check_following(response) is False + + +def test_paragraph_first_word_compares_a_multi_token_value_in_order(): + # A multi-token value cannot equal `paragraph.split()[0]`, so upstream + # returns False for *every* possible response -- the slot measures nothing. + fixed, upstream = _paragraph_pair(num_paragraphs=2, nth=2, first_word="once upon") + good = "alpha one\n\nonce upon a time" + assert upstream.check_following(good) is False + assert fixed.check_following(good) is True + + # Still an opening, still in order: neither a bag of words nor a prefix. + assert fixed.check_following("alpha one\n\nupon once a time") is False + assert fixed.check_following("alpha one\n\nonce a time") is False + assert fixed.check_following("alpha one\n\na once upon time") is False + + +def test_paragraph_first_word_normalises_every_token_of_a_multi_token_value(): + # `_leading_word` is upstream's own normalisation, applied per token rather + # than to the first token only -- quotes and trailing punctuation are handled + # the same way at position 2 as at position 1. + fixed, _upstream = _paragraph_pair(num_paragraphs=1, nth=1, first_word="once upon") + assert fixed.check_following('"Once upon" a time') is True + + +def test_paragraph_first_word_with_a_blank_value_stays_ungradeable(): + # Three Multi-IF slots ship an empty `first_word`. A constraint with no value + # states nothing to check, so inventing a rule for it would be worse than + # upstream's behaviour: the repair deliberately leaves these failing, exactly + # as upstream does, rather than passing everything. + fixed, upstream = _paragraph_pair(num_paragraphs=1, nth=1, first_word="") + for response in ("alpha one", "", " "): + assert fixed.check_following(response) == upstream.check_following(response) + assert fixed.check_following(response) is False + + +# --------------------------------------------------------------------------- +# keywords:letter_frequency +# --------------------------------------------------------------------------- + + +def _frequency_pair(letter, frequency=2, relation="at least"): + fixed = IFEvalLetterFrequencyCheckerFixed(LETTER_FREQUENCY) + upstream = ifeval_upstream.LetterFrequencyChecker(LETTER_FREQUENCY) + descriptions = [ + checker.build_description( + letter=letter, let_frequency=frequency, let_relation=relation + ) + for checker in (fixed, upstream) + ] + return fixed, upstream, descriptions + + +@pytest.mark.parametrize("letter", ["a", "z", "Q"]) +@pytest.mark.parametrize("relation", ["at least", "less than"]) +def test_letter_frequency_reduces_to_upstream_for_an_ascii_letter(letter, relation): + fixed, upstream, (fixed_desc, upstream_desc) = _frequency_pair( + letter, relation=relation + ) + assert fixed_desc == upstream_desc + assert fixed._letter == upstream._letter + for response in ("aardvark", "quiz", "", "ZZZ"): + assert fixed.check_following(response) == upstream.check_following(response) + + +@pytest.mark.parametrize("character", ["#", "!", "1"]) +def test_letter_frequency_keeps_the_character_the_item_names(character): + fixed, upstream, (fixed_desc, _upstream_desc) = _frequency_pair(character) + assert fixed._letter == character + assert character in fixed_desc + # Upstream silently graded a different character -- one it drew itself. + assert upstream._letter != character + assert fixed.check_following(f"{character}{character} and more") is True + assert fixed.check_following("no such character here") is False + + +def test_letter_frequency_grades_the_same_letter_every_time(): + # Upstream draws freshly *per call*, so the same item is graded against a + # different letter each time it is scored. Seeded so the comparison is a + # fact about the two implementations rather than about today's RNG. + random.seed(20260814) + upstream_letters = set() + fixed_letters = set() + for _ in range(20): + fixed, upstream, _desc = _frequency_pair("#") + upstream_letters.add(upstream._letter) + fixed_letters.add(fixed._letter) + assert fixed_letters == {"#"} + assert len(upstream_letters) > 1 + + +def test_letter_frequency_consumes_no_random_draw(): + # A grading fix has no business perturbing the global RNG stream: every other + # checker that defaults an argument draws from it, so consuming a value here + # would shift verdicts in constraints this module does not touch. (Built + # alone, not through `_frequency_pair` -- upstream draws, which is the whole + # point.) + random.seed(20260814) + before = random.getstate() + checker = IFEvalLetterFrequencyCheckerFixed(LETTER_FREQUENCY) + checker.build_description(letter="#", let_frequency=2, let_relation="at least") + assert checker._letter == "#" + assert random.getstate() == before + + +@pytest.mark.parametrize("letter", [None, "", "ab", " "]) +def test_letter_frequency_defers_to_upstream_for_an_unusable_value(letter): + # Not one character after stripping: the item states nothing gradable, so + # upstream's fallback stands rather than this module inventing a rule. + fixed, _upstream, (fixed_desc, _upstream_desc) = _frequency_pair(letter) + assert len(fixed._letter) == 1 + assert "a" <= fixed._letter <= "z" + assert fixed._letter in fixed_desc + + +def test_letter_frequency_lowercases_a_cased_substitute_like_upstream(): + fixed, upstream, _desc = _frequency_pair("Q") + assert fixed._letter == "q" == upstream._letter + + +# --------------------------------------------------------------------------- +# change_case:english_capital +# --------------------------------------------------------------------------- + + +def _capital_pair(): + fixed = IFEvalCapitalLettersEnglishCheckerFixed(ENGLISH_CAPITAL) + upstream = ifeval_upstream.CapitalLettersEnglishChecker(ENGLISH_CAPITAL) + for checker in (fixed, upstream): + checker.build_description() + return fixed, upstream + + +def test_english_capital_detects_on_a_case_folded_copy(monkeypatch): + seen: list[str] = [] + + def _detect(text): + seen.append(text) + return "en" + + monkeypatch.setattr(langdetect, "detect", _detect) + fixed, upstream = _capital_pair() + response = "THIS ENTIRE RESPONSE IS IN CAPITAL LETTERS." + + assert upstream.check_following(response) is True + assert fixed.check_following(response) is True + # Upstream hands the detector ALL-CAPS text, which every profile it ships is + # off-distribution for; the repair hands it text the profiles were built on. + assert seen == [response, response.lower()] + + +def test_english_capital_does_not_modify_the_response(monkeypatch): + monkeypatch.setattr(langdetect, "detect", lambda _text: "en") + fixed, _upstream = _capital_pair() + response = "ALL CAPS HERE." + fixed.check_following(response) + assert response == "ALL CAPS HERE." + + +def test_english_capital_leaves_the_capitals_requirement_to_upstream(monkeypatch): + # `isupper()` is upstream's, still first, and still short-circuiting: the set + # of responses that reach the detector is exactly the set upstream sends + # there, so this repair cannot turn a mixed-case response into a pass. + calls: list[str] = [] + monkeypatch.setattr(langdetect, "detect", lambda text: calls.append(text) or "en") + fixed, upstream = _capital_pair() + for response in ("Mixed Case Text.", "lower case text.", "1234 !!"): + assert fixed.check_following(response) is False + assert upstream.check_following(response) is False + assert calls == [] + + +def test_english_capital_fails_capitals_in_another_language(monkeypatch): + monkeypatch.setattr(langdetect, "detect", lambda _text: "de") + fixed, _upstream = _capital_pair() + assert fixed.check_following("DIES IST EIN DEUTSCHER SATZ.") is False + + +def test_english_capital_counts_an_undetectable_text_as_following(monkeypatch): + # Parity with upstream, kept deliberately: `isupper()` above makes it + # near-unreachable, and a repair that quietly tightened it would be a second + # divergence riding along with the measured one. + def _raise(_text): + raise LangDetectException(ErrorCode.CantDetectError, "no features") + + monkeypatch.setattr(langdetect, "detect", _raise) + fixed, upstream = _capital_pair() + response = "ABC DEF." + assert fixed.check_following(response) is True + assert upstream.check_following(response) is True + + +def test_english_capital_matches_upstream_whenever_detection_agrees(monkeypatch): + # The only input whose handling changes is the *argument* to the detector. + # Hold the detector constant and the two implementations are the same + # function -- which is what makes the measured delta attributable to + # detection quality rather than to a changed rule. + monkeypatch.setattr(langdetect, "detect", lambda _text: "en") + fixed, upstream = _capital_pair() + for response in ("ALL CAPS.", "Mixed.", "", " ", "123"): + assert fixed.check_following(response) == upstream.check_following(response) diff --git a/tests/unit/tasks/test_ifeval_0shot_gen_fixed.py b/tests/unit/tasks/test_ifeval_0shot_gen_fixed.py new file mode 100644 index 00000000..ba92155a --- /dev/null +++ b/tests/unit/tasks/test_ifeval_0shot_gen_fixed.py @@ -0,0 +1,231 @@ +"""Registration and seam contract for the corrected IFEval task. + +The repairs themselves are tested in +``tests/unit/community/test_instruction_following_eval_fixed.py``. What this +module pins is the claim the variant is built on: that it is the unqualified +task plus *one* overridden method, so any measured delta is the repaired +checkers and cannot be a second change riding along. + +Fixtures avoid ``change_case:*`` constraints on purpose — those route through +``langdetect``, which upstream leaves unseeded, so a verdict built on them can +flip between runs. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import asyncio +import subprocess +import sys + +import pytest + +from sieval.core.tasks.meta import get_task_meta +from sieval.tasks.ifeval_0shot_gen import IFEvalZeroShotGenTask +from sieval.tasks.ifeval_0shot_gen_fixed import IFEvalZeroShotGenFixedTask + +_NTH = "length_constraints:nth_paragraph_first_word" +_NO_COMMA = "punctuation:no_comma" + + +def test_import_does_not_pull_evaluation_lib(): + # Same contract as the unqualified task's, and one more: the repair module + # reaches the vendored checkers and langdetect, so importing it at module + # scope would make *registration* -- which imports every task module -- + # pay for them. + code = ( + "import sys\n" + "import sieval.tasks.ifeval_0shot_gen_fixed\n" + "assert 'sieval.community.instruction_following_eval.evaluation_lib' " + "not in sys.modules, 'evaluation_lib must be lazy-imported'\n" + "assert 'sieval.community.instruction_following_eval_fixed' " + "not in sys.modules, 'the repair module must be lazy-imported'\n" + "assert 'langdetect' not in sys.modules, 'langdetect must be lazy-imported'\n" + ) + # Run in a fresh interpreter so pytest's already-loaded modules + # don't mask the check. + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + + +def test_meta(): + meta = get_task_meta(IFEvalZeroShotGenFixedTask) + base = get_task_meta(IFEvalZeroShotGenTask) + assert meta.name == "ifeval_0shot_gen_fixed" + # The dataset FK resolves through the MRO: the subclass never re-declares a + # generic base, so this is what proves it inherited one rather than silently + # registering against a different sample type. + assert meta.dataset == base.dataset + assert meta.eval_mode == base.eval_mode + assert meta.n_shot == base.n_shot == 0 + assert meta.deps_group == base.deps_group == "ifeval" + # Not gated on reproducing a published number: the divergence is carried by + # the name, and what `_fixed` owes is the quantified delta in `notes`. + assert meta.status == "stable" + + +def test_reference_impl_quantifies_the_divergence(): + # `_fixed` is licensed by a defect, not a preference, and owes two things: + # every divergence enumerated, and a measured score impact. An unmeasured + # fork is not a fix. + reference_impl = get_task_meta(IFEvalZeroShotGenFixedTask).reference_impl + assert reference_impl is not None + notes = reference_impl.notes + for instruction_id in ( + "length_constraints:nth_paragraph_first_word", + "keywords:letter_frequency", + "change_case:english_capital", + ): + assert instruction_id in notes + assert "SCORE IMPACT" in notes + assert "92.79→95.38" in notes + + +def _task(cls): + # `__new__`: the constructor requires a model with a bound dialect_id, and + # nothing below reaches the model. + return cls.__new__(cls) + + +class _Ctx: + def __init__(self, raw): + self.raw_sample = raw + + +def test_the_base_task_still_grades_through_the_vendored_registry(): + # The unqualified name tracks upstream, bugs included. `None` is what makes + # the graders fall back to the vendored `INSTRUCTION_DICT`, so this is the + # assertion that the repairs are unreachable from it. + assert _task(IFEvalZeroShotGenTask)._instruction_dict() is None + + +def test_the_fixed_task_returns_the_repaired_registry(): + from sieval.community.instruction_following_eval_fixed import ( + fixed_ifeval_registry, + ) + + registry = _task(IFEvalZeroShotGenFixedTask)._instruction_dict() + assert registry == fixed_ifeval_registry() + + +def test_the_fixed_task_hands_out_a_fresh_registry_each_call(): + # Samples grade concurrently; a shared dict would let one sample's grading + # change another's. + task = _task(IFEvalZeroShotGenFixedTask) + assert task._instruction_dict() is not task._instruction_dict() + + +def test_exactly_one_method_differs_from_the_unqualified_task(): + # The whole evidential weight of the measured delta rests on this: prompt, + # graders, records and report are inherited, so the two tasks cannot diverge + # anywhere except at the registry. + overrides = { + name + for name, value in vars(IFEvalZeroShotGenFixedTask).items() + if callable(value) and hasattr(IFEvalZeroShotGenTask, name) + } + assert overrides == {"_instruction_dict"} + for name in ("preprocess", "infer", "postprocess", "feedback", "report"): + assert getattr(IFEvalZeroShotGenFixedTask, name) is getattr( + IFEvalZeroShotGenTask, name + ) + + +def _sample(instruction_ids, kwargs) -> dict: + return { + "key": 1, + "prompt": "write something", + "instruction_id_list": list(instruction_ids), + "kwargs": list(kwargs), + } + + +def _judge(cls, raw, response): + post = {"rollouts": [{"index": 0, "extracted": True, "prediction": response}]} + _final, judgement = asyncio.run(_task(cls).feedback(post, _Ctx(raw))) + return judgement + + +def test_the_two_tasks_agree_on_a_sample_with_no_repaired_constraint(): + # Not "the scores match" but "the records match": 22 of the 25 checkers are + # the same objects in both registries, so a sample built only from those must + # produce a byte-identical judgement. + raw = _sample([_NO_COMMA], [{}]) + base = _judge(IFEvalZeroShotGenTask, raw, "no commas here") + fixed = _judge(IFEvalZeroShotGenFixedTask, raw, "no commas here") + assert base == fixed + + +def test_the_fixed_task_reads_the_paragraph_the_prompt_asked_for(): + # The response opens with a blank line, which is what whole runs do; upstream + # then reads paragraph 1 in slot 2 and fails a compliant answer. + raw = _sample( + [_NTH], + [{"num_paragraphs": 3, "nth_paragraph": 2, "first_word": "beta"}], + ) + response = "\n\nalpha one\n\nbeta two\n\ngamma three" + base = _judge(IFEvalZeroShotGenTask, raw, response) + fixed = _judge(IFEvalZeroShotGenFixedTask, raw, response) + assert base["metrics"]["strict_follow_all"] is False + assert fixed["metrics"]["strict_follow_all"] is True + # The headline is derived from the same metric, so it moves with it. + assert base["rollouts"][0]["correct"] is False + assert fixed["rollouts"][0]["correct"] is True + + +def test_the_loose_reading_was_already_masking_the_paragraph_defect(): + # Why loose moves less than strict, as a test rather than a claim: loose + # re-tries the response with its first line stripped, which deletes the very + # blank chunk that shifts the index. It was accidentally hiding the defect, + # not immune to it. + raw = _sample( + [_NTH], + [{"num_paragraphs": 3, "nth_paragraph": 2, "first_word": "beta"}], + ) + response = "\n\nalpha one\n\nbeta two\n\ngamma three" + base = _judge(IFEvalZeroShotGenTask, raw, response) + assert base["metrics"]["strict_follow_all"] is False + assert base["metrics"]["loose_follow_all"] is True + + +def test_the_fixed_task_does_not_pass_a_response_that_missed_the_constraint(): + raw = _sample( + [_NTH], + [{"num_paragraphs": 3, "nth_paragraph": 2, "first_word": "beta"}], + ) + fixed = _judge( + IFEvalZeroShotGenFixedTask, + raw, + "\n\nalpha one\n\nWRONG two\n\ngamma three", + ) + assert fixed["metrics"]["strict_follow_all"] is False + + +def test_a_mixed_sample_moves_only_at_the_repaired_constraint(): + # Two constraints, one repaired and one not. The unrepaired verdict must be + # identical in both arms -- that is what makes an instruction-level delta + # attributable rather than merely correlated. + raw = _sample( + [_NO_COMMA, _NTH], + [{}, {"num_paragraphs": 3, "nth_paragraph": 2, "first_word": "beta"}], + ) + response = "\n\nalpha one\n\nbeta two\n\ngamma three" + base = _judge(IFEvalZeroShotGenTask, raw, response) + fixed = _judge(IFEvalZeroShotGenFixedTask, raw, response) + base_followed = base["extra"]["strict"]["follow_instruction_list"] + fixed_followed = fixed["extra"]["strict"]["follow_instruction_list"] + assert base_followed == [True, False] + assert fixed_followed == [True, True] + assert base["metrics"]["strict_instruction_level"] == pytest.approx(0.5) + assert fixed["metrics"]["strict_instruction_level"] == pytest.approx(1.0) + + +def test_report_is_inherited_verbatim(): + # Deliberately not overridden: `check_report_declarations` treats a + # `super().report()` call as a new definition rather than a delegate, and the + # pooled report has nothing variant-specific in it anyway. + assert IFEvalZeroShotGenFixedTask.report is IFEvalZeroShotGenTask.report diff --git a/tests/unit/tasks/test_multi_if_0shot_gen_fixed.py b/tests/unit/tasks/test_multi_if_0shot_gen_fixed.py new file mode 100644 index 00000000..d17dd870 --- /dev/null +++ b/tests/unit/tasks/test_multi_if_0shot_gen_fixed.py @@ -0,0 +1,256 @@ +"""Registration and seam contract for the corrected Multi-IF task. + +The repairs themselves are tested in +``tests/unit/community/test_instruction_following_eval_fixed.py``. What this +module pins is that the variant is the unqualified task plus *one* overridden +method — conversation walking, per-turn grading, the cumulative constraint lists +and the per-language pooling all inherited — plus the one defect that is +Multi-IF-specific: a ``first_word`` spanning more than one token, which upstream +cannot pass with any response at all. + +Every fixture is synthetic. Nothing here quotes Multi-IF's prompts or kwargs. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import asyncio +import subprocess +import sys + +import pytest + +from sieval.core.tasks.meta import get_task_meta +from sieval.tasks.multi_if_0shot_gen import MultiIFZeroShotGenTask +from sieval.tasks.multi_if_0shot_gen_fixed import MultiIFZeroShotGenFixedTask + +_NTH = "length_constraints:nth_paragraph_first_word" +_NO_COMMA = "punctuation:no_comma" + + +def test_import_does_not_pull_evaluation_lib(): + code = ( + "import sys\n" + "import sieval.tasks.multi_if_0shot_gen_fixed\n" + "assert 'sieval.community.multi_if.evaluation_lib' not in sys.modules, " + "'evaluation_lib must be lazy-imported'\n" + "assert 'sieval.community.instruction_following_eval_fixed' " + "not in sys.modules, 'the repair module must be lazy-imported'\n" + "assert 'langdetect' not in sys.modules, 'langdetect must be lazy-imported'\n" + "assert 'nltk' not in sys.modules, 'nltk must be lazy-imported'\n" + ) + # Run in a fresh interpreter so pytest's already-loaded modules + # don't mask the check. + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + + +def test_meta(): + meta = get_task_meta(MultiIFZeroShotGenFixedTask) + base = get_task_meta(MultiIFZeroShotGenTask) + assert meta.name == "multi_if_0shot_gen_fixed" + # Resolved through the MRO, since the subclass re-declares no generic base. + assert meta.dataset == base.dataset + assert meta.eval_mode == base.eval_mode + assert meta.deps_group == base.deps_group == "multi-if" + assert meta.status == "stable" + + +def test_reference_impl_quantifies_the_divergence(): + reference_impl = get_task_meta(MultiIFZeroShotGenFixedTask).reference_impl + assert reference_impl is not None + notes = reference_impl.notes + for instruction_id in ( + "length_constraints:nth_paragraph_first_word", + "keywords:letter_frequency", + "change_case:english_capital", + ): + assert instruction_id in notes + assert "SCORE IMPACT" in notes + assert "69.07→70.52" in notes + + +def _task(cls): + # `__new__`: the constructor requires a model with a bound dialect_id, and + # nothing below reaches the model. + return cls.__new__(cls) + + +class _Ctx: + def __init__(self, raw): + self.raw_sample = raw + + +def test_the_base_task_still_grades_through_the_vendored_registry(): + assert _task(MultiIFZeroShotGenTask)._instruction_dict() is None + + +def test_the_fixed_task_returns_the_repaired_registry(): + from sieval.community.instruction_following_eval_fixed import ( + fixed_multi_if_registry, + ) + + registry = _task(MultiIFZeroShotGenFixedTask)._instruction_dict() + assert registry == fixed_multi_if_registry() + + +def test_the_fixed_task_uses_multi_ifs_own_registry_not_ifevals(): + # The two vendored copies are logic-identical for the three repaired + # checkers, but not for the other 22: routing Multi-IF through IFEval's + # registry would quietly change the multilingual checkers. + from sieval.community.instruction_following_eval_fixed import ( + fixed_ifeval_registry, + ) + from sieval.community.multi_if import ifeval as multi_if_upstream + + registry = _task(MultiIFZeroShotGenFixedTask)._instruction_dict() + assert registry != fixed_ifeval_registry() + unrepaired = set(registry) - { + _NTH, + "keywords:letter_frequency", + "change_case:english_capital", + } + for instruction_id in unrepaired: + assert ( + registry[instruction_id] + is multi_if_upstream.INSTRUCTION_DICT[instruction_id] + ) + + +def test_the_fixed_task_hands_out_a_fresh_registry_each_call(): + task = _task(MultiIFZeroShotGenFixedTask) + assert task._instruction_dict() is not task._instruction_dict() + + +def test_exactly_one_method_differs_from_the_unqualified_task(): + overrides = { + name + for name, value in vars(MultiIFZeroShotGenFixedTask).items() + if callable(value) and hasattr(MultiIFZeroShotGenTask, name) + } + assert overrides == {"_instruction_dict"} + for name in ("preprocess", "infer", "postprocess", "feedback", "report"): + assert getattr(MultiIFZeroShotGenFixedTask, name) is getattr( + MultiIFZeroShotGenTask, name + ) + + +def _sample(instruction_ids, kwargs) -> dict: + # One turn is enough: what differs between the two tasks is per-response + # grading, and the conversation walk is inherited and tested elsewhere. + return { + "key": "k:1:en", + "language": "English", + "turns": [ + { + "prompt": "write something", + "instruction_id_list": list(instruction_ids), + # JSON-encoded in the dataset and decoded by `feedback`. + "kwargs": list(kwargs), + } + ], + } + + +def _judge(cls, raw, response): + post = { + "rollouts": [ + { + "index": 0, + "extracted": True, + "prediction": [{"turn": 1, "response": response}], + } + ] + } + _final, judgement = asyncio.run(_task(cls).feedback(post, _Ctx(raw))) + return judgement + + +def test_the_two_tasks_agree_on_a_turn_with_no_repaired_constraint(): + raw = _sample([_NO_COMMA], ["{}"]) + base = _judge(MultiIFZeroShotGenTask, raw, "no commas here") + fixed = _judge(MultiIFZeroShotGenFixedTask, raw, "no commas here") + assert base == fixed + + +def test_the_fixed_task_reads_the_paragraph_the_prompt_asked_for(): + raw = _sample( + [_NTH], + ['{"num_paragraphs": 3, "nth_paragraph": 2, "first_word": "beta"}'], + ) + response = "\n\nalpha one\n\nbeta two\n\ngamma three" + base = _judge(MultiIFZeroShotGenTask, raw, response) + fixed = _judge(MultiIFZeroShotGenFixedTask, raw, response) + assert base["metrics"]["turn_1_strict_follow_all"] is False + assert fixed["metrics"]["turn_1_strict_follow_all"] is True + + +def test_upstream_cannot_pass_a_multi_token_first_word_at_all(): + # The Multi-IF-only defect, and the reason it is worth a repair rather than a + # note: upstream compares against one whitespace-delimited token, so a slot + # whose value spans two returns FAIL for *every* response. A check that + # cannot pass measures nothing -- it is not a hard constraint, it is a + # constant. + raw = _sample( + [_NTH], + ['{"num_paragraphs": 2, "nth_paragraph": 2, "first_word": "once upon"}'], + ) + for response in ( + "alpha one\n\nonce upon a time", + "alpha one\n\nonce a time", + "alpha one\n\nupon once a time", + ): + assert ( + _judge(MultiIFZeroShotGenTask, raw, response)["metrics"][ + "turn_1_strict_follow_all" + ] + is False + ) + + # The repair grades it: the phrase must open the paragraph, in order. + passed = _judge(MultiIFZeroShotGenFixedTask, raw, "alpha one\n\nonce upon a time") + assert passed["metrics"]["turn_1_strict_follow_all"] is True + for response in ("alpha one\n\nonce a time", "alpha one\n\nupon once a time"): + assert ( + _judge(MultiIFZeroShotGenFixedTask, raw, response)["metrics"][ + "turn_1_strict_follow_all" + ] + is False + ) + + +def test_a_blank_first_word_stays_ungradeable_in_both_tasks(): + # Three slots in the pinned set ship an empty value. A constraint that states + # nothing to check is not this repair's to invent a rule for, so it keeps + # failing exactly as upstream does -- the repair is not a backfill. + raw = _sample( + [_NTH], + ['{"num_paragraphs": 1, "nth_paragraph": 1, "first_word": ""}'], + ) + for response in ("alpha one", "\n\nalpha one"): + base = _judge(MultiIFZeroShotGenTask, raw, response) + fixed = _judge(MultiIFZeroShotGenFixedTask, raw, response) + assert base["metrics"]["turn_1_strict_follow_all"] is False + assert fixed["metrics"]["turn_1_strict_follow_all"] is False + + +def test_a_mixed_turn_moves_only_at_the_repaired_constraint(): + raw = _sample( + [_NO_COMMA, _NTH], + ["{}", '{"num_paragraphs": 3, "nth_paragraph": 2, "first_word": "beta"}'], + ) + response = "\n\nalpha one\n\nbeta two\n\ngamma three" + base = _judge(MultiIFZeroShotGenTask, raw, response) + fixed = _judge(MultiIFZeroShotGenFixedTask, raw, response) + assert base["extra"]["turn_1"]["strict"]["follow_instruction_list"] == [True, False] + assert fixed["extra"]["turn_1"]["strict"]["follow_instruction_list"] == [True, True] + assert base["metrics"]["turn_1_strict_instruction_level"] == pytest.approx(0.5) + assert fixed["metrics"]["turn_1_strict_instruction_level"] == pytest.approx(1.0) + + +def test_report_is_inherited_verbatim(): + assert MultiIFZeroShotGenFixedTask.report is MultiIFZeroShotGenTask.report