feat(tasks): ifeval/multi_if _fixed — three repaired constraint checkers

pyproject.toml

@@ -61,13 +61,21 @@ ifeval = [
 iheval = ["sieval[ifeval]", "rouge-score>=0.1.2"]
 multi-if = [
   # Multi-IF vendors its own multilingual fork of the IFEval checkers, so this
-  # group is deliberately not `ifeval`'s: no absl/immutabledict (the fork uses
-  # stdlib logging and MappingProxyType), and emoji is required rather than
-  # optional because the CJK word counter behind every Chinese length
-  # constraint counts emoji as words.
+  # group is deliberately not `ifeval`'s: it restates what the fork reaches
+  # rather than reusing a list built for the other copy, and emoji is required
+  # rather than optional because the CJK word counter behind every Chinese
+  # length constraint counts emoji as words.
   "emoji>=2.15.0",
   "langdetect>=1.0.9",
   "nltk>=3.9.2",
+  # Not reached by the fork itself, but `multi_if_0shot_gen_fixed` grades
+  # through `sieval.community.instruction_following_eval_fixed`, whose repairs
+  # are shared with the ifeval family and so import google-research's
+  # `instructions` module -- these two are that module's. Needed at grading
+  # time, not import time, so omitting them fails inside `feedback()` with
+  # inference already paid for.
+  "absl-py>=2.3.1",
+  "immutabledict>=4.2.2",
   # Only reached when langdetect reports Thai for a model *response* (Thai is
   # not one of the dataset's eight languages). Declared anyway so that path
   # cannot die on ImportError mid-run; the import itself is deferred.

sieval/community/instruction_following_eval/evaluation_lib.py

@@ -19,6 +19,13 @@
 """
 
 # adapted from https://github.com/google-research/google-research/blob/f97f6adab57bd3065b24169bcfc559dc34d0db84/instruction_following_eval/evaluation_lib.py
+#
+# Local adaptation: the two graders take a keyword-only `instruction_dict`,
+# defaulting to the vendored registry, so `ifeval_0shot_gen_fixed` can grade
+# through the repaired checkers in
+# `sieval.community.instruction_following_eval_fixed` without mutating a global
+# that concurrently-graded samples share. Omitting the argument reproduces
+# upstream's behaviour exactly, which is what the unqualified task does.
 import collections
 import dataclasses
 import json
@@ -82,14 +89,18 @@ def write_outputs(output_jsonl_filename, outputs):
 def test_instruction_following_strict(
     inp,
     prompt_to_response,
+    *,
+    instruction_dict=None,
 ):
     """Tests response to see if instrutions are followed."""
+    if instruction_dict is None:
+        instruction_dict = instructions_registry.INSTRUCTION_DICT
     response = prompt_to_response[inp.prompt]
     instruction_list = inp.instruction_id_list
     is_following_list = []
 
     for index, instruction_id in enumerate(instruction_list):
-        instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id]
+        instruction_cls = instruction_dict[instruction_id]
         instruction = instruction_cls(instruction_id)
 
         instruction.build_description(**inp.kwargs[index])
@@ -114,8 +125,12 @@ def test_instruction_following_strict(
 def test_instruction_following_loose(
     inp,
     prompt_to_response,
+    *,
+    instruction_dict=None,
 ):
     """Tests response for an upper bound for following instructions."""
+    if instruction_dict is None:
+        instruction_dict = instructions_registry.INSTRUCTION_DICT
     response = prompt_to_response[inp.prompt]
     r = response.split("\n")
     response_remove_first = "\n".join(r[1:]).strip()
@@ -139,7 +154,7 @@ def test_instruction_following_loose(
     is_following_list = []
 
     for index, instruction_id in enumerate(instruction_list):
-        instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id]
+        instruction_cls = instruction_dict[instruction_id]
         instruction = instruction_cls(instruction_id)
 
         instruction.build_description(**inp.kwargs[index])