refactor(tasks): share auxiliary model-role resolution on Task

sieval/core/tasks/task.py

@@ -5,7 +5,7 @@
 
 import re
 from abc import ABC, abstractmethod
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from collections.abc import Set as AbstractSet
 from dataclasses import replace
 from typing import ClassVar, Literal, Protocol, cast
@@ -168,8 +168,6 @@ def _bind_model_requirements(
     ) -> tuple[TaskModelRequirement, ...]:
         if not isinstance(context, RequirementContext):
             raise TypeError("context must be a RequirementContext")
-        if not isinstance(requires, TaskRequirements):
-            raise TypeError("requires must be TaskRequirements")
 
         bindings = context.model_bindings
         if not bindings:
@@ -208,8 +206,6 @@ def _bind_role_requirement(
             raise TypeError("context must be a RequirementContext")
         if not isinstance(role, str) or not role:
             raise TypeError("role must be a non-empty string")
-        if not isinstance(requires, TaskRequirements):
-            raise TypeError("requires must be TaskRequirements")
         try:
             binding = context.model_bindings[role]
         except KeyError as exc:
@@ -228,6 +224,35 @@ def _bind_role_requirement(
             ),
         )
 
+    @classmethod
+    def _resolve_role_model[T](
+        cls,
+        role: str,
+        configured: T,
+        models_by_role: Mapping[str, Model] | None,
+        *,
+        build: Callable[[T], Model],
+    ) -> Model:
+        """Resolve one auxiliary model role to the Model the task will call.
+
+        ``models_by_role`` is the pooled path a YAML run takes; otherwise
+        ``build`` turns ``configured`` into the Model, so each task keeps its
+        own "you must supply one" message. Supplying both is an error rather
+        than a precedence question: silently preferring either would let a run
+        score against a model the config did not name.
+        """
+
+        if models_by_role is not None:
+            if configured is not None:
+                raise ValueError(f"{role} and models_by_role cannot both be supplied")
+            try:
+                return models_by_role[role]
+            except KeyError as exc:
+                raise ValueError(
+                    f"models_by_role is missing the {role!r} model"
+                ) from exc
+        return build(configured)
+
     @classmethod
     def _bind_top_logprobs_requirements(
         cls,

sieval/tasks/aa_lcr_0shot_gen.py

@@ -175,24 +175,12 @@ def __init__(
     ):
         super().__init__(dataset=dataset, model=model, name=name)
         self._n = n
-        self._grader = self._resolve_grader(grader, models_by_role)
-
-    @classmethod
-    def _resolve_grader(
-        cls,
-        grader: Mapping | Model | None,
-        models_by_role: Mapping[str, Model] | None,
-    ) -> Model:
-        if models_by_role is not None:
-            if grader is not None:
-                raise ValueError("grader and models_by_role cannot both be supplied")
-            try:
-                return models_by_role["grader"]
-            except KeyError as exc:
-                raise ValueError(
-                    "models_by_role is missing the 'grader' model"
-                ) from exc
-        return cls._build_grader(grader)
+        self._grader = self._resolve_role_model(
+            "grader",
+            grader,
+            models_by_role,
+            build=self._build_grader,
+        )
 
     @staticmethod
     def _build_grader(grader: Mapping | Model | None) -> Model:

sieval/tasks/advanced_if_0shot_gen.py

@@ -219,31 +219,19 @@ def __init__(
     ):
         super().__init__(dataset=dataset, model=model, name=name)
         self._n = n
-        self._grader = self._resolve_grader(grader, models_by_role)
+        self._grader = self._resolve_role_model(
+            "grader",
+            grader,
+            models_by_role,
+            build=self._build_grader,
+        )
         # Validate the checkout here, not at the first grade. Discovered in
         # feedback() it costs a whole generation pass to learn: every grade then
         # raises, and a wholly failed grading stage still reports 0.0 -- the
         # floor that reads as a score. `@cache`d, so this is one read per run,
         # and construction already fails without a grader anyway.
         load_judge_prompts()
 
-    @classmethod
-    def _resolve_grader(
-        cls,
-        grader: Mapping | Model | None,
-        models_by_role: Mapping[str, Model] | None,
-    ) -> Model:
-        if models_by_role is not None:
-            if grader is not None:
-                raise ValueError("grader and models_by_role cannot both be supplied")
-            try:
-                return models_by_role["grader"]
-            except KeyError as exc:
-                raise ValueError(
-                    "models_by_role is missing the 'grader' model"
-                ) from exc
-        return cls._build_grader(grader)
-
     @staticmethod
     def _build_grader(grader: Mapping | Model | None) -> Model:
         """Resolve the ``grader`` task arg into a Model.

sieval/tasks/agieval_0shot_gen.py

@@ -247,30 +247,13 @@ def __init__(
         models_by_role: Mapping[str, Model] | None = None,
     ):
         super().__init__(dataset=dataset, model=model, name=name)
-        self._extractor = self._resolve_extractor(
+        self._extractor = self._resolve_role_model(
+            "extractor",
             extractor,
-            model,
             models_by_role,
+            build=lambda cfg: self._build_extractor(cfg, model),
         )
 
-    @classmethod
-    def _resolve_extractor(
-        cls,
-        extractor: Mapping | Model | str | None,
-        model: Model,
-        models_by_role: Mapping[str, Model] | None,
-    ) -> Model:
-        if models_by_role is not None:
-            if extractor is not None:
-                raise ValueError("extractor and models_by_role cannot both be supplied")
-            try:
-                return models_by_role["extractor"]
-            except KeyError as exc:
-                raise ValueError(
-                    "models_by_role is missing the 'extractor' model"
-                ) from exc
-        return cls._build_extractor(extractor, model)
-
     @staticmethod
     def _build_extractor(
         extractor: Mapping | Model | str | None, model: Model

sieval/tasks/browsecomp_0shot_gen.py

@@ -144,24 +144,12 @@ def __init__(
     ):
         super().__init__(dataset=dataset, model=model, name=name)
         self._n = n
-        self._grader = self._resolve_grader(grader, models_by_role)
-
-    @classmethod
-    def _resolve_grader(
-        cls,
-        grader: Mapping | Model | None,
-        models_by_role: Mapping[str, Model] | None,
-    ) -> Model:
-        if models_by_role is not None:
-            if grader is not None:
-                raise ValueError("grader and models_by_role cannot both be supplied")
-            try:
-                return models_by_role["grader"]
-            except KeyError as exc:
-                raise ValueError(
-                    "models_by_role is missing the 'grader' model"
-                ) from exc
-        return cls._build_grader(grader)
+        self._grader = self._resolve_role_model(
+            "grader",
+            grader,
+            models_by_role,
+            build=self._build_grader,
+        )
 
     @staticmethod
     def _build_grader(grader: Mapping | Model | None) -> Model:

sieval/tasks/complex_constraints_0shot_gen.py

@@ -248,24 +248,12 @@ def __init__(
     ):
         super().__init__(dataset=dataset, model=model, name=name)
         self._n = n
-        self._grader = self._resolve_grader(grader, models_by_role)
-
-    @classmethod
-    def _resolve_grader(
-        cls,
-        grader: Mapping | Model | None,
-        models_by_role: Mapping[str, Model] | None,
-    ) -> Model:
-        if models_by_role is not None:
-            if grader is not None:
-                raise ValueError("grader and models_by_role cannot both be supplied")
-            try:
-                return models_by_role["grader"]
-            except KeyError as exc:
-                raise ValueError(
-                    "models_by_role is missing the 'grader' model"
-                ) from exc
-        return cls._build_grader(grader)
+        self._grader = self._resolve_role_model(
+            "grader",
+            grader,
+            models_by_role,
+            build=self._build_grader,
+        )
 
     @staticmethod
     def _build_grader(grader: Mapping | Model | None) -> Model:

sieval/tasks/hle_0shot_gen.py

@@ -193,24 +193,12 @@ def __init__(
         # Which subset was loaded is the dataset's decision; read it back so
         # `report()` can record it.
         self._text_only = dataset.text_only
-        self._grader = self._resolve_grader(grader, models_by_role)
-
-    @classmethod
-    def _resolve_grader(
-        cls,
-        grader: Mapping | Model | None,
-        models_by_role: Mapping[str, Model] | None,
-    ) -> Model:
-        if models_by_role is not None:
-            if grader is not None:
-                raise ValueError("grader and models_by_role cannot both be supplied")
-            try:
-                return models_by_role["grader"]
-            except KeyError as exc:
-                raise ValueError(
-                    "models_by_role is missing the 'grader' model"
-                ) from exc
-        return cls._build_grader(grader)
+        self._grader = self._resolve_role_model(
+            "grader",
+            grader,
+            models_by_role,
+            build=self._build_grader,
+        )
 
     @staticmethod
     def _build_grader(grader: Mapping | Model | None) -> Model:

sieval/tasks/inverse_ifeval_0shot_gen.py

@@ -173,24 +173,12 @@ def __init__(
             raise ValueError(f"k must be <= n, got k={k}, n={n}")
         self._n = n
         self._k = k
-        self._grader = self._resolve_grader(grader, models_by_role)
-
-    @classmethod
-    def _resolve_grader(
-        cls,
-        grader: Mapping | Model | None,
-        models_by_role: Mapping[str, Model] | None,
-    ) -> Model:
-        if models_by_role is not None:
-            if grader is not None:
-                raise ValueError("grader and models_by_role cannot both be supplied")
-            try:
-                return models_by_role["grader"]
-            except KeyError as exc:
-                raise ValueError(
-                    "models_by_role is missing the 'grader' model"
-                ) from exc
-        return cls._build_grader(grader)
+        self._grader = self._resolve_role_model(
+            "grader",
+            grader,
+            models_by_role,
+            build=self._build_grader,
+        )
 
     @staticmethod
     def _build_grader(grader: Mapping | Model | None) -> Model:

sieval/tasks/simpleqa_verified_0shot_gen.py

@@ -135,24 +135,12 @@ def __init__(
     ):
         super().__init__(dataset=dataset, model=model, name=name)
         self._n = n
-        self._grader = self._resolve_grader(grader, models_by_role)
-
-    @classmethod
-    def _resolve_grader(
-        cls,
-        grader: Mapping | Model | None,
-        models_by_role: Mapping[str, Model] | None,
-    ) -> Model:
-        if models_by_role is not None:
-            if grader is not None:
-                raise ValueError("grader and models_by_role cannot both be supplied")
-            try:
-                return models_by_role["grader"]
-            except KeyError as exc:
-                raise ValueError(
-                    "models_by_role is missing the 'grader' model"
-                ) from exc
-        return cls._build_grader(grader)
+        self._grader = self._resolve_role_model(
+            "grader",
+            grader,
+            models_by_role,
+            build=self._build_grader,
+        )
 
     @staticmethod
     def _build_grader(grader: Mapping | Model | None) -> Model: