@@ -35,29 +35,80 @@ def _section_present(response: str, name: str) -> bool:
3535 return bool (label .search (response or "" ))
3636
3737
38- def _check (op : str , arg : Any , response : str , tools_called : List [str ]) -> bool :
38+ def _check (op : str , arg : Any , response : str ,
39+ tools_called : List [str ]) -> Tuple [bool , str ]:
40+ """Evaluate one check.
41+
42+ Returns ``(passed, problem)``. ``problem`` is non-empty only when the check
43+ itself is malformed (e.g. an unparseable regex) rather than simply unmet —
44+ the two need opposite fixes, so they must not look alike in the rationale.
45+ """
3946 r = response or ""
4047 if op == "section_present" :
41- return _section_present (r , str (arg ))
48+ return _section_present (r , str (arg )), ""
4249 if op == "regex" :
4350 try :
44- return bool (re .search (str (arg ), r ))
45- except re .error :
46- return False
51+ return bool (re .search (str (arg ), r )), ""
52+ except re .error as exc :
53+ # A malformed pattern can never match, so it would fail every
54+ # rollout forever and read exactly like a model that never
55+ # complies. Surface it instead of hiding it behind a False.
56+ return False , f"invalid regex ({ exc }
4757 if op == "max_chars" :
48- return len (r ) <= int (arg )
58+ return len (r ) <= int (arg ), ""
4959 if op == "min_chars" :
50- return len (r ) >= int (arg )
60+ return len (r ) >= int (arg ), ""
5161 if op == "contains" :
52- return str (arg ).lower () in r .lower ()
62+ return str (arg ).lower () in r .lower (), ""
5363 if op == "tool_called" :
5464 name = str (arg ).lower ()
5565 if any (name == t .lower () for t in tools_called ):
56- return True
66+ return True , ""
5767 # single-shot approximation: the agent emits an explicit marker
58- return bool (re .search (r"(?i)\btool_call\s*:\s*%s\b" % re .escape (name ), r ))
68+ return bool (re .search (r"(?i)\btool_call\s*:\s*%s\b" % re .escape (name ), r )), ""
5969 # unknown op: do not block
60- return True
70+ return True , ""
71+
72+
73+ KNOWN_OPS = frozenset ({
74+ "section_present" , "regex" , "max_chars" , "min_chars" , "contains" , "tool_called" ,
75+ })
76+
77+
78+ def validate_checks (judge : Dict [str , Any ]) -> Tuple [List [str ], List [str ]]:
79+ """Return ``(errors, warnings)`` for a rule judge's checks.
80+
81+ An *error* means the check can never behave as written — a regex that does
82+ not compile always scores 0.0, which is indistinguishable from a model that
83+ never complies. A *warning* means the check is accepted but toothless, e.g.
84+ an unknown op, which :func:`_check` deliberately lets pass.
85+ """
86+ errors : List [str ] = []
87+ warnings : List [str ] = []
88+ if judge and not isinstance (judge , dict ):
89+ return [f"judge must be an object, got { type (judge ).__name__ } ], warnings
90+ checks = (judge or {}).get ("checks" , []) or []
91+ if not isinstance (checks , list ):
92+ return [f"judge 'checks' must be an array, got { type (checks ).__name__ } ], warnings
93+ for i , c in enumerate (checks ):
94+ if not isinstance (c , dict ):
95+ errors .append (f"check #{ i } )
96+ continue
97+ op = c .get ("op" , "" )
98+ arg = c .get ("arg" )
99+ if op == "regex" :
100+ try :
101+ re .compile (str (arg ))
102+ except re .error as exc :
103+ errors .append (f"check #{ i } { exc } { arg !r} )
104+ elif op in {"max_chars" , "min_chars" }:
105+ try :
106+ int (arg )
107+ except (TypeError , ValueError ):
108+ errors .append (f"check #{ i } { op } { arg !r} )
109+ elif op not in KNOWN_OPS :
110+ warnings .append (f"check #{ i } { op !r} )
111+ return errors , warnings
61112
62113
63114def score_rule_judge (
@@ -73,11 +124,14 @@ def score_rule_judge(
73124 passed = 0
74125 failed_desc : List [str ] = []
75126 for c in checks :
76- ok = _check (c .get ("op" , "" ), c .get ("arg" ), response , tools_called )
127+ ok , problem = _check (c .get ("op" , "" ), c .get ("arg" ), response , tools_called )
77128 if ok :
78129 passed += 1
79130 else :
80- failed_desc .append (f"{ c .get ('op' )} { c .get ('arg' )} )
131+ desc = f"{ c .get ('op' )} { c .get ('arg' )}
132+ if problem :
133+ desc += f" [{ problem }
134+ failed_desc .append (desc )
81135 soft = passed / len (checks )
82136 hard = 1.0 if passed == len (checks ) else 0.0
83137 rationale = "all checks passed" if hard else "failed: " + ", " .join (failed_desc )
0 commit comments