3131 received (``metadata.request_id`` aside) and the warning classes; and
3232 the core must have been reached. A difference passes only under an
3333 approved register entry naming the vector with its transition
34- (``contract/sampled-null-key-vectors.json#<id or glob> <from> -> <to>``);
35- ``register.required`` (derived: ``mixed-null-decimal``, INTERNAL_ERROR at
36- materialize) excuses nothing by itself.
34+ (``contract/sampled-null-key-vectors.json#<id or glob> <from> -> <to>``),
35+ as the other vector tests read it: a ref covers exactly the transition
36+ it declares, a proposed entry excuses nothing, and ``success ->
37+ success`` is a transition like any other, so a request that differs on
38+ a success passes only under an approved entry declaring that for the
39+ vector. ``two-nulls-text`` is the case in point: two null keys reached
40+ for one example, where P1B-13 (approved, re-approved by the owner on
41+ 2026-10-13: the reference corrupts null-key requests) makes a null in
42+ the walk no link, so the core's request differs from the reference's;
43+ it passes once P1B-13 carries
44+ ``contract/sampled-null-key-vectors.json#two-nulls-text success ->
45+ success``. ``register.required`` (derived: ``mixed-null-decimal``,
46+ INTERNAL_ERROR at materialize) excuses nothing by itself.
47+ - The singletons (:data:`EXACT`: one null key sampled alone, text and
48+ decimal, the review's reproduction among them) are held to the
49+ reference's exact answer and request: no entry excuses them, whatever
50+ it declares, since a single null in the walk is the reference's clean
51+ case (np.unique sorts it) and P1B-13 changes nothing there.
3752- The case format holds neither ``num_context_samples`` nor
3853 ``random_seed``, so these vectors are not cases and ``rust-host`` does
3954 not run them; the core's answer is held through the Python host.
@@ -113,14 +128,25 @@ def transitions() -> tuple[list[tuple[str, str, str, str, str]], list[str]]:
113128 return out , malformed
114129
115130
131+ #: The vectors held to the reference's exact answer, request included,
132+ #: which no register entry excuses: one null key sampled alone.
133+ EXACT = frozenset ({'singleton-null-text' , 'singleton-null-decimal' })
134+
135+
116136def problem_of (vector : dict [str , Any ], got : dict [str , Any ]) -> str | None :
117137 r"""Why ``got`` fails the vector: None when it is the reference's
118- answer, or a difference an approved entry declares exactly."""
138+ answer, or, outside :data:`EXACT`, a difference an approved entry
139+ declares exactly (``success -> success`` included)."""
119140 want = vector ['reference' ]
120141 if got == want :
121142 return None
122143 old = R .outcome_class (want ['outcome' ])
123144 new = R .outcome_class (got ['outcome' ])
145+ if vector ['id' ] in EXACT :
146+ return (
147+ f'{ vector ["id" ]} : got { got } , the reference { want } ({ old } -> '
148+ f'{ new } ); held exactly, no entry excuses it'
149+ )
124150 for _ , status , glob , frm , to in transitions ()[0 ]:
125151 if (
126152 status == 'approved'
@@ -241,9 +267,10 @@ def test_the_comparison_fails_closed(monkeypatch: pytest.MonkeyPatch) -> None:
241267 module = sys .modules [__name__ ]
242268 entries : list [tuple [str , str , str , str , str ]] = []
243269 monkeypatch .setattr (module , 'transitions' , lambda : (entries , []))
244- singleton = BY_ID ['singleton-null-text' ]
245- want = singleton ['reference' ]
246- assert problem_of (singleton , copy .deepcopy (want )) is None
270+ plain = BY_ID ['null-text-not-sampled' ]
271+ want = plain ['reference' ]
272+ assert plain ['id' ] not in EXACT
273+ assert problem_of (plain , copy .deepcopy (want )) is None
247274 refused = copy .deepcopy (want )
248275 refused ['outcome' ] = {
249276 'result' : 'refusal' ,
@@ -252,23 +279,101 @@ def test_the_comparison_fails_closed(monkeypatch: pytest.MonkeyPatch) -> None:
252279 'code' : 'INVALID_REQUEST' ,
253280 }
254281 refused ['request' ] = None
255- assert problem_of (singleton , refused ) is not None
282+ assert problem_of (plain , refused ) is not None
256283 entries [:] = [
257- ('P-1' , 'proposed' , 'singleton -*' , 'success' , 'refusal INVALID_REQUEST' )
284+ ('P-1' , 'proposed' , 'null-text -*' , 'success' , 'refusal INVALID_REQUEST' )
258285 ]
259- assert problem_of (singleton , refused ) is not None
286+ assert problem_of (plain , refused ) is not None
287+ entries [:] = [
288+ ('A-1' , 'approved' , 'null-text-*' , 'success' , 'refusal INVALID_REQUEST' )
289+ ]
290+ assert problem_of (plain , refused ) is None
291+ # The same approved transition, for the singleton: held exactly.
260292 entries [:] = [
261293 ('A-1' , 'approved' , 'singleton-*' , 'success' , 'refusal INVALID_REQUEST' )
262294 ]
263- assert problem_of (singleton , refused ) is None
295+ singleton = BY_ID ['singleton-null-text' ]
296+ refused_singleton = copy .deepcopy (refused )
297+ refused_singleton ['keys' ] = singleton ['reference' ]['keys' ]
298+ refused_singleton ['warnings' ] = singleton ['reference' ]['warnings' ]
299+ assert problem_of (singleton , refused_singleton ) is not None
264300 entries [:] = []
265301 # Another request is a difference, however small.
266- other = copy .deepcopy (want )
267- other ['request' ]['metadata' ]['num_context_examples' ] = {'number' : '2' }
268- assert problem_of (singleton , other ) is not None
269- other = copy .deepcopy (want )
270- other ['keys' ]['payments' ] = None
271- assert problem_of (singleton , other ) is not None
302+ for vector in (plain , singleton ):
303+ other = copy .deepcopy (vector ['reference' ])
304+ other ['request' ]['metadata' ]['num_context_examples' ] = {'number' : '2' }
305+ assert problem_of (vector , other ) is not None
306+ other = copy .deepcopy (vector ['reference' ])
307+ other ['keys' ]['payments' ] = None
308+ assert problem_of (vector , other ) is not None
309+
310+
311+ def test_an_approved_success_to_success_ref_excuses_a_request_alone (
312+ monkeypatch : pytest .MonkeyPatch ,
313+ ) -> None :
314+ r"""``two-nulls-text``: another request on a success passes only
315+ under an approved ``success -> success`` ref naming it; a proposed
316+ one, another vector's, or another transition excuses nothing; and no
317+ ref excuses a singleton."""
318+ module = sys .modules [__name__ ]
319+ entries : list [tuple [str , str , str , str , str ]] = []
320+ monkeypatch .setattr (module , 'transitions' , lambda : (entries , []))
321+ two = BY_ID ['two-nulls-text' ]
322+ assert R .outcome_class (two ['reference' ]['outcome' ]) == 'success'
323+ other = copy .deepcopy (two ['reference' ])
324+ other ['request' ]['metadata' ]['num_context_examples' ] = {'number' : '9' }
325+ assert problem_of (two , other ) is not None
326+ entries [:] = [
327+ ('P1B-13' , 'proposed' , 'two-nulls-text' , 'success' , 'success' )
328+ ]
329+ assert problem_of (two , other ) is not None
330+ entries [:] = [
331+ ('P1B-13' , 'approved' , 'mixed-null-text' , 'success' , 'success' ),
332+ ('P1B-13' , 'approved' , 'two-nulls-text' , 'success' , 'refusal X' ),
333+ ]
334+ assert problem_of (two , other ) is not None
335+ entries [:] = [
336+ ('P1B-13' , 'approved' , 'two-nulls-text' , 'success' , 'success' )
337+ ]
338+ assert problem_of (two , other ) is None
339+ # The same ref excuses no refusal in its place.
340+ refused = copy .deepcopy (two ['reference' ])
341+ refused ['outcome' ] = dict (
342+ refused ['outcome' ],
343+ result = 'refusal' ,
344+ stage = 'materialize' ,
345+ error = 'StructuredDataError' ,
346+ code = 'INVALID_REQUEST' ,
347+ )
348+ refused ['request' ] = None
349+ assert problem_of (two , refused ) is not None
350+ # The singletons are held exactly, under any entry.
351+ entries [:] = [
352+ ('A-1' , 'approved' , '*' , 'success' , 'success' ),
353+ ('A-2' , 'approved' , '*' , 'success' , 'refusal INVALID_REQUEST' ),
354+ ]
355+ for vector_id in sorted (EXACT ):
356+ singleton = BY_ID [vector_id ]
357+ assert (
358+ problem_of (singleton , copy .deepcopy (singleton ['reference' ])) is None
359+ )
360+ changed = copy .deepcopy (singleton ['reference' ])
361+ changed ['request' ]['metadata' ]['num_context_examples' ] = {'number' : '9' }
362+ assert 'held exactly' in (problem_of (singleton , changed ) or '' )
363+ refused = copy .deepcopy (singleton ['reference' ])
364+ refused ['outcome' ] = dict (
365+ refused ['outcome' ],
366+ result = 'refusal' ,
367+ stage = 'materialize' ,
368+ error = 'StructuredDataError' ,
369+ code = 'INVALID_REQUEST' ,
370+ )
371+ refused ['request' ] = None
372+ assert 'held exactly' in (problem_of (singleton , refused ) or '' )
373+ # Only singletons are exact: both sampled alone, one null each.
374+ assert {
375+ v ['id' ] for v in RECORDED ['vectors' ] if v ['id' ].startswith ('singleton-' )
376+ } == EXACT
272377
273378
274379def test_every_vector_ref_names_its_transition () -> None :
0 commit comments