Skip to content

Commit fdc6cf0

Browse files
committed
T1.32e tests: an approved success -> success ref excuses a sampled request; the singletons held exactly
two-nulls-text (two null keys reached for one example) succeeds on the core with a request other than the reference's: P1B-13's approved behaviour, a null in the walk no link, re-approved by the owner on 2026-10-13 because the reference corrupts null-key requests. The sampled-null-key test already read approved binding-transition refs; this makes that explicit and seeded: a different request on a success passes only under an approved entry whose ref names the vector with success -> success (contract/sampled-null-key-vectors.json#two-nulls-text success -> success, which the implementer adds to P1B-13); a proposed entry, another vector's ref, or another transition excuses nothing, and the ref excuses no refusal in the success's place. The singletons (singleton-null-text, the review's reproduction, and singleton-null-decimal) are held to the reference's exact answer and request: no entry excuses them, whatever it declares. The fail-closed seed that showed an approved transition excusing a singleton now shows it on null-text-not-sampled, and that the same entry leaves the singleton failing. No pending id changes: two-nulls-text stays pending until P1B-13 carries its ref. Signed-off-by: Adi Wadaskar <awadaskar@nvidia.com>
1 parent 8d72f8a commit fdc6cf0

1 file changed

Lines changed: 122 additions & 17 deletions

File tree

‎contract/tests/test_t132e_sampled_null_keys.py‎

Lines changed: 122 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,24 @@
3131
received (``metadata.request_id`` aside) and the warning classes; and
3232
the core must have been reached. A difference passes only under an
3333
approved register entry naming the vector with its transition
34-
(``contract/sampled-null-key-vectors.json#<id or glob> <from> -> <to>``);
35-
``register.required`` (derived: ``mixed-null-decimal``, INTERNAL_ERROR at
36-
materialize) excuses nothing by itself.
34+
(``contract/sampled-null-key-vectors.json#<id or glob> <from> -> <to>``),
35+
as the other vector tests read it: a ref covers exactly the transition
36+
it declares, a proposed entry excuses nothing, and ``success ->
37+
success`` is a transition like any other, so a request that differs on
38+
a success passes only under an approved entry declaring that for the
39+
vector. ``two-nulls-text`` is the case in point: two null keys reached
40+
for one example, where P1B-13 (approved, re-approved by the owner on
41+
2026-10-13: the reference corrupts null-key requests) makes a null in
42+
the walk no link, so the core's request differs from the reference's;
43+
it passes once P1B-13 carries
44+
``contract/sampled-null-key-vectors.json#two-nulls-text success ->
45+
success``. ``register.required`` (derived: ``mixed-null-decimal``,
46+
INTERNAL_ERROR at materialize) excuses nothing by itself.
47+
- The singletons (:data:`EXACT`: one null key sampled alone, text and
48+
decimal, the review's reproduction among them) are held to the
49+
reference's exact answer and request: no entry excuses them, whatever
50+
it declares, since a single null in the walk is the reference's clean
51+
case (np.unique sorts it) and P1B-13 changes nothing there.
3752
- The case format holds neither ``num_context_samples`` nor
3853
``random_seed``, so these vectors are not cases and ``rust-host`` does
3954
not run them; the core's answer is held through the Python host.
@@ -113,14 +128,25 @@ def transitions() -> tuple[list[tuple[str, str, str, str, str]], list[str]]:
113128
return out, malformed
114129

115130

131+
#: The vectors held to the reference's exact answer, request included,
132+
#: which no register entry excuses: one null key sampled alone.
133+
EXACT = frozenset({'singleton-null-text', 'singleton-null-decimal'})
134+
135+
116136
def problem_of(vector: dict[str, Any], got: dict[str, Any]) -> str | None:
117137
r"""Why ``got`` fails the vector: None when it is the reference's
118-
answer, or a difference an approved entry declares exactly."""
138+
answer, or, outside :data:`EXACT`, a difference an approved entry
139+
declares exactly (``success -> success`` included)."""
119140
want = vector['reference']
120141
if got == want:
121142
return None
122143
old = R.outcome_class(want['outcome'])
123144
new = R.outcome_class(got['outcome'])
145+
if vector['id'] in EXACT:
146+
return (
147+
f'{vector["id"]}: got {got}, the reference {want} ({old} -> '
148+
f'{new}); held exactly, no entry excuses it'
149+
)
124150
for _, status, glob, frm, to in transitions()[0]:
125151
if (
126152
status == 'approved'
@@ -241,9 +267,10 @@ def test_the_comparison_fails_closed(monkeypatch: pytest.MonkeyPatch) -> None:
241267
module = sys.modules[__name__]
242268
entries: list[tuple[str, str, str, str, str]] = []
243269
monkeypatch.setattr(module, 'transitions', lambda: (entries, []))
244-
singleton = BY_ID['singleton-null-text']
245-
want = singleton['reference']
246-
assert problem_of(singleton, copy.deepcopy(want)) is None
270+
plain = BY_ID['null-text-not-sampled']
271+
want = plain['reference']
272+
assert plain['id'] not in EXACT
273+
assert problem_of(plain, copy.deepcopy(want)) is None
247274
refused = copy.deepcopy(want)
248275
refused['outcome'] = {
249276
'result': 'refusal',
@@ -252,23 +279,101 @@ def test_the_comparison_fails_closed(monkeypatch: pytest.MonkeyPatch) -> None:
252279
'code': 'INVALID_REQUEST',
253280
}
254281
refused['request'] = None
255-
assert problem_of(singleton, refused) is not None
282+
assert problem_of(plain, refused) is not None
256283
entries[:] = [
257-
('P-1', 'proposed', 'singleton-*', 'success', 'refusal INVALID_REQUEST')
284+
('P-1', 'proposed', 'null-text-*', 'success', 'refusal INVALID_REQUEST')
258285
]
259-
assert problem_of(singleton, refused) is not None
286+
assert problem_of(plain, refused) is not None
287+
entries[:] = [
288+
('A-1', 'approved', 'null-text-*', 'success', 'refusal INVALID_REQUEST')
289+
]
290+
assert problem_of(plain, refused) is None
291+
# The same approved transition, for the singleton: held exactly.
260292
entries[:] = [
261293
('A-1', 'approved', 'singleton-*', 'success', 'refusal INVALID_REQUEST')
262294
]
263-
assert problem_of(singleton, refused) is None
295+
singleton = BY_ID['singleton-null-text']
296+
refused_singleton = copy.deepcopy(refused)
297+
refused_singleton['keys'] = singleton['reference']['keys']
298+
refused_singleton['warnings'] = singleton['reference']['warnings']
299+
assert problem_of(singleton, refused_singleton) is not None
264300
entries[:] = []
265301
# Another request is a difference, however small.
266-
other = copy.deepcopy(want)
267-
other['request']['metadata']['num_context_examples'] = {'number': '2'}
268-
assert problem_of(singleton, other) is not None
269-
other = copy.deepcopy(want)
270-
other['keys']['payments'] = None
271-
assert problem_of(singleton, other) is not None
302+
for vector in (plain, singleton):
303+
other = copy.deepcopy(vector['reference'])
304+
other['request']['metadata']['num_context_examples'] = {'number': '2'}
305+
assert problem_of(vector, other) is not None
306+
other = copy.deepcopy(vector['reference'])
307+
other['keys']['payments'] = None
308+
assert problem_of(vector, other) is not None
309+
310+
311+
def test_an_approved_success_to_success_ref_excuses_a_request_alone(
312+
monkeypatch: pytest.MonkeyPatch,
313+
) -> None:
314+
r"""``two-nulls-text``: another request on a success passes only
315+
under an approved ``success -> success`` ref naming it; a proposed
316+
one, another vector's, or another transition excuses nothing; and no
317+
ref excuses a singleton."""
318+
module = sys.modules[__name__]
319+
entries: list[tuple[str, str, str, str, str]] = []
320+
monkeypatch.setattr(module, 'transitions', lambda: (entries, []))
321+
two = BY_ID['two-nulls-text']
322+
assert R.outcome_class(two['reference']['outcome']) == 'success'
323+
other = copy.deepcopy(two['reference'])
324+
other['request']['metadata']['num_context_examples'] = {'number': '9'}
325+
assert problem_of(two, other) is not None
326+
entries[:] = [
327+
('P1B-13', 'proposed', 'two-nulls-text', 'success', 'success')
328+
]
329+
assert problem_of(two, other) is not None
330+
entries[:] = [
331+
('P1B-13', 'approved', 'mixed-null-text', 'success', 'success'),
332+
('P1B-13', 'approved', 'two-nulls-text', 'success', 'refusal X'),
333+
]
334+
assert problem_of(two, other) is not None
335+
entries[:] = [
336+
('P1B-13', 'approved', 'two-nulls-text', 'success', 'success')
337+
]
338+
assert problem_of(two, other) is None
339+
# The same ref excuses no refusal in its place.
340+
refused = copy.deepcopy(two['reference'])
341+
refused['outcome'] = dict(
342+
refused['outcome'],
343+
result='refusal',
344+
stage='materialize',
345+
error='StructuredDataError',
346+
code='INVALID_REQUEST',
347+
)
348+
refused['request'] = None
349+
assert problem_of(two, refused) is not None
350+
# The singletons are held exactly, under any entry.
351+
entries[:] = [
352+
('A-1', 'approved', '*', 'success', 'success'),
353+
('A-2', 'approved', '*', 'success', 'refusal INVALID_REQUEST'),
354+
]
355+
for vector_id in sorted(EXACT):
356+
singleton = BY_ID[vector_id]
357+
assert (
358+
problem_of(singleton, copy.deepcopy(singleton['reference'])) is None
359+
)
360+
changed = copy.deepcopy(singleton['reference'])
361+
changed['request']['metadata']['num_context_examples'] = {'number': '9'}
362+
assert 'held exactly' in (problem_of(singleton, changed) or '')
363+
refused = copy.deepcopy(singleton['reference'])
364+
refused['outcome'] = dict(
365+
refused['outcome'],
366+
result='refusal',
367+
stage='materialize',
368+
error='StructuredDataError',
369+
code='INVALID_REQUEST',
370+
)
371+
refused['request'] = None
372+
assert 'held exactly' in (problem_of(singleton, refused) or '')
373+
# Only singletons are exact: both sampled alone, one null each.
374+
assert {
375+
v['id'] for v in RECORDED['vectors'] if v['id'].startswith('singleton-')
376+
} == EXACT
272377

273378

274379
def test_every_vector_ref_names_its_transition() -> None:

0 commit comments

Comments
 (0)