Skip to content

Commit 1935b6c

Browse files
committed
T1.28a tests: rust-host's classification payload and entity limit against the reference's
The package's independent test author's acceptance tests on rust-host, from the vectors t128a_payload_rules_record.py recorded from the reference (t128a-payload-rules.json), which python-core is held to as well, so the two hosts agree through them: - base.binary and base.multiclass, from the real payload: the task (kind, classes, their order, positive class), the context's and predicted rows, each related table's columns and kept instance ids (the context that answers the question taken out, the static target's column left out); - base.regression with 1,001 entities refused UNSUPPORTED_REQUEST as the reference refuses it, by the core's own limit, not a rule not held yet. At 81001a8 both classification tests fail on the core's classification_task refusal; the recording and entity-limit tests pass. Signed-off-by: Adi Wadaskar <awadaskar@nvidia.com>
1 parent d496f2f commit 1935b6c

1 file changed

Lines changed: 216 additions & 0 deletions

File tree

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
2+
// All rights reserved.
3+
// SPDX-License-Identifier: Apache-2.0
4+
5+
//! T1.28a's exits on `rust-host`, held from outside the implementation (the
6+
//! package's independent test author), against what the reference gave:
7+
//! the vectors are `t128a-payload-rules.json`, recorded from the reference
8+
//! by `t128a_payload_rules_record.py` beside it, which the Python test
9+
//! holds `python-core` to as well, so the two hosts agree through them.
10+
//!
11+
//! - Exit "on both core hosts, over `base.binary` and `base.multiclass`,
12+
//! the classes, their order, the positive class and the context-leakage
13+
//! payload fields match the reference", from the real payload (not the
14+
//! test-only labels operation): the task, the context's and predicted
15+
//! rows, and each related table's columns and the instance ids it keeps
16+
//! (the context that answers the question taken out; a static target's
17+
//! column left out).
18+
//! - Exit "python-core and rust-host public outcomes match for context
19+
//! leakage and limits": the leakage fields above, and the entity count's
20+
//! limit, 1,001 entities refused as the reference refuses them
21+
//! (`UNSUPPORTED_REQUEST`), which `python-core` is held to by the same
22+
//! recorded case.
23+
//!
24+
//! Before T1.28a the core refuses both classification cases by its rule
25+
//! `classification_task`, and these tests say so.
26+
27+
use std::path::PathBuf;
28+
29+
use sdm_testkit::case::Case;
30+
use sdm_testkit::host::{self, Answer};
31+
use sdm_testkit::{bases, case};
32+
use serde_json::{Map, Value, json};
33+
34+
fn root() -> PathBuf {
35+
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../..")
36+
}
37+
38+
fn recorded() -> Value {
39+
let path = root().join("packages/structured-data-models-client/tests/t128a-payload-rules.json");
40+
serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()
41+
}
42+
43+
fn expected(name: &str) -> Value {
44+
recorded()["cases"]
45+
.as_array()
46+
.unwrap()
47+
.iter()
48+
.find(|c| c["name"] == name)
49+
.unwrap_or_else(|| panic!("{name} is recorded"))["expected"]
50+
.clone()
51+
}
52+
53+
fn the_case(id: &str) -> (Case, bases::Bases) {
54+
let contract = root().join("contract");
55+
let loaded = case::load(&[contract.join("cases")]).unwrap();
56+
let found = loaded
57+
.iter()
58+
.find(|l| l.case.id == id)
59+
.unwrap_or_else(|| panic!("{id} is a case"))
60+
.case
61+
.clone();
62+
(found, bases::load(&contract.join("bases.yaml")).unwrap())
63+
}
64+
65+
/// The inverse of `host::numbers_as_text`: `{"number": text}` read back as
66+
/// the number written.
67+
fn numbers(value: Value) -> Value {
68+
match value {
69+
Value::Object(members) => match members.get("number") {
70+
Some(Value::String(text)) if members.len() == 1 => {
71+
serde_json::from_str(text).expect("a number's text")
72+
}
73+
_ => Value::Object(members.into_iter().map(|(k, v)| (k, numbers(v))).collect()),
74+
},
75+
Value::Array(items) => Value::Array(items.into_iter().map(numbers).collect()),
76+
other => other,
77+
}
78+
}
79+
80+
fn rows(table: &Value) -> Value {
81+
json!({"columns": table["columns"], "rows": table["rows"]})
82+
}
83+
84+
/// Each related table's columns and the sorted instance ids its rows hang
85+
/// off, as the recorder keeps them.
86+
fn instances(related: &Value) -> Value {
87+
let mut out = Map::new();
88+
let Some(tables) = related.as_object() else {
89+
return Value::Null;
90+
};
91+
for (name, table) in tables {
92+
let columns = table["columns"].as_array().cloned().unwrap_or_default();
93+
let at = columns.iter().position(|c| c == "instance_id");
94+
let mut ids: Vec<Value> = Vec::new();
95+
if let Some(at) = at {
96+
for row in table["rows"].as_array().into_iter().flatten() {
97+
let id = row[at].clone();
98+
if !ids.contains(&id) {
99+
ids.push(id);
100+
}
101+
}
102+
}
103+
ids.sort_by_key(|v| v.as_i64());
104+
out.insert(
105+
name.clone(),
106+
json!({"columns": columns, "instance_ids": ids}),
107+
);
108+
}
109+
Value::Object(out)
110+
}
111+
112+
/// The payload `rust-host` materializes for `id`, or the refusal's rule.
113+
fn payload(id: &str) -> Value {
114+
let (case, bases) = the_case(id);
115+
let observed = host::run(&case, &bases).unwrap();
116+
match observed.materialized {
117+
Some(state) => numbers(state["payload"].clone()),
118+
None => match observed.answer {
119+
Answer::Error { code, fields, .. } => panic!(
120+
"{id}: rust-host refused it ({code}, rule {:?}); T1.28a lifts \
121+
classification_task and target_leakage",
122+
fields.get("rule")
123+
),
124+
other => panic!("{id}: no payload ({other:?})"),
125+
},
126+
}
127+
}
128+
129+
fn classification(id: &str) {
130+
let want = expected(id);
131+
let got = payload(id);
132+
assert_eq!(got["task"], want["task"], "{id}: the task");
133+
assert_eq!(
134+
rows(&got["context"]["instance_table"]),
135+
want["context"],
136+
"{id}: the context's rows"
137+
);
138+
assert_eq!(
139+
instances(&got["context"]["related_tables"]),
140+
want["context_related"],
141+
"{id}: the context's related tables"
142+
);
143+
assert_eq!(
144+
rows(&got["predict"]["instance_table"]),
145+
want["predict"],
146+
"{id}: the predicted rows"
147+
);
148+
assert_eq!(
149+
instances(&got["predict"]["related_tables"]),
150+
want["predict_related"],
151+
"{id}: the predicted rows' related tables"
152+
);
153+
}
154+
155+
#[test]
156+
fn the_recording_names_the_two_base_cases() {
157+
// The vectors are the reference's, and hold what the exit compares.
158+
let all = recorded();
159+
assert_eq!(all["recorded_by"], "t128a_payload_rules_record.py");
160+
assert_eq!(
161+
expected("base.binary")["task"]["kind"],
162+
"binary_classification"
163+
);
164+
assert_eq!(
165+
expected("base.binary")["task"]["target"]["positive_class"],
166+
"true"
167+
);
168+
assert_eq!(
169+
expected("base.multiclass")["task"]["kind"],
170+
"multiclass_classification"
171+
);
172+
// base.multiclass's static target leaks: the reference took rows out.
173+
let leaked = expected("base.multiclass")["notices"]
174+
.as_array()
175+
.unwrap()
176+
.iter()
177+
.any(|n| n.as_str().unwrap().contains("rows sampled as context"));
178+
assert!(leaked);
179+
}
180+
181+
#[test]
182+
fn base_binary_s_classification_payload_is_the_reference_s() {
183+
classification("base.binary");
184+
}
185+
186+
#[test]
187+
fn base_multiclass_s_classification_payload_and_leakage_are_the_reference_s() {
188+
classification("base.multiclass");
189+
}
190+
191+
#[test]
192+
fn more_entities_than_the_endpoint_scores_are_refused_as_the_reference_refuses_them() {
193+
// The entity count's limit (1,000), on base.regression with 1,001
194+
// entities: the reference's UNSUPPORTED_REQUEST, which python-core is
195+
// held to by the same recorded case.
196+
let want = expected("base.regression.entities-1001");
197+
assert_eq!(want["outcome"], "refused");
198+
let (mut case, bases) = the_case("base.regression");
199+
let materialize = case
200+
.core
201+
.as_mut()
202+
.and_then(|core| core.materialize.as_mut())
203+
.unwrap();
204+
materialize.indices = (0..1001)
205+
.map(|i| json!({"t": "i64", "v": i.to_string()}))
206+
.collect();
207+
let observed = host::run(&case, &bases).unwrap();
208+
assert!(observed.materialized.is_none(), "1,001 entities were taken");
209+
let Answer::Error { code, fields, .. } = &observed.answer else {
210+
panic!("not refused: {:?}", observed.answer);
211+
};
212+
// The core's own limit, not a rule it does not hold yet (`not_yet`
213+
// names one), which would refuse with the same code for another reason.
214+
assert_eq!(fields.get("rule"), None, "refused by a rule not held yet");
215+
assert_eq!(Some(code.as_str()), want["code"].as_str());
216+
}

0 commit comments

Comments
 (0)