|
| 1 | +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. |
| 2 | +// All rights reserved. |
| 3 | +// SPDX-License-Identifier: Apache-2.0 |
| 4 | + |
| 5 | +//! T1.28a's exits on `rust-host`, held from outside the implementation (the |
| 6 | +//! package's independent test author), against what the reference gave: |
| 7 | +//! the vectors are `t128a-payload-rules.json`, recorded from the reference |
| 8 | +//! by `t128a_payload_rules_record.py` beside it, which the Python test |
| 9 | +//! holds `python-core` to as well, so the two hosts agree through them. |
| 10 | +//! |
| 11 | +//! - Exit "on both core hosts, over `base.binary` and `base.multiclass`, |
| 12 | +//! the classes, their order, the positive class and the context-leakage |
| 13 | +//! payload fields match the reference", from the real payload (not the |
| 14 | +//! test-only labels operation): the task, the context's and predicted |
| 15 | +//! rows, and each related table's columns and the instance ids it keeps |
| 16 | +//! (the context that answers the question taken out; a static target's |
| 17 | +//! column left out). |
| 18 | +//! - Exit "python-core and rust-host public outcomes match for context |
| 19 | +//! leakage and limits": the leakage fields above, and the entity count's |
| 20 | +//! limit, 1,001 entities refused as the reference refuses them |
| 21 | +//! (`UNSUPPORTED_REQUEST`), which `python-core` is held to by the same |
| 22 | +//! recorded case. |
| 23 | +//! |
| 24 | +//! Before T1.28a the core refuses both classification cases by its rule |
| 25 | +//! `classification_task`, and these tests say so. |
| 26 | +
|
| 27 | +use std::path::PathBuf; |
| 28 | + |
| 29 | +use sdm_testkit::case::Case; |
| 30 | +use sdm_testkit::host::{self, Answer}; |
| 31 | +use sdm_testkit::{bases, case}; |
| 32 | +use serde_json::{Map, Value, json}; |
| 33 | + |
| 34 | +fn root() -> PathBuf { |
| 35 | + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../..") |
| 36 | +} |
| 37 | + |
| 38 | +fn recorded() -> Value { |
| 39 | + let path = root().join("packages/structured-data-models-client/tests/t128a-payload-rules.json"); |
| 40 | + serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() |
| 41 | +} |
| 42 | + |
| 43 | +fn expected(name: &str) -> Value { |
| 44 | + recorded()["cases"] |
| 45 | + .as_array() |
| 46 | + .unwrap() |
| 47 | + .iter() |
| 48 | + .find(|c| c["name"] == name) |
| 49 | + .unwrap_or_else(|| panic!("{name} is recorded"))["expected"] |
| 50 | + .clone() |
| 51 | +} |
| 52 | + |
| 53 | +fn the_case(id: &str) -> (Case, bases::Bases) { |
| 54 | + let contract = root().join("contract"); |
| 55 | + let loaded = case::load(&[contract.join("cases")]).unwrap(); |
| 56 | + let found = loaded |
| 57 | + .iter() |
| 58 | + .find(|l| l.case.id == id) |
| 59 | + .unwrap_or_else(|| panic!("{id} is a case")) |
| 60 | + .case |
| 61 | + .clone(); |
| 62 | + (found, bases::load(&contract.join("bases.yaml")).unwrap()) |
| 63 | +} |
| 64 | + |
| 65 | +/// The inverse of `host::numbers_as_text`: `{"number": text}` read back as |
| 66 | +/// the number written. |
| 67 | +fn numbers(value: Value) -> Value { |
| 68 | + match value { |
| 69 | + Value::Object(members) => match members.get("number") { |
| 70 | + Some(Value::String(text)) if members.len() == 1 => { |
| 71 | + serde_json::from_str(text).expect("a number's text") |
| 72 | + } |
| 73 | + _ => Value::Object(members.into_iter().map(|(k, v)| (k, numbers(v))).collect()), |
| 74 | + }, |
| 75 | + Value::Array(items) => Value::Array(items.into_iter().map(numbers).collect()), |
| 76 | + other => other, |
| 77 | + } |
| 78 | +} |
| 79 | + |
| 80 | +fn rows(table: &Value) -> Value { |
| 81 | + json!({"columns": table["columns"], "rows": table["rows"]}) |
| 82 | +} |
| 83 | + |
| 84 | +/// Each related table's columns and the sorted instance ids its rows hang |
| 85 | +/// off, as the recorder keeps them. |
| 86 | +fn instances(related: &Value) -> Value { |
| 87 | + let mut out = Map::new(); |
| 88 | + let Some(tables) = related.as_object() else { |
| 89 | + return Value::Null; |
| 90 | + }; |
| 91 | + for (name, table) in tables { |
| 92 | + let columns = table["columns"].as_array().cloned().unwrap_or_default(); |
| 93 | + let at = columns.iter().position(|c| c == "instance_id"); |
| 94 | + let mut ids: Vec<Value> = Vec::new(); |
| 95 | + if let Some(at) = at { |
| 96 | + for row in table["rows"].as_array().into_iter().flatten() { |
| 97 | + let id = row[at].clone(); |
| 98 | + if !ids.contains(&id) { |
| 99 | + ids.push(id); |
| 100 | + } |
| 101 | + } |
| 102 | + } |
| 103 | + ids.sort_by_key(|v| v.as_i64()); |
| 104 | + out.insert( |
| 105 | + name.clone(), |
| 106 | + json!({"columns": columns, "instance_ids": ids}), |
| 107 | + ); |
| 108 | + } |
| 109 | + Value::Object(out) |
| 110 | +} |
| 111 | + |
| 112 | +/// The payload `rust-host` materializes for `id`, or the refusal's rule. |
| 113 | +fn payload(id: &str) -> Value { |
| 114 | + let (case, bases) = the_case(id); |
| 115 | + let observed = host::run(&case, &bases).unwrap(); |
| 116 | + match observed.materialized { |
| 117 | + Some(state) => numbers(state["payload"].clone()), |
| 118 | + None => match observed.answer { |
| 119 | + Answer::Error { code, fields, .. } => panic!( |
| 120 | + "{id}: rust-host refused it ({code}, rule {:?}); T1.28a lifts \ |
| 121 | + classification_task and target_leakage", |
| 122 | + fields.get("rule") |
| 123 | + ), |
| 124 | + other => panic!("{id}: no payload ({other:?})"), |
| 125 | + }, |
| 126 | + } |
| 127 | +} |
| 128 | + |
| 129 | +fn classification(id: &str) { |
| 130 | + let want = expected(id); |
| 131 | + let got = payload(id); |
| 132 | + assert_eq!(got["task"], want["task"], "{id}: the task"); |
| 133 | + assert_eq!( |
| 134 | + rows(&got["context"]["instance_table"]), |
| 135 | + want["context"], |
| 136 | + "{id}: the context's rows" |
| 137 | + ); |
| 138 | + assert_eq!( |
| 139 | + instances(&got["context"]["related_tables"]), |
| 140 | + want["context_related"], |
| 141 | + "{id}: the context's related tables" |
| 142 | + ); |
| 143 | + assert_eq!( |
| 144 | + rows(&got["predict"]["instance_table"]), |
| 145 | + want["predict"], |
| 146 | + "{id}: the predicted rows" |
| 147 | + ); |
| 148 | + assert_eq!( |
| 149 | + instances(&got["predict"]["related_tables"]), |
| 150 | + want["predict_related"], |
| 151 | + "{id}: the predicted rows' related tables" |
| 152 | + ); |
| 153 | +} |
| 154 | + |
| 155 | +#[test] |
| 156 | +fn the_recording_names_the_two_base_cases() { |
| 157 | + // The vectors are the reference's, and hold what the exit compares. |
| 158 | + let all = recorded(); |
| 159 | + assert_eq!(all["recorded_by"], "t128a_payload_rules_record.py"); |
| 160 | + assert_eq!( |
| 161 | + expected("base.binary")["task"]["kind"], |
| 162 | + "binary_classification" |
| 163 | + ); |
| 164 | + assert_eq!( |
| 165 | + expected("base.binary")["task"]["target"]["positive_class"], |
| 166 | + "true" |
| 167 | + ); |
| 168 | + assert_eq!( |
| 169 | + expected("base.multiclass")["task"]["kind"], |
| 170 | + "multiclass_classification" |
| 171 | + ); |
| 172 | + // base.multiclass's static target leaks: the reference took rows out. |
| 173 | + let leaked = expected("base.multiclass")["notices"] |
| 174 | + .as_array() |
| 175 | + .unwrap() |
| 176 | + .iter() |
| 177 | + .any(|n| n.as_str().unwrap().contains("rows sampled as context")); |
| 178 | + assert!(leaked); |
| 179 | +} |
| 180 | + |
| 181 | +#[test] |
| 182 | +fn base_binary_s_classification_payload_is_the_reference_s() { |
| 183 | + classification("base.binary"); |
| 184 | +} |
| 185 | + |
| 186 | +#[test] |
| 187 | +fn base_multiclass_s_classification_payload_and_leakage_are_the_reference_s() { |
| 188 | + classification("base.multiclass"); |
| 189 | +} |
| 190 | + |
| 191 | +#[test] |
| 192 | +fn more_entities_than_the_endpoint_scores_are_refused_as_the_reference_refuses_them() { |
| 193 | + // The entity count's limit (1,000), on base.regression with 1,001 |
| 194 | + // entities: the reference's UNSUPPORTED_REQUEST, which python-core is |
| 195 | + // held to by the same recorded case. |
| 196 | + let want = expected("base.regression.entities-1001"); |
| 197 | + assert_eq!(want["outcome"], "refused"); |
| 198 | + let (mut case, bases) = the_case("base.regression"); |
| 199 | + let materialize = case |
| 200 | + .core |
| 201 | + .as_mut() |
| 202 | + .and_then(|core| core.materialize.as_mut()) |
| 203 | + .unwrap(); |
| 204 | + materialize.indices = (0..1001) |
| 205 | + .map(|i| json!({"t": "i64", "v": i.to_string()})) |
| 206 | + .collect(); |
| 207 | + let observed = host::run(&case, &bases).unwrap(); |
| 208 | + assert!(observed.materialized.is_none(), "1,001 entities were taken"); |
| 209 | + let Answer::Error { code, fields, .. } = &observed.answer else { |
| 210 | + panic!("not refused: {:?}", observed.answer); |
| 211 | + }; |
| 212 | + // The core's own limit, not a rule it does not hold yet (`not_yet` |
| 213 | + // names one), which would refuse with the same code for another reason. |
| 214 | + assert_eq!(fields.get("rule"), None, "refused by a rule not held yet"); |
| 215 | + assert_eq!(Some(code.as_str()), want["code"].as_str()); |
| 216 | +} |
0 commit comments