T2.1i: the case format carries rows_per_call and the failed call

T2.8c's cases need two inputs no case could carry: the caller's
rows_per_call (main's 1fc5a68) and the call the scripted endpoint
fails (T2.1a built the endpoint's fail_call for these cases, serve and
answer_call, but no case field reached it).

- contract/case.schema.json: input.materialize.rows_per_call, any JSON
  scalar but null, passed as written, so the values the reference
  refuses (0, -1, 2.5, 3.0, true, "3") are a case's input; and
  input.fail_call, a whole number from 0 to 2**64 - 1, the call's
  number from zero.
- run_python.py passes rows_per_call to materialize as written and
  serves the endpoint with the case's fail_call; its loader refuses a
  fail_call written with a fraction, as JSON Schema alone does not.
- sdm-testkit's loader reads both (rows_per_call kept as the text the
  case wrote, so 2**64 and 3.0 pass through nothing that changes them;
  fail_call as a u64). The test host's use of them (rows_per_call in
  the call's values, predict answered through an Endpoint failing the
  case's fail_call) stays with multi-call predict, which lands after:
  the gate builds a candidate with its checkpoint's case.rs, so the case
  format lands alone first (lesson 13).
- malformed-cases.json: the refused and accepted values of both, which
  both loaders' tests run.

The host's reading of RequestState.batches and CoreError.answered is
the implementer's (T2.1b fixed them in the contract).

Signed-off-by: Adi Wadaskar <awadaskar@nvidia.com>
(cherry picked from commit 6577580758d764116f1090716398e3a64da8777a)

Author: adi-kumo

contract/case.schema.json

@@ -54,7 +54,13 @@
         "from_source": {"$ref": "#/$defs/from_source"},
         "edits": {"$ref": "#/$defs/edits"},
         "pql": {"type": "string", "minLength": 1},
-        "materialize": {"$ref": "#/$defs/materialize"}
+        "materialize": {"$ref": "#/$defs/materialize"},
+        "fail_call": {
+          "description": "The call the scripted endpoint fails, numbered from zero in the order the calls are received (T2.1a's serve(fail_call=), for main's multi-call predict, T2.8c): that call is answered 500 scripted_failure and every other by the rule; absent, no call fails. A JSON integer written as one, below 2**64.",
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 18446744073709551615
+        }
       }
     },
     "until": {
@@ -264,6 +270,10 @@
         "top_k": {
           "description": "How many of the candidates a ranking or forecast answers, a JSON integer written as one; a value the reference refuses (0) is the case's input.",
           "type": "integer"
+        },
+        "rows_per_call": {
+          "description": "How many entities one model call scores (main's 1fc5a686, T2.8c), passed as written: a JSON integer as an integer of any size, a number written with a fraction or an exponent as a float, true or false as a bool, a string as text, so a value the reference refuses is the case's input.",
+          "type": ["integer", "number", "boolean", "string"]
         }
       }
     },

contract/runners/run_python.py

@@ -153,7 +153,8 @@ def _invalid(validate: Any, document: Any) -> str | None:
 
 def _indices(given: dict[str, Any]) -> str | None:
     r"""An entity key is a JSON integer literal of any size, or a string,
-    in ``indices`` and ``candidates`` alike; ``top_k`` is an integer.
+    in ``indices`` and ``candidates`` alike; ``top_k`` and ``fail_call``
+    are integers (``rows_per_call`` is passed as written, whatever it is).
 
     JSON Schema counts ``1.0`` as an integer; ``json`` reads it as a float,
     and a bool is no index either, so the type read is checked too, as
@@ -166,6 +167,8 @@ def _indices(given: dict[str, Any]) -> str | None:
                 return f'input/materialize/{key}/{position} is not an integer'
     if 'top_k' in spec and type(spec['top_k']) is not int:
         return 'input/materialize/top_k is not an integer'
+    if 'fail_call' in given and type(given['fail_call']) is not int:
+        return 'input/fail_call is not an integer'
     return None
 
 
@@ -2432,6 +2435,10 @@ def _materialize_arguments(spec: dict[str, Any]) -> dict[str, Any]:
         arguments['candidates'] = list(spec['candidates'])
     if 'top_k' in spec:
         arguments['top_k'] = spec['top_k']
+    if 'rows_per_call' in spec:
+        # As written (T2.8c's cases): a value the reference refuses is the
+        # case's input.
+        arguments['rows_per_call'] = spec['rows_per_call']
     return arguments
 
 
@@ -2651,7 +2658,7 @@ def _call(
     if log is not None:
         log.stage = stage
     with (
-        serve() as (url, received),
+        serve(fail_call=given.get('fail_call')) as (url, received),
         warnings.catch_warnings(record=True) as shown,
     ):
         warnings.simplefilter('always')

contract/tests/malformed-cases.json

@@ -1,5 +1,5 @@
 {
-  "about": "Changes to base.adventureworks's case that both case loaders must refuse, and ones both must take: contract/runners/run_python.py's validator (tests/test_contract_cases.py) and sdm-testkit's case::load (tests/s0_cases.rs). Each refused or accepted change sets the value at path, and each omitted one removes the key at path (which both must take), a list of keys and positions into the case (T1.21b; from T1.21c the schema's minProperties, minItems and minLength, core.materialize.seed required, and expect's if/then/else; from T1.21d core.materialize naming a schema, refused; from T1.24b the edits set_primary_key, set_time_column, set_stype and set_relationships; from T2.1a materialize's candidates and top_k, while a databases or exclusion change, which needs its partner, is held in tests/test_t21a_case_format.py through both keys).",
+  "about": "Changes to base.adventureworks's case that both case loaders must refuse, and ones both must take: contract/runners/run_python.py's validator (tests/test_contract_cases.py) and sdm-testkit's case::load (tests/s0_cases.rs). Each refused or accepted change sets the value at path, and each omitted one removes the key at path (which both must take), a list of keys and positions into the case (T1.21b; from T1.21c the schema's minProperties, minItems and minLength, core.materialize.seed required, and expect's if/then/else; from T1.21d core.materialize naming a schema, refused; from T1.24b the edits set_primary_key, set_time_column, set_stype and set_relationships; from T2.1a materialize's candidates and top_k; from T2.8c's test author materialize's rows_per_call, any JSON scalar but null, and input.fail_call, a whole number from 0 to 2**64 - 1, while a databases or exclusion change, which needs its partner, is held in tests/test_t21a_case_format.py through both keys).",
   "case": "contract/cases/s1/base.adventureworks.json",
   "refused": [
     {
@@ -915,6 +915,85 @@
         "top_k"
       ],
       "value": null
+    },
+    {
+      "why": "rows_per_call null",
+      "path": [
+        "input",
+        "materialize",
+        "rows_per_call"
+      ],
+      "value": null
+    },
+    {
+      "why": "rows_per_call a list",
+      "path": [
+        "input",
+        "materialize",
+        "rows_per_call"
+      ],
+      "value": [
+        3
+      ]
+    },
+    {
+      "why": "rows_per_call an object",
+      "path": [
+        "input",
+        "materialize",
+        "rows_per_call"
+      ],
+      "value": {
+        "n": 3
+      }
+    },
+    {
+      "why": "fail_call as text",
+      "path": [
+        "input",
+        "fail_call"
+      ],
+      "value": "2"
+    },
+    {
+      "why": "fail_call written with a fraction",
+      "path": [
+        "input",
+        "fail_call"
+      ],
+      "value": 2.0
+    },
+    {
+      "why": "fail_call as a boolean",
+      "path": [
+        "input",
+        "fail_call"
+      ],
+      "value": true
+    },
+    {
+      "why": "fail_call below zero",
+      "path": [
+        "input",
+        "fail_call"
+      ],
+      "value": -1
+    },
+    {
+      "why": "fail_call null",
+      "path": [
+        "input",
+        "fail_call"
+      ],
+      "value": null
+    },
+    {
+      "why": "fail_call past what a u64 holds",
+      "path": [
+        "input",
+        "fail_call"
+      ],
+      "value": 18446744073709551616
     }
   ],
   "accepted": [
@@ -1158,6 +1237,76 @@
         "top_k"
       ],
       "value": -1
+    },
+    {
+      "why": "rows_per_call of 3",
+      "path": [
+        "input",
+        "materialize",
+        "rows_per_call"
+      ],
+      "value": 3
+    },
+    {
+      "why": "rows_per_call of 0, an input the reference refuses, carried as a case",
+      "path": [
+        "input",
+        "materialize",
+        "rows_per_call"
+      ],
+      "value": 0
+    },
+    {
+      "why": "rows_per_call written with a fraction, an input the reference refuses",
+      "path": [
+        "input",
+        "materialize",
+        "rows_per_call"
+      ],
+      "value": 2.5
+    },
+    {
+      "why": "rows_per_call as a boolean, an input the reference refuses",
+      "path": [
+        "input",
+        "materialize",
+        "rows_per_call"
+      ],
+      "value": true
+    },
+    {
+      "why": "rows_per_call as text, an input the reference refuses",
+      "path": [
+        "input",
+        "materialize",
+        "rows_per_call"
+      ],
+      "value": "3"
+    },
+    {
+      "why": "rows_per_call past int64, a JSON integer all the same",
+      "path": [
+        "input",
+        "materialize",
+        "rows_per_call"
+      ],
+      "value": 18446744073709551616
+    },
+    {
+      "why": "fail_call of 0, the first call",
+      "path": [
+        "input",
+        "fail_call"
+      ],
+      "value": 0
+    },
+    {
+      "why": "fail_call of 2",
+      "path": [
+        "input",
+        "fail_call"
+      ],
+      "value": 2
     }
   ],
   "omitted": [

rust/crates/sdm-testkit/src/case.rs

@@ -128,6 +128,12 @@ pub struct Input {
     #[serde(deserialize_with = "text")]
     pub pql: String,
     pub materialize: Materialize,
+    /// The call the scripted endpoint fails, numbered from zero in the
+    /// order received (T2.1a, for `main`'s multi-call `predict`, T2.8c); a
+    /// JSON integer written as one, `1.0` refused as the Python loader
+    /// refuses it.
+    #[serde(default, deserialize_with = "present")]
+    pub fail_call: Option<u64>,
 }
 
 /// Keyword arguments to `Schema.from_source`, each as the case gives it.
@@ -434,6 +440,29 @@ pub struct Materialize {
     /// refuses, is a case's input.
     #[serde(default, deserialize_with = "present")]
     pub top_k: Option<Integer>,
+    /// How many entities one call scores (`main`'s 1fc5a686, T2.8c), kept
+    /// as written, whatever JSON scalar it is, so a value the reference
+    /// refuses is the case's input.
+    #[serde(default, deserialize_with = "present")]
+    pub rows_per_call: Option<Written>,
+}
+
+/// A JSON scalar other than `null` (a number, a bool or a string), kept as
+/// the text the case wrote: a number of any size or form passes through
+/// nothing that would change it.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Written(pub String);
+
+impl<'de> Deserialize<'de> for Written {
+    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Written, D::Error> {
+        let raw = Box::<RawValue>::deserialize(deserializer)?;
+        let text = raw.get().trim();
+        match text.as_bytes().first() {
+            Some(b'"' | b'-' | b'0'..=b'9') => Ok(Written(text.to_owned())),
+            _ if text == "true" || text == "false" => Ok(Written(text.to_owned())),
+            _ => Err(D::Error::custom("not a number, a bool or a string")),
+        }
+    }
 }
 
 /// One entity key: a JSON integer literal of any size, kept as its