diff --git a/CHANGELOG.md b/CHANGELOG.md index fa383366d..cc3a98d61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,18 @@ these packages. All five distributions share one version and one entry. ## Unreleased +### Added + +- `kumo-tabular` accepts `explanation` in `outputs`, over one scored row at a + time, and returns it on `result.explanation`. Asking for one over more rows + is refused before sending. + ### Changed +- A `kumo-relational` explanation is no longer refused above the 1,000-row + context budget. That limit belonged to an earlier deployment; the Kumo + Relational SDM NIM explains under either budget. + - The Databricks example notebook is now a get-started notebook, `examples/databricks/kumo_relational_quickstart.ipynb`, in place of `adventureworks_relational.ipynb`. It deploys the model to a serving diff --git a/docs/reference/limits.md b/docs/reference/limits.md index 6a224862b..058ab7e48 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -24,7 +24,7 @@ Both endpoints enforce sizes, and most of these are the relational one's. | context rows | tabular | 10,000 | sampled at `materialize()`, refused before sending on a direct `Request` | | rows scored per call | relational | 1,000, or 200 for link prediction | at `materialize()` | | an explanation | relational | exactly one entity | before sending | -| an explanation | relational | only within the 1,000-row context budget | before sending | +| an explanation | tabular | exactly one row | before sending | | hops | relational | 6 | at `materialize()` | | tables | relational | 15 | against the graph the request carries, not the schema it was built from | diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/relational.py b/packages/structured-data-models-client/src/sdm_client/adapters/relational.py index ce1dce06f..e6e831524 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/relational.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/relational.py @@ -340,8 +340,7 @@ def _preflight( """Refuse what the endpoint enforces, before spending the round trip. An explanation is the narrow case: the model explains one entity at a - time, and only in the mode the smaller context budget selects. Sending - more gets a 500 that names neither limit, so both are checked here. + time, and the endpoint refuses more, so that limit is checked here. `sends` names the halves this call puts on the wire. A session pins its context at `fit()` and a scoring call carries only rows, so judging the @@ -373,19 +372,6 @@ def _preflight( details={'model': self.spec.model_id, 'scored': scored}, ) - cap = limits['context_rows_fast'] - if 'context' in sends and _asks_for_the_larger_context(payloads, cap): - raise UnsupportedRequestError( - f'an explanation is only produced in the mode the smaller ' - f'context budget selects, and this request was gathered with ' - f'num_context_samples above {cap:,}. Re-materialize with ' - f'num_context_samples={cap} or fewer', - details={ - 'model': self.spec.model_id, - 'context_rows_fast': cap, - }, - ) - def _check_class_count( self, payloads: tuple[dict[str, Any], ...], limits: Mapping[str, int] ) -> None: @@ -607,41 +593,6 @@ def _columns_over(payload: Mapping[str, Any], limit: int) -> list[str]: ] -def _asks_for_the_larger_context( - payloads: tuple[dict[str, Any], ...], cap: int -) -> bool: - r"""Whether this request selected the mode that will not explain. - - The endpoint gates explaining on ``inference.run_mode``, which the client - derives from the *requested* ``num_context_samples``. Counting the gathered - rows instead misses every graph that supplies less context than was asked - for: 120 rows gathered under a request for 1,500 passed this guard and the - endpoint's refusal escaped, naming a knob the caller never set. - - A payload the engine did not gather carries no ``run_mode``, and for that - one the rows are all there is to read. - """ - modes = { - str(payload['inference']['run_mode']) - for payload in payloads - if isinstance(payload.get('inference'), Mapping) - and payload['inference'].get('run_mode') is not None - } - if modes: - return modes != {'fast'} - return ( - sum( - len( - payload.get('context', {}) - .get('instance_table', {}) - .get('rows', []) - ) - for payload in payloads - ) - > cap - ) - - _WIRE_TARGET = 'TARGET' _A_PREDICTED_COLUMN = re.compile( r'^\s*PREDICT\s+([A-Za-z_]\w*\.[A-Za-z_]\w*)\s', re.IGNORECASE diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py index ce1e2bf1c..8d267e879 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py @@ -90,7 +90,9 @@ def prediction_plan( have to carry the same context, so splitting would multiply the upload without raising the cap. """ - yield self._envelope(request, outputs) + envelope = self._envelope(request, outputs) + self._check_explanation_rows(request, outputs) + yield envelope def session_plan( self, request: Request, outputs: tuple[str, ...] @@ -119,6 +121,7 @@ def session_prediction_plan( raise _a_row_the_pinned_schema_cannot_hold( request, declared, error ) from error + self._check_explanation_rows(request, outputs) yield { key: envelope[key] for key in _SESSION_SCORING if key in envelope } @@ -140,6 +143,14 @@ def normalize( f'match the contract: {error}', code='INVALID_RESPONSE', ) from error + explanation = next( + ( + item.explanation + for item in parsed.predictions + if item.explanation + ), + None, + ) return PredictionResult( model=parsed.model, request_id=parsed.id, @@ -149,6 +160,7 @@ def normalize( ), metadata=parsed.metadata, outputs=outputs, + explanation=explanation, ) def explain_error( @@ -162,6 +174,27 @@ def close(self) -> None: r"""Nothing to release: this adapter sends through the client's own transport and opens nothing of its own.""" + def _check_explanation_rows( + self, request: Request, outputs: tuple[str, ...] + ) -> None: + r"""Refuse an explanation over more rows than the endpoint explains. + + The endpoint explains one row at a time and refuses more with + ``UNSUPPORTED_OUTPUT``, so the limit is named here before anything is + sent. Only the rows being scored count: a session's pinned context is + never judged by it. + """ + if 'explanation' not in outputs: + return + allowed = self.spec.runtime.limits['explanation_predict_rows'] + scored = len(request.predict.instance_table) + if scored != allowed: + raise UnsupportedRequestError( + f'an explanation covers exactly {allowed} row, and this ' + f'request scores {scored}; ask for one at a time', + details={'model': self.spec.model_id, 'scored': scored}, + ) + def _envelope( self, request: Request, diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index ae67cb6e7..23e378e88 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -77,6 +77,7 @@ # large context to be told the limit at the far end. TABULAR_LIMITS = { 'context_rows_max': 10_000, + 'explanation_predict_rows': 1, } KUMO_TABULAR = ModelSpec( @@ -86,7 +87,9 @@ # The endpoint takes the coarse kind only and decides binary versus # multiclass itself, so this client never narrows it here. tasks=frozenset({'classification', 'regression'}), - outputs=frozenset({'prediction', 'probabilities', 'quantiles'}), + outputs=frozenset( + {'prediction', 'probabilities', 'quantiles', 'explanation'} + ), limits=TABULAR_LIMITS, ), ) diff --git a/packages/structured-data-models-client/tests/conftest.py b/packages/structured-data-models-client/tests/conftest.py new file mode 100644 index 000000000..7246350a5 --- /dev/null +++ b/packages/structured-data-models-client/tests/conftest.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Fixtures shared across the client's test modules.""" + +from __future__ import annotations + +from collections.abc import Callable + +import pandas as pd +import pytest + +from sdm_client import Request + + +@pytest.fixture +def tabular_request() -> Callable[..., Request]: + """Build a single-table request: 20 labelled context rows, ``rows`` to score. + + A classification target alternates 0/1; a regression target is a float. + """ + + def build(rows: int = 1, task: str = 'classification') -> Request: + labels = [ + i % 2 if task == 'classification' else float(i) for i in range(20) + ] + context = pd.DataFrame( + { + 'row_id': range(20), + 'age': [20.0 + i for i in range(20)], + 'churn': labels, + } + ) + predict = pd.DataFrame( + { + 'row_id': range(90, 90 + rows), + 'age': [33.0 + i for i in range(rows)], + } + ) + return Request( + context=context, predict=predict, target='churn', task=task + ) + + return build diff --git a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py index 6fba94865..f4ce97362 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py +++ b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py @@ -324,24 +324,29 @@ def explanation_plan(request): ) -def test_an_explanation_is_refused_on_the_mode_the_endpoint_refuses_it_on(): - """A graph that supplies 120 rows for a request of 1,500 still selects the - larger-context mode, and that is what the endpoint gates explaining on.""" - with pytest.raises(StructuredDataError) as raised: - explanation_plan(gathered(context_rows=120, run_mode='best')) - assert 'num_context_samples' in str(raised.value) +def test_explains_larger_context(): + """An explanation is sent under the larger-context run mode. + + Assumptions: + - The Kumo Relational SDM NIM explains under either context budget. + - The payload scores one entity. + """ + assert explanation_plan(gathered(context_rows=120, run_mode='best')) def test_a_small_context_asked_for_smally_still_explains(): assert explanation_plan(gathered(context_rows=120, run_mode='fast')) -def test_a_payload_naming_no_mode_is_judged_on_the_rows_it_carries(): - """Nothing the engine gathers arrives without a run_mode, but a hand-built - payload can, and the rows are then all there is to read.""" +def test_explains_without_mode(): + """An explanation is sent whatever the context size when no run mode is + named. + + Assumptions: + - The payload scores one entity and names no run_mode. + """ assert explanation_plan(gathered(context_rows=120, run_mode=None)) - with pytest.raises(StructuredDataError): - explanation_plan(gathered(context_rows=1_001, run_mode=None)) + assert explanation_plan(gathered(context_rows=1_001, run_mode=None)) # --- a context that teaches nothing, on both routes ------------------------- diff --git a/packages/structured-data-models-client/tests/test_limits_and_sessions.py b/packages/structured-data-models-client/tests/test_limits_and_sessions.py index 36a227ecf..6384e0f26 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_sessions.py +++ b/packages/structured-data-models-client/tests/test_limits_and_sessions.py @@ -83,18 +83,20 @@ def test_an_explanation_covering_more_than_one_entity_is_refused(): assert 'one at a time' in str(error.value) -def test_an_explanation_over_the_smaller_context_budget_is_refused(): - """Explaining forces the smaller budget, which caps context at 1,000.""" +def test_explains_large_context(): + """An explanation over a 5,000-row context is sent. + + Assumptions: + - The payload scores one entity. + - The SDM NIM has no 1,000-row explanation limit. + """ request = _Gathered(payload(predict_rows=1, context_rows=5_000)) - with pytest.raises(UnsupportedRequestError) as error: - list( - RelationalAdapter().prediction_plan( - request, ('prediction', 'explanation') - ) + plans = list( + RelationalAdapter().prediction_plan( + request, ('prediction', 'explanation') ) - message = str(error.value) - assert 'num_context_samples' in message - assert '1,000' in message + ) + assert plans[0]['output']['fields'] == ['prediction', 'explanation'] def test_an_explanation_within_both_limits_is_allowed(): diff --git a/packages/structured-data-models-client/tests/test_request_and_dispatch.py b/packages/structured-data-models-client/tests/test_request_and_dispatch.py index be422884f..6274092a9 100644 --- a/packages/structured-data-models-client/tests/test_request_and_dispatch.py +++ b/packages/structured-data-models-client/tests/test_request_and_dispatch.py @@ -177,14 +177,20 @@ def test_a_task_kind_the_model_does_not_serve_is_refused(): check_request_against_model(request, KUMO_RELATIONAL, ('prediction',)) -def test_an_output_field_the_model_cannot_produce_is_refused_by_name(): +def test_unserved_output_named(): + """An output the model cannot produce is refused, naming it and the + outputs the model does serve. + + Assumptions: + - kumo-tabular does not serve rankings. + """ request = simple_request() with pytest.raises(UnsupportedRequestError) as error: check_request_against_model( - request, KUMO_TABULAR, ('prediction', 'explanation') + request, KUMO_TABULAR, ('prediction', 'rankings') ) message = str(error.value) - assert 'explanation' in message + assert 'rankings' in message assert 'probabilities' in message diff --git a/packages/structured-data-models-client/tests/test_tabular_explanations.py b/packages/structured-data-models-client/tests/test_tabular_explanations.py new file mode 100644 index 000000000..a4e969b47 --- /dev/null +++ b/packages/structured-data-models-client/tests/test_tabular_explanations.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Asking ``kumo-tabular`` for an explanation. + +The endpoint explains one scored row at a time, so the client refuses more +before sending, and hands the explanation it gets back to the caller. +""" + +from __future__ import annotations + +import pytest + +from sdm_client import UnsupportedRequestError +from sdm_client.adapters.tabular import TabularAdapter +from sdm_client.base import check_request_against_model +from sdm_client.models import KUMO_TABULAR + +EXPLAIN = ('prediction', 'explanation') + + +def response(**record) -> dict: + """A one-record ``kumo-tabular`` response carrying ``record``'s fields.""" + return { + 'id': 'r', + 'model': 'kumo-tabular', + 'predictions': [{'row_index': 0, 'prediction': 1, **record}], + 'metadata': {}, + } + + +@pytest.mark.parametrize('task', ['classification', 'regression']) +def test_allowed(tabular_request, task): + """kumo-tabular accepts explanation as an output on both of its tasks. + + Assumptions: + - The request scores one row. + """ + check_request_against_model( + tabular_request(task=task), KUMO_TABULAR, EXPLAIN + ) + + +def test_one_row_sent(tabular_request): + """An explanation over one row is sent with explanation in its fields. + + Assumptions: + - The request scores exactly one row. + """ + (envelope,) = TabularAdapter().prediction_plan(tabular_request(), EXPLAIN) + assert envelope['output']['fields'] == list(EXPLAIN) + + +def test_rows_refused(tabular_request): + """An explanation over two rows is refused before sending, naming the count. + + Assumptions: + - The endpoint explains one row at a time. + """ + with pytest.raises(UnsupportedRequestError) as error: + list(TabularAdapter().prediction_plan(tabular_request(rows=2), EXPLAIN)) + assert 'one at a time' in str(error.value) + assert error.value.details['scored'] == 2 + + +def test_rows_without_explanation(tabular_request): + """Several rows are sent when no explanation is asked for. + + Assumptions: + - The one-row limit applies only to explanations. + """ + (envelope,) = TabularAdapter().prediction_plan( + tabular_request(rows=2), ('prediction',) + ) + assert envelope['output']['fields'] == ['prediction'] + + +def test_pin_not_judged(tabular_request): + """Pinning a session context is not refused by the row limit. + + Assumptions: + - Pinning sends no predict rows, so the limit has nothing to count. + """ + pinned = TabularAdapter().session_plan(tabular_request(rows=2), EXPLAIN) + assert 'predict' not in pinned + + +def test_session_rows_refused(tabular_request): + """Session scoring refuses an explanation over two rows. + + Assumptions: + - Scoring sends the predict rows, so the limit applies. + """ + with pytest.raises(UnsupportedRequestError): + list( + TabularAdapter().session_prediction_plan( + tabular_request(rows=2), EXPLAIN + ) + ) + + +def test_result_carries_explanation(): + """The explanation in the response reaches the result unchanged. + + Assumptions: + - The endpoint returns one record with an explanation dictionary. + """ + explanation = {'format': 'sdm_gradient_v1', 'details': {'outputs': []}} + result = TabularAdapter().normalize( + response(explanation=explanation), EXPLAIN + ) + assert result.explanation == explanation + + +def test_result_without_explanation(): + """A response without an explanation leaves the result's explanation None. + + Assumptions: + - The record carries no explanation field. + """ + result = TabularAdapter().normalize(response(), ('prediction',)) + assert result.explanation is None diff --git a/qa/probe_tabular.py b/qa/probe_tabular.py index 56d0ae9dc..a903cadb4 100644 --- a/qa/probe_tabular.py +++ b/qa/probe_tabular.py @@ -331,11 +331,9 @@ def run(client): task='classification', ), model=KumoTabular(), - outputs=['explanation'], - ) - fail( - e, 'explanation was accepted by a model that does not serve it' + outputs=['rankings'], ) + fail(e, 'rankings was accepted by a model that does not serve it') except StructuredDataError as exc: note(e, str(exc)[:120]) diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index 9306e1b29..b7c4dc9ac 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -163,8 +163,7 @@ would break rather than letting the endpoint return a 500. **The table limit is on the graph a request carries, not on the schema.** A schema may describe far more than fifteen tables; what counts is how many the -query actually reaches. An explanation has a second limit besides its one row: -it must fit the 1,000-row context budget. +query actually reaches. ## Asking for more than one answer each diff --git a/skills/tabular/SKILL.md b/skills/tabular/SKILL.md index 1038c758f..7d9434634 100644 --- a/skills/tabular/SKILL.md +++ b/skills/tabular/SKILL.md @@ -29,16 +29,17 @@ with SDMClient(url='http://localhost:8000') as client: | | `kumo-tabular` | |---|---| | Tasks | `classification`, `regression` | -| Outputs | `prediction`, `probabilities`, `quantiles` | +| Outputs | `prediction`, `probabilities`, `quantiles`, `explanation` | | Several tables | no | | Relationships | no | -| Explanations | no | +| Explanations | yes | | Sessions | yes | | Context rows | 10,000 | Anything outside that list belongs to the relational model. Asking for -`rankings`, an `explanation`, or a forecast here is refused before a request -is sent. +`rankings` or a forecast here is refused before a request is sent. An +`explanation` covers one scored row at a time, and asking for one over more +rows is refused before sending. Outputs narrow again by task, and that narrowing happens at the endpoint rather than in the client: `probabilities` on a classification, `quantiles` on a @@ -205,7 +206,8 @@ names what to change. | Refusal | Cause | |---|---| | a task the model does not serve | `forecasting`, `temporal_link_prediction` and the rest are relational | -| an output it does not produce | `rankings`, `explanation`, `embeddings`, `scores` | +| an output it does not produce | `rankings`, `embeddings`, `scores` | +| an explanation over more than one row | the endpoint explains one row at a time | | a context above the cap | more than 10,000 labelled rows in a direct `Request` | | relationships | this model reads one table; use `kumo-relational` |