Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,18 @@ these packages. All five distributions share one version and one entry.

## Unreleased

### Added

- `kumo-tabular` accepts `explanation` in `outputs`, over one scored row at a
time, and returns it on `result.explanation`. Asking for one over more rows
is refused before sending.

### Changed

- A `kumo-relational` explanation is no longer refused above the 1,000-row
context budget. That limit belonged to an earlier deployment; the Kumo
Relational SDM NIM explains under either budget.

- The Databricks example notebook is now a get-started notebook,
`examples/databricks/kumo_relational_quickstart.ipynb`, in place of
`adventureworks_relational.ipynb`. It deploys the model to a serving
Expand Down
2 changes: 1 addition & 1 deletion docs/reference/limits.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ Both endpoints enforce sizes, and most of these are the relational one's.
| context rows | tabular | 10,000 | sampled at `materialize()`, refused before sending on a direct `Request` |
| rows scored per call | relational | 1,000, or 200 for link prediction | at `materialize()` |
| an explanation | relational | exactly one entity | before sending |
| an explanation | relational | only within the 1,000-row context budget | before sending |
| an explanation | tabular | exactly one row | before sending |
| hops | relational | 6 | at `materialize()` |
| tables | relational | 15 | against the graph the request carries, not the schema it was built from |

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -340,8 +340,7 @@ def _preflight(
"""Refuse what the endpoint enforces, before spending the round trip.

An explanation is the narrow case: the model explains one entity at a
time, and only in the mode the smaller context budget selects. Sending
more gets a 500 that names neither limit, so both are checked here.
time, and the endpoint refuses more, so that limit is checked here.

`sends` names the halves this call puts on the wire. A session pins its
context at `fit()` and a scoring call carries only rows, so judging the
Expand Down Expand Up @@ -373,19 +372,6 @@ def _preflight(
details={'model': self.spec.model_id, 'scored': scored},
)

cap = limits['context_rows_fast']
if 'context' in sends and _asks_for_the_larger_context(payloads, cap):
raise UnsupportedRequestError(
f'an explanation is only produced in the mode the smaller '
f'context budget selects, and this request was gathered with '
f'num_context_samples above {cap:,}. Re-materialize with '
f'num_context_samples={cap} or fewer',
details={
'model': self.spec.model_id,
'context_rows_fast': cap,
},
)

def _check_class_count(
self, payloads: tuple[dict[str, Any], ...], limits: Mapping[str, int]
) -> None:
Expand Down Expand Up @@ -607,41 +593,6 @@ def _columns_over(payload: Mapping[str, Any], limit: int) -> list[str]:
]


def _asks_for_the_larger_context(
payloads: tuple[dict[str, Any], ...], cap: int
) -> bool:
r"""Whether this request selected the mode that will not explain.

The endpoint gates explaining on ``inference.run_mode``, which the client
derives from the *requested* ``num_context_samples``. Counting the gathered
rows instead misses every graph that supplies less context than was asked
for: 120 rows gathered under a request for 1,500 passed this guard and the
endpoint's refusal escaped, naming a knob the caller never set.

A payload the engine did not gather carries no ``run_mode``, and for that
one the rows are all there is to read.
"""
modes = {
str(payload['inference']['run_mode'])
for payload in payloads
if isinstance(payload.get('inference'), Mapping)
and payload['inference'].get('run_mode') is not None
}
if modes:
return modes != {'fast'}
return (
sum(
len(
payload.get('context', {})
.get('instance_table', {})
.get('rows', [])
)
for payload in payloads
)
> cap
)


_WIRE_TARGET = 'TARGET'
_A_PREDICTED_COLUMN = re.compile(
r'^\s*PREDICT\s+([A-Za-z_]\w*\.[A-Za-z_]\w*)\s', re.IGNORECASE
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,9 @@ def prediction_plan(
have to carry the same context, so splitting would multiply the upload
without raising the cap.
"""
yield self._envelope(request, outputs)
envelope = self._envelope(request, outputs)
self._check_explanation_rows(request, outputs)
yield envelope

def session_plan(
self, request: Request, outputs: tuple[str, ...]
Expand Down Expand Up @@ -119,6 +121,7 @@ def session_prediction_plan(
raise _a_row_the_pinned_schema_cannot_hold(
request, declared, error
) from error
self._check_explanation_rows(request, outputs)
yield {
key: envelope[key] for key in _SESSION_SCORING if key in envelope
}
Expand All @@ -140,6 +143,14 @@ def normalize(
f'match the contract: {error}',
code='INVALID_RESPONSE',
) from error
explanation = next(
(
item.explanation
for item in parsed.predictions
if item.explanation
),
None,
)
return PredictionResult(
model=parsed.model,
request_id=parsed.id,
Expand All @@ -149,6 +160,7 @@ def normalize(
),
metadata=parsed.metadata,
outputs=outputs,
explanation=explanation,
)

def explain_error(
Expand All @@ -162,6 +174,27 @@ def close(self) -> None:
r"""Nothing to release: this adapter sends through the client's own
transport and opens nothing of its own."""

def _check_explanation_rows(
self, request: Request, outputs: tuple[str, ...]
) -> None:
r"""Refuse an explanation over more rows than the endpoint explains.

The endpoint explains one row at a time and refuses more with
``UNSUPPORTED_OUTPUT``, so the limit is named here before anything is
sent. Only the rows being scored count: a session's pinned context is
never judged by it.
"""
if 'explanation' not in outputs:
return
allowed = self.spec.runtime.limits['explanation_predict_rows']
scored = len(request.predict.instance_table)
if scored != allowed:
raise UnsupportedRequestError(
f'an explanation covers exactly {allowed} row, and this '
f'request scores {scored}; ask for one at a time',
details={'model': self.spec.model_id, 'scored': scored},
)

def _envelope(
self,
request: Request,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
# large context to be told the limit at the far end.
TABULAR_LIMITS = {
'context_rows_max': 10_000,
'explanation_predict_rows': 1,
}

KUMO_TABULAR = ModelSpec(
Expand All @@ -86,7 +87,9 @@
# The endpoint takes the coarse kind only and decides binary versus
# multiclass itself, so this client never narrows it here.
tasks=frozenset({'classification', 'regression'}),
outputs=frozenset({'prediction', 'probabilities', 'quantiles'}),
outputs=frozenset(
{'prediction', 'probabilities', 'quantiles', 'explanation'}
),
limits=TABULAR_LIMITS,
),
)
Expand Down
45 changes: 45 additions & 0 deletions packages/structured-data-models-client/tests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Fixtures shared across the client's test modules."""

from __future__ import annotations

from collections.abc import Callable

import pandas as pd
import pytest

from sdm_client import Request


@pytest.fixture
def tabular_request() -> Callable[..., Request]:
"""Build a single-table request: 20 labelled context rows, ``rows`` to score.

A classification target alternates 0/1; a regression target is a float.
"""

def build(rows: int = 1, task: str = 'classification') -> Request:
labels = [
i % 2 if task == 'classification' else float(i) for i in range(20)
]
context = pd.DataFrame(
{
'row_id': range(20),
'age': [20.0 + i for i in range(20)],
'churn': labels,
}
)
predict = pd.DataFrame(
{
'row_id': range(90, 90 + rows),
'age': [33.0 + i for i in range(rows)],
}
)
return Request(
context=context, predict=predict, target='churn', task=task
)

return build
Original file line number Diff line number Diff line change
Expand Up @@ -324,24 +324,29 @@ def explanation_plan(request):
)


def test_an_explanation_is_refused_on_the_mode_the_endpoint_refuses_it_on():
"""A graph that supplies 120 rows for a request of 1,500 still selects the
larger-context mode, and that is what the endpoint gates explaining on."""
with pytest.raises(StructuredDataError) as raised:
explanation_plan(gathered(context_rows=120, run_mode='best'))
assert 'num_context_samples' in str(raised.value)
def test_explains_larger_context():
"""An explanation is sent under the larger-context run mode.

Assumptions:
- The Kumo Relational SDM NIM explains under either context budget.
- The payload scores one entity.
"""
assert explanation_plan(gathered(context_rows=120, run_mode='best'))


def test_a_small_context_asked_for_smally_still_explains():
assert explanation_plan(gathered(context_rows=120, run_mode='fast'))


def test_a_payload_naming_no_mode_is_judged_on_the_rows_it_carries():
"""Nothing the engine gathers arrives without a run_mode, but a hand-built
payload can, and the rows are then all there is to read."""
def test_explains_without_mode():
"""An explanation is sent whatever the context size when no run mode is
named.

Assumptions:
- The payload scores one entity and names no run_mode.
"""
assert explanation_plan(gathered(context_rows=120, run_mode=None))
with pytest.raises(StructuredDataError):
explanation_plan(gathered(context_rows=1_001, run_mode=None))
assert explanation_plan(gathered(context_rows=1_001, run_mode=None))


# --- a context that teaches nothing, on both routes -------------------------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,18 +83,20 @@ def test_an_explanation_covering_more_than_one_entity_is_refused():
assert 'one at a time' in str(error.value)


def test_an_explanation_over_the_smaller_context_budget_is_refused():
"""Explaining forces the smaller budget, which caps context at 1,000."""
def test_explains_large_context():
"""An explanation over a 5,000-row context is sent.

Assumptions:
- The payload scores one entity.
- The SDM NIM has no 1,000-row explanation limit.
"""
request = _Gathered(payload(predict_rows=1, context_rows=5_000))
with pytest.raises(UnsupportedRequestError) as error:
list(
RelationalAdapter().prediction_plan(
request, ('prediction', 'explanation')
)
plans = list(
RelationalAdapter().prediction_plan(
request, ('prediction', 'explanation')
)
message = str(error.value)
assert 'num_context_samples' in message
assert '1,000' in message
)
assert plans[0]['output']['fields'] == ['prediction', 'explanation']


def test_an_explanation_within_both_limits_is_allowed():
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -177,14 +177,20 @@ def test_a_task_kind_the_model_does_not_serve_is_refused():
check_request_against_model(request, KUMO_RELATIONAL, ('prediction',))


def test_an_output_field_the_model_cannot_produce_is_refused_by_name():
def test_unserved_output_named():
"""An output the model cannot produce is refused, naming it and the
outputs the model does serve.

Assumptions:
- kumo-tabular does not serve rankings.
"""
request = simple_request()
with pytest.raises(UnsupportedRequestError) as error:
check_request_against_model(
request, KUMO_TABULAR, ('prediction', 'explanation')
request, KUMO_TABULAR, ('prediction', 'rankings')
)
message = str(error.value)
assert 'explanation' in message
assert 'rankings' in message
assert 'probabilities' in message


Expand Down
Loading
Loading