fix(devserver): honor request max_concurrency (#1072)

Follow-up to braintrustdata/braintrust#24075,
which made the playground and "+ Experiment" dialog send
`max_concurrency` to remote eval dev servers. This PR makes the dev
server honor it.

Companion JS SDK PR:
braintrustdata/braintrust-sdk-javascript#2730

### Context

The playground **Max concurrency** slider and the "+ Experiment" **Max
concurrency** field had no effect on remote eval tasks.
braintrustdata/braintrust#24075 (merged) made the UI send
`max_concurrency` in the `POST /eval` request to the dev server, but
`parse_eval_body` never read the field and `run_eval` built the
`EvalAsync` kwargs only from the registered evaluator, so it was
ignored. The value set on the Eval in code always won. A customer
lowered the slider to avoid Azure 429s and saw no change (Pylon 20534,
COR-288).

### Description

Parse `max_concurrency` from the eval request and pass it to
`EvalAsync`.

**The request value can lower the evaluator's limit but never raise it**
(`min(request, evaluator)`; if the evaluator sets no limit, the request
value applies). This is deliberate: the playground always sends a value
and the slider defaults to 10, so letting the request override outright
would silently raise every code-set limit below 10 (e.g.
`max_concurrency=5` → 10) as soon as users upgrade the SDK. The
trade-off is that the UI can't push concurrency *above* what's in code.

Other details:
- `null` or a missing value keeps the evaluator's limit, which also
covers older UIs that don't send the field.
- Fractional values are rounded down (the "+ Experiment" field is a free
number input validated only as `>= 1`). Values below 1 or non-numbers
are rejected with a 400.

### Testing

- New tests cover: request lowers the limit, request can't raise it,
request applies when the evaluator has no limit, null or omitted request
keeps the evaluator's limit, rounding down, and invalid values. They
fail without the fix and pass with it.
- `nox -s test_cli` passes (43 passed, 2 skipped); `make lint` is clean.
- End to end: ran the real dev server (`eval --dev`) from this branch
against a Braintrust org, with a 12-row eval whose task records how many
calls are in flight, and sent `POST /eval` requests shaped like the
playground's. Same run against the released Python 0.49.2 (PyPI) for
comparison:

| Evaluator limit in code | Request `max_concurrency` | Expected peak |
This branch | Released |
|---|---|---|---|---|
| 5 | 2 | 2 | **2** | 5 |
| 5 | 10 | 5 | 5 | 5 |
| 5 | `null` | 5 | 5 | 5 |
| 5 | omitted | 5 | 5 | 5 |
| 5 | 2.5 | 2 | **2** | 5 |
| none | 3 | 3 | **3** | 12 |
| none | omitted | 12 | 12 | 12 |

Refs: COR-288

Co-authored-by: Paul Tancre <paultancre@p-tancre.home>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Author: 3 people

py/src/braintrust/devserver/schemas.py

@@ -42,6 +42,7 @@ class ParsedEvalBody(TypedDict, total=False):
     project_id: str
     parent: str | ParsedParent
     stream: bool
+    max_concurrency: int
 
 
 def validate_typed_dict(data: Any, typed_dict_class: type, path: str = "") -> dict[str, Any]:
@@ -260,4 +261,10 @@ def parse_eval_body(request_data: str | bytes | dict) -> ParsedEvalBody:
             raise ValidationError("stream must be a boolean")
         parsed["stream"] = data["stream"]
 
+    if data.get("max_concurrency") is not None:
+        max_concurrency = data["max_concurrency"]
+        if not isinstance(max_concurrency, (int, float)) or isinstance(max_concurrency, bool) or max_concurrency < 1:
+            raise ValidationError("max_concurrency must be a number greater than or equal to 1")
+        parsed["max_concurrency"] = int(max_concurrency)
+
     return parsed

py/src/braintrust/devserver/server.py

@@ -234,6 +234,16 @@ def stream_fn(event: SSEProgressEvent):
     # registered against.
     project_id = eval_data.get("project_id") or evaluator.project_id
 
+    # The request can lower the evaluator's max_concurrency but never raise it,
+    # so the playground's default slider value can't exceed a limit set in code.
+    request_max_concurrency = eval_data.get("max_concurrency")
+    if request_max_concurrency is not None:
+        eval_kwargs["max_concurrency"] = (
+            request_max_concurrency
+            if evaluator.max_concurrency is None
+            else min(request_max_concurrency, evaluator.max_concurrency)
+        )
+
     try:
         eval_task = asyncio.create_task(
             EvalAsync(

py/src/braintrust/devserver/test_server_integration.py

@@ -480,3 +480,93 @@ async def fake_eval_async(*, project_id, **_kwargs):
 
     assert response.status_code == 200
     assert captured["project_id"] == "request-explicit-project-id"
+
+
+_OMIT = object()
+
+
+@pytest.mark.parametrize(
+    "evaluator_max_concurrency,request_max_concurrency,expected",
+    [
+        pytest.param(5, 2, 2, id="request-lowers-evaluator"),
+        pytest.param(5, 10, 5, id="request-cannot-raise-evaluator"),
+        pytest.param(None, 3, 3, id="request-applies-when-evaluator-unset"),
+        pytest.param(5, None, 5, id="null-request-keeps-evaluator"),
+        pytest.param(5, _OMIT, 5, id="omitted-request-keeps-evaluator"),
+        pytest.param(None, _OMIT, None, id="both-unset"),
+    ],
+)
+def test_eval_request_max_concurrency_caps_evaluator(
+    api_key, org_name, monkeypatch, evaluator_max_concurrency, request_max_concurrency, expected
+):
+    """The playground's max concurrency can lower, but never raise, the evaluator's own limit."""
+    from braintrust import Evaluator
+    from braintrust.devserver import server as devserver_module
+    from braintrust.devserver.server import create_app
+    from braintrust.logger import BraintrustState
+    from starlette.testclient import TestClient
+
+    evaluator = Evaluator(
+        project_name="test-max-concurrency",
+        eval_name="max-concurrency-eval",
+        data=lambda: [{"input": "ping", "expected": "pong"}],
+        task=lambda input, _hooks: "pong",
+        scores=[],
+        experiment_name=None,
+        metadata=None,
+        max_concurrency=evaluator_max_concurrency,
+    )
+
+    captured: dict[str, Any] = {}
+
+    async def fake_cached_login(**_kwargs):
+        return BraintrustState()
+
+    class FakeSummary:
+        def as_dict(self):
+            return {"experiment_name": evaluator.eval_name, "project_name": "", "scores": {}}
+
+    class FakeResult:
+        summary = FakeSummary()
+
+    async def fake_eval_async(*, max_concurrency, **_kwargs):
+        captured["max_concurrency"] = max_concurrency
+        return FakeResult()
+
+    monkeypatch.setattr(devserver_module, "cached_login", fake_cached_login)
+    monkeypatch.setattr(devserver_module, "EvalAsync", fake_eval_async)
+
+    eval_request: dict[str, Any] = {
+        "name": "max-concurrency-eval",
+        "stream": False,
+        "data": [{"input": "ping", "expected": "pong"}],
+    }
+    if request_max_concurrency is not _OMIT:
+        eval_request["max_concurrency"] = request_max_concurrency
+
+    response = TestClient(create_app([evaluator])).post(
+        "/eval",
+        headers={
+            "x-bt-auth-token": api_key,
+            "x-bt-org-name": org_name,
+            "Content-Type": "application/json",
+        },
+        json=eval_request,
+    )
+
+    assert response.status_code == 200
+    assert captured["max_concurrency"] == expected
+
+
+def test_parse_eval_body_rounds_down_fractional_max_concurrency():
+    from braintrust.devserver.schemas import parse_eval_body
+
+    assert parse_eval_body({"name": "eval", "max_concurrency": 2.5})["max_concurrency"] == 2
+
+
+@pytest.mark.parametrize("max_concurrency", [0, -1, 0.5, "2", True])
+def test_parse_eval_body_rejects_invalid_max_concurrency(max_concurrency):
+    from braintrust.devserver.schemas import ValidationError, parse_eval_body
+
+    with pytest.raises(ValidationError, match="max_concurrency must be a number greater than or equal to 1"):
+        parse_eval_body({"name": "eval", "max_concurrency": max_concurrency})