fix(dev): honor request max_concurrency

.changeset/devserver-request-max-concurrency.md

@@ -0,0 +1,5 @@
+---
+"braintrust": patch
+---
+
+fix(dev): honor max_concurrency from remote eval requests, capped by the evaluator's maxConcurrency

js/dev/server.test.ts

@@ -19,3 +19,56 @@ describe("remote eval data", () => {
     });
   });
 });
+
+describe("remote eval max concurrency", () => {
+  test.each([
+    { evaluator: 5, request: 2, expected: 2 },
+    { evaluator: 5, request: 10, expected: 5 },
+    { evaluator: undefined, request: 3, expected: 3 },
+    { evaluator: 5, request: null, expected: 5 },
+    { evaluator: 5, request: undefined, expected: 5 },
+    { evaluator: undefined, request: undefined, expected: undefined },
+  ])(
+    "evaluator $evaluator with request $request resolves to $expected",
+    ({ evaluator, request, expected }) => {
+      expect(
+        _exportsForTestingOnly.resolveMaxConcurrency(evaluator, request),
+      ).toBe(expected);
+    },
+  );
+
+  test("parses max_concurrency from the request body", () => {
+    expect(
+      evalBodySchema.parse({
+        name: "remote-evaluator",
+        data: { experiment_name: "source-experiment" },
+        max_concurrency: 2,
+      }).max_concurrency,
+    ).toBe(2);
+  });
+
+  test("rounds down a fractional max_concurrency", () => {
+    expect(
+      evalBodySchema.parse({
+        name: "remote-evaluator",
+        data: { experiment_name: "source-experiment" },
+        max_concurrency: 2.5,
+      }).max_concurrency,
+    ).toBe(2);
+  });
+
+  test.each([0, -1, 0.5, "2"])(
+    "rejects invalid max_concurrency %s",
+    (maxConcurrency) => {
+      const result = evalBodySchema.safeParse({
+        name: "remote-evaluator",
+        data: { experiment_name: "source-experiment" },
+        max_concurrency: maxConcurrency,
+      });
+      expect(result.success).toBe(false);
+      expect(result.error?.issues.map((issue) => issue.path)).toEqual([
+        ["max_concurrency"],
+      ]);
+    },
+  );
+});

js/dev/server.ts

@@ -146,6 +146,7 @@ export function runDevServer(
         data,
         scores,
         stream,
+        max_concurrency,
       } = evalBodySchema.parse(req.body);
 
       if (!req.ctx?.state) {
@@ -229,6 +230,10 @@ export function runDevServer(
             state,
             experimentName: experiment_name ?? undefined,
             projectId: project_id ?? undefined,
+            maxConcurrency: resolveMaxConcurrency(
+              evaluator.maxConcurrency,
+              max_concurrency,
+            ),
           },
           {
             // Avoid printing the bar to the console.
@@ -340,6 +345,21 @@ async function getDataset(
   }
 }
 
+// The request can lower the evaluator's maxConcurrency but never raise it, so
+// the playground's default slider value can't exceed a limit set in code.
+function resolveMaxConcurrency(
+  evaluatorMaxConcurrency: number | undefined,
+  requestMaxConcurrency: number | null | undefined,
+): number | undefined {
+  if (requestMaxConcurrency == null) {
+    return evaluatorMaxConcurrency;
+  }
+  if (evaluatorMaxConcurrency === undefined) {
+    return requestMaxConcurrency;
+  }
+  return Math.min(evaluatorMaxConcurrency, requestMaxConcurrency);
+}
+
 function makeScorer(
   state: BraintrustState,
   name: string,
@@ -379,4 +399,5 @@ function makeScorer(
 
 export const _exportsForTestingOnly = {
   getDataset,
+  resolveMaxConcurrency,
 };

js/dev/types.ts

@@ -24,6 +24,7 @@ export const evalBodySchema = z.object({
   project_id: z.string().nullish(),
   parent: invokeParentSchema.optional(),
   stream: z.boolean().optional(),
+  max_concurrency: z.number().min(1).transform(Math.floor).nullish(),
 });
 
 export type EvaluatorManifest = Record<