From bc0e88a6d9d9a0bbcf35ff6e83289289caad3c3d Mon Sep 17 00:00:00 2001
From: Edward Irby <e.irby@pm.me>
Date: Wed, 6 May 2026 19:59:12 -0700
Subject: [PATCH] fix: run command graders from trial cwd

---
 src/eval.ts            |  4 +++-
 src/tests/eval.spec.ts | 50 +++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/src/eval.ts b/src/eval.ts
index 70a7efd..d050ed3 100644
--- a/src/eval.ts
+++ b/src/eval.ts
@@ -208,6 +208,7 @@ const toInvocationEvidence = (params: {
 const runCommand = async (params: {
   commandSpec: z.infer<typeof RunEvalInputSchema.shape.adapter>
   stdinJson: string
+  cwd?: string
 }): Promise<{ invocation: CommandInvocation; stdout: string; stderr: string; timedOut: boolean }> => {
   const timeoutMs = params.commandSpec.timeoutMs ?? DEFAULT_TIMEOUT_MS
   const maxOutputBytes = params.commandSpec.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES
@@ -221,7 +222,7 @@ const runCommand = async (params: {
     stdin: new TextEncoder().encode(params.stdinJson),
     stdout: 'pipe',
     stderr: 'pipe',
-    cwd: process.cwd(),
+    cwd: params.cwd ?? process.cwd(),
   })
 
   let timedOut = false
@@ -616,6 +617,7 @@ const runCommandGrader = async (params: {
     commandRun = await runCommand({
       commandSpec: params.grader.options,
       stdinJson,
+      cwd: params.cwd,
     })
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error)
diff --git a/src/tests/eval.spec.ts b/src/tests/eval.spec.ts
index 4874186..4ae6ad1 100644
--- a/src/tests/eval.spec.ts
+++ b/src/tests/eval.spec.ts
@@ -1,5 +1,5 @@
 import { describe, expect, test } from 'bun:test'
-import { mkdir, rm } from 'node:fs/promises'
+import { mkdir, realpath, rm } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { EvalInputSchema, TaskSchema } from '../eval.schemas.ts'
@@ -533,6 +533,54 @@ describe('grade mode', () => {
     }
   })
 
+  test('runs command graders from the trial cwd', async () => {
+    const dir = join(tmpdir(), `eval-grader-cwd-${Date.now()}`)
+    await mkdir(dir, { recursive: true })
+    try {
+      const cwd = await realpath(dir)
+      const row = createCompletedRow()
+      row.trial.cwd = cwd
+      const payload = JSON.stringify({
+        mode: 'grade',
+        graders: [
+          {
+            id: 'cwd-grader',
+            type: 'command',
+            options: {
+              command: [
+                'bun',
+                '-e',
+                'if (process.cwd() !== Bun.argv[1]) { process.stderr.write(process.cwd()); process.exit(1) }',
+                cwd,
+              ],
+              output: 'exit_code',
+            },
+          },
+        ],
+      })
+
+      const cliPath = `${import.meta.dir}/../cli.ts`
+      const proc = Bun.spawn(['bun', cliPath, 'eval', payload], {
+        stdin: new TextEncoder().encode(`${JSON.stringify(row)}\n`),
+        stdout: 'pipe',
+        stderr: 'pipe',
+      })
+      const [stdout, , exitCode] = await Promise.all([
+        new Response(proc.stdout).text(),
+        new Response(proc.stderr).text(),
+        proc.exited,
+      ])
+
+      expect(exitCode).toBe(0)
+      const graded = JSON.parse(stdout.trim())
+      const result = graded.graderResults[0]
+      expect(result.pass).toBe(true)
+      expect(result.outcome.invocation.exitCode).toBe(0)
+    } finally {
+      await rm(dir, { force: true, recursive: true })
+    }
+  })
+
   test('grader_json success includes normalized invocation without retaining stdout text', async () => {
     const dir = join(tmpdir(), `eval-grader-json-success-${Date.now()}`)
     await mkdir(dir, { recursive: true })