From 1d1a54d162cb0fd808963691556f524f0fb3408c Mon Sep 17 00:00:00 2001 From: lixenstrand Date: Fri, 21 Aug 2026 17:45:45 +0200 Subject: [PATCH] fix(codex): read text-block tool output --- src/cli.test.ts | 38 +++++++++++++++++++ src/vendors/codex/transcript.test.ts | 12 ++++++ src/vendors/codex/transcript.ts | 13 ++++++- .../codex-custom-tool-output-formats.jsonl | 4 ++ .../transcripts/codex-tdd-test-failed.jsonl | 4 +- test/integration/enforce-tdd-codex.test.ts | 2 +- 6 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/transcripts/codex-custom-tool-output-formats.jsonl diff --git a/src/cli.test.ts b/src/cli.test.ts index 23fa6ac..8e0bbce 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -5,6 +5,7 @@ import { describe, it, expect } from 'vitest' import { failClosedResponse, run, type RunResult, type Vendor } from './cli.js' import type { Config } from './config.js' import type { Agent, SessionEvent } from './types.js' +import { enforceTdd } from './rules/enforce-tdd.js' import { enforceFilenameCasing } from './rules/enforce-filename-casing.js' import { forbidContentPattern } from './rules/forbid-content-pattern.js' import type { FileContent, Rule } from './rules/contract.js' @@ -150,6 +151,43 @@ describe('cli', () => { expect(captured?.some((e) => e.kind === 'command')).toBe(true) }) + it('passes Codex Vitest text-block output to enforceTdd as red evidence', async () => { + let capturedPrompt = '' + const redAwareAgent: Agent = { + reason: (prompt) => { + capturedPrompt = prompt + const observedRed = + prompt.includes('AssertionError: expected undefined to be 5') && + /Test Files\s+1 failed/.test(prompt) + return Promise.resolve({ + kind: observedRed ? 'pass' : 'violation', + reason: observedRed ? '' : 'Vitest red evidence was not observed', + }) + }, + } + const payload = JSON.stringify({ + transcript_path: 'test/fixtures/transcripts/codex-tdd-test-failed.jsonl', + cwd: '/workspaces/probity', + tool_name: 'apply_patch', + tool_input: { + command: + '*** Begin Patch\n' + + '*** Add File: src/calculator.ts\n' + + '+export const add = (a: number, b: number) => a + b\n' + + '*** End Patch\n', + }, + }) + + const { response } = await setup({ + vendor: 'codex', + payload, + config: { rules: [enforceTdd({ fastPath: false })], ai: redAwareAgent }, + }) + + expect(capturedPrompt).toContain('npx vitest run src/calculator.test.ts') + expect(response).toBe('') + }) + it('threads a ctx.readFile capability into the RuleContext for write actions', async () => { let captured: FileContent | undefined const captureRule: Rule = async (_action, ctx) => { diff --git a/src/vendors/codex/transcript.test.ts b/src/vendors/codex/transcript.test.ts index 310d618..552e6c5 100644 --- a/src/vendors/codex/transcript.test.ts +++ b/src/vendors/codex/transcript.test.ts @@ -41,6 +41,18 @@ describe('codex transcript', () => { } }) + it('reads custom tool output from legacy strings and current text blocks', async () => { + const events = await readTranscript( + 'test/fixtures/transcripts/codex-custom-tool-output-formats.jsonl', + ) + + const actions = events.filter((event) => event.kind === 'action') + expect(actions.map((event) => event.output)).toEqual([ + 'legacy output', + 'current output\nsecond block', + ]) + }) + it('parses JSON-encoded function_call arguments into objects at the boundary', async () => { const events = await readTranscript( 'test/fixtures/transcripts/codex-basic.jsonl', diff --git a/src/vendors/codex/transcript.ts b/src/vendors/codex/transcript.ts index acfc408..29e9fbe 100644 --- a/src/vendors/codex/transcript.ts +++ b/src/vendors/codex/transcript.ts @@ -42,12 +42,23 @@ const CustomToolCallSchema = z.object({ }), }) +const CustomToolOutputSchema = z + .union([ + z.string(), + z.array(z.object({ type: z.literal('text'), text: z.string() })), + ]) + .transform((output) => + typeof output === 'string' + ? output + : output.map((block) => block.text).join('\n'), + ) + const CustomToolCallOutputSchema = z.object({ type: z.literal('response_item'), payload: z.object({ type: z.literal('custom_tool_call_output'), call_id: z.string(), - output: z.string(), + output: CustomToolOutputSchema, }), }) diff --git a/test/fixtures/transcripts/codex-custom-tool-output-formats.jsonl b/test/fixtures/transcripts/codex-custom-tool-output-formats.jsonl new file mode 100644 index 0000000..715d3b6 --- /dev/null +++ b/test/fixtures/transcripts/codex-custom-tool-output-formats.jsonl @@ -0,0 +1,4 @@ +{"type":"response_item","payload":{"type":"custom_tool_call","name":"legacy_tool","input":"legacy input","call_id":"call_legacy"}} +{"type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call_legacy","output":"legacy output"}} +{"type":"response_item","payload":{"type":"custom_tool_call","name":"current_tool","input":"current input","call_id":"call_current"}} +{"type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call_current","output":[{"type":"text","text":"current output"},{"type":"text","text":"second block"}]}} diff --git a/test/fixtures/transcripts/codex-tdd-test-failed.jsonl b/test/fixtures/transcripts/codex-tdd-test-failed.jsonl index a867271..0a5457b 100644 --- a/test/fixtures/transcripts/codex-tdd-test-failed.jsonl +++ b/test/fixtures/transcripts/codex-tdd-test-failed.jsonl @@ -1,5 +1,5 @@ {"timestamp":"2026-01-01T00:00:00.000Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"text":"Write a simple add function in src/calculator.ts using TDD."}]}} {"timestamp":"2026-01-01T00:00:01.000Z","type":"response_item","payload":{"type":"custom_tool_call","status":"completed","call_id":"call_test_write","name":"apply_patch","input":"*** Begin Patch\n*** Add File: /workspaces/probity/src/calculator.test.ts\n+import { describe, expect, it } from 'vitest'\n+\n+import { add } from './calculator.js'\n+\n+describe('calculator', () => {\n+ it('adds two numbers', () => {\n+ expect(add(2, 3)).toBe(5)\n+ })\n+})\n*** End Patch\n"}} {"timestamp":"2026-01-01T00:00:01.500Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call_test_write","output":"Success."}} -{"timestamp":"2026-01-01T00:00:02.000Z","type":"response_item","payload":{"type":"function_call","name":"shell","arguments":"{\"command\":\"npx vitest run src/calculator.test.ts\"}","call_id":"call_test_run"}} -{"timestamp":"2026-01-01T00:00:03.000Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call_test_run","output":" FAIL src/calculator.test.ts > calculator > adds two numbers\nAssertionError: expected undefined to be 5 // Object.is equality\n\n- Expected: 5\n+ Received: undefined\n\n at src/calculator.test.ts:6:25\n\nTest Files 1 failed (1)\n Tests 1 failed (1)"}} +{"timestamp":"2026-01-01T00:00:02.000Z","type":"response_item","payload":{"type":"custom_tool_call","name":"exec_command","input":"{\"cmd\":\"npx vitest run src/calculator.test.ts\"}","call_id":"call_test_run"}} +{"timestamp":"2026-01-01T00:00:03.000Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call_test_run","output":[{"type":"text","text":" FAIL src/calculator.test.ts > calculator > adds two numbers\nAssertionError: expected undefined to be 5 // Object.is equality\n\n- Expected: 5\n+ Received: undefined\n\n at src/calculator.test.ts:6:25"},{"type":"text","text":"Test Files 1 failed (1)\n Tests 1 failed (1)"}]}} diff --git a/test/integration/enforce-tdd-codex.test.ts b/test/integration/enforce-tdd-codex.test.ts index 238315c..3207e07 100644 --- a/test/integration/enforce-tdd-codex.test.ts +++ b/test/integration/enforce-tdd-codex.test.ts @@ -49,7 +49,7 @@ const it = baseTest describe.concurrent( 'enforce-tdd + codex', () => { - it('allows clean TDD with minimal implementation', async ({ + it('uses Vitest text-block output as red evidence for minimal implementation', async ({ runScenario, }) => { const result = await runScenario({