Skip to content
Closed
2 changes: 2 additions & 0 deletions docs/reference/api.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ Every media endpoint on this page, plus web search, web fetch, and embeddings be

Image generation accepts JSON `{ model, prompt }`, optional `Accept: text/event-stream`, and `?response_format=binary`. Image edits accept multipart or provider-native JSON on supported models. xAI accepts `image: { url }` or `image: { file_id }`, or up to five sources in `images`; the adapter preserves `storage_options` and converts multipart `n` to a number. `sdwebui` and `comfyui` need no stored key.

Responses `input_image` accepts `image_url` (URL or `data:` base64). A bare `file_id` is not resolved when either the registered Responses translator or worker converter converts a request to Chat Completions; it returns HTTP 400 instead of being sent as an image URL. This applies to message images and images inside `function_call_output`, with either stream setting. When both references are present, `image_url` takes precedence. The file reference is preserved on native-compatible, same-format Responses transports, while gateway normalizations still apply.

TTS is JSON `{ model, input }` plus optional `language`. `?response_format=mp3` (default) or `json`. Voices: `GET /v1/audio/voices?provider={p}&lang=xx` with `p` in `elevenlabs`, `deepgram`, `inworld`, `minimax`, `minimax-cn`, `edge-tts`, `local-device`. Each voice row includes `model` ready for `/v1/audio/speech`.

STT and translation are multipart: `model`, `file`. Translation forwards to the provider `/audio/translations` path. Only OpenAI-format STT providers (OpenAI, Groq, Local Whisper) have one; the others return 400. The transcription and translation handlers declare a maximum route duration of 300 seconds. This is a server execution setting, not a 300-second audio-file limit.
Expand Down
10 changes: 8 additions & 2 deletions open-sse/handlers/responsesHandler.js
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,14 @@ export async function handleResponsesCore({ body, modelInfo, credentials, log, o
}
}

// Convert Responses API format to Chat Completions format
const convertedBody = convertResponsesApiFormat(body);
// Keep converter validation failures inside the same HTTP result contract as chatCore.
let convertedBody;
try {
convertedBody = convertResponsesApiFormat(body);
} catch (error) {
if (error?.statusCode !== HTTP_STATUS.BAD_REQUEST) throw error;
return createErrorResult(HTTP_STATUS.BAD_REQUEST, error.message);
}

// Preserve client's stream preference (matches OpenClaw behavior)
// Default to false if omitted: Boolean(undefined) = false
Expand Down
21 changes: 20 additions & 1 deletion open-sse/translator/formats/responsesApi.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,17 @@
import { ROLE, OPENAI_BLOCK, RESPONSES_ITEM } from "../schema/index.js";
import { isObject, isString } from "../../../src/shared/utils/typeChecks.js";
import { HTTP_STATUS } from "../../config/runtimeConfig.js";

/** Lower image references only when a URL is available; scoped file IDs need native Responses. */
export function responsesImageUrl(part) {
if (part.image_url) return part.image_url;
if (part.file_id) {
const error = new Error("input_image.file_id is not supported for this provider; send image_url (URL or data: base64) instead");
error.statusCode = HTTP_STATUS.BAD_REQUEST;
throw error;
}
return "";
}

const STORED_ITEM_REFERENCE_PATTERN = /^(?:at|msg|amsg|rs|lsh|fc|tsc|fco|ctc|ctco|tso|ws|ig|cmp|resp)_/;
const STATELESS_CALL_ITEM_TYPES = new Set([
Expand Down Expand Up @@ -298,7 +310,7 @@ export function convertResponsesApiFormat(body) {
if (c.type === RESPONSES_ITEM.INPUT_TEXT) return { type: OPENAI_BLOCK.TEXT, text: c.text };
if (c.type === RESPONSES_ITEM.OUTPUT_TEXT) return { type: OPENAI_BLOCK.TEXT, text: c.text };
if (c.type === RESPONSES_ITEM.INPUT_IMAGE) {
const url = c.image_url || c.file_id || "";
const url = responsesImageUrl(c);
return { type: OPENAI_BLOCK.IMAGE_URL, image_url: { url, detail: c.detail || "auto" } };
}
return c;
Expand Down Expand Up @@ -337,6 +349,13 @@ export function convertResponsesApiFormat(body) {
// a genuine orphan (no call to answer) and is dropped rather than emitting
// an unpairable tool message.
if (item.call_id) {
// This worker converter keeps its existing tool-output serialization,
// but scoped image IDs must fail before their block type becomes text.
if (Array.isArray(item.output)) {
for (const part of item.output) {
if (part?.type === RESPONSES_ITEM.INPUT_IMAGE) responsesImageUrl(part);
}
}
pendingToolResults.push({
role: ROLE.TOOL,
tool_call_id: item.call_id,
Expand Down
6 changes: 3 additions & 3 deletions open-sse/translator/request/openai-responses.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
repairMissingResponsesCallIds,
buildDeclaredToolTypes,
resolveDeclaredCustom,
responsesImageUrl,
} from "../formats/responsesApi.js";
import { ROLE, OPENAI_BLOCK, RESPONSES_ITEM, VALID_OPENAI_CONTENT_TYPES } from "../schema/index.js";
import { collapseTextParts } from "../concerns/message.js";
Expand Down Expand Up @@ -88,7 +89,7 @@ function splitToolOutputImages(output) {
}
const images = output.
filter((c) => c?.type === RESPONSES_ITEM.INPUT_IMAGE).
map((c) => ({ type: OPENAI_BLOCK.IMAGE_URL, image_url: { url: c.image_url || c.file_id || "", detail: c.detail || "auto" } }));
map((c) => ({ type: OPENAI_BLOCK.IMAGE_URL, image_url: { url: responsesImageUrl(c), detail: c.detail || "auto" } }));
const text = coerceResponsesOutput(output.filter((c) => c?.type !== RESPONSES_ITEM.INPUT_IMAGE));
const note = "[tool returned an image; see attached]";
return { content: text ? `${text}\n${note}` : note, images };
Expand Down Expand Up @@ -260,8 +261,7 @@ export function openaiResponsesToOpenAIRequest(model, body, stream, credentials)
if (c.type === RESPONSES_ITEM.INPUT_TEXT) return { type: OPENAI_BLOCK.TEXT, text: c.text };
if (c.type === RESPONSES_ITEM.OUTPUT_TEXT) return { type: OPENAI_BLOCK.TEXT, text: c.text };
if (c.type === RESPONSES_ITEM.INPUT_IMAGE) {
const url = c.image_url || c.file_id || "";
return { type: OPENAI_BLOCK.IMAGE_URL, image_url: { url, detail: c.detail || "auto" } };
return { type: OPENAI_BLOCK.IMAGE_URL, image_url: { url: responsesImageUrl(c), detail: c.detail || "auto" } };
}
return c;
}) :
Expand Down
90 changes: 78 additions & 12 deletions tests/translator/bugs-codexCli-responses.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { describe, it, expect } from "vitest";
import "./registerAll.js";
import { translateRequest } from "../../open-sse/translator/index.js";
import { FORMATS } from "../../open-sse/translator/formats.js";
import { convertResponsesApiFormat } from "../../open-sse/translator/formats/responsesApi.js";

const R2O = (body) => translateRequest(FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, "m", body, true, null, null);
const O2R = (body) => translateRequest(FORMATS.OPENAI, FORMATS.OPENAI_RESPONSES, "m", body, true, null, null);
Expand Down Expand Up @@ -32,18 +33,83 @@ describe("Codex CLI Responses → OpenAI", () => {
expect(typeof asst.tool_calls[0].function.arguments).toBe("string");
});

// openai-responses.js:75-77 — input_image uses file_id as raw url
// KNOWN BUG
it.fails("input_image with file_id is not used as a raw url", () => {
const out = R2O({
input: [{ type: "message", role: "user", content: [
{ type: "input_image", file_id: "file-abc" },
] }],
});
const userMsg = out.messages.find((m) => m.role === "user");
const img = Array.isArray(userMsg?.content) ? userMsg.content.find((c) => c.type === "image_url") : null;
// A bare file_id is not a valid image URL
expect(img?.image_url?.url === "file-abc").toBe(false);
// input_image file_id cannot be resolved here: reject with 400, never emit it as a URL.
it.each([true, false])("input_image with file_id is rejected with statusCode 400 (stream=%s)", (stream) => {
const body = { stream, input: [{ type: "message", role: "user", content: [
{ type: "input_image", file_id: "file-abc" },
] }] };
expect(() => translateRequest(FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, "m", body, stream, null, null))
.toThrowError(expect.objectContaining({
statusCode: 400,
message: expect.stringMatching(/file_id is not supported/),
}));
});

it.each([true, false])("function_call_output input_image with file_id is rejected with statusCode 400 (stream=%s)", (stream) => {
const body = { stream, input: [
{ type: "function_call", call_id: "c1", name: "f", arguments: "{}" },
{ type: "function_call_output", call_id: "c1", output: [{ type: "input_image", file_id: "file-abc" }] },
] };
expect(() => translateRequest(FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, "m", body, stream, null, null))
.toThrowError(expect.objectContaining({
statusCode: 400,
message: expect.stringMatching(/file_id is not supported/),
}));
});

it.each([true, false])("same-format Responses preserves input_image file references (stream=%s)", (stream) => {
const body = { stream, input: [
{ type: "message", role: "user", content: [{ type: "input_image", file_id: "file-user" }] },
{ type: "function_call", call_id: "c1", name: "f", arguments: "{}" },
{ type: "function_call_output", call_id: "c1", output: [{ type: "input_image", file_id: "file-tool" }] },
] };
const out = translateRequest(FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI_RESPONSES, "m", body, stream, null, null);
expect(out.input[0].content).toEqual([{ type: "input_image", file_id: "file-user" }]);
expect(out.input[2].output).toEqual([{ type: "input_image", file_id: "file-tool" }]);
});

it.each([true, false])("input_image with image_url (URL and base64) is preserved; image_url wins over file_id (stream=%s)", (stream) => {
for (const url of ["https://example.com/a.png", "data:image/png;base64,AAAA"]) {
const body = { stream, input: [{ type: "message", role: "user", content: [
{ type: "input_image", image_url: url, file_id: "file-abc" },
] }] };
const out = translateRequest(FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, "m", body, stream, null, null);
const img = out.messages.find((m) => m.role === "user").content.find((c) => c.type === "image_url");
expect(img.image_url.url).toBe(url);
}
});

it.each([true, false])("converter rejects bare file IDs and preserves URL priority (stream=%s)", (stream) => {
const body = (reference) => ({ stream, input: [{ type: "message", role: "user", content: [
{ type: "input_image", ...reference },
] }] });
expect(() => convertResponsesApiFormat(body({ file_id: "file-abc" })))
.toThrowError(expect.objectContaining({ statusCode: 400, message: expect.stringContaining("send image_url") }));
for (const url of ["https://example.com/a.png", "data:image/png;base64,AAAA"]) {
expect(convertResponsesApiFormat(body({ image_url: url, file_id: "file-abc" })).messages[0].content)
.toEqual([{ type: "image_url", image_url: { url, detail: "auto" } }]);
}
});

it.each([true, false])("preserves tool-output text and image URLs without splitting parallel results (stream=%s)", (stream) => {
for (const url of ["https://example.com/a.png", "data:image/png;base64,AAAA"]) {
const body = { stream, input: [
{ type: "function_call", call_id: "c1", name: "capture", arguments: "{}" },
{ type: "function_call", call_id: "c2", name: "inspect", arguments: "{}" },
{ type: "function_call_output", call_id: "c1", output: [
{ type: "input_text", text: "captured" },
{ type: "input_image", image_url: url, file_id: "file-abc", detail: "high" },
] },
{ type: "function_call_output", call_id: "c2", output: "inspected" },
] };
const out = translateRequest(FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, "m", body, stream, null, null);
expect(out.messages.map((message) => message.role)).toEqual(["assistant", "tool", "tool", "user"]);
expect(out.messages[1]).toMatchObject({ tool_call_id: "c1", content: "captured\n[tool returned an image; see attached]" });
expect(out.messages[2]).toMatchObject({ tool_call_id: "c2", content: "inspected" });
expect(out.messages[3].content).toEqual([
{ type: "image_url", image_url: { url, detail: "high" } },
]);
}
});

// The unit-level openaiResponsesToOpenAIRequest test only checks the merged
Expand Down
Loading