From b8a8ef85f7d568eae79066467be216bd6e3a8ceb Mon Sep 17 00:00:00 2001 From: Maximo Guk <62088388+Maximo-Guk@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:20:21 -0500 Subject: [PATCH 1/4] Add Workers AI binding transport for AI Gateway with CF_AI_GATEWAY_USE_BINDING opt-out --- docs/ai-gateway-billing.md | 22 +- docs/public-server.md | 34 ++- .../ai-gateway-binding-fetch.test.ts | 275 ++++++++++++++++++ .../__tests__/ai-gateway.test.ts | 96 ++++++ .../__tests__/ai-models.test.ts | 179 +++++++++++- .../src/ai-gateway-binding-fetch.ts | 188 ++++++++++++ packages/workshop-backend/src/ai-gateway.ts | 54 +++- packages/workshop-backend/src/ai-models.ts | 46 ++- packages/workshop-backend/src/env.d.ts | 16 +- scripts/release/manifest-lib.ts | 9 +- scripts/run-dev-server.ts | 10 +- 11 files changed, 878 insertions(+), 51 deletions(-) create mode 100644 packages/workshop-backend/__tests__/ai-gateway-binding-fetch.test.ts create mode 100644 packages/workshop-backend/src/ai-gateway-binding-fetch.ts diff --git a/docs/ai-gateway-billing.md b/docs/ai-gateway-billing.md index 9f2983e9..ea7cab35 100644 --- a/docs/ai-gateway-billing.md +++ b/docs/ai-gateway-billing.md @@ -54,19 +54,29 @@ CLOUDFLARE_OAUTH_CLIENT_SECRET=... CF_AI_GATEWAY=your-gateway CF_AI_GATEWAY_PROVIDERS=anthropic,openai,google -# Required whenever CF_AI_GATEWAY is set (all inference goes over HTTPS with tokens): +# Required whenever CF_AI_GATEWAY is set: CF_AI_GATEWAY_ACCOUNT_ID=... +# Required unless the WORKERS_AI binding carries gateway traffic; always required for the +# google provider and for CF_AI_GATEWAY_WAI_DIRECT: CF_AI_GATEWAY_API_TOKEN=... # To send Workers AI straight to its REST endpoint (no gateway, no cost logs): CF_AI_GATEWAY_WAI_DIRECT=true ``` -Gateway mode always requires `CF_AI_GATEWAY_ACCOUNT_ID` and an API token with AI Gateway Run and -Read permissions; Read access lets Gadgets retrieve each log's cost for user-visible accounting. -Workers AI uses `CF_AI_GATEWAY` as its Gateway ID by default; set `CF_AI_GATEWAY_WAI` to select -another Gateway, or `CF_AI_GATEWAY_WAI_DIRECT=true` to call the Workers AI REST endpoint directly -(same credentials, no gateway cost logs). +Gateway mode always requires `CF_AI_GATEWAY_ACCOUNT_ID` plus a transport: the `WORKERS_AI` +binding when present (binding requests are pre-authenticated, and cost-log reads work through +the binding too), or otherwise an API token with AI Gateway Run and Read permissions — Read +access lets Gadgets retrieve each log's cost for user-visible accounting. The binding transport +only works when the Gateway lives in the Worker's own account, which the Worker can't verify at +runtime — a deployment whose Gateway is in a different account must set +`CF_AI_GATEWAY_USE_BINDING=false` to opt out and use the token transport. The token stays +required for the `google` provider and for `CF_AI_GATEWAY_WAI_DIRECT` even when the binding +transport applies — and since `CF_AI_GATEWAY_WAI_DIRECT` calls the Workers AI REST endpoint, its +token additionally needs Workers AI Read permission. Workers AI uses `CF_AI_GATEWAY` as its +Gateway ID by default; set `CF_AI_GATEWAY_WAI` to select another Gateway, or +`CF_AI_GATEWAY_WAI_DIRECT=true` to call the Workers AI REST endpoint directly (token +credentials, no gateway cost logs). The Cloudflare dashboard OAuth endpoints and scopes are **hardcoded** in the Cloudflare gatekeeper (`packages/gatekeeper-cloudflare/src/oauth.ts`): diff --git a/docs/public-server.md b/docs/public-server.md index aa2fd033..acb77880 100644 --- a/docs/public-server.md +++ b/docs/public-server.md @@ -40,25 +40,39 @@ CLOUDFLARE_OAUTH_CLIENT_SECRET=... CF_AI_GATEWAY=your-gateway CF_AI_GATEWAY_PROVIDERS=anthropic,openai,google -# Required whenever CF_AI_GATEWAY is set (all inference goes over HTTPS with tokens): +# Required whenever CF_AI_GATEWAY is set: CF_AI_GATEWAY_ACCOUNT_ID=... +# Required unless the WORKERS_AI binding carries gateway traffic (see below); always required +# for the google provider and for CF_AI_GATEWAY_WAI_DIRECT: CF_AI_GATEWAY_API_TOKEN=... # To send Workers AI straight to its REST endpoint (no gateway, no cost logs): CF_AI_GATEWAY_WAI_DIRECT=true ``` -Gateway mode always requires `CF_AI_GATEWAY_ACCOUNT_ID` and `CF_AI_GATEWAY_API_TOKEN`; the token -needs AI Gateway Run and Read permissions so Gadgets can execute models and report their costs -(the Gateway may live in the Worker's own account or a different one). Workers AI defaults to the -same Gateway ID; set `CF_AI_GATEWAY_WAI` to route it through a different Gateway in the same -account, or `CF_AI_GATEWAY_WAI_DIRECT=true` to bypass gateways and call the Workers AI REST -endpoint directly (using the same account/token pair; such requests produce no cost logs). +Gateway mode always requires `CF_AI_GATEWAY_ACCOUNT_ID`, plus a transport for gateway requests. +When the `WORKERS_AI` binding is present, the binding is that transport by default: its requests +are pre-authenticated in-account, so inference and cost-log reads need no API token. This is only +valid when the Gateway lives in the Worker's **own** account — binding requests can't reach +another account's Gateway, and the Worker cannot verify where the Gateway lives at runtime — so +deployments whose Gateway is in a different account must set `CF_AI_GATEWAY_USE_BINDING=false` to +opt out and route over HTTPS instead. Without the binding transport, set +`CF_AI_GATEWAY_API_TOKEN` — a token with AI Gateway Run and Read permissions so Gadgets can +execute models and report their costs (over HTTPS the Gateway may live in the Worker's own +account or a different one). The token stays required in two cases regardless of the binding: the +`google` provider (its SDK can't ride the binding transport — note the platform config above +enables it, so the platform server itself still needs the token) and `CF_AI_GATEWAY_WAI_DIRECT` +(which calls the Workers AI REST endpoint, so its token additionally needs Workers AI Read +permission). Workers AI defaults to the same Gateway ID; set `CF_AI_GATEWAY_WAI` to route it +through a different Gateway in the same account, or `CF_AI_GATEWAY_WAI_DIRECT=true` to bypass +gateways and call the Workers AI REST endpoint directly (using the account/token pair; such +requests produce no cost logs). When using `CF_AI_GATEWAY*` in local development, start the server with -`pnpm run dev-server -- --use-workers-ai-binding` so the webFetch tool's document-to-Markdown -conversion still has a `WORKERS_AI` binding. (Inference itself no longer uses the binding; it goes -over HTTPS with the tokens above.) +`pnpm run dev-server -- --use-workers-ai-binding` so the server has a `WORKERS_AI` binding for +the webFetch tool's document-to-Markdown conversion and for the gateway transport above (without +it, gateway traffic falls back to HTTPS with `CF_AI_GATEWAY_API_TOKEN`). If your dev Gateway +lives in a different account than the binding, also set `CF_AI_GATEWAY_USE_BINDING=false`. Each gatekeeper's OAuth app must be registered with that gatekeeper's redirect URI (replace the host with `PUBLIC_BASE_URL`): diff --git a/packages/workshop-backend/__tests__/ai-gateway-binding-fetch.test.ts b/packages/workshop-backend/__tests__/ai-gateway-binding-fetch.test.ts new file mode 100644 index 00000000..9dd0a559 --- /dev/null +++ b/packages/workshop-backend/__tests__/ai-gateway-binding-fetch.test.ts @@ -0,0 +1,275 @@ +import { describe, expect, it } from "vitest"; +import { + type AiGatewayUniversalRequestLike, + CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL, + createGatewayBindingFetch, +} from "../src/ai-gateway-binding-fetch.js"; + +const BASE_URL = "https://gateway.ai.cloudflare.com/v1/account-id/my-gateway"; + +type CapturedRun = { + gatewayId: string; + data: AiGatewayUniversalRequestLike; + options: { signal?: AbortSignal } | undefined; +}; + +function fakeBinding(response?: Response) { + const runs: CapturedRun[] = []; + const binding = { + gateway: (gatewayId: string) => ({ + run: (data: AiGatewayUniversalRequestLike, options?: { signal?: AbortSignal }) => { + runs.push({ gatewayId, data, options }); + return Promise.resolve(response ?? new Response("{}")); + }, + }), + }; + return { binding, runs }; +} + +describe("createGatewayBindingFetch", () => { + it("derives provider and endpoint from gateway passthrough URLs", async () => { + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + + await fetchFn(`${BASE_URL}/anthropic/v1/messages`, { + method: "POST", + body: JSON.stringify({ model: "claude" }), + }); + await fetchFn(`${BASE_URL}/openai/responses`, { + method: "POST", + body: JSON.stringify({ model: "gpt" }), + }); + await fetchFn(`${BASE_URL}/workers-ai/v1/chat/completions`, { + method: "POST", + body: JSON.stringify({ model: "@cf/meta/llama" }), + }); + + expect(runs.map((run) => [run.data.provider, run.data.endpoint])).toEqual([ + ["anthropic", "v1/messages"], + ["openai", "responses"], + ["workers-ai", "v1/chat/completions"], + ]); + expect(runs.map((run) => run.gatewayId)).toEqual(["my-gateway", "my-gateway", "my-gateway"]); + expect(runs[0].data.query).toEqual({ model: "claude" }); + }); + + it("keeps the query string in the endpoint", async () => { + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + + await fetchFn(`${BASE_URL}/openai/responses?beta=true`, { + method: "POST", + body: "{}", + }); + + expect(runs[0].data.endpoint).toBe("responses?beta=true"); + }); + + it("lowercases header names so case-variant duplicates collapse", async () => { + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + + await fetchFn(`${BASE_URL}/anthropic/v1/messages`, { + method: "POST", + headers: { "Anthropic-Version": "2023-06-01" }, + body: "{}", + }); + + expect(runs[0].data.headers).toEqual({ "anthropic-version": "2023-06-01" }); + }); + + it("lets init headers replace a Request input's headers, per the fetch spec", async () => { + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + + await fetchFn( + new Request(`${BASE_URL}/anthropic/v1/messages`, { + method: "POST", + headers: { "x-from-request": "yes" }, + body: "{}", + }), + { headers: { "x-from-init": "yes" } }); + + expect(runs[0].data.headers["x-from-init"]).toBe("yes"); + expect(runs[0].data.headers["x-from-request"]).toBeUndefined(); + }); + + it("strips gateway auth and derived headers, forwards the rest", async () => { + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + + await fetchFn(`${BASE_URL}/anthropic/v1/messages`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": "17", + "CF-AIG-Authorization": `Bearer ${CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL}`, + "cf-aig-metadata": '{"user":"42"}', + "anthropic-version": "2023-06-01", + "x-api-key": "provider-key", + }, + body: "{}", + }); + + const headers = Object.fromEntries( + Object.entries(runs[0].data.headers).map(([key, value]) => [key.toLowerCase(), value])); + expect(headers["cf-aig-authorization"]).toBeUndefined(); + expect(headers["content-length"]).toBeUndefined(); + expect(headers["cf-aig-metadata"]).toBe('{"user":"42"}'); + expect(headers["anthropic-version"]).toBe("2023-06-01"); + // Provider auth headers pass through: that is how request-supplied (BYOK) keys ride. + expect(headers["x-api-key"]).toBe("provider-key"); + }); + + it("accepts Request inputs and forwards their headers and body", async () => { + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + + await fetchFn(new Request(`${BASE_URL}/openai/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ stream: true }), + })); + + expect(runs).toHaveLength(1); + expect(runs[0].data.provider).toBe("openai"); + expect(runs[0].data.endpoint).toBe("chat/completions"); + expect(runs[0].data.query).toEqual({ stream: true }); + expect(runs[0].data.headers["content-type"]).toBe("application/json"); + }); + + it("forwards the abort signal", async () => { + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + const controller = new AbortController(); + + await fetchFn(`${BASE_URL}/anthropic/v1/messages`, { + method: "POST", + body: "{}", + signal: controller.signal, + }); + + expect(runs[0].options?.signal).toBe(controller.signal); + }); + + it("lets an explicit `signal: null` in init clear a Request input's signal, per the fetch spec", + async () => { + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + const controller = new AbortController(); + + await fetchFn( + new Request(`${BASE_URL}/anthropic/v1/messages`, { + method: "POST", + body: "{}", + signal: controller.signal, + }), + { signal: null }); + + expect(runs).toHaveLength(1); + expect(runs[0].options?.signal).toBeUndefined(); + }); + + it("returns the binding response untouched, including streaming bodies", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("data: {}\n\n")); + controller.close(); + }, + }); + const bindingResponse = new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream", "cf-aig-log-id": "log-1" }, + }); + const { binding } = fakeBinding(bindingResponse); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + + const response = await fetchFn(`${BASE_URL}/workers-ai/v1/chat/completions`, { + method: "POST", + body: "{}", + }); + + expect(response).toBe(bindingResponse); + expect(response.headers.get("cf-aig-log-id")).toBe("log-1"); + expect(await response.text()).toBe("data: {}\n\n"); + }); + + it("rejects in-prefix requests the universal endpoint cannot express", async () => { + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + + await expect(fetchFn(`${BASE_URL}/anthropic/v1/messages`, { method: "GET" })) + .rejects.toThrow("cannot express GET"); + await expect(fetchFn(`${BASE_URL}/anthropic/v1/messages`, { method: "POST", body: "not json" })) + .rejects.toThrow("non-JSON body"); + await expect(fetchFn(`${BASE_URL}/anthropic`, { method: "POST", body: "{}" })) + .rejects.toThrow("missing provider/endpoint path"); + expect(runs).toHaveLength(0); + }); + + it("rejects URLs outside the gateway prefix: transport selection is the caller's", async () => { + // Silent passthrough would ship the auth sentinel to whatever host the URL names; a + // misconfigured baseUrl must fail loudly instead. + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + + await expect(fetchFn("https://api.openai.com/v1/chat/completions", { method: "POST", body: "{}" })) + .rejects.toThrow("outside the configured gateway prefix"); + // Same origin, different path (another account's gateway) is just as out-of-prefix. + await expect(fetchFn( + "https://gateway.ai.cloudflare.com/v1/other-account/my-gateway/anthropic/v1/messages", + { method: "POST", body: "{}" })) + .rejects.toThrow("outside the configured gateway prefix"); + expect(runs).toHaveLength(0); + }); + + it("matches and splits on the URL-normalized path, as real fetch would send it", async () => { + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + + // Dot segments normalize away before the provider/endpoint split, so a lexical variant + // routes exactly like its normal form (raw string prefixing would split it differently). + await fetchFn(`${BASE_URL}/anthropic/../anthropic/v1/./messages`, { + method: "POST", + body: JSON.stringify({ model: "claude" }), + }); + expect(runs.map((run) => [run.data.provider, run.data.endpoint])).toEqual([ + ["anthropic", "v1/messages"], + ]); + + // A dot-segment URL that resolves outside the prefix is rejected even though it starts + // with the prefix as a raw string. + await expect(fetchFn(`${BASE_URL}/../other-gateway/anthropic/v1/messages`, + { method: "POST", body: "{}" })) + .rejects.toThrow("outside the configured gateway prefix"); + expect(runs).toHaveLength(1); + }); + + it("consumes a one-shot stream body for the JSON probe", async () => { + const { binding, runs } = fakeBinding(); + const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); + const streamOf = (text: string) => new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(text)); + controller.close(); + }, + }); + + // JSON stream body: consumed once, reaches the binding as the parsed query. + await fetchFn(`${BASE_URL}/anthropic/v1/messages`, { + method: "POST", + body: streamOf('{"model":"claude"}'), + duplex: "half", + } as RequestInit); + expect(runs).toHaveLength(1); + expect(runs[0].data.query).toEqual({ model: "claude" }); + + // Non-JSON stream body: rejects like any other non-JSON body (never replayed). + await expect(fetchFn(`${BASE_URL}/anthropic/v1/messages`, { + method: "POST", + body: streamOf("not json"), + duplex: "half", + } as RequestInit)).rejects.toThrow("non-JSON body"); + expect(runs).toHaveLength(1); + }); +}); diff --git a/packages/workshop-backend/__tests__/ai-gateway.test.ts b/packages/workshop-backend/__tests__/ai-gateway.test.ts index 54a5996e..cb8aed69 100644 --- a/packages/workshop-backend/__tests__/ai-gateway.test.ts +++ b/packages/workshop-backend/__tests__/ai-gateway.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { + AiGatewayConfig, AiGatewayLogRetryableError, getAiGatewayLogCost, } from "../src/ai-gateway.js"; @@ -13,6 +14,101 @@ function env(overrides: Partial = {}): Cloudflare.Env { } as Cloudflare.Env; } +describe("AiGatewayConfig transport selection", () => { + const binding = { gateway: () => ({}) } as unknown as Ai; + // google needs the HTTPS+token transport, so token-less configs must not enable it. + const bindingOnly = env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_PROVIDERS: "anthropic,openai,cloudflare", + WORKERS_AI: binding, + }); + + it("uses the binding for every provider except google", () => { + const config = new AiGatewayConfig(bindingOnly); + expect(config.apiToken).toBeUndefined(); + expect(config.bindingFor("anthropic")).toBe(binding); + expect(config.bindingFor("openai")).toBe(binding); + expect(config.bindingFor("cloudflare")).toBe(binding); + expect(config.bindingFor("google")).toBeUndefined(); + }); + + it("falls back to HTTPS with the token when the binding is absent", () => { + const config = new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_API_TOKEN: "gateway-token", + WORKERS_AI: undefined, + })); + expect(config.apiToken).toBe("gateway-token"); + expect(config.bindingFor("anthropic")).toBeUndefined(); + }); + + it("ignores the binding when CF_AI_GATEWAY_USE_BINDING=false opts out", () => { + // The cross-account shape (e.g. the internal production Workshop): WORKERS_AI is injected + // for webFetch, but the gateway lives in a different account, so the deployment opts out + // and gateway traffic rides HTTPS with the token. + const config = new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_API_TOKEN: "gateway-token", + CF_AI_GATEWAY_USE_BINDING: "false", + WORKERS_AI: binding, + })); + expect(config.binding).toBeUndefined(); + expect(config.apiToken).toBe("gateway-token"); + expect(config.bindingFor("anthropic")).toBeUndefined(); + expect(config.bindingFor("openai")).toBeUndefined(); + }); + + it("still requires a transport when the opt-out leaves no token", () => { + expect(() => new AiGatewayConfig({ + ...bindingOnly, + CF_AI_GATEWAY_USE_BINDING: "false", + })).toThrow("AI Gateway mode needs a transport"); + }); + + it("rejects an explicit CF_AI_GATEWAY_USE_BINDING=true without the WORKERS_AI binding", () => { + expect(() => new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_API_TOKEN: "gateway-token", + CF_AI_GATEWAY_USE_BINDING: "true", + WORKERS_AI: undefined, + }))).toThrow("CF_AI_GATEWAY_USE_BINDING requires the WORKERS_AI binding"); + }); + + it("requires the account id", () => { + expect(() => new AiGatewayConfig(env({ CF_AI_GATEWAY_ACCOUNT_ID: undefined }))) + .toThrow("CF_AI_GATEWAY_ACCOUNT_ID is required when CF_AI_GATEWAY is set."); + }); + + it("requires a transport", () => { + expect(() => new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + WORKERS_AI: undefined, + }))).toThrow("AI Gateway mode needs a transport"); + }); + + it("requires the token when google is enabled", () => { + expect(() => new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + WORKERS_AI: binding, + }))).toThrow("enabling the google provider requires CF_AI_GATEWAY_API_TOKEN"); + }); + + it("requires the token for WAI_DIRECT", () => { + expect(() => new AiGatewayConfig({ + ...bindingOnly, + CF_AI_GATEWAY_WAI_DIRECT: "true", + })).toThrow("CF_AI_GATEWAY_WAI_DIRECT bypasses the gateway"); + }); + + it("keeps rejecting conflicting Workers AI routing", () => { + expect(() => new AiGatewayConfig({ + ...bindingOnly, + CF_AI_GATEWAY_WAI: "workers-ai-gateway", + CF_AI_GATEWAY_WAI_DIRECT: "true", + })).toThrow("CF_AI_GATEWAY_WAI and CF_AI_GATEWAY_WAI_DIRECT cannot be configured together."); + }); +}); + describe("getAiGatewayLogCost", () => { afterEach(() => vi.unstubAllGlobals()); diff --git a/packages/workshop-backend/__tests__/ai-models.test.ts b/packages/workshop-backend/__tests__/ai-models.test.ts index cfad0d65..16dee769 100644 --- a/packages/workshop-backend/__tests__/ai-models.test.ts +++ b/packages/workshop-backend/__tests__/ai-models.test.ts @@ -139,13 +139,15 @@ describe("getModel AI Gateway routing", () => { }); }, 15000); - it.each([ - { CF_AI_GATEWAY_ACCOUNT_ID: undefined }, - { CF_AI_GATEWAY_API_TOKEN: undefined }, - ])("requires gateway credentials whenever gateway mode is enabled", (overrides) => { - expect(() => getModel(env(overrides), ANTHROPIC_CONFIG, INITIATOR)).toThrow( - "CF_AI_GATEWAY_ACCOUNT_ID and CF_AI_GATEWAY_API_TOKEN (a Run + Read token) are required " + - "when CF_AI_GATEWAY is set."); + it("requires the gateway account id whenever gateway mode is enabled", () => { + expect(() => getModel(env({ CF_AI_GATEWAY_ACCOUNT_ID: undefined }), ANTHROPIC_CONFIG, + INITIATOR)).toThrow("CF_AI_GATEWAY_ACCOUNT_ID is required when CF_AI_GATEWAY is set."); + }); + + it("requires a transport: the Workers AI binding or an API token", () => { + // Without the binding (local dev without --use-workers-ai-binding), the token is required. + expect(() => getModel(env({ CF_AI_GATEWAY_API_TOKEN: undefined }), ANTHROPIC_CONFIG, + INITIATOR)).toThrow("AI Gateway mode needs a transport"); }); it("rejects conflicting Workers AI routing configuration", () => { @@ -261,6 +263,169 @@ describe("getModel AI Gateway routing", () => { }); }); +describe("getModel AI Gateway binding transport", () => { + // Universal-endpoint entries captured by the fake Workers AI binding. In binding mode the + // handle's requests never hit HTTP: pi's SDK fetch is the pi gateway-binding shim, which + // translates each request into binding.gateway(gw).run(entry). + type CapturedEntry = { + gatewayId: string; + provider: string; + endpoint: string; + headers: Record; + query: unknown; + }; + const capturedEntries: CapturedEntry[] = []; + + const fakeBinding = { + gateway: (gatewayId: string) => ({ + run: async (data: Omit) => { + capturedEntries.push({ gatewayId, ...data }); + // Same non-retryable client error as the HTTP fetch stub: pi surfaces an error-stop + // message and the entry stays captured for assertions. + return Response.json( + { error: { type: "bad_request", message: "stubbed" } }, { status: 400 }); + }, + }), + } as unknown as Ai; + + // Binding transport selects by default: binding present, no API token (in-account gateways; + // CF_AI_GATEWAY_USE_BINDING=false is the cross-account opt-out). google must not be an + // enabled provider in this mode (its transport still needs the token). + function bindingEnv(overrides: Partial = {}): Cloudflare.Env { + return env({ + CF_AI_GATEWAY_API_TOKEN: undefined, + CF_AI_GATEWAY_PROVIDERS: "anthropic,openai,cloudflare", + WORKERS_AI: fakeBinding, + ...overrides, + }); + } + + async function captureEntry(handle: ModelHandle): Promise { + const stream = handle.stream(handle.model, { + messages: [{ role: "user", content: "hello", timestamp: 0 }], + }, { maxRetries: 0 }); + const message = await stream.result(); + expect(message.stopReason).toBe("error"); + expect(capturedEntries.length).toBeGreaterThan(0); + return capturedEntries[0]; + } + + beforeEach(() => { + capturedEntries.length = 0; + capturedRequests.length = 0; + }); + + it("drives Anthropic through the binding with no API token", async () => { + const handle = getModel(bindingEnv(), ANTHROPIC_CONFIG, INITIATOR, { + metadata: { source: "chat", gadgetId: "gadget-123", chatId: 7 }, + }); + + expect(handle.model.api).toBe("anthropic-messages"); + expect(handle.model.baseUrl).toBe( + "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/anthropic"); + // Same-account log reads ride the binding too: no account id or token in the route. + expect(handle.aiGatewayLogRoute).toEqual({ gateway: "platform-gateway" }); + + const entry = await captureEntry(handle); + expect(entry.gatewayId).toBe("platform-gateway"); + expect(entry.provider).toBe("anthropic"); + expect(entry.endpoint).toBe("v1/messages"); + // The sentinel auth header satisfies pi's request-auth check but must never reach the + // gateway; the SDK's own auth headers stay suppressed. + const headerNames = Object.keys(entry.headers).map((name) => name.toLowerCase()); + expect(headerNames).not.toContain("cf-aig-authorization"); + expect(headerNames).not.toContain("x-api-key"); + expect(headerNames).not.toContain("authorization"); + const metadataHeader = Object.entries(entry.headers) + .find(([name]) => name.toLowerCase() === "cf-aig-metadata")?.[1]; + expect(JSON.parse(metadataHeader!)).toEqual({ + user: "user-123", + source: "chat", + gadgetId: "gadget-123", + chatId: 7, + }); + expect((entry.query as { model: string }).model).toBe("claude-sonnet-4-5"); + }, 15000); + + it("drives Workers AI through the binding via its gateway route", async () => { + const handle = getModel(bindingEnv({ CF_AI_GATEWAY_WAI: "workers-ai-gateway" }), + WORKERS_AI_CONFIG, INITIATOR); + + expect(handle.model.baseUrl).toBe( + "https://gateway.ai.cloudflare.com/v1/gateway-account-id/workers-ai-gateway/" + + "workers-ai/v1"); + expect(handle.aiGatewayLogRoute).toEqual({ gateway: "workers-ai-gateway" }); + + const entry = await captureEntry(handle); + expect(entry.gatewayId).toBe("workers-ai-gateway"); + expect(entry.provider).toBe("workers-ai"); + expect(entry.endpoint).toBe("v1/chat/completions"); + expect((entry.query as { model: string }).model) + .toBe("@cf/meta/llama-3.3-70b-instruct-fp8-fast"); + // openai-completions adapters inject `Authorization: Bearer unused` under header-owned + // auth; the gatewayAuthHeaders nulls must delete it before the entry is built, else the + // gateway would treat it as a request-supplied provider key overriding stored keys. + const headerNames = Object.keys(entry.headers).map((name) => name.toLowerCase()); + expect(headerNames).not.toContain("authorization"); + expect(headerNames).not.toContain("x-api-key"); + expect(headerNames).not.toContain("cf-aig-authorization"); + }, 15000); + + it("requires the token for direct Workers AI REST routing", () => { + expect(() => getModel(bindingEnv({ CF_AI_GATEWAY_WAI_DIRECT: "true" }), + WORKERS_AI_CONFIG, INITIATOR)).toThrow( + "CF_AI_GATEWAY_WAI_DIRECT bypasses the gateway"); + }); + + it("lets a per-call fetch override the binding transport", async () => { + // Tests and diagnostics inject options.fetch; it must win over the handle's binding shim. + // The raw request still carries the sentinel (stripping is the shim's job). + const handle = getModel(bindingEnv(), ANTHROPIC_CONFIG, INITIATOR); + + const request = await captureRequest(handle); + expect(capturedEntries).toHaveLength(0); + expect(request.url).toBe( + "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/anthropic/" + + "v1/messages"); + expect(request.headers.get("cf-aig-authorization")) + .toBe("Bearer cloudflare-gateway-binding"); + }, 15000); + + it("keeps Google on HTTPS with the token while other providers use the binding", async () => { + // Hybrid mode: binding and token both present. pi's Google adapter rejects a custom fetch, + // so Google inference rides HTTPS with the gateway token -- but same-account log reads + // still use the binding. + const hybridEnv = env({ + CF_AI_GATEWAY_PROVIDERS: "anthropic,openai,google,cloudflare", + WORKERS_AI: fakeBinding, + }); + + const googleHandle = getModel(hybridEnv, { + provider: "google", + model: "gemini-2.5-flash", + apiToken: "ignored-in-gateway-mode", + }, INITIATOR); + expect(googleHandle.model.baseUrl).toBe( + "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/" + + "google-ai-studio/v1beta"); + expect(googleHandle.aiGatewayLogRoute).toEqual({ gateway: "platform-gateway" }); + + const anthropicHandle = getModel(hybridEnv, ANTHROPIC_CONFIG, INITIATOR); + const entry = await captureEntry(anthropicHandle); + expect(entry.provider).toBe("anthropic"); + const headerNames = Object.keys(entry.headers).map((name) => name.toLowerCase()); + expect(headerNames).not.toContain("cf-aig-authorization"); + }, 15000); + + it("requires the token when google is an enabled provider", () => { + expect(() => getModel( + bindingEnv({ CF_AI_GATEWAY_PROVIDERS: "anthropic,google" }), + ANTHROPIC_CONFIG, INITIATOR)).toThrow( + "enabling the google provider requires CF_AI_GATEWAY_API_TOKEN"); + }); + +}); + describe("getModel direct routing (no gateway)", () => { beforeEach(() => { capturedRequests.length = 0; diff --git a/packages/workshop-backend/src/ai-gateway-binding-fetch.ts b/packages/workshop-backend/src/ai-gateway-binding-fetch.ts new file mode 100644 index 00000000..a53d2798 --- /dev/null +++ b/packages/workshop-backend/src/ai-gateway-binding-fetch.ts @@ -0,0 +1,188 @@ +// Cloudflare AI Gateway over the Workers AI binding. +// +// `createGatewayBindingFetch` returns a fetch that translates requests bound for an AI Gateway +// HTTPS endpoint (https://gateway.ai.cloudflare.com/v1/{account}/{gateway}/...) into calls +// through the Workers AI binding's universal endpoint +// (env.WORKERS_AI.gateway(id).run({provider, endpoint, headers, query})), which returns the +// provider's native wire format as a regular streaming Response. Binding calls are +// pre-authenticated in-account, so no cf-aig-authorization token is needed -- pass +// CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL where an auth header is required to satisfy pi's +// request-auth checks; the shim strips it before dispatch. Pair the sentinel with explicit +// `Authorization: null` / `x-api-key: null` provider headers (a null value deletes the header +// in the SDKs): with header-owned auth the SDKs otherwise inject a placeholder auth header +// (e.g. `Authorization: Bearer unused`), which the shim forwards like any provider auth header +// -- and the gateway treats a request-supplied provider auth header as a BYOK key that +// overrides its stored keys, the same as it would over HTTPS. + +import type { FetchFunction } from "@earendil-works/pi-ai"; + +// Structural type for the Workers AI binding's gateway surface (`env.WORKERS_AI`); any real +// `Ai` binding satisfies it. +export interface AiGatewayBinding { + gateway(id: string): AiGatewayBindingGateway; +} + +export interface AiGatewayBindingGateway { + run(data: AiGatewayUniversalRequestLike, options?: { signal?: AbortSignal }): Promise; +} + +/** One universal-endpoint request entry, as accepted by `AiGateway.run()`. */ +export interface AiGatewayUniversalRequestLike { + provider: string; + endpoint: string; + headers: Record; + query: unknown; +} + +/** + * Placeholder value for auth headers on binding-routed requests. pi's API implementations + * require an API key or a recognized auth header (`authorization`, `x-api-key`, + * `cf-aig-authorization`) before dispatch; binding calls are pre-authenticated, so pass + * `cf-aig-authorization: Bearer ${CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL}` to satisfy the + * check. The shim strips `cf-aig-authorization` before calling the binding. Pair it with + * `Authorization: null` / `x-api-key: null` so the SDKs' placeholder auth headers never reach + * the gateway (see the module docs). + */ +export const CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL = "cloudflare-gateway-binding"; + +export interface GatewayBindingFetchOptions { + // The Workers AI binding (e.g. env.WORKERS_AI). + binding: AiGatewayBinding; + // Gateway HTTPS prefix every request must fall under, without a trailing slash: + // https://gateway.ai.cloudflare.com/v1/{accountId}/{gatewayName}. + baseUrl: string; + // Gateway name passed to binding.gateway(). Must match the baseUrl gateway. + gateway: string; +} + +// Never forwarded to the binding: hop-by-hop/derived headers, and gateway auth (binding calls +// are pre-authenticated; the sentinel must not reach the wire, and a real gateway token is +// meaningless on this path and would only end up in request logs). +const STRIP_HEADERS = new Set(["content-length", "host", "cf-aig-authorization"]); + +type FetchInput = Parameters[0]; + +/** + * Create a `fetch` that routes AI Gateway requests through the Workers AI binding. This is + * the transport for one gateway-bound client, not a general-purpose fetch: URLs outside the + * configured gateway prefix, and in-prefix requests the binding cannot express (non-POST, + * non-JSON body), reject with a descriptive error. Transport selection is the caller's job, + * per client -- route such traffic over HTTPS with real gateway auth instead. + */ +export function createGatewayBindingFetch(options: GatewayBindingFetchOptions): FetchFunction { + const { binding, gateway } = options; + // Prefix matching runs on URL-normalized components (origin + pathname), not raw strings: + // dot segments resolve away and fragments drop, matching what real fetch would put on the + // wire, so a lexical variant can't split provider/endpoint differently than HTTPS would. + const base = new URL(options.baseUrl); + const basePath = base.pathname.endsWith("/") ? base.pathname : `${base.pathname}/`; + + return async (input: FetchInput, init?: RequestInit): Promise => { + const request = input instanceof Request ? input : undefined; + const url = request ? request.url : input.toString(); + const method = (init?.method ?? request?.method ?? "GET").toUpperCase(); + let parsed: URL | undefined; + try { + parsed = new URL(url); + } catch { + parsed = undefined; + } + // Out-of-prefix URLs are a configuration bug, not passthrough traffic: silently + // forwarding would ship the auth sentinel to whatever host the URL names. + if (parsed === undefined || parsed.origin !== base.origin || + !parsed.pathname.startsWith(basePath)) { + throw new Error( + `createGatewayBindingFetch: ${method} ${url} is outside the configured gateway ` + + `prefix (${base.origin}${basePath}); this fetch only serves its gateway-bound client`); + } + + // In-prefix requests the universal endpoint cannot express always reject: forwarding + // them over HTTPS would send the sentinel to the gateway and fail with a misleading + // auth error instead of naming the real problem. Callers that need such endpoints + // route them over HTTPS with real gateway auth themselves. + const unexpressible = (reason: string): never => { + throw new Error( + `createGatewayBindingFetch: cannot express ${method} ${url} as a universal ` + + `gateway request (${reason}); route it over HTTPS with gateway auth instead`); + }; + if (method !== "POST") return unexpressible("only POST is supported"); + + const rest = parsed.pathname.slice(basePath.length); + const slash = rest.indexOf("/"); + if (slash <= 0) { + return unexpressible("missing provider/endpoint path"); + } + const provider = rest.slice(0, slash); + // Keep the query string on the endpoint -- it's part of what HTTPS would have sent. + const endpoint = rest.slice(slash + 1) + parsed.search; + + const bodyText = await readBodyText(request, init); + let query: unknown; + try { + query = bodyText === undefined ? undefined : JSON.parse(bodyText); + } catch { + return unexpressible("non-JSON body"); + } + if (query === undefined) { + return unexpressible("missing body"); + } + + const headers = collectHeaders(request, init); + // Per the fetch spec an explicit `signal: null` in init clears a Request input's signal. + const signal = init?.signal ?? + (init && "signal" in init && init.signal === null ? undefined : request?.signal); + return binding.gateway(gateway).run( + { provider, endpoint, headers, query }, signal ? { signal } : {}); + }; +} + +async function readBodyText(request: Request | undefined, init?: RequestInit) + : Promise { + const body = init?.body; + if (body === undefined || body === null) { + // Per the fetch spec an explicit `body: null` in init clears a Request input's body. + if (init && "body" in init && body === null) return undefined; + if (request && request.body !== null) return request.clone().text(); + return undefined; + } + if (typeof body === "string") return body; + if (body instanceof Uint8Array) return new TextDecoder().decode(body); + if (body instanceof ArrayBuffer) return new TextDecoder().decode(new Uint8Array(body)); + // URLSearchParams, FormData, Blob, ReadableStream in init: read via a Request wrapper. + // Consuming a one-shot stream here is fine -- unexpressible requests reject rather than + // replay, so nothing downstream needs the body again. + return new Request("http://body.local", { + method: "POST", + body, + // The fetch spec requires `duplex: "half"` to construct a Request with a stream body + // (Node's undici enforces it; it is ignored for the replayable body types). TypeScript's + // RequestInit does not declare the field yet, hence the cast. + duplex: "half", + } as RequestInit).text(); +} + +// Entry header names are lowercased so case-variant duplicates collapse and stripping is +// uniform. Per the fetch spec, `init.headers` replaces a Request input's headers entirely. +function collectHeaders(request: Request | undefined, init?: RequestInit) + : Record { + const result: Record = {}; + const add = (key: string, value: string) => { + const name = key.toLowerCase(); + if (!STRIP_HEADERS.has(name)) result[name] = value; + }; + const headers = init?.headers; + if (headers === undefined) { + if (request) { + for (const [key, value] of request.headers) add(key, value); + } + } else if (headers instanceof Headers) { + for (const [key, value] of headers) add(key, value); + } else if (Array.isArray(headers)) { + for (const [key, value] of headers) add(key, value); + } else { + for (const [key, value] of Object.entries(headers)) { + if (value !== undefined) add(key, String(value)); + } + } + return result; +} diff --git a/packages/workshop-backend/src/ai-gateway.ts b/packages/workshop-backend/src/ai-gateway.ts index b0656408..3ef3045a 100644 --- a/packages/workshop-backend/src/ai-gateway.ts +++ b/packages/workshop-backend/src/ai-gateway.ts @@ -11,31 +11,67 @@ export class AiGatewayConfig { readonly gateway: string; readonly workersAiGateway?: string; readonly accountId: string; - readonly apiToken: string; + readonly apiToken?: string; + // Workers AI binding, used as the gateway transport whenever present unless + // CF_AI_GATEWAY_USE_BINDING=false opts out: binding requests are pre-authenticated in-account, + // so inference and cost-log reads need no API token. Binding requests only reach gateways in + // the Worker's own account, and the Worker can't verify that itself (it can't discover its own + // account ID), so deployments whose gateway lives in a DIFFERENT account must set the opt-out + // and use CF_AI_GATEWAY_API_TOKEN over HTTPS. Absent in local dev unless run-dev-server is + // started with --use-workers-ai-binding. + readonly binding?: Ai; readonly providers: Set; constructor(env: Cloudflare.Env) { this.gateway = env.CF_AI_GATEWAY!; - // Inference now goes over HTTPS with tokens (pi has no Workers-binding transport), so the - // account/token pair is required whenever gateway mode is enabled. The token-less - // same-account mode existed only because of the Workers binding. - if (!env.CF_AI_GATEWAY_ACCOUNT_ID || !env.CF_AI_GATEWAY_API_TOKEN) { - throw new Error( - "CF_AI_GATEWAY_ACCOUNT_ID and CF_AI_GATEWAY_API_TOKEN (a Run + Read token) are " + - "required when CF_AI_GATEWAY is set."); + if (!env.CF_AI_GATEWAY_ACCOUNT_ID) { + throw new Error("CF_AI_GATEWAY_ACCOUNT_ID is required when CF_AI_GATEWAY is set."); } this.accountId = env.CF_AI_GATEWAY_ACCOUNT_ID; - this.apiToken = env.CF_AI_GATEWAY_API_TOKEN; + this.apiToken = env.CF_AI_GATEWAY_API_TOKEN || undefined; + this.binding = env.CF_AI_GATEWAY_USE_BINDING === "false" + ? undefined + : (env as { WORKERS_AI?: Ai }).WORKERS_AI; + if (env.CF_AI_GATEWAY_USE_BINDING === "true" && !this.binding) { + throw new Error( + "CF_AI_GATEWAY_USE_BINDING requires the WORKERS_AI binding; without it the config " + + "would silently fall back to the HTTPS transport."); + } + if (!this.apiToken && !this.binding) { + throw new Error( + "AI Gateway mode needs a transport: bind Workers AI (WORKERS_AI; in local dev start " + + "with --use-workers-ai-binding) or set CF_AI_GATEWAY_API_TOKEN (a Run + Read token)."); + } if (env.CF_AI_GATEWAY_WAI_DIRECT === "true" && env.CF_AI_GATEWAY_WAI) { throw new Error( "CF_AI_GATEWAY_WAI and CF_AI_GATEWAY_WAI_DIRECT cannot be configured together."); } + if (env.CF_AI_GATEWAY_WAI_DIRECT === "true" && !this.apiToken) { + throw new Error( + "CF_AI_GATEWAY_WAI_DIRECT bypasses the gateway and calls the Workers AI REST " + + "endpoint, which requires CF_AI_GATEWAY_API_TOKEN."); + } this.workersAiGateway = env.CF_AI_GATEWAY_WAI_DIRECT === "true" ? undefined : env.CF_AI_GATEWAY_WAI || this.gateway; this.providers = new Set( (env.CF_AI_GATEWAY_PROVIDERS || "").split(",").map(s => s.trim()).filter(s => s !== "") ); + if (this.providers.has("google") && !this.apiToken) { + throw new Error( + "Google models cannot use the Workers AI binding transport (the @google/genai SDK " + + "does not support a custom fetch), so enabling the google provider requires " + + "CF_AI_GATEWAY_API_TOKEN."); + } + } + + /** + * Transport for a provider's gateway inference: the Workers AI binding when present, except + * for Google (pi's Google adapter rejects a custom fetch, so Google rides HTTPS with the + * token; the constructor guarantees a token whenever google is an enabled provider). + */ + bindingFor(provider: string): Ai | undefined { + return provider === "google" ? undefined : this.binding; } /** diff --git a/packages/workshop-backend/src/ai-models.ts b/packages/workshop-backend/src/ai-models.ts index 7f7ea8de..6c40ff61 100644 --- a/packages/workshop-backend/src/ai-models.ts +++ b/packages/workshop-backend/src/ai-models.ts @@ -1,10 +1,13 @@ import { DurableObject, RpcStub, RpcTarget } from "cloudflare:workers"; import { validateRpc } from "capnweb-validate"; import type { - AnthropicMessagesCompat, Api, AssistantMessageEventStream, Context, Model, ModelCost, - OpenAICompletionsCompat, ProviderHeaders, SimpleStreamOptions, StreamFunction, + AnthropicMessagesCompat, Api, AssistantMessageEventStream, Context, FetchFunction, Model, + ModelCost, OpenAICompletionsCompat, ProviderHeaders, SimpleStreamOptions, StreamFunction, } from "@earendil-works/pi-ai"; import { stream as anthropicMessagesStream } from "@earendil-works/pi-ai/api/anthropic-messages"; +import { + CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL, createGatewayBindingFetch, +} from "./ai-gateway-binding-fetch.js"; import { stream as googleGenerativeAiStream } from "@earendil-works/pi-ai/api/google-generative-ai"; import { stream as openaiCompletionsStream } from "@earendil-works/pi-ai/api/openai-completions"; import { stream as openaiResponsesStream } from "@earendil-works/pi-ai/api/openai-responses"; @@ -268,6 +271,9 @@ type HandleArgs = { gatewayMetadata?: GatewayMetadata; sessionAffinity?: string; aiGatewayLogRoute?: AiGatewayLogRoute; + // Transport override for every request on this handle (e.g. the AI Gateway binding shim). + // A per-call options.fetch still wins, which tests rely on to capture requests. + fetch?: FetchFunction; }; function makeHandle(args: HandleArgs): ModelHandle { @@ -315,6 +321,7 @@ function makeHandle(args: HandleArgs): ModelHandle { ...(thinking ? apiExtras : args.model.api === "anthropic-messages" ? { thinkingEnabled: false } : {}), + ...(args.fetch !== undefined ? { fetch: args.fetch } : {}), ...options, ...(args.apiKey !== undefined ? { apiKey: args.apiKey } : {}), ...(Object.keys(headers).length > 0 ? { headers } : {}), @@ -333,9 +340,6 @@ function makeHandle(args: HandleArgs): ModelHandle { const replaced = await options.onPayload?.(payload, payloadModel); return bridgePdfAttachments(args.model.api, replaced ?? payload) ?? replaced; }, - // NOTE(binding-transport): pi passes `options.fetch` into its SDK clients on all paths. - // If Workers-binding-backed inference returns (upstream ask filed), inject a - // fetch-to-binding shim here and relax the token requirements in ai-gateway.ts. }; return streamFn(model, context, merged); }, @@ -420,23 +424,34 @@ function getModelViaGateway( options: ModelRoutingOptions, ): ModelHandle { const metadata = buildMetadata(initiator, options.metadata); + // Binding transport when available (all providers except Google): requests go through + // env.WORKERS_AI.gateway().run(), pre-authenticated in-account. pi's API impls require a + // recognized auth header before dispatch, so binding-routed requests carry a sentinel + // cf-aig-authorization that the shim strips before it reaches the wire. + const binding = gwConfig.bindingFor(config.provider); const gatewayAuthHeaders: ProviderHeaders = { // pi's API impls explicitly recognize cf-aig-authorization and skip SDK auth; the null // values suppress the SDKs' own auth headers so the gateway's server-managed provider keys // apply. - "cf-aig-authorization": `Bearer ${gwConfig.apiToken}`, + "cf-aig-authorization": + `Bearer ${binding ? CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL : gwConfig.apiToken}`, Authorization: null, "x-api-key": null, }; const gatewayBase = `https://gateway.ai.cloudflare.com/v1/${gwConfig.accountId}`; - const logRoute = (gateway: string): AiGatewayLogRoute => - ({ gateway, accountId: gwConfig.accountId, apiToken: gwConfig.apiToken }); + // Cost-log reads are same-account, so the binding arm applies whenever the binding transport + // is active (gwConfig.binding is unset when CF_AI_GATEWAY_USE_BINDING=false opts out) -- + // even for Google inference, which itself rides HTTPS (see AiGatewayConfig.bindingFor). + const logRoute = (gateway: string): AiGatewayLogRoute => gwConfig.binding + ? { gateway } + : { gateway, accountId: gwConfig.accountId, apiToken: gwConfig.apiToken! }; if (config.provider === "cloudflare" && !gwConfig.workersAiGateway) { // CF_AI_GATEWAY_WAI_DIRECT: the plain Workers AI REST endpoint -- no gateway, no log route, // no gateway metadata (mirroring the old direct-binding path, which had no - // aiGatewayLogRoute). Reuses the CF_AI_GATEWAY_* account/token pair. + // aiGatewayLogRoute). Reuses the CF_AI_GATEWAY_* account/token pair (the config constructor + // guarantees the token in this mode). const catalog = catalogModel(config.provider, config.model); const model: Model = { id: config.model, @@ -457,11 +472,19 @@ function getModelViaGateway( }); } + if (config.provider === "google" && !gwConfig.apiToken) { + // Unreachable when google is an enabled provider (the config constructor throws), but a + // stored model config can still name google directly. + throw new Error("Google models require CF_AI_GATEWAY_API_TOKEN (the Workers AI binding " + + "transport cannot carry them)."); + } + // Workers AI may be routed through a different gateway than the other providers // (CF_AI_GATEWAY_WAI); either way, gateway log route and attribution metadata apply. const gateway = config.provider === "cloudflare" ? gwConfig.workersAiGateway! : gwConfig.gateway; - const model = gatewayNativeModel(config, `${gatewayBase}/${gateway}`); + const gatewayUrl = `${gatewayBase}/${gateway}`; + const model = gatewayNativeModel(config, gatewayUrl); if (!model) { throw new Error( `Provider "${config.provider}" is not supported through AI Gateway. ` + @@ -479,6 +502,9 @@ function getModelViaGateway( // the gateway recognizes its own token there and applies the stored Google key instead. ...(config.provider === "google" ? { apiKey: gwConfig.apiToken } : {}), headers: gatewayAuthHeaders, + ...(binding + ? { fetch: createGatewayBindingFetch({ binding, baseUrl: gatewayUrl, gateway }) } + : {}), gatewayMetadata: metadata, sessionAffinity: options.sessionAffinity, aiGatewayLogRoute: logRoute(gateway), diff --git a/packages/workshop-backend/src/env.d.ts b/packages/workshop-backend/src/env.d.ts index e0443836..fd51cb1c 100644 --- a/packages/workshop-backend/src/env.d.ts +++ b/packages/workshop-backend/src/env.d.ts @@ -15,12 +15,22 @@ declare global { // AI Gateway mode: when CF_AI_GATEWAY is set, supported providers are routed through // Cloudflare AI Gateway with server-managed keys. Users don't need their own keys. - // Inference goes over HTTPS with tokens (there is no Workers-binding transport), so the - // ACCOUNT_ID/API_TOKEN pair is REQUIRED whenever CF_AI_GATEWAY is set. + // Transport: the WORKERS_AI binding when present (pre-authenticated in-account -- no API + // token; in local dev the binding needs --use-workers-ai-binding) unless + // CF_AI_GATEWAY_USE_BINDING=false opts out, HTTPS with CF_AI_GATEWAY_API_TOKEN otherwise. + // The token stays REQUIRED for the google provider (pi's Google adapter can't use the + // binding transport) and for CF_AI_GATEWAY_WAI_DIRECT. CF_AI_GATEWAY?: string; // Gateway name (enables gateway mode) CF_AI_GATEWAY_PROVIDERS?: string; // Comma-separated list: "anthropic,openai,google,cloudflare" CF_AI_GATEWAY_ACCOUNT_ID?: string; // Gateway owner account ID (required with CF_AI_GATEWAY) - CF_AI_GATEWAY_API_TOKEN?: string; // Run + Read token for inference and cost-log reads + CF_AI_GATEWAY_API_TOKEN?: string; // Run + Read token; optional when the binding transport + // applies (still required for google / WAI_DIRECT) + // "false" = never use the WORKERS_AI binding as the gateway transport. Binding requests + // only reach gateways in the Worker's own account, and the Worker can't verify where the + // gateway lives (it can't discover its own account ID at runtime) -- so deployments whose + // gateway is in a DIFFERENT account (e.g. the internal production Workshop) must set this + // opt-out and route over HTTPS with the token. Unset/"true" = binding when present. + CF_AI_GATEWAY_USE_BINDING?: string; CF_AI_GATEWAY_WAI?: string; // Optional Workers AI gateway override CF_AI_GATEWAY_WAI_DIRECT?: string; // "true" to route Workers AI to its plain REST endpoint // (no gateway, no cost logs) instead of a named Gateway diff --git a/scripts/release/manifest-lib.ts b/scripts/release/manifest-lib.ts index be83b6dd..9d18c5fe 100644 --- a/scripts/release/manifest-lib.ts +++ b/scripts/release/manifest-lib.ts @@ -400,9 +400,12 @@ export function buildWorkerEntry( // the deploy service's backendExtraVars at PUT time, never manifest-templated. vars.PUBLIC_BASE_URL = "$PUBLIC_BASE_URL"; // Every deployed backend gets the Workers AI binding (hardcoded like PUBLIC_BASE_URL, not - // read from wrangler.jsonc): webFetch's toMarkdown conversion depends on it, and it costs - // nothing when unused. (Inference does not — Workers AI models are reached over HTTPS like - // every other provider.) No placeholders — the deploy renderer passes it through. + // read from wrangler.jsonc): webFetch's toMarkdown conversion depends on it, and it is also + // the backend's default AI Gateway transport (the deploy service creates the gateway in the + // user's own account, so the in-account requirement holds; CF_AI_GATEWAY_USE_BINDING=false + // is the cross-account opt-out) — binding requests are pre-authenticated, so inference and + // cost-log reads need no CF_AI_GATEWAY_API_TOKEN (google provider and WAI_DIRECT excepted). + // No placeholders — the deploy renderer passes it through. bindings.push({ type: "ai", name: "WORKERS_AI" }); // Installed gatekeepers are called through GATEKEEPER_* service bindings with the // GatekeeperVendor entrypoint (same shape run-dev-server.ts generates for dev). diff --git a/scripts/run-dev-server.ts b/scripts/run-dev-server.ts index 71cde0ae..b40b6820 100644 --- a/scripts/run-dev-server.ts +++ b/scripts/run-dev-server.ts @@ -479,11 +479,15 @@ for (const gk of gatekeepers) { const OPTIONAL_FEATURE_VARS = [ "DISABLE_PASSWORD_AUTH", "AUTH_GATEKEEPERS", "ENABLE_CLOUDFLARE_LIMITS", "PUBLIC_BASE_URL", "DAILY_LLM_CALL_LIMIT", "MINIMUM_CLOUDFLARE_BALANCE", - // Platform AI Gateway — makes the cross-provider model catalog available. The - // ACCOUNT_ID/API_TOKEN pair is required whenever CF_AI_GATEWAY is set (all inference goes - // over HTTPS with tokens). + // Platform AI Gateway — makes the cross-provider model catalog available. CF_AI_GATEWAY + // always needs CF_AI_GATEWAY_ACCOUNT_ID plus one transport: the WORKERS_AI binding + // (start with --use-workers-ai-binding; CF_AI_GATEWAY_USE_BINDING=false opts out, e.g. + // when the gateway lives in a different account than the dev binding) or + // CF_AI_GATEWAY_API_TOKEN over HTTPS. Two cases can't ride the binding and need the + // token even when it's present: the google provider and CF_AI_GATEWAY_WAI_DIRECT. "CF_AI_GATEWAY", "CF_AI_GATEWAY_PROVIDERS", "CF_AI_GATEWAY_ACCOUNT_ID", "CF_AI_GATEWAY_API_TOKEN", "CF_AI_GATEWAY_WAI", "CF_AI_GATEWAY_WAI_DIRECT", + "CF_AI_GATEWAY_USE_BINDING", ]; // OAuth app credentials (GOOGLE_/GITHUB_/CLOUDFLARE_OAUTH_*) are NOT passed to the backend anymore; // they are injected into the gatekeeper Workers (see SHARED_GATEKEEPER_CREDS below). From 5a853ef379d335b3dcf8957432b1d96b0d8ef386 Mon Sep 17 00:00:00 2001 From: Maximo Guk <62088388+Maximo-Guk@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:56:42 -0500 Subject: [PATCH 2/4] Remove CF_AI_GATEWAY_WAI and CF_AI_GATEWAY_WAI_DIRECT: Workers AI always rides the gateway The WAI knobs existed because Workers AI originally ran on the account-local WORKERS_AI binding, which can only name same-account gateways; a cross-account CF_AI_GATEWAY (the internal production shape) therefore needed an escape hatch. Workers AI has since moved to ordinary gateway routes (.../{account}/{gateway}/workers-ai/v1), which work cross-account over HTTPS+token exactly like anthropic/openai -- so the escape hatch no longer pays for its config surface. --- docs/ai-gateway-billing.md | 20 ++---- docs/public-server.md | 17 ++--- .../__tests__/ai-gateway.test.ts | 21 +++--- .../__tests__/ai-models.test.ts | 70 ++++--------------- .../__tests__/web-fetch.test.ts | 6 +- packages/workshop-backend/src/ai-gateway.ts | 19 ++--- packages/workshop-backend/src/ai-models.ts | 32 +-------- packages/workshop-backend/src/env.d.ts | 15 ++-- packages/workshop-backend/src/web-fetch.ts | 9 +-- scripts/release/manifest-lib.ts | 2 +- scripts/run-dev-server.ts | 7 +- 11 files changed, 61 insertions(+), 157 deletions(-) diff --git a/docs/ai-gateway-billing.md b/docs/ai-gateway-billing.md index ea7cab35..ce32d83d 100644 --- a/docs/ai-gateway-billing.md +++ b/docs/ai-gateway-billing.md @@ -13,10 +13,9 @@ turn, the overseer calls `checkUsageAndBalance`: - **Connected, balance ≥ `$2`** → allowed, routed through the user's own account so usage bills their Cloudflare credits — even while free-tier allowance remains. The platform is never charged for funded users, and their daily free-tier counter is left untouched. -- **Otherwise, within the free tier** → allowed, served via the platform's configured AI Gateway. - Workers AI uses the same Gateway ID unless `CF_AI_GATEWAY_WAI_DIRECT=true` sends it straight to - the Workers AI REST endpoint or `CF_AI_GATEWAY_WAI` selects another Gateway. This includes - connected users whose balance is below `$2` (incl. $0). +- **Otherwise, within the free tier** → allowed, served via the platform's configured AI Gateway + (all providers, Workers AI included). This includes connected users whose balance is below `$2` + (incl. $0). - **Free tier exhausted, no Cloudflare account connected** → blocked, with a prompt to connect. - **Free tier exhausted, connected but balance below `$2`** → blocked, with a prompt to add credits. @@ -57,11 +56,8 @@ CF_AI_GATEWAY_PROVIDERS=anthropic,openai,google # Required whenever CF_AI_GATEWAY is set: CF_AI_GATEWAY_ACCOUNT_ID=... # Required unless the WORKERS_AI binding carries gateway traffic; always required for the -# google provider and for CF_AI_GATEWAY_WAI_DIRECT: +# google provider: CF_AI_GATEWAY_API_TOKEN=... - -# To send Workers AI straight to its REST endpoint (no gateway, no cost logs): -CF_AI_GATEWAY_WAI_DIRECT=true ``` Gateway mode always requires `CF_AI_GATEWAY_ACCOUNT_ID` plus a transport: the `WORKERS_AI` @@ -71,12 +67,8 @@ access lets Gadgets retrieve each log's cost for user-visible accounting. The bi only works when the Gateway lives in the Worker's own account, which the Worker can't verify at runtime — a deployment whose Gateway is in a different account must set `CF_AI_GATEWAY_USE_BINDING=false` to opt out and use the token transport. The token stays -required for the `google` provider and for `CF_AI_GATEWAY_WAI_DIRECT` even when the binding -transport applies — and since `CF_AI_GATEWAY_WAI_DIRECT` calls the Workers AI REST endpoint, its -token additionally needs Workers AI Read permission. Workers AI uses `CF_AI_GATEWAY` as its -Gateway ID by default; set `CF_AI_GATEWAY_WAI` to select another Gateway, or -`CF_AI_GATEWAY_WAI_DIRECT=true` to call the Workers AI REST endpoint directly (token -credentials, no gateway cost logs). +required for the `google` provider even when the binding transport applies. Every provider, +Workers AI included, routes through the same Gateway. The Cloudflare dashboard OAuth endpoints and scopes are **hardcoded** in the Cloudflare gatekeeper (`packages/gatekeeper-cloudflare/src/oauth.ts`): diff --git a/docs/public-server.md b/docs/public-server.md index acb77880..d4cefa4d 100644 --- a/docs/public-server.md +++ b/docs/public-server.md @@ -43,11 +43,8 @@ CF_AI_GATEWAY_PROVIDERS=anthropic,openai,google # Required whenever CF_AI_GATEWAY is set: CF_AI_GATEWAY_ACCOUNT_ID=... # Required unless the WORKERS_AI binding carries gateway traffic (see below); always required -# for the google provider and for CF_AI_GATEWAY_WAI_DIRECT: +# for the google provider: CF_AI_GATEWAY_API_TOKEN=... - -# To send Workers AI straight to its REST endpoint (no gateway, no cost logs): -CF_AI_GATEWAY_WAI_DIRECT=true ``` Gateway mode always requires `CF_AI_GATEWAY_ACCOUNT_ID`, plus a transport for gateway requests. @@ -59,14 +56,10 @@ deployments whose Gateway is in a different account must set `CF_AI_GATEWAY_USE_ opt out and route over HTTPS instead. Without the binding transport, set `CF_AI_GATEWAY_API_TOKEN` — a token with AI Gateway Run and Read permissions so Gadgets can execute models and report their costs (over HTTPS the Gateway may live in the Worker's own -account or a different one). The token stays required in two cases regardless of the binding: the -`google` provider (its SDK can't ride the binding transport — note the platform config above -enables it, so the platform server itself still needs the token) and `CF_AI_GATEWAY_WAI_DIRECT` -(which calls the Workers AI REST endpoint, so its token additionally needs Workers AI Read -permission). Workers AI defaults to the same Gateway ID; set `CF_AI_GATEWAY_WAI` to route it -through a different Gateway in the same account, or `CF_AI_GATEWAY_WAI_DIRECT=true` to bypass -gateways and call the Workers AI REST endpoint directly (using the account/token pair; such -requests produce no cost logs). +account or a different one). The token stays required for the `google` provider regardless of the +binding (its SDK can't ride the binding transport — note the platform config above enables it, so +the platform server itself still needs the token). Every provider, Workers AI included, routes +through the same Gateway. When using `CF_AI_GATEWAY*` in local development, start the server with `pnpm run dev-server -- --use-workers-ai-binding` so the server has a `WORKERS_AI` binding for diff --git a/packages/workshop-backend/__tests__/ai-gateway.test.ts b/packages/workshop-backend/__tests__/ai-gateway.test.ts index cb8aed69..343af41f 100644 --- a/packages/workshop-backend/__tests__/ai-gateway.test.ts +++ b/packages/workshop-backend/__tests__/ai-gateway.test.ts @@ -93,19 +93,14 @@ describe("AiGatewayConfig transport selection", () => { }))).toThrow("enabling the google provider requires CF_AI_GATEWAY_API_TOKEN"); }); - it("requires the token for WAI_DIRECT", () => { - expect(() => new AiGatewayConfig({ - ...bindingOnly, - CF_AI_GATEWAY_WAI_DIRECT: "true", - })).toThrow("CF_AI_GATEWAY_WAI_DIRECT bypasses the gateway"); - }); - - it("keeps rejecting conflicting Workers AI routing", () => { - expect(() => new AiGatewayConfig({ - ...bindingOnly, - CF_AI_GATEWAY_WAI: "workers-ai-gateway", - CF_AI_GATEWAY_WAI_DIRECT: "true", - })).toThrow("CF_AI_GATEWAY_WAI and CF_AI_GATEWAY_WAI_DIRECT cannot be configured together."); + it("resolves the same-account gateway for binding-based callers (webFetch)", () => { + expect(new AiGatewayConfig(bindingOnly).sameAccountGateway).toBe("platform-gateway"); + expect(new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_API_TOKEN: "gateway-token", + CF_AI_GATEWAY_USE_BINDING: "false", + WORKERS_AI: binding, + })).sameAccountGateway).toBeUndefined(); }); }); diff --git a/packages/workshop-backend/__tests__/ai-models.test.ts b/packages/workshop-backend/__tests__/ai-models.test.ts index 16dee769..7cd75eaa 100644 --- a/packages/workshop-backend/__tests__/ai-models.test.ts +++ b/packages/workshop-backend/__tests__/ai-models.test.ts @@ -150,14 +150,6 @@ describe("getModel AI Gateway routing", () => { INITIATOR)).toThrow("AI Gateway mode needs a transport"); }); - it("rejects conflicting Workers AI routing configuration", () => { - expect(() => getModel(env({ - CF_AI_GATEWAY_WAI: "workers-ai-gateway", - CF_AI_GATEWAY_WAI_DIRECT: "true", - }), WORKERS_AI_CONFIG, INITIATOR)).toThrow( - "CF_AI_GATEWAY_WAI and CF_AI_GATEWAY_WAI_DIRECT cannot be configured together."); - }); - it("prioritizes a connected user's Gateway over platform routing", async () => { const handle = getModel(env(), WORKERS_AI_CONFIG, INITIATOR, { userGateway: { accountId: "user-account-id", apiKey: "user-token" }, @@ -212,32 +204,10 @@ describe("getModel AI Gateway routing", () => { expect(request.headers.get("authorization")).toBeNull(); }, 15000); - it("routes Workers AI to its REST endpoint when explicitly configured direct", async () => { - const handle = getModel( - env({ CF_AI_GATEWAY_WAI_DIRECT: "true" }), - WORKERS_AI_CONFIG, - INITIATOR, + it("routes Workers AI through the platform gateway like every other provider", async () => { + const handle = getModel(env(), WORKERS_AI_CONFIG, INITIATOR, { sessionAffinity: "session-a" }); - expect(handle.model.api).toBe("openai-completions"); - expect(handle.model.id).toBe("@cf/meta/llama-3.3-70b-instruct-fp8-fast"); - expect(handle.model.baseUrl).toBe( - "https://api.cloudflare.com/client/v4/accounts/gateway-account-id/ai/v1"); - // No gateway in the path: no log route (and no gateway metadata). - expect(handle.aiGatewayLogRoute).toBeUndefined(); - - const request = await captureRequest(handle); - expect(request.url).toBe( - "https://api.cloudflare.com/client/v4/accounts/gateway-account-id/ai/v1/chat/completions"); - expect(request.headers.get("authorization")).toBe("Bearer gateway-token"); - expect(request.headers.get("cf-aig-metadata")).toBeNull(); - // Session affinity flows through (Workers AI models opt in to the affinity headers). - expect(request.headers.get("x-session-affinity")).toBe("session-a"); - }, 15000); - - it("routes same-account Workers AI through the platform gateway by default", () => { - const handle = getModel(env(), WORKERS_AI_CONFIG, INITIATOR); - expect(handle.model.api).toBe("openai-completions"); expect(handle.model.id).toBe("@cf/meta/llama-3.3-70b-instruct-fp8-fast"); expect(handle.model.baseUrl).toBe( @@ -247,20 +217,15 @@ describe("getModel AI Gateway routing", () => { accountId: "gateway-account-id", apiToken: "gateway-token", }); - }); - - it("uses an explicit Workers AI gateway override", () => { - const handle = getModel( - env({ CF_AI_GATEWAY_WAI: "workers-ai-gateway" }), WORKERS_AI_CONFIG, INITIATOR); - expect(handle.model.baseUrl).toBe( - "https://gateway.ai.cloudflare.com/v1/gateway-account-id/workers-ai-gateway/workers-ai/v1"); - expect(handle.aiGatewayLogRoute).toEqual({ - gateway: "workers-ai-gateway", - accountId: "gateway-account-id", - apiToken: "gateway-token", - }); - }); + const request = await captureRequest(handle); + expect(request.url).toBe( + "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/workers-ai/" + + "v1/chat/completions"); + expect(request.headers.get("cf-aig-authorization")).toBe("Bearer gateway-token"); + // Session affinity flows through (Workers AI models opt in to the affinity headers). + expect(request.headers.get("x-session-affinity")).toBe("session-a"); + }, 15000); }); describe("getModel AI Gateway binding transport", () => { @@ -348,16 +313,15 @@ describe("getModel AI Gateway binding transport", () => { }, 15000); it("drives Workers AI through the binding via its gateway route", async () => { - const handle = getModel(bindingEnv({ CF_AI_GATEWAY_WAI: "workers-ai-gateway" }), - WORKERS_AI_CONFIG, INITIATOR); + const handle = getModel(bindingEnv(), WORKERS_AI_CONFIG, INITIATOR); expect(handle.model.baseUrl).toBe( - "https://gateway.ai.cloudflare.com/v1/gateway-account-id/workers-ai-gateway/" + + "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/" + "workers-ai/v1"); - expect(handle.aiGatewayLogRoute).toEqual({ gateway: "workers-ai-gateway" }); + expect(handle.aiGatewayLogRoute).toEqual({ gateway: "platform-gateway" }); const entry = await captureEntry(handle); - expect(entry.gatewayId).toBe("workers-ai-gateway"); + expect(entry.gatewayId).toBe("platform-gateway"); expect(entry.provider).toBe("workers-ai"); expect(entry.endpoint).toBe("v1/chat/completions"); expect((entry.query as { model: string }).model) @@ -371,12 +335,6 @@ describe("getModel AI Gateway binding transport", () => { expect(headerNames).not.toContain("cf-aig-authorization"); }, 15000); - it("requires the token for direct Workers AI REST routing", () => { - expect(() => getModel(bindingEnv({ CF_AI_GATEWAY_WAI_DIRECT: "true" }), - WORKERS_AI_CONFIG, INITIATOR)).toThrow( - "CF_AI_GATEWAY_WAI_DIRECT bypasses the gateway"); - }); - it("lets a per-call fetch override the binding transport", async () => { // Tests and diagnostics inject options.fetch; it must win over the handle's binding shim. // The raw request still carries the sentinel (stripping is the shim's job). diff --git a/packages/workshop-backend/__tests__/web-fetch.test.ts b/packages/workshop-backend/__tests__/web-fetch.test.ts index 275ef6a5..d13ad33f 100644 --- a/packages/workshop-backend/__tests__/web-fetch.test.ts +++ b/packages/workshop-backend/__tests__/web-fetch.test.ts @@ -134,7 +134,7 @@ describe("webFetch document conversion", () => { }); }); - it("does not pass a gateway to toMarkdown when direct Workers AI is configured", async () => { + it("does not pass a gateway to toMarkdown when the gateway is cross-account", async () => { mockResponse("

Title

", "text/html"); const toMarkdown = vi.fn(async (doc: { name: string; blob: Blob }) => ({ @@ -145,11 +145,13 @@ describe("webFetch document conversion", () => { tokens: 1, data: "# Title", })); + // CF_AI_GATEWAY_USE_BINDING=false marks the platform gateway as living in a different + // account; the binding-based toMarkdown call can't log through it. const gateway = new AiGatewayConfig({ CF_AI_GATEWAY: "platform-gateway", CF_AI_GATEWAY_ACCOUNT_ID: "gateway-account-id", CF_AI_GATEWAY_API_TOKEN: "gateway-token", - CF_AI_GATEWAY_WAI_DIRECT: "true", + CF_AI_GATEWAY_USE_BINDING: "false", } as Cloudflare.Env); await webFetch(makeEnv(toMarkdown, gateway), { url: "https://example.com/page" }); diff --git a/packages/workshop-backend/src/ai-gateway.ts b/packages/workshop-backend/src/ai-gateway.ts index 3ef3045a..a640b8ff 100644 --- a/packages/workshop-backend/src/ai-gateway.ts +++ b/packages/workshop-backend/src/ai-gateway.ts @@ -9,7 +9,10 @@ const QUICK_MODEL_ID = "@cf/meta/llama-3.3-70b-instruct-fp8-fast"; export class AiGatewayConfig { readonly gateway: string; - readonly workersAiGateway?: string; + // The gateway name for Workers-AI-binding calls (webFetch's toMarkdown): binding calls only + // reach gateways in the Worker's own account, so this is the platform gateway unless + // CF_AI_GATEWAY_USE_BINDING=false marks it cross-account. + readonly sameAccountGateway?: string; readonly accountId: string; readonly apiToken?: string; // Workers AI binding, used as the gateway transport whenever present unless @@ -42,18 +45,8 @@ export class AiGatewayConfig { "AI Gateway mode needs a transport: bind Workers AI (WORKERS_AI; in local dev start " + "with --use-workers-ai-binding) or set CF_AI_GATEWAY_API_TOKEN (a Run + Read token)."); } - if (env.CF_AI_GATEWAY_WAI_DIRECT === "true" && env.CF_AI_GATEWAY_WAI) { - throw new Error( - "CF_AI_GATEWAY_WAI and CF_AI_GATEWAY_WAI_DIRECT cannot be configured together."); - } - if (env.CF_AI_GATEWAY_WAI_DIRECT === "true" && !this.apiToken) { - throw new Error( - "CF_AI_GATEWAY_WAI_DIRECT bypasses the gateway and calls the Workers AI REST " + - "endpoint, which requires CF_AI_GATEWAY_API_TOKEN."); - } - this.workersAiGateway = env.CF_AI_GATEWAY_WAI_DIRECT === "true" - ? undefined - : env.CF_AI_GATEWAY_WAI || this.gateway; + this.sameAccountGateway = + env.CF_AI_GATEWAY_USE_BINDING === "false" ? undefined : this.gateway; this.providers = new Set( (env.CF_AI_GATEWAY_PROVIDERS || "").split(",").map(s => s.trim()).filter(s => s !== "") ); diff --git a/packages/workshop-backend/src/ai-models.ts b/packages/workshop-backend/src/ai-models.ts index 6c40ff61..8a1a556a 100644 --- a/packages/workshop-backend/src/ai-models.ts +++ b/packages/workshop-backend/src/ai-models.ts @@ -447,31 +447,6 @@ function getModelViaGateway( ? { gateway } : { gateway, accountId: gwConfig.accountId, apiToken: gwConfig.apiToken! }; - if (config.provider === "cloudflare" && !gwConfig.workersAiGateway) { - // CF_AI_GATEWAY_WAI_DIRECT: the plain Workers AI REST endpoint -- no gateway, no log route, - // no gateway metadata (mirroring the old direct-binding path, which had no - // aiGatewayLogRoute). Reuses the CF_AI_GATEWAY_* account/token pair (the config constructor - // guarantees the token in this mode). - const catalog = catalogModel(config.provider, config.model); - const model: Model = { - id: config.model, - name: catalog?.name ?? config.model, - api: "openai-completions", - provider: "cloudflare-workers-ai", - baseUrl: `https://api.cloudflare.com/client/v4/accounts/${gwConfig.accountId}/ai/v1`, - reasoning: catalog?.reasoning ?? false, - input: catalog?.input ?? ["text"], - cost: catalog?.cost ?? ZERO_COST, - ...modelTokenWindow(config, catalog), - compat: workersAiCompat(catalog), - }; - return makeHandle({ - model, - apiKey: gwConfig.apiToken, - sessionAffinity: options.sessionAffinity, - }); - } - if (config.provider === "google" && !gwConfig.apiToken) { // Unreachable when google is an enabled provider (the config constructor throws), but a // stored model config can still name google directly. @@ -479,10 +454,9 @@ function getModelViaGateway( "transport cannot carry them)."); } - // Workers AI may be routed through a different gateway than the other providers - // (CF_AI_GATEWAY_WAI); either way, gateway log route and attribution metadata apply. - const gateway = config.provider === "cloudflare" - ? gwConfig.workersAiGateway! : gwConfig.gateway; + // Every provider -- Workers AI included -- rides the same gateway, with the same log route + // and attribution metadata. + const gateway = gwConfig.gateway; const gatewayUrl = `${gatewayBase}/${gateway}`; const model = gatewayNativeModel(config, gatewayUrl); if (!model) { diff --git a/packages/workshop-backend/src/env.d.ts b/packages/workshop-backend/src/env.d.ts index fd51cb1c..66ba26df 100644 --- a/packages/workshop-backend/src/env.d.ts +++ b/packages/workshop-backend/src/env.d.ts @@ -15,25 +15,22 @@ declare global { // AI Gateway mode: when CF_AI_GATEWAY is set, supported providers are routed through // Cloudflare AI Gateway with server-managed keys. Users don't need their own keys. - // Transport: the WORKERS_AI binding when present (pre-authenticated in-account -- no API - // token; in local dev the binding needs --use-workers-ai-binding) unless - // CF_AI_GATEWAY_USE_BINDING=false opts out, HTTPS with CF_AI_GATEWAY_API_TOKEN otherwise. - // The token stays REQUIRED for the google provider (pi's Google adapter can't use the - // binding transport) and for CF_AI_GATEWAY_WAI_DIRECT. + // Transport: the WORKERS_AI binding when present (pre-authenticated + // in-account -- no API token; in local dev the binding needs --use-workers-ai-binding) + // unless CF_AI_GATEWAY_USE_BINDING=false opts out, HTTPS with CF_AI_GATEWAY_API_TOKEN + // otherwise. The token stays REQUIRED for the google provider (pi's Google adapter can't + // use the binding transport). CF_AI_GATEWAY?: string; // Gateway name (enables gateway mode) CF_AI_GATEWAY_PROVIDERS?: string; // Comma-separated list: "anthropic,openai,google,cloudflare" CF_AI_GATEWAY_ACCOUNT_ID?: string; // Gateway owner account ID (required with CF_AI_GATEWAY) CF_AI_GATEWAY_API_TOKEN?: string; // Run + Read token; optional when the binding transport - // applies (still required for google / WAI_DIRECT) + // applies (still required for google) // "false" = never use the WORKERS_AI binding as the gateway transport. Binding requests // only reach gateways in the Worker's own account, and the Worker can't verify where the // gateway lives (it can't discover its own account ID at runtime) -- so deployments whose // gateway is in a DIFFERENT account (e.g. the internal production Workshop) must set this // opt-out and route over HTTPS with the token. Unset/"true" = binding when present. CF_AI_GATEWAY_USE_BINDING?: string; - CF_AI_GATEWAY_WAI?: string; // Optional Workers AI gateway override - CF_AI_GATEWAY_WAI_DIRECT?: string; // "true" to route Workers AI to its plain REST endpoint - // (no gateway, no cost logs) instead of a named Gateway // Note: outside gateway mode, Workers AI (provider "cloudflare") is BYOK like every other // provider -- the account ID and API token live in the user's model config, not in env. diff --git a/packages/workshop-backend/src/web-fetch.ts b/packages/workshop-backend/src/web-fetch.ts index 3ad43b63..252c0550 100644 --- a/packages/workshop-backend/src/web-fetch.ts +++ b/packages/workshop-backend/src/web-fetch.ts @@ -177,14 +177,15 @@ const TO_MARKDOWN_MIME_TYPES = new Set([ "application/vnd.apple.numbers", // .numbers ]); -// `toMarkdown()` uses the Workers AI binding, so only apply the same-account Workers AI gateway -// resolved by AiGatewayConfig. A cross-account platform gateway cannot be used by this binding. +// `toMarkdown()` uses the Workers AI binding, and binding calls only reach gateways in the +// Worker's own account -- so apply the platform gateway only when AiGatewayConfig resolves it +// as same-account (CF_AI_GATEWAY_USE_BINDING=false marks it cross-account). function buildGatewayOptions( gateway: AiGatewayConfig | null, ): GatewayOptions | undefined { if (!gateway) return undefined; - if (!gateway.workersAiGateway) return undefined; - return { id: gateway.workersAiGateway, metadata: { tool: "webFetch", automated: true } }; + if (!gateway.sameAccountGateway) return undefined; + return { id: gateway.sameAccountGateway, metadata: { tool: "webFetch", automated: true } }; } // Attempt to convert a document to Markdown using the Workers AI binding. Returns the diff --git a/scripts/release/manifest-lib.ts b/scripts/release/manifest-lib.ts index 9d18c5fe..42cdae94 100644 --- a/scripts/release/manifest-lib.ts +++ b/scripts/release/manifest-lib.ts @@ -404,7 +404,7 @@ export function buildWorkerEntry( // the backend's default AI Gateway transport (the deploy service creates the gateway in the // user's own account, so the in-account requirement holds; CF_AI_GATEWAY_USE_BINDING=false // is the cross-account opt-out) — binding requests are pre-authenticated, so inference and - // cost-log reads need no CF_AI_GATEWAY_API_TOKEN (google provider and WAI_DIRECT excepted). + // cost-log reads need no CF_AI_GATEWAY_API_TOKEN (google provider excepted). // No placeholders — the deploy renderer passes it through. bindings.push({ type: "ai", name: "WORKERS_AI" }); // Installed gatekeepers are called through GATEKEEPER_* service bindings with the diff --git a/scripts/run-dev-server.ts b/scripts/run-dev-server.ts index b40b6820..86c81afb 100644 --- a/scripts/run-dev-server.ts +++ b/scripts/run-dev-server.ts @@ -483,11 +483,10 @@ for (const gk of gatekeepers) { // always needs CF_AI_GATEWAY_ACCOUNT_ID plus one transport: the WORKERS_AI binding // (start with --use-workers-ai-binding; CF_AI_GATEWAY_USE_BINDING=false opts out, e.g. // when the gateway lives in a different account than the dev binding) or - // CF_AI_GATEWAY_API_TOKEN over HTTPS. Two cases can't ride the binding and need the - // token even when it's present: the google provider and CF_AI_GATEWAY_WAI_DIRECT. + // CF_AI_GATEWAY_API_TOKEN over HTTPS. The google provider can't ride the binding and + // needs the token even when the binding is present. "CF_AI_GATEWAY", "CF_AI_GATEWAY_PROVIDERS", "CF_AI_GATEWAY_ACCOUNT_ID", - "CF_AI_GATEWAY_API_TOKEN", "CF_AI_GATEWAY_WAI", "CF_AI_GATEWAY_WAI_DIRECT", - "CF_AI_GATEWAY_USE_BINDING", + "CF_AI_GATEWAY_API_TOKEN", "CF_AI_GATEWAY_USE_BINDING", ]; // OAuth app credentials (GOOGLE_/GITHUB_/CLOUDFLARE_OAUTH_*) are NOT passed to the backend anymore; // they are injected into the gatekeeper Workers (see SHARED_GATEKEEPER_CREDS below). From 945eb5e2587f40143cd843f97c6703735c136dcd Mon Sep 17 00:00:00 2001 From: Maximo Guk <62088388+Maximo-Guk@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:40:42 -0500 Subject: [PATCH 3/4] Route the AI Gateway binding transport straight through env.AI.fetch() Binding-routed gateway traffic went through gateway().run(), the universal endpoint's [{provider, endpoint, headers, query}] envelope, which a vendored shim built by splicing the request body into a JSON string. The gateway also serves its provider-native passthrough over the AI binding at /ai-gateway/gateways/{gateway}/{provider}/... -- the HTTPS path minus the account id, since the binding channel carries identity -- and that route accepts exactly the requests pi's API impls already produce. So there is nothing left to translate. Binding-routed models take the binding host as their gateway root and pi's fetch option is the binding's own fetch, unwrapped; the envelope, the splice, the JSON-object scanner and the whole vendored ai-gateway-binding-fetch module go away. Method, headers, query string and the body stream ride through untouched, so multi-MB prompt bodies are never copied in the isolate. Google keeps its HTTPS root automatically: `binding` is resolved per provider, and bindingFor() returns undefined for google (its adapter can't take a custom fetch). cf-aig-authorization now reaches the gateway, which recognizes the pre-authentication sentinel and strips it. Co-Authored-By: Claude Opus 5 (1M context) --- .../ai-gateway-binding-fetch.test.ts | 275 ------------------ .../__tests__/ai-models.test.ts | 96 +++--- .../src/ai-gateway-binding-fetch.ts | 188 ------------ packages/workshop-backend/src/ai-gateway.ts | 24 +- packages/workshop-backend/src/ai-models.ts | 52 +++- packages/workshop-backend/src/env.d.ts | 10 - 6 files changed, 100 insertions(+), 545 deletions(-) delete mode 100644 packages/workshop-backend/__tests__/ai-gateway-binding-fetch.test.ts delete mode 100644 packages/workshop-backend/src/ai-gateway-binding-fetch.ts diff --git a/packages/workshop-backend/__tests__/ai-gateway-binding-fetch.test.ts b/packages/workshop-backend/__tests__/ai-gateway-binding-fetch.test.ts deleted file mode 100644 index 9dd0a559..00000000 --- a/packages/workshop-backend/__tests__/ai-gateway-binding-fetch.test.ts +++ /dev/null @@ -1,275 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - type AiGatewayUniversalRequestLike, - CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL, - createGatewayBindingFetch, -} from "../src/ai-gateway-binding-fetch.js"; - -const BASE_URL = "https://gateway.ai.cloudflare.com/v1/account-id/my-gateway"; - -type CapturedRun = { - gatewayId: string; - data: AiGatewayUniversalRequestLike; - options: { signal?: AbortSignal } | undefined; -}; - -function fakeBinding(response?: Response) { - const runs: CapturedRun[] = []; - const binding = { - gateway: (gatewayId: string) => ({ - run: (data: AiGatewayUniversalRequestLike, options?: { signal?: AbortSignal }) => { - runs.push({ gatewayId, data, options }); - return Promise.resolve(response ?? new Response("{}")); - }, - }), - }; - return { binding, runs }; -} - -describe("createGatewayBindingFetch", () => { - it("derives provider and endpoint from gateway passthrough URLs", async () => { - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - - await fetchFn(`${BASE_URL}/anthropic/v1/messages`, { - method: "POST", - body: JSON.stringify({ model: "claude" }), - }); - await fetchFn(`${BASE_URL}/openai/responses`, { - method: "POST", - body: JSON.stringify({ model: "gpt" }), - }); - await fetchFn(`${BASE_URL}/workers-ai/v1/chat/completions`, { - method: "POST", - body: JSON.stringify({ model: "@cf/meta/llama" }), - }); - - expect(runs.map((run) => [run.data.provider, run.data.endpoint])).toEqual([ - ["anthropic", "v1/messages"], - ["openai", "responses"], - ["workers-ai", "v1/chat/completions"], - ]); - expect(runs.map((run) => run.gatewayId)).toEqual(["my-gateway", "my-gateway", "my-gateway"]); - expect(runs[0].data.query).toEqual({ model: "claude" }); - }); - - it("keeps the query string in the endpoint", async () => { - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - - await fetchFn(`${BASE_URL}/openai/responses?beta=true`, { - method: "POST", - body: "{}", - }); - - expect(runs[0].data.endpoint).toBe("responses?beta=true"); - }); - - it("lowercases header names so case-variant duplicates collapse", async () => { - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - - await fetchFn(`${BASE_URL}/anthropic/v1/messages`, { - method: "POST", - headers: { "Anthropic-Version": "2023-06-01" }, - body: "{}", - }); - - expect(runs[0].data.headers).toEqual({ "anthropic-version": "2023-06-01" }); - }); - - it("lets init headers replace a Request input's headers, per the fetch spec", async () => { - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - - await fetchFn( - new Request(`${BASE_URL}/anthropic/v1/messages`, { - method: "POST", - headers: { "x-from-request": "yes" }, - body: "{}", - }), - { headers: { "x-from-init": "yes" } }); - - expect(runs[0].data.headers["x-from-init"]).toBe("yes"); - expect(runs[0].data.headers["x-from-request"]).toBeUndefined(); - }); - - it("strips gateway auth and derived headers, forwards the rest", async () => { - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - - await fetchFn(`${BASE_URL}/anthropic/v1/messages`, { - method: "POST", - headers: { - "Content-Type": "application/json", - "Content-Length": "17", - "CF-AIG-Authorization": `Bearer ${CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL}`, - "cf-aig-metadata": '{"user":"42"}', - "anthropic-version": "2023-06-01", - "x-api-key": "provider-key", - }, - body: "{}", - }); - - const headers = Object.fromEntries( - Object.entries(runs[0].data.headers).map(([key, value]) => [key.toLowerCase(), value])); - expect(headers["cf-aig-authorization"]).toBeUndefined(); - expect(headers["content-length"]).toBeUndefined(); - expect(headers["cf-aig-metadata"]).toBe('{"user":"42"}'); - expect(headers["anthropic-version"]).toBe("2023-06-01"); - // Provider auth headers pass through: that is how request-supplied (BYOK) keys ride. - expect(headers["x-api-key"]).toBe("provider-key"); - }); - - it("accepts Request inputs and forwards their headers and body", async () => { - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - - await fetchFn(new Request(`${BASE_URL}/openai/chat/completions`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ stream: true }), - })); - - expect(runs).toHaveLength(1); - expect(runs[0].data.provider).toBe("openai"); - expect(runs[0].data.endpoint).toBe("chat/completions"); - expect(runs[0].data.query).toEqual({ stream: true }); - expect(runs[0].data.headers["content-type"]).toBe("application/json"); - }); - - it("forwards the abort signal", async () => { - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - const controller = new AbortController(); - - await fetchFn(`${BASE_URL}/anthropic/v1/messages`, { - method: "POST", - body: "{}", - signal: controller.signal, - }); - - expect(runs[0].options?.signal).toBe(controller.signal); - }); - - it("lets an explicit `signal: null` in init clear a Request input's signal, per the fetch spec", - async () => { - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - const controller = new AbortController(); - - await fetchFn( - new Request(`${BASE_URL}/anthropic/v1/messages`, { - method: "POST", - body: "{}", - signal: controller.signal, - }), - { signal: null }); - - expect(runs).toHaveLength(1); - expect(runs[0].options?.signal).toBeUndefined(); - }); - - it("returns the binding response untouched, including streaming bodies", async () => { - const stream = new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode("data: {}\n\n")); - controller.close(); - }, - }); - const bindingResponse = new Response(stream, { - status: 200, - headers: { "content-type": "text/event-stream", "cf-aig-log-id": "log-1" }, - }); - const { binding } = fakeBinding(bindingResponse); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - - const response = await fetchFn(`${BASE_URL}/workers-ai/v1/chat/completions`, { - method: "POST", - body: "{}", - }); - - expect(response).toBe(bindingResponse); - expect(response.headers.get("cf-aig-log-id")).toBe("log-1"); - expect(await response.text()).toBe("data: {}\n\n"); - }); - - it("rejects in-prefix requests the universal endpoint cannot express", async () => { - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - - await expect(fetchFn(`${BASE_URL}/anthropic/v1/messages`, { method: "GET" })) - .rejects.toThrow("cannot express GET"); - await expect(fetchFn(`${BASE_URL}/anthropic/v1/messages`, { method: "POST", body: "not json" })) - .rejects.toThrow("non-JSON body"); - await expect(fetchFn(`${BASE_URL}/anthropic`, { method: "POST", body: "{}" })) - .rejects.toThrow("missing provider/endpoint path"); - expect(runs).toHaveLength(0); - }); - - it("rejects URLs outside the gateway prefix: transport selection is the caller's", async () => { - // Silent passthrough would ship the auth sentinel to whatever host the URL names; a - // misconfigured baseUrl must fail loudly instead. - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - - await expect(fetchFn("https://api.openai.com/v1/chat/completions", { method: "POST", body: "{}" })) - .rejects.toThrow("outside the configured gateway prefix"); - // Same origin, different path (another account's gateway) is just as out-of-prefix. - await expect(fetchFn( - "https://gateway.ai.cloudflare.com/v1/other-account/my-gateway/anthropic/v1/messages", - { method: "POST", body: "{}" })) - .rejects.toThrow("outside the configured gateway prefix"); - expect(runs).toHaveLength(0); - }); - - it("matches and splits on the URL-normalized path, as real fetch would send it", async () => { - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - - // Dot segments normalize away before the provider/endpoint split, so a lexical variant - // routes exactly like its normal form (raw string prefixing would split it differently). - await fetchFn(`${BASE_URL}/anthropic/../anthropic/v1/./messages`, { - method: "POST", - body: JSON.stringify({ model: "claude" }), - }); - expect(runs.map((run) => [run.data.provider, run.data.endpoint])).toEqual([ - ["anthropic", "v1/messages"], - ]); - - // A dot-segment URL that resolves outside the prefix is rejected even though it starts - // with the prefix as a raw string. - await expect(fetchFn(`${BASE_URL}/../other-gateway/anthropic/v1/messages`, - { method: "POST", body: "{}" })) - .rejects.toThrow("outside the configured gateway prefix"); - expect(runs).toHaveLength(1); - }); - - it("consumes a one-shot stream body for the JSON probe", async () => { - const { binding, runs } = fakeBinding(); - const fetchFn = createGatewayBindingFetch({ binding, baseUrl: BASE_URL, gateway: "my-gateway" }); - const streamOf = (text: string) => new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode(text)); - controller.close(); - }, - }); - - // JSON stream body: consumed once, reaches the binding as the parsed query. - await fetchFn(`${BASE_URL}/anthropic/v1/messages`, { - method: "POST", - body: streamOf('{"model":"claude"}'), - duplex: "half", - } as RequestInit); - expect(runs).toHaveLength(1); - expect(runs[0].data.query).toEqual({ model: "claude" }); - - // Non-JSON stream body: rejects like any other non-JSON body (never replayed). - await expect(fetchFn(`${BASE_URL}/anthropic/v1/messages`, { - method: "POST", - body: streamOf("not json"), - duplex: "half", - } as RequestInit)).rejects.toThrow("non-JSON body"); - expect(runs).toHaveLength(1); - }); -}); diff --git a/packages/workshop-backend/__tests__/ai-models.test.ts b/packages/workshop-backend/__tests__/ai-models.test.ts index 7cd75eaa..11e1cc2f 100644 --- a/packages/workshop-backend/__tests__/ai-models.test.ts +++ b/packages/workshop-backend/__tests__/ai-models.test.ts @@ -229,28 +229,33 @@ describe("getModel AI Gateway routing", () => { }); describe("getModel AI Gateway binding transport", () => { - // Universal-endpoint entries captured by the fake Workers AI binding. In binding mode the - // handle's requests never hit HTTP: pi's SDK fetch is the pi gateway-binding shim, which - // translates each request into binding.gateway(gw).run(entry). - type CapturedEntry = { - gatewayId: string; - provider: string; - endpoint: string; + // Provider-native requests captured by the fake Workers AI binding. In binding mode the + // handle's requests never hit HTTP: pi's SDK fetch is the gateway-binding shim, which only + // rewrites the URL onto the gateway's provider passthrough + // (workers-binding.ai/ai-gateway/gateways/{gateway}/{provider}/...) and hands the request to + // binding.fetch() otherwise unchanged. + type CapturedBindingRequest = { + url: string; + method: string; headers: Record; - query: unknown; + body: string; }; - const capturedEntries: CapturedEntry[] = []; + const capturedEntries: CapturedBindingRequest[] = []; const fakeBinding = { - gateway: (gatewayId: string) => ({ - run: async (data: Omit) => { - capturedEntries.push({ gatewayId, ...data }); - // Same non-retryable client error as the HTTP fetch stub: pi surfaces an error-stop - // message and the entry stays captured for assertions. - return Response.json( - { error: { type: "bad_request", message: "stubbed" } }, { status: 400 }); - }, - }), + fetch: async (input: Request | string | URL, init?: RequestInit) => { + const request = input instanceof Request ? input : new Request(input, init); + capturedEntries.push({ + url: request.url, + method: request.method, + headers: Object.fromEntries(request.headers), + body: await request.text(), + }); + // Same non-retryable client error as the HTTP fetch stub: pi surfaces an error-stop + // message and the request stays captured for assertions. + return Response.json( + { error: { type: "bad_request", message: "stubbed" } }, { status: 400 }); + }, } as unknown as Ai; // Binding transport selects by default: binding present, no API token (in-account gateways; @@ -265,7 +270,7 @@ describe("getModel AI Gateway binding transport", () => { }); } - async function captureEntry(handle: ModelHandle): Promise { + async function captureEntry(handle: ModelHandle): Promise { const stream = handle.stream(handle.model, { messages: [{ role: "user", content: "hello", timestamp: 0 }], }, { maxRetries: 0 }); @@ -286,65 +291,63 @@ describe("getModel AI Gateway binding transport", () => { }); expect(handle.model.api).toBe("anthropic-messages"); + // Binding-routed models address the gateway on the binding's host, which takes no account + // id -- the binding channel carries identity. expect(handle.model.baseUrl).toBe( - "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/anthropic"); + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/anthropic"); // Same-account log reads ride the binding too: no account id or token in the route. expect(handle.aiGatewayLogRoute).toEqual({ gateway: "platform-gateway" }); const entry = await captureEntry(handle); - expect(entry.gatewayId).toBe("platform-gateway"); - expect(entry.provider).toBe("anthropic"); - expect(entry.endpoint).toBe("v1/messages"); - // The sentinel auth header satisfies pi's request-auth check but must never reach the - // gateway; the SDK's own auth headers stay suppressed. + expect(entry.url).toBe( + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/anthropic/v1/messages"); + expect(entry.method).toBe("POST"); + // The sentinel auth header satisfies pi's request-auth check; the gateway recognizes and + // strips it on binding-routed requests, so the shim forwards it. The SDK's own auth + // headers stay suppressed. + expect(entry.headers["cf-aig-authorization"]).toBe("Bearer cloudflare-gateway-binding"); const headerNames = Object.keys(entry.headers).map((name) => name.toLowerCase()); - expect(headerNames).not.toContain("cf-aig-authorization"); expect(headerNames).not.toContain("x-api-key"); expect(headerNames).not.toContain("authorization"); - const metadataHeader = Object.entries(entry.headers) - .find(([name]) => name.toLowerCase() === "cf-aig-metadata")?.[1]; - expect(JSON.parse(metadataHeader!)).toEqual({ + expect(JSON.parse(entry.headers["cf-aig-metadata"])).toEqual({ user: "user-123", source: "chat", gadgetId: "gadget-123", chatId: 7, }); - expect((entry.query as { model: string }).model).toBe("claude-sonnet-4-5"); + expect((JSON.parse(entry.body) as { model: string }).model).toBe("claude-sonnet-4-5"); }, 15000); it("drives Workers AI through the binding via its gateway route", async () => { const handle = getModel(bindingEnv(), WORKERS_AI_CONFIG, INITIATOR); expect(handle.model.baseUrl).toBe( - "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/" + - "workers-ai/v1"); + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/workers-ai/v1"); expect(handle.aiGatewayLogRoute).toEqual({ gateway: "platform-gateway" }); const entry = await captureEntry(handle); - expect(entry.gatewayId).toBe("platform-gateway"); - expect(entry.provider).toBe("workers-ai"); - expect(entry.endpoint).toBe("v1/chat/completions"); - expect((entry.query as { model: string }).model) + expect(entry.url).toBe( + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/workers-ai/" + + "v1/chat/completions"); + expect((JSON.parse(entry.body) as { model: string }).model) .toBe("@cf/meta/llama-3.3-70b-instruct-fp8-fast"); // openai-completions adapters inject `Authorization: Bearer unused` under header-owned - // auth; the gatewayAuthHeaders nulls must delete it before the entry is built, else the - // gateway would treat it as a request-supplied provider key overriding stored keys. + // auth; the gatewayAuthHeaders nulls must delete it before dispatch, else the gateway + // would treat it as a request-supplied provider key overriding stored keys. const headerNames = Object.keys(entry.headers).map((name) => name.toLowerCase()); expect(headerNames).not.toContain("authorization"); expect(headerNames).not.toContain("x-api-key"); - expect(headerNames).not.toContain("cf-aig-authorization"); }, 15000); it("lets a per-call fetch override the binding transport", async () => { - // Tests and diagnostics inject options.fetch; it must win over the handle's binding shim. - // The raw request still carries the sentinel (stripping is the shim's job). + // Tests and diagnostics inject options.fetch; it must win over the handle's binding fetch. + // The URL is the model's, so it still names the binding route -- only the transport swaps. const handle = getModel(bindingEnv(), ANTHROPIC_CONFIG, INITIATOR); const request = await captureRequest(handle); expect(capturedEntries).toHaveLength(0); expect(request.url).toBe( - "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/anthropic/" + - "v1/messages"); + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/anthropic/v1/messages"); expect(request.headers.get("cf-aig-authorization")) .toBe("Bearer cloudflare-gateway-binding"); }, 15000); @@ -370,9 +373,10 @@ describe("getModel AI Gateway binding transport", () => { const anthropicHandle = getModel(hybridEnv, ANTHROPIC_CONFIG, INITIATOR); const entry = await captureEntry(anthropicHandle); - expect(entry.provider).toBe("anthropic"); - const headerNames = Object.keys(entry.headers).map((name) => name.toLowerCase()); - expect(headerNames).not.toContain("cf-aig-authorization"); + expect(entry.url).toBe( + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/anthropic/v1/messages"); + // The binding arm carries the sentinel, never the real gateway token. + expect(entry.headers["cf-aig-authorization"]).toBe("Bearer cloudflare-gateway-binding"); }, 15000); it("requires the token when google is an enabled provider", () => { diff --git a/packages/workshop-backend/src/ai-gateway-binding-fetch.ts b/packages/workshop-backend/src/ai-gateway-binding-fetch.ts deleted file mode 100644 index a53d2798..00000000 --- a/packages/workshop-backend/src/ai-gateway-binding-fetch.ts +++ /dev/null @@ -1,188 +0,0 @@ -// Cloudflare AI Gateway over the Workers AI binding. -// -// `createGatewayBindingFetch` returns a fetch that translates requests bound for an AI Gateway -// HTTPS endpoint (https://gateway.ai.cloudflare.com/v1/{account}/{gateway}/...) into calls -// through the Workers AI binding's universal endpoint -// (env.WORKERS_AI.gateway(id).run({provider, endpoint, headers, query})), which returns the -// provider's native wire format as a regular streaming Response. Binding calls are -// pre-authenticated in-account, so no cf-aig-authorization token is needed -- pass -// CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL where an auth header is required to satisfy pi's -// request-auth checks; the shim strips it before dispatch. Pair the sentinel with explicit -// `Authorization: null` / `x-api-key: null` provider headers (a null value deletes the header -// in the SDKs): with header-owned auth the SDKs otherwise inject a placeholder auth header -// (e.g. `Authorization: Bearer unused`), which the shim forwards like any provider auth header -// -- and the gateway treats a request-supplied provider auth header as a BYOK key that -// overrides its stored keys, the same as it would over HTTPS. - -import type { FetchFunction } from "@earendil-works/pi-ai"; - -// Structural type for the Workers AI binding's gateway surface (`env.WORKERS_AI`); any real -// `Ai` binding satisfies it. -export interface AiGatewayBinding { - gateway(id: string): AiGatewayBindingGateway; -} - -export interface AiGatewayBindingGateway { - run(data: AiGatewayUniversalRequestLike, options?: { signal?: AbortSignal }): Promise; -} - -/** One universal-endpoint request entry, as accepted by `AiGateway.run()`. */ -export interface AiGatewayUniversalRequestLike { - provider: string; - endpoint: string; - headers: Record; - query: unknown; -} - -/** - * Placeholder value for auth headers on binding-routed requests. pi's API implementations - * require an API key or a recognized auth header (`authorization`, `x-api-key`, - * `cf-aig-authorization`) before dispatch; binding calls are pre-authenticated, so pass - * `cf-aig-authorization: Bearer ${CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL}` to satisfy the - * check. The shim strips `cf-aig-authorization` before calling the binding. Pair it with - * `Authorization: null` / `x-api-key: null` so the SDKs' placeholder auth headers never reach - * the gateway (see the module docs). - */ -export const CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL = "cloudflare-gateway-binding"; - -export interface GatewayBindingFetchOptions { - // The Workers AI binding (e.g. env.WORKERS_AI). - binding: AiGatewayBinding; - // Gateway HTTPS prefix every request must fall under, without a trailing slash: - // https://gateway.ai.cloudflare.com/v1/{accountId}/{gatewayName}. - baseUrl: string; - // Gateway name passed to binding.gateway(). Must match the baseUrl gateway. - gateway: string; -} - -// Never forwarded to the binding: hop-by-hop/derived headers, and gateway auth (binding calls -// are pre-authenticated; the sentinel must not reach the wire, and a real gateway token is -// meaningless on this path and would only end up in request logs). -const STRIP_HEADERS = new Set(["content-length", "host", "cf-aig-authorization"]); - -type FetchInput = Parameters[0]; - -/** - * Create a `fetch` that routes AI Gateway requests through the Workers AI binding. This is - * the transport for one gateway-bound client, not a general-purpose fetch: URLs outside the - * configured gateway prefix, and in-prefix requests the binding cannot express (non-POST, - * non-JSON body), reject with a descriptive error. Transport selection is the caller's job, - * per client -- route such traffic over HTTPS with real gateway auth instead. - */ -export function createGatewayBindingFetch(options: GatewayBindingFetchOptions): FetchFunction { - const { binding, gateway } = options; - // Prefix matching runs on URL-normalized components (origin + pathname), not raw strings: - // dot segments resolve away and fragments drop, matching what real fetch would put on the - // wire, so a lexical variant can't split provider/endpoint differently than HTTPS would. - const base = new URL(options.baseUrl); - const basePath = base.pathname.endsWith("/") ? base.pathname : `${base.pathname}/`; - - return async (input: FetchInput, init?: RequestInit): Promise => { - const request = input instanceof Request ? input : undefined; - const url = request ? request.url : input.toString(); - const method = (init?.method ?? request?.method ?? "GET").toUpperCase(); - let parsed: URL | undefined; - try { - parsed = new URL(url); - } catch { - parsed = undefined; - } - // Out-of-prefix URLs are a configuration bug, not passthrough traffic: silently - // forwarding would ship the auth sentinel to whatever host the URL names. - if (parsed === undefined || parsed.origin !== base.origin || - !parsed.pathname.startsWith(basePath)) { - throw new Error( - `createGatewayBindingFetch: ${method} ${url} is outside the configured gateway ` + - `prefix (${base.origin}${basePath}); this fetch only serves its gateway-bound client`); - } - - // In-prefix requests the universal endpoint cannot express always reject: forwarding - // them over HTTPS would send the sentinel to the gateway and fail with a misleading - // auth error instead of naming the real problem. Callers that need such endpoints - // route them over HTTPS with real gateway auth themselves. - const unexpressible = (reason: string): never => { - throw new Error( - `createGatewayBindingFetch: cannot express ${method} ${url} as a universal ` + - `gateway request (${reason}); route it over HTTPS with gateway auth instead`); - }; - if (method !== "POST") return unexpressible("only POST is supported"); - - const rest = parsed.pathname.slice(basePath.length); - const slash = rest.indexOf("/"); - if (slash <= 0) { - return unexpressible("missing provider/endpoint path"); - } - const provider = rest.slice(0, slash); - // Keep the query string on the endpoint -- it's part of what HTTPS would have sent. - const endpoint = rest.slice(slash + 1) + parsed.search; - - const bodyText = await readBodyText(request, init); - let query: unknown; - try { - query = bodyText === undefined ? undefined : JSON.parse(bodyText); - } catch { - return unexpressible("non-JSON body"); - } - if (query === undefined) { - return unexpressible("missing body"); - } - - const headers = collectHeaders(request, init); - // Per the fetch spec an explicit `signal: null` in init clears a Request input's signal. - const signal = init?.signal ?? - (init && "signal" in init && init.signal === null ? undefined : request?.signal); - return binding.gateway(gateway).run( - { provider, endpoint, headers, query }, signal ? { signal } : {}); - }; -} - -async function readBodyText(request: Request | undefined, init?: RequestInit) - : Promise { - const body = init?.body; - if (body === undefined || body === null) { - // Per the fetch spec an explicit `body: null` in init clears a Request input's body. - if (init && "body" in init && body === null) return undefined; - if (request && request.body !== null) return request.clone().text(); - return undefined; - } - if (typeof body === "string") return body; - if (body instanceof Uint8Array) return new TextDecoder().decode(body); - if (body instanceof ArrayBuffer) return new TextDecoder().decode(new Uint8Array(body)); - // URLSearchParams, FormData, Blob, ReadableStream in init: read via a Request wrapper. - // Consuming a one-shot stream here is fine -- unexpressible requests reject rather than - // replay, so nothing downstream needs the body again. - return new Request("http://body.local", { - method: "POST", - body, - // The fetch spec requires `duplex: "half"` to construct a Request with a stream body - // (Node's undici enforces it; it is ignored for the replayable body types). TypeScript's - // RequestInit does not declare the field yet, hence the cast. - duplex: "half", - } as RequestInit).text(); -} - -// Entry header names are lowercased so case-variant duplicates collapse and stripping is -// uniform. Per the fetch spec, `init.headers` replaces a Request input's headers entirely. -function collectHeaders(request: Request | undefined, init?: RequestInit) - : Record { - const result: Record = {}; - const add = (key: string, value: string) => { - const name = key.toLowerCase(); - if (!STRIP_HEADERS.has(name)) result[name] = value; - }; - const headers = init?.headers; - if (headers === undefined) { - if (request) { - for (const [key, value] of request.headers) add(key, value); - } - } else if (headers instanceof Headers) { - for (const [key, value] of headers) add(key, value); - } else if (Array.isArray(headers)) { - for (const [key, value] of headers) add(key, value); - } else { - for (const [key, value] of Object.entries(headers)) { - if (value !== undefined) add(key, String(value)); - } - } - return result; -} diff --git a/packages/workshop-backend/src/ai-gateway.ts b/packages/workshop-backend/src/ai-gateway.ts index a640b8ff..cdb093cd 100644 --- a/packages/workshop-backend/src/ai-gateway.ts +++ b/packages/workshop-backend/src/ai-gateway.ts @@ -9,19 +9,23 @@ const QUICK_MODEL_ID = "@cf/meta/llama-3.3-70b-instruct-fp8-fast"; export class AiGatewayConfig { readonly gateway: string; - // The gateway name for Workers-AI-binding calls (webFetch's toMarkdown): binding calls only - // reach gateways in the Worker's own account, so this is the platform gateway unless - // CF_AI_GATEWAY_USE_BINDING=false marks it cross-account. + /** + * The gateway name for Workers-AI-binding calls (webFetch's toMarkdown): binding calls only + * reach gateways in the Worker's own account, so this is the platform gateway unless + * CF_AI_GATEWAY_USE_BINDING=false marks it cross-account. + */ readonly sameAccountGateway?: string; readonly accountId: string; readonly apiToken?: string; - // Workers AI binding, used as the gateway transport whenever present unless - // CF_AI_GATEWAY_USE_BINDING=false opts out: binding requests are pre-authenticated in-account, - // so inference and cost-log reads need no API token. Binding requests only reach gateways in - // the Worker's own account, and the Worker can't verify that itself (it can't discover its own - // account ID), so deployments whose gateway lives in a DIFFERENT account must set the opt-out - // and use CF_AI_GATEWAY_API_TOKEN over HTTPS. Absent in local dev unless run-dev-server is - // started with --use-workers-ai-binding. + /** + * Workers AI binding, used as the gateway transport whenever present unless + * CF_AI_GATEWAY_USE_BINDING=false opts out: binding requests are pre-authenticated in-account, + * so inference and cost-log reads need no API token. Binding requests only reach gateways in + * the Worker's own account, and the Worker can't verify that itself (it can't discover its own + * account ID), so deployments whose gateway lives in a DIFFERENT account must set the opt-out + * and use CF_AI_GATEWAY_API_TOKEN over HTTPS. Absent in local dev unless run-dev-server is + * started with --use-workers-ai-binding. + */ readonly binding?: Ai; readonly providers: Set; diff --git a/packages/workshop-backend/src/ai-models.ts b/packages/workshop-backend/src/ai-models.ts index 8a1a556a..d82d3708 100644 --- a/packages/workshop-backend/src/ai-models.ts +++ b/packages/workshop-backend/src/ai-models.ts @@ -5,9 +5,6 @@ import type { ModelCost, OpenAICompletionsCompat, ProviderHeaders, SimpleStreamOptions, StreamFunction, } from "@earendil-works/pi-ai"; import { stream as anthropicMessagesStream } from "@earendil-works/pi-ai/api/anthropic-messages"; -import { - CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL, createGatewayBindingFetch, -} from "./ai-gateway-binding-fetch.js"; import { stream as googleGenerativeAiStream } from "@earendil-works/pi-ai/api/google-generative-ai"; import { stream as openaiCompletionsStream } from "@earendil-works/pi-ai/api/openai-completions"; import { stream as openaiResponsesStream } from "@earendil-works/pi-ai/api/openai-responses"; @@ -168,9 +165,11 @@ function workersAiCompat(catalog: Model | undefined): OpenAICompletionsComp } // Build the pi model descriptor for reaching a provider's own native API through an AI Gateway -// (the platform's or a user's). `gatewayUrl` is a gateway root -// (https://gateway.ai.cloudflare.com/v1/{accountId}/{gateway}); each provider's native API is -// exposed under a per-provider path on it. AI Gateway also offers a unified OpenAI-compat +// (the platform's or a user's). `gatewayUrl` is a gateway root -- over HTTPS +// (https://gateway.ai.cloudflare.com/v1/{accountId}/{gateway}) or, for binding-routed requests, +// over the AI binding (https://workers-binding.ai/ai-gateway/gateways/{gateway}); each +// provider's native API is exposed under the same per-provider path on either. AI Gateway also +// offers a unified OpenAI-compat // translation layer (/compat), which we deliberately never use: we already speak every // provider's native API, and the translation drops provider features pi relies on (extended // thinking, Anthropic cache_control prompt caching, the OpenAI Responses API). Billing -- @@ -271,7 +270,8 @@ type HandleArgs = { gatewayMetadata?: GatewayMetadata; sessionAffinity?: string; aiGatewayLogRoute?: AiGatewayLogRoute; - // Transport override for every request on this handle (e.g. the AI Gateway binding shim). + // Transport override for every request on this handle: how a binding-routed model reaches the + // gateway over env.WORKERS_AI.fetch() instead of the global fetch (see bindingFetch). // A per-call options.fetch still wins, which tests rely on to capture requests. fetch?: FetchFunction; }; @@ -415,6 +415,28 @@ function getModelViaUserGateway( }); } +/** + * Placeholder auth value for binding-routed requests. pi's API impls require an API key or a + * recognized auth header (authorization, x-api-key, cf-aig-authorization) before dispatch; + * binding calls are pre-authenticated in-account, so this satisfies the check and the gateway + * recognizes and strips it rather than treating it as a BYOK provider key. + */ +const CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL = "cloudflare-gateway-binding"; + +/** + * `Ai#fetch` exists at runtime but @cloudflare/workers-types' `Ai` doesn't declare it, so the + * binding is cast structurally to reach the passthrough. + */ +type AiFetchBinding = { + fetch(input: Request | string | URL, init?: RequestInit): Promise; +}; + +// pi drives the model's baseUrl, which already names the gateway route on the binding's host, +// so the binding's fetch passes through unchanged -- no URL rewriting needed. +function bindingFetch(binding: Ai): FetchFunction { + return (input, init) => (binding as unknown as AiFetchBinding).fetch(input, init); +} + // Platform free-tier path: route through the deployment's configured AI Gateway (platform-funded). // Used only for requests that are NOT billed to a connected user's account. function getModelViaGateway( @@ -424,10 +446,6 @@ function getModelViaGateway( options: ModelRoutingOptions, ): ModelHandle { const metadata = buildMetadata(initiator, options.metadata); - // Binding transport when available (all providers except Google): requests go through - // env.WORKERS_AI.gateway().run(), pre-authenticated in-account. pi's API impls require a - // recognized auth header before dispatch, so binding-routed requests carry a sentinel - // cf-aig-authorization that the shim strips before it reaches the wire. const binding = gwConfig.bindingFor(config.provider); const gatewayAuthHeaders: ProviderHeaders = { // pi's API impls explicitly recognize cf-aig-authorization and skip SDK auth; the null @@ -455,9 +473,13 @@ function getModelViaGateway( } // Every provider -- Workers AI included -- rides the same gateway, with the same log route - // and attribution metadata. + // and attribution metadata. Binding-routed providers address it on the binding's host, which + // takes no account id (the binding channel carries identity); the paths are otherwise the + // same, so the model descriptors are built identically from either root. const gateway = gwConfig.gateway; - const gatewayUrl = `${gatewayBase}/${gateway}`; + const gatewayUrl = binding + ? `https://workers-binding.ai/ai-gateway/gateways/${gateway}` + : `${gatewayBase}/${gateway}`; const model = gatewayNativeModel(config, gatewayUrl); if (!model) { throw new Error( @@ -476,9 +498,7 @@ function getModelViaGateway( // the gateway recognizes its own token there and applies the stored Google key instead. ...(config.provider === "google" ? { apiKey: gwConfig.apiToken } : {}), headers: gatewayAuthHeaders, - ...(binding - ? { fetch: createGatewayBindingFetch({ binding, baseUrl: gatewayUrl, gateway }) } - : {}), + ...(binding ? { fetch: bindingFetch(binding) } : {}), gatewayMetadata: metadata, sessionAffinity: options.sessionAffinity, aiGatewayLogRoute: logRoute(gateway), diff --git a/packages/workshop-backend/src/env.d.ts b/packages/workshop-backend/src/env.d.ts index 66ba26df..9ad6c001 100644 --- a/packages/workshop-backend/src/env.d.ts +++ b/packages/workshop-backend/src/env.d.ts @@ -15,21 +15,11 @@ declare global { // AI Gateway mode: when CF_AI_GATEWAY is set, supported providers are routed through // Cloudflare AI Gateway with server-managed keys. Users don't need their own keys. - // Transport: the WORKERS_AI binding when present (pre-authenticated - // in-account -- no API token; in local dev the binding needs --use-workers-ai-binding) - // unless CF_AI_GATEWAY_USE_BINDING=false opts out, HTTPS with CF_AI_GATEWAY_API_TOKEN - // otherwise. The token stays REQUIRED for the google provider (pi's Google adapter can't - // use the binding transport). CF_AI_GATEWAY?: string; // Gateway name (enables gateway mode) CF_AI_GATEWAY_PROVIDERS?: string; // Comma-separated list: "anthropic,openai,google,cloudflare" CF_AI_GATEWAY_ACCOUNT_ID?: string; // Gateway owner account ID (required with CF_AI_GATEWAY) CF_AI_GATEWAY_API_TOKEN?: string; // Run + Read token; optional when the binding transport // applies (still required for google) - // "false" = never use the WORKERS_AI binding as the gateway transport. Binding requests - // only reach gateways in the Worker's own account, and the Worker can't verify where the - // gateway lives (it can't discover its own account ID at runtime) -- so deployments whose - // gateway is in a DIFFERENT account (e.g. the internal production Workshop) must set this - // opt-out and route over HTTPS with the token. Unset/"true" = binding when present. CF_AI_GATEWAY_USE_BINDING?: string; // Note: outside gateway mode, Workers AI (provider "cloudflare") is BYOK like every other // provider -- the account ID and API token live in the user's model config, not in env. From 5acb81936920131425e48908052d2dd43b5d970f Mon Sep 17 00:00:00 2001 From: Maximo Guk <62088388+Maximo-Guk@users.noreply.github.com> Date: Tue, 18 Aug 2026 08:22:51 -0500 Subject: [PATCH 4/4] Make our workers previews use AI binding --- .github/workflows/preview.yml | 15 +++++-- scripts/env-passthrough.test.ts | 2 +- scripts/preview/staging-config.test.ts | 45 +++++++++++++++----- scripts/preview/staging-config.ts | 57 ++++++++++++++++++-------- 4 files changed, 87 insertions(+), 32 deletions(-) diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 01b5d03d..6292ebf2 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -106,13 +106,22 @@ jobs: CF_ACCESS_ISS: ${{ secrets.CF_ACCESS_ISS }} # AI Gateway, so a preview's chats use server-managed keys rather than asking each user # for their own. Optional as a group: with CF_AI_GATEWAY unset the preview is BYOK, but - # once it is set the account id and the Run + Read token are required. + # once it is set the account id is required. CF_AI_GATEWAY: ${{ secrets.CF_AI_GATEWAY }} # This is delibaretely set to CF_OS_AI_GATEWAY_ACCOUNT_ID (gets uploaded as CF_AI_GATEWAY_ACCOUNT_ID) CF_AI_GATEWAY_ACCOUNT_ID: ${{ secrets.CF_OS_AI_GATEWAY_ACCOUNT_ID }} - CF_AI_GATEWAY_API_TOKEN: ${{ secrets.CF_AI_GATEWAY_API_TOKEN }} CF_AI_GATEWAY_PROVIDERS: ${{ secrets.CF_AI_GATEWAY_PROVIDERS }} - CF_AI_GATEWAY_WAI_DIRECT: ${{ secrets.CF_AI_GATEWAY_WAI_DIRECT }} + # The gateway above lives in the account these previews deploy to, so the backend's + # WORKERS_AI binding reaches it: inference and cost-log reads are pre-authenticated + # in-account, which is why no CF_AI_GATEWAY_API_TOKEN is passed at all. `true` rather than + # unset so a preview that somehow lost the binding fails outright instead of quietly + # falling back to an HTTPS transport it has no token for. + # + # Two configurations cannot use the binding and so are unreachable from here until a + # `CF_AI_GATEWAY_API_TOKEN: ${{ secrets.CF_AI_GATEWAY_API_TOKEN }}` line is added back: + # a gateway in a *different* account (which sets this secret to `false`), and the google + # provider, whose SDK cannot take the binding's fetch. Both fail the deploy, by name. + CF_AI_GATEWAY_USE_BINDING: ${{ secrets.CF_AI_GATEWAY_USE_BINDING || 'true' }} # Together these are the preview name, which is the first label of its hostname: a preview # reads as `pr123-my-branch-router..workers.dev`. The number is what keeps two # branches that slugify alike from sharing one instance, and is how the nightly sweep diff --git a/scripts/env-passthrough.test.ts b/scripts/env-passthrough.test.ts index 3fd1989c..88203d51 100644 --- a/scripts/env-passthrough.test.ts +++ b/scripts/env-passthrough.test.ts @@ -71,7 +71,7 @@ const EXPECTED: Record = { forwarded: ["VITE_FRONTEND_ERROR_REPORTING"], external: [ "CF_ACCESS_AUD", "CF_ACCESS_ISS", "CF_AI_GATEWAY", "CF_AI_GATEWAY_ACCOUNT_ID", - "CF_AI_GATEWAY_API_TOKEN", "CF_AI_GATEWAY_PROVIDERS", "CF_AI_GATEWAY_WAI_DIRECT", + "CF_AI_GATEWAY_API_TOKEN", "CF_AI_GATEWAY_PROVIDERS", "CF_AI_GATEWAY_USE_BINDING", "CI_COMMIT_SHA", "CI_PIPELINE_IID", "CLOUDFLARE_ACCOUNT_ID", "CLOUDFLARE_API_TOKEN", "GITHUB_REPOSITORY", "GITHUB_TOKEN", "PREVIEW_ADMINS", "PREVIEW_NAME", "PREVIEW_PR_NUMBER", "PREVIEW_WORKERS_DEV_HOST", "PREVIEW_WRANGLER", "VITE_BACKEND_HOST", diff --git a/scripts/preview/staging-config.test.ts b/scripts/preview/staging-config.test.ts index 58f22e26..c91ea9b9 100644 --- a/scripts/preview/staging-config.test.ts +++ b/scripts/preview/staging-config.test.ts @@ -41,7 +41,7 @@ const AI_GATEWAY = { accountId: "1".repeat(32), apiToken: "example-run-and-read-token", providers: "cloudflare", - waiDirect: "true", + useBinding: "false", }; // Every input is passed explicitly: each resolver defaults to reading the environment, so a machine // with any of these set would otherwise change what the tests assert. @@ -269,7 +269,7 @@ test("the backend's secrets are the admin list, the Access pair and the AI gatew CF_AI_GATEWAY_ACCOUNT_ID: AI_GATEWAY.accountId, CF_AI_GATEWAY_API_TOKEN: AI_GATEWAY.apiToken, CF_AI_GATEWAY_PROVIDERS: AI_GATEWAY.providers, - CF_AI_GATEWAY_WAI_DIRECT: AI_GATEWAY.waiDirect, + CF_AI_GATEWAY_USE_BINDING: AI_GATEWAY.useBinding, }); }); @@ -278,20 +278,45 @@ test("the AI gateway is optional as a group, but not half-configured", () => { assert.deepEqual(resolveAiGateway({}), {}); assert.deepEqual(resolveAiGateway({ accountId: AI_GATEWAY.accountId }), {}, "orphans are ignored"); - // With one, the account and token are what AiGatewayConfig demands: without them it throws on the + // With one, the account is what AiGatewayConfig demands: without it the backend throws on the // first chat, so the deploy has to be the thing that fails instead. assert.throws(() => resolveAiGateway({ gateway: "g" }), - /CF_AI_GATEWAY_ACCOUNT_ID and CF_AI_GATEWAY_API_TOKEN must be set when CF_AI_GATEWAY is/); - assert.throws(() => resolveAiGateway({ gateway: "g", apiToken: "t" }), - /CF_AI_GATEWAY_ACCOUNT_ID must be set/); - assert.throws(() => resolveAiGateway({ gateway: "g", accountId: "a" }), - /CF_AI_GATEWAY_API_TOKEN must be set/); + /CF_AI_GATEWAY_ACCOUNT_ID must be set when CF_AI_GATEWAY is/); - // The two knobs below the required trio are each independently optional. + // The gateway name and its account are the whole requirement: a preview binds Workers AI, and + // the binding transport is pre-authenticated in-account, so a tokenless gateway is a complete + // configuration. Everything below them is independently optional. + assert.deepEqual(resolveAiGateway({ gateway: "g", accountId: "a" }), + { CF_AI_GATEWAY: "g", CF_AI_GATEWAY_ACCOUNT_ID: "a" }); assert.deepEqual(resolveAiGateway({ gateway: "g", accountId: "a", apiToken: "t" }), { CF_AI_GATEWAY: "g", CF_AI_GATEWAY_ACCOUNT_ID: "a", CF_AI_GATEWAY_API_TOKEN: "t" }); }); +test("a preview that cannot use the binding transport needs the gateway token", () => { + // Both mirror an AiGatewayConfig throw: opting out of the binding leaves only HTTPS, and the + // google SDK cannot take the binding's fetch. Each is a deploy failure rather than a chat one. + assert.throws( + () => resolveAiGateway({ gateway: "g", accountId: "a", useBinding: "false" }), + /CF_AI_GATEWAY_API_TOKEN must be set when CF_AI_GATEWAY_USE_BINDING is false/); + assert.throws( + () => resolveAiGateway({ gateway: "g", accountId: "a", providers: "cloudflare,google" }), + /CF_AI_GATEWAY_API_TOKEN must be set when the google provider is enabled/); + + // With the token, both are configurations rather than errors. + assert.deepEqual( + resolveAiGateway({ gateway: "g", accountId: "a", apiToken: "t", useBinding: "false" }), + { CF_AI_GATEWAY: "g", CF_AI_GATEWAY_ACCOUNT_ID: "a", CF_AI_GATEWAY_API_TOKEN: "t", + CF_AI_GATEWAY_USE_BINDING: "false" }); + // Requiring the binding is the other half of the same knob, and needs no token at all. + assert.deepEqual(resolveAiGateway({ gateway: "g", accountId: "a", useBinding: "true" }), + { CF_AI_GATEWAY: "g", CF_AI_GATEWAY_ACCOUNT_ID: "a", CF_AI_GATEWAY_USE_BINDING: "true" }); + + // The backend compares against the two strings and reads anything else as unset, so a value it + // would silently ignore fails here instead. + assert.throws(() => resolveAiGateway({ gateway: "g", accountId: "a", useBinding: "False" }), + /CF_AI_GATEWAY_USE_BINDING must be "true" or "false"/); +}); + test("no generated config declares a secret's variable", () => { for (const [name, config] of buildAll().configs) { const halves: [string, Record | undefined][] = [ @@ -317,7 +342,7 @@ test("no generated config carries a secret's value anywhere", () => { // Every value that identifies this deployment — the two whose values are ordinary words are left // to the name check above. The bare email as well as its JSON form, so that seeding the admins as // anything other than the secret's exact encoding — a comma-joined var, say — is caught too. - const generic = new Set(["CF_AI_GATEWAY_PROVIDERS", "CF_AI_GATEWAY_WAI_DIRECT"]); + const generic = new Set(["CF_AI_GATEWAY_PROVIDERS", "CF_AI_GATEWAY_USE_BINDING"]); const sensitive = [ ...Object.entries(SECRETS).filter(([key]) => !generic.has(key)), ["an admin's email", ADMIN], diff --git a/scripts/preview/staging-config.ts b/scripts/preview/staging-config.ts index 8d65cad6..b12fb09a 100644 --- a/scripts/preview/staging-config.ts +++ b/scripts/preview/staging-config.ts @@ -558,11 +558,16 @@ export function resolveAccess({ * Optional as a group, unlike {@link resolveAccess}: with CF_AI_GATEWAY unset a preview is BYOK, * exactly like a deployment that never configured a gateway, and the agent still works. Set, it * routes inference through Cloudflare AI Gateway with server-managed keys — and then the gateway - * account and its Run + Read token are required, because `AiGatewayConfig` (ai-gateway.ts) throws - * without them. That throw would otherwise land in a chat rather than in this deploy. + * account is required, because `AiGatewayConfig` (ai-gateway.ts) throws without it. That throw + * would otherwise land in a chat rather than in this deploy. * - * CF_AI_GATEWAY_WAI is deliberately not offered: it cannot be combined with CF_AI_GATEWAY_WAI_DIRECT - * (the backend rejects the pair), and one knob for where Workers AI inference goes is enough. + * The Run + Read token is not required with it. Every preview backend binds Workers AI (see + * {@link applyBackend}), and the binding is the gateway transport whenever it is present: those + * requests are pre-authenticated in-account, so neither inference nor a cost-log read needs a + * token. Two configurations still do, and each is the deploy-time mirror of a constructor throw: + * CF_AI_GATEWAY_USE_BINDING=false, which marks the gateway as living in a *different* account and + * so forces the HTTPS transport, and the google provider, whose SDK cannot take the binding's + * fetch. * * The environment is read in the parameter defaults rather than the body so that * env-passthrough.test.js, whose discovery is textual, can see every name. @@ -572,16 +577,16 @@ export function resolveAiGateway({ accountId = process.env.CF_AI_GATEWAY_ACCOUNT_ID, apiToken = process.env.CF_AI_GATEWAY_API_TOKEN, providers = process.env.CF_AI_GATEWAY_PROVIDERS, - waiDirect = process.env.CF_AI_GATEWAY_WAI_DIRECT, + useBinding = process.env.CF_AI_GATEWAY_USE_BINDING, }: { gateway?: string; accountId?: string; apiToken?: string; providers?: string; - waiDirect?: string; + useBinding?: string; } = {}): Record { const rest = { CF_AI_GATEWAY_ACCOUNT_ID: accountId, CF_AI_GATEWAY_API_TOKEN: apiToken, - CF_AI_GATEWAY_PROVIDERS: providers, CF_AI_GATEWAY_WAI_DIRECT: waiDirect }; + CF_AI_GATEWAY_PROVIDERS: providers, CF_AI_GATEWAY_USE_BINDING: useBinding }; if (!gateway) { // Every one of these does nothing without a gateway name, so a set of them without it is a // half-finished configuration rather than a deliberate BYOK preview. @@ -592,22 +597,38 @@ export function resolveAiGateway({ } return {}; } - if (!accountId || !apiToken) { - const missing = [ - ...(accountId ? [] : ["CF_AI_GATEWAY_ACCOUNT_ID"]), - ...(apiToken ? [] : ["CF_AI_GATEWAY_API_TOKEN"]), - ]; - throw new Error(`${missing.join(" and ")} must be set when CF_AI_GATEWAY is: inference goes ` + - "over HTTPS with a Run + Read token, so the backend refuses to start a chat without it"); + if (!accountId) { + throw new Error("CF_AI_GATEWAY_ACCOUNT_ID must be set when CF_AI_GATEWAY is: the backend " + + "cannot discover its own account, and refuses to start a chat without it"); + } + if (useBinding !== undefined && useBinding !== "true" && useBinding !== "false") { + // The backend compares against those two strings and treats anything else as unset, which for + // an intended "false" is the opposite of what was asked for -- silently, and in a preview + // nobody is reading the logs of. + throw new Error(`CF_AI_GATEWAY_USE_BINDING must be "true" or "false", not ` + + `"${useBinding}": the backend reads any other value as unset.`); + } + // The two AiGatewayConfig throws the Workers AI binding does not cover. Raised here so a + // half-configured preview fails its deploy rather than its first chat. + if (!apiToken && useBinding === "false") { + throw new Error("CF_AI_GATEWAY_API_TOKEN must be set when CF_AI_GATEWAY_USE_BINDING is " + + "false: opting out of the binding leaves the HTTPS transport, which needs a Run + Read " + + "token. Drop the opt-out unless the gateway is in another account."); + } + if (!apiToken && providers?.split(",").some(p => p.trim() === "google")) { + throw new Error("CF_AI_GATEWAY_API_TOKEN must be set when the google provider is enabled: " + + "the @google/genai SDK does not support a custom fetch, so Google inference cannot ride " + + "the Workers AI binding."); } return { CF_AI_GATEWAY: gateway, CF_AI_GATEWAY_ACCOUNT_ID: accountId, - CF_AI_GATEWAY_API_TOKEN: apiToken, - // Both are optional on their own: no providers means the gateway offers no server-keyed model, - // and no WAI_DIRECT routes Workers AI through the gateway itself. + // Each of the three is optional on its own: no token rides the binding transport, no providers + // means the gateway offers no server-keyed model, and no USE_BINDING takes the binding + // whenever it is bound -- which, for a preview, is always. + ...(apiToken ? { CF_AI_GATEWAY_API_TOKEN: apiToken } : {}), ...(providers ? { CF_AI_GATEWAY_PROVIDERS: providers } : {}), - ...(waiDirect ? { CF_AI_GATEWAY_WAI_DIRECT: waiDirect } : {}), + ...(useBinding ? { CF_AI_GATEWAY_USE_BINDING: useBinding } : {}), }; }