diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 01b5d03d..1c172243 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -106,13 +106,20 @@ jobs: CF_ACCESS_ISS: ${{ secrets.CF_ACCESS_ISS }} # AI Gateway, so a preview's chats use server-managed keys rather than asking each user # for their own. Optional as a group: with CF_AI_GATEWAY unset the preview is BYOK, but - # once it is set the account id and the Run + Read token are required. + # once it is set the account id is required. CF_AI_GATEWAY: ${{ secrets.CF_AI_GATEWAY }} # This is delibaretely set to CF_OS_AI_GATEWAY_ACCOUNT_ID (gets uploaded as CF_AI_GATEWAY_ACCOUNT_ID) CF_AI_GATEWAY_ACCOUNT_ID: ${{ secrets.CF_OS_AI_GATEWAY_ACCOUNT_ID }} - CF_AI_GATEWAY_API_TOKEN: ${{ secrets.CF_AI_GATEWAY_API_TOKEN }} CF_AI_GATEWAY_PROVIDERS: ${{ secrets.CF_AI_GATEWAY_PROVIDERS }} - CF_AI_GATEWAY_WAI_DIRECT: ${{ secrets.CF_AI_GATEWAY_WAI_DIRECT }} + # The gateway above lives in the account these previews deploy to, so the backend's + # WORKERS_AI binding reaches it: inference and cost-log reads are pre-authenticated + # in-account, which is why no CF_AI_GATEWAY_API_TOKEN is passed at all. + # + # Two configurations cannot use the binding and so are unreachable from here until a + # `CF_AI_GATEWAY_API_TOKEN: ${{ secrets.CF_AI_GATEWAY_API_TOKEN }}` line is added back: + # a gateway in a *different* account (which sets this secret to `false`), and the google + # provider, whose SDK cannot take the binding's fetch. Both fail the deploy. + CF_AI_GATEWAY_USE_BINDING: ${{ secrets.CF_AI_GATEWAY_USE_BINDING || 'true' }} # Together these are the preview name, which is the first label of its hostname: a preview # reads as `pr123-my-branch-router..workers.dev`. The number is what keeps two # branches that slugify alike from sharing one instance, and is how the nightly sweep diff --git a/docs/ai-gateway-billing.md b/docs/ai-gateway-billing.md index 9f2983e9..3c12aeab 100644 --- a/docs/ai-gateway-billing.md +++ b/docs/ai-gateway-billing.md @@ -13,10 +13,9 @@ turn, the overseer calls `checkUsageAndBalance`: - **Connected, balance ≥ `$2`** → allowed, routed through the user's own account so usage bills their Cloudflare credits — even while free-tier allowance remains. The platform is never charged for funded users, and their daily free-tier counter is left untouched. -- **Otherwise, within the free tier** → allowed, served via the platform's configured AI Gateway. - Workers AI uses the same Gateway ID unless `CF_AI_GATEWAY_WAI_DIRECT=true` sends it straight to - the Workers AI REST endpoint or `CF_AI_GATEWAY_WAI` selects another Gateway. This includes - connected users whose balance is below `$2` (incl. $0). +- **Otherwise, within the free tier** → allowed, served via the platform's configured AI Gateway + (all providers, Workers AI included). This includes connected users whose balance is below `$2` + (incl. $0). - **Free tier exhausted, no Cloudflare account connected** → blocked, with a prompt to connect. - **Free tier exhausted, connected but balance below `$2`** → blocked, with a prompt to add credits. @@ -54,19 +53,24 @@ CLOUDFLARE_OAUTH_CLIENT_SECRET=... CF_AI_GATEWAY=your-gateway CF_AI_GATEWAY_PROVIDERS=anthropic,openai,google -# Required whenever CF_AI_GATEWAY is set (all inference goes over HTTPS with tokens): +# Required whenever CF_AI_GATEWAY is set: CF_AI_GATEWAY_ACCOUNT_ID=... +# Required unless the WORKERS_AI binding carries gateway traffic; always required for the +# google provider: CF_AI_GATEWAY_API_TOKEN=... - -# To send Workers AI straight to its REST endpoint (no gateway, no cost logs): -CF_AI_GATEWAY_WAI_DIRECT=true ``` -Gateway mode always requires `CF_AI_GATEWAY_ACCOUNT_ID` and an API token with AI Gateway Run and -Read permissions; Read access lets Gadgets retrieve each log's cost for user-visible accounting. -Workers AI uses `CF_AI_GATEWAY` as its Gateway ID by default; set `CF_AI_GATEWAY_WAI` to select -another Gateway, or `CF_AI_GATEWAY_WAI_DIRECT=true` to call the Workers AI REST endpoint directly -(same credentials, no gateway cost logs). +Gateway mode always requires `CF_AI_GATEWAY_ACCOUNT_ID` plus a transport: the `WORKERS_AI` +binding when present (binding requests are pre-authenticated, and cost-log reads work through +the binding too), or otherwise an API token with AI Gateway Run and Read permissions — Read +access lets Gadgets retrieve each log's cost for user-visible accounting. The binding transport +only works when the Gateway lives in the Worker's own account, which the Worker can't verify at +runtime — a deployment whose Gateway is in a different account must set +`CF_AI_GATEWAY_USE_BINDING=false` to opt out and use the token transport. That is a flag rather +than an unbinding because `WORKERS_AI` also backs the webFetch tool's document-to-Markdown +conversion (and is hardcoded for every released backend), so removing it would break that instead +of just moving gateway traffic. The token stays required for the `google` provider even when the +binding transport applies. Every provider, Workers AI included, routes through the same Gateway. The Cloudflare dashboard OAuth endpoints and scopes are **hardcoded** in the Cloudflare gatekeeper (`packages/gatekeeper-cloudflare/src/oauth.ts`): diff --git a/docs/public-server.md b/docs/public-server.md index aa2fd033..8c8f00f2 100644 --- a/docs/public-server.md +++ b/docs/public-server.md @@ -40,25 +40,35 @@ CLOUDFLARE_OAUTH_CLIENT_SECRET=... CF_AI_GATEWAY=your-gateway CF_AI_GATEWAY_PROVIDERS=anthropic,openai,google -# Required whenever CF_AI_GATEWAY is set (all inference goes over HTTPS with tokens): +# Required whenever CF_AI_GATEWAY is set: CF_AI_GATEWAY_ACCOUNT_ID=... +# Required unless the WORKERS_AI binding carries gateway traffic (see below); always required +# for the google provider: CF_AI_GATEWAY_API_TOKEN=... - -# To send Workers AI straight to its REST endpoint (no gateway, no cost logs): -CF_AI_GATEWAY_WAI_DIRECT=true ``` -Gateway mode always requires `CF_AI_GATEWAY_ACCOUNT_ID` and `CF_AI_GATEWAY_API_TOKEN`; the token -needs AI Gateway Run and Read permissions so Gadgets can execute models and report their costs -(the Gateway may live in the Worker's own account or a different one). Workers AI defaults to the -same Gateway ID; set `CF_AI_GATEWAY_WAI` to route it through a different Gateway in the same -account, or `CF_AI_GATEWAY_WAI_DIRECT=true` to bypass gateways and call the Workers AI REST -endpoint directly (using the same account/token pair; such requests produce no cost logs). +Gateway mode always requires `CF_AI_GATEWAY_ACCOUNT_ID`, plus a transport for gateway requests. +When the `WORKERS_AI` binding is present, the binding is that transport by default: its requests +are pre-authenticated in-account, so inference and cost-log reads need no API token. This is only +valid when the Gateway lives in the Worker's **own** account — binding requests can't reach +another account's Gateway, and the Worker cannot verify where the Gateway lives at runtime — so +deployments whose Gateway is in a different account must set `CF_AI_GATEWAY_USE_BINDING=false` to +opt out and route over HTTPS instead. Keep `WORKERS_AI` bound when you do: it is also what the +webFetch tool's document-to-Markdown conversion runs on, so unbinding it opts out of far more than +the gateway transport. Without the binding transport, set +`CF_AI_GATEWAY_API_TOKEN` — a token with AI Gateway Run and Read permissions so Gadgets can +execute models and report their costs (over HTTPS the Gateway may live in the Worker's own +account or a different one). The token stays required for the `google` provider regardless of the +binding (the model SDK adapter refuses the binding's fetch — note the platform config above enables +it, so the platform server itself still needs the token). Every provider, Workers AI included, +routes through the same Gateway. When using `CF_AI_GATEWAY*` in local development, start the server with -`pnpm run dev-server -- --use-workers-ai-binding` so the webFetch tool's document-to-Markdown -conversion still has a `WORKERS_AI` binding. (Inference itself no longer uses the binding; it goes -over HTTPS with the tokens above.) +`pnpm run dev-server -- --use-workers-ai-binding` so the server has a `WORKERS_AI` binding for +the webFetch tool's document-to-Markdown conversion and for the gateway transport above (without +it, gateway traffic falls back to HTTPS with `CF_AI_GATEWAY_API_TOKEN`). If your dev Gateway +lives in a different account than the binding, also set `CF_AI_GATEWAY_USE_BINDING=false` — keep +`--use-workers-ai-binding` on, since the Markdown conversion still needs the binding. Each gatekeeper's OAuth app must be registered with that gatekeeper's redirect URI (replace the host with `PUBLIC_BASE_URL`): diff --git a/packages/workshop-backend/__tests__/ai-gateway.test.ts b/packages/workshop-backend/__tests__/ai-gateway.test.ts index 54a5996e..1ca10066 100644 --- a/packages/workshop-backend/__tests__/ai-gateway.test.ts +++ b/packages/workshop-backend/__tests__/ai-gateway.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { + AiGatewayConfig, AiGatewayLogRetryableError, getAiGatewayLogCost, } from "../src/ai-gateway.js"; @@ -13,6 +14,114 @@ function env(overrides: Partial = {}): Cloudflare.Env { } as Cloudflare.Env; } +describe("AiGatewayConfig transport selection", () => { + const binding = { gateway: () => ({}) } as unknown as Ai; + // google needs the HTTPS+token transport, so token-less configs must not enable it. + const bindingOnly = env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_PROVIDERS: "anthropic,openai,cloudflare", + WORKERS_AI: binding, + }); + + it("uses the binding for every provider except google", () => { + const config = new AiGatewayConfig(bindingOnly); + expect(config.apiToken).toBeUndefined(); + expect(config.bindingFor("anthropic")).toBe(binding); + expect(config.bindingFor("openai")).toBe(binding); + expect(config.bindingFor("cloudflare")).toBe(binding); + expect(config.bindingFor("google")).toBeUndefined(); + }); + + it("falls back to HTTPS with the token when the binding is absent", () => { + const config = new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_API_TOKEN: "gateway-token", + WORKERS_AI: undefined, + })); + expect(config.apiToken).toBe("gateway-token"); + expect(config.bindingFor("anthropic")).toBeUndefined(); + }); + + it("ignores the binding when CF_AI_GATEWAY_USE_BINDING=false opts out", () => { + // The cross-account shape (e.g. the internal production Workshop): WORKERS_AI is injected + // for webFetch, but the gateway lives in a different account, so the deployment opts out + // and gateway traffic rides HTTPS with the token. + const config = new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_API_TOKEN: "gateway-token", + CF_AI_GATEWAY_USE_BINDING: "false", + WORKERS_AI: binding, + })); + expect(config.binding).toBeUndefined(); + expect(config.apiToken).toBe("gateway-token"); + expect(config.bindingFor("anthropic")).toBeUndefined(); + expect(config.bindingFor("openai")).toBeUndefined(); + }); + + it("opts out on a padded, mixed-case CF_AI_GATEWAY_USE_BINDING", () => { + const config = new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_API_TOKEN: "gateway-token", + CF_AI_GATEWAY_USE_BINDING: " False ", + WORKERS_AI: binding, + })); + expect(config.binding).toBeUndefined(); + expect(config.bindingFor("anthropic")).toBeUndefined(); + }); + + it("still requires a transport when the opt-out leaves no token", () => { + expect(() => new AiGatewayConfig({ + ...bindingOnly, + CF_AI_GATEWAY_USE_BINDING: "false", + })).toThrow("AI Gateway mode needs a transport"); + }); + + it("rejects an explicit CF_AI_GATEWAY_USE_BINDING=true without the WORKERS_AI binding", () => { + expect(() => new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_API_TOKEN: "gateway-token", + CF_AI_GATEWAY_USE_BINDING: "true", + WORKERS_AI: undefined, + }))).toThrow("CF_AI_GATEWAY_USE_BINDING requires the WORKERS_AI binding"); + }); + + it("requires the account id", () => { + expect(() => new AiGatewayConfig(env({ CF_AI_GATEWAY_ACCOUNT_ID: undefined }))) + .toThrow("CF_AI_GATEWAY_ACCOUNT_ID is required when CF_AI_GATEWAY is set."); + }); + + it("requires a transport", () => { + expect(() => new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + WORKERS_AI: undefined, + }))).toThrow("AI Gateway mode needs a transport"); + }); + + it("requires the token when google is enabled", () => { + expect(() => new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + WORKERS_AI: binding, + }))).toThrow("enabling the google provider requires CF_AI_GATEWAY_API_TOKEN"); + }); + + it("resolves the same-account gateway for binding-based callers (webFetch)", () => { + expect(new AiGatewayConfig(bindingOnly).sameAccountGateway).toBe("platform-gateway"); + expect(new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_API_TOKEN: "gateway-token", + CF_AI_GATEWAY_USE_BINDING: "false", + WORKERS_AI: binding, + })).sameAccountGateway).toBeUndefined(); + // It tracks the binding rather than the opt-out, so an HTTPS-only deployment that never had a + // binding to opt out of resolves no same-account gateway either. + expect(new AiGatewayConfig(env({ + CF_AI_GATEWAY_ACCOUNT_ID: "account-id", + CF_AI_GATEWAY_API_TOKEN: "gateway-token", + WORKERS_AI: undefined, + })).sameAccountGateway).toBeUndefined(); + }); +}); + describe("getAiGatewayLogCost", () => { afterEach(() => vi.unstubAllGlobals()); diff --git a/packages/workshop-backend/__tests__/ai-models.test.ts b/packages/workshop-backend/__tests__/ai-models.test.ts index cfad0d65..c6e99145 100644 --- a/packages/workshop-backend/__tests__/ai-models.test.ts +++ b/packages/workshop-backend/__tests__/ai-models.test.ts @@ -139,21 +139,15 @@ describe("getModel AI Gateway routing", () => { }); }, 15000); - it.each([ - { CF_AI_GATEWAY_ACCOUNT_ID: undefined }, - { CF_AI_GATEWAY_API_TOKEN: undefined }, - ])("requires gateway credentials whenever gateway mode is enabled", (overrides) => { - expect(() => getModel(env(overrides), ANTHROPIC_CONFIG, INITIATOR)).toThrow( - "CF_AI_GATEWAY_ACCOUNT_ID and CF_AI_GATEWAY_API_TOKEN (a Run + Read token) are required " + - "when CF_AI_GATEWAY is set."); + it("requires the gateway account id whenever gateway mode is enabled", () => { + expect(() => getModel(env({ CF_AI_GATEWAY_ACCOUNT_ID: undefined }), ANTHROPIC_CONFIG, + INITIATOR)).toThrow("CF_AI_GATEWAY_ACCOUNT_ID is required when CF_AI_GATEWAY is set."); }); - it("rejects conflicting Workers AI routing configuration", () => { - expect(() => getModel(env({ - CF_AI_GATEWAY_WAI: "workers-ai-gateway", - CF_AI_GATEWAY_WAI_DIRECT: "true", - }), WORKERS_AI_CONFIG, INITIATOR)).toThrow( - "CF_AI_GATEWAY_WAI and CF_AI_GATEWAY_WAI_DIRECT cannot be configured together."); + it("requires a transport: the Workers AI binding or an API token", () => { + // Without the binding (local dev without --use-workers-ai-binding), the token is required. + expect(() => getModel(env({ CF_AI_GATEWAY_API_TOKEN: undefined }), ANTHROPIC_CONFIG, + INITIATOR)).toThrow("AI Gateway mode needs a transport"); }); it("prioritizes a connected user's Gateway over platform routing", async () => { @@ -210,55 +204,197 @@ describe("getModel AI Gateway routing", () => { expect(request.headers.get("authorization")).toBeNull(); }, 15000); - it("routes Workers AI to its REST endpoint when explicitly configured direct", async () => { - const handle = getModel( - env({ CF_AI_GATEWAY_WAI_DIRECT: "true" }), - WORKERS_AI_CONFIG, - INITIATOR, + it("routes Workers AI through the platform gateway like every other provider", async () => { + const handle = getModel(env(), WORKERS_AI_CONFIG, INITIATOR, { sessionAffinity: "session-a" }); expect(handle.model.api).toBe("openai-completions"); expect(handle.model.id).toBe("@cf/meta/llama-3.3-70b-instruct-fp8-fast"); expect(handle.model.baseUrl).toBe( - "https://api.cloudflare.com/client/v4/accounts/gateway-account-id/ai/v1"); - // No gateway in the path: no log route (and no gateway metadata). - expect(handle.aiGatewayLogRoute).toBeUndefined(); + "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/workers-ai/v1"); + expect(handle.aiGatewayLogRoute).toEqual({ + gateway: "platform-gateway", + accountId: "gateway-account-id", + apiToken: "gateway-token", + }); const request = await captureRequest(handle); expect(request.url).toBe( - "https://api.cloudflare.com/client/v4/accounts/gateway-account-id/ai/v1/chat/completions"); - expect(request.headers.get("authorization")).toBe("Bearer gateway-token"); - expect(request.headers.get("cf-aig-metadata")).toBeNull(); + "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/workers-ai/" + + "v1/chat/completions"); + expect(request.headers.get("cf-aig-authorization")).toBe("Bearer gateway-token"); // Session affinity flows through (Workers AI models opt in to the affinity headers). expect(request.headers.get("x-session-affinity")).toBe("session-a"); }, 15000); +}); - it("routes same-account Workers AI through the platform gateway by default", () => { - const handle = getModel(env(), WORKERS_AI_CONFIG, INITIATOR); +describe("getModel AI Gateway binding transport", () => { + // Provider-native requests captured by the fake Workers AI binding. In binding mode the + // handle's requests never hit HTTP: pi's SDK fetch is the gateway-binding shim, which only + // rewrites the URL onto the gateway's provider passthrough + // (workers-binding.ai/ai-gateway/gateways/{gateway}/{provider}/...) and hands the request to + // binding.fetch() otherwise unchanged. + type CapturedBindingRequest = { + url: string; + method: string; + headers: Record; + body: string; + }; + const capturedEntries: CapturedBindingRequest[] = []; + + const fakeBinding = { + fetch: async (input: Request | string | URL, init?: RequestInit) => { + const request = input instanceof Request ? input : new Request(input, init); + capturedEntries.push({ + url: request.url, + method: request.method, + headers: Object.fromEntries(request.headers), + body: await request.text(), + }); + // Same non-retryable client error as the HTTP fetch stub: pi surfaces an error-stop + // message and the request stays captured for assertions. + return Response.json( + { error: { type: "bad_request", message: "stubbed" } }, { status: 400 }); + }, + } as unknown as Ai; + + // Binding transport selects by default: binding present, no API token (in-account gateways; + // CF_AI_GATEWAY_USE_BINDING=false is the cross-account opt-out). google must not be an + // enabled provider in this mode (its transport still needs the token). + function bindingEnv(overrides: Partial = {}): Cloudflare.Env { + return env({ + CF_AI_GATEWAY_API_TOKEN: undefined, + CF_AI_GATEWAY_PROVIDERS: "anthropic,openai,cloudflare", + WORKERS_AI: fakeBinding, + ...overrides, + }); + } - expect(handle.model.api).toBe("openai-completions"); - expect(handle.model.id).toBe("@cf/meta/llama-3.3-70b-instruct-fp8-fast"); + async function captureEntry(handle: ModelHandle): Promise { + const stream = handle.stream(handle.model, { + messages: [{ role: "user", content: "hello", timestamp: 0 }], + }, { maxRetries: 0 }); + const message = await stream.result(); + expect(message.stopReason).toBe("error"); + expect(capturedEntries.length).toBeGreaterThan(0); + return capturedEntries[0]; + } + + beforeEach(() => { + capturedEntries.length = 0; + capturedRequests.length = 0; + }); + + it("drives Anthropic through the binding with no API token", async () => { + const handle = getModel(bindingEnv(), ANTHROPIC_CONFIG, INITIATOR, { + metadata: { source: "chat", gadgetId: "gadget-123", chatId: 7 }, + }); + + expect(handle.model.api).toBe("anthropic-messages"); + // Binding-routed models address the gateway on the binding's host, which takes no account + // id -- the binding channel carries identity. expect(handle.model.baseUrl).toBe( - "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/workers-ai/v1"); - expect(handle.aiGatewayLogRoute).toEqual({ - gateway: "platform-gateway", - accountId: "gateway-account-id", - apiToken: "gateway-token", + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/anthropic"); + // Same-account log reads ride the binding too: no account id or token in the route. + expect(handle.aiGatewayLogRoute).toEqual({ gateway: "platform-gateway" }); + + const entry = await captureEntry(handle); + expect(entry.url).toBe( + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/anthropic/v1/messages"); + expect(entry.method).toBe("POST"); + // The sentinel auth header satisfies pi's request-auth check; the gateway recognizes and + // strips it on binding-routed requests, so the shim forwards it. The SDK's own auth + // headers stay suppressed. + expect(entry.headers["cf-aig-authorization"]).toBe("Bearer cloudflare-gateway-binding"); + const headerNames = Object.keys(entry.headers).map((name) => name.toLowerCase()); + expect(headerNames).not.toContain("x-api-key"); + expect(headerNames).not.toContain("authorization"); + expect(JSON.parse(entry.headers["cf-aig-metadata"])).toEqual({ + user: "user-123", + source: "chat", + gadgetId: "gadget-123", + chatId: 7, }); - }); + expect((JSON.parse(entry.body) as { model: string }).model).toBe("claude-sonnet-4-5"); + }, 15000); - it("uses an explicit Workers AI gateway override", () => { - const handle = getModel( - env({ CF_AI_GATEWAY_WAI: "workers-ai-gateway" }), WORKERS_AI_CONFIG, INITIATOR); + it("drives Workers AI through the binding via its gateway route", async () => { + const handle = getModel(bindingEnv(), WORKERS_AI_CONFIG, INITIATOR); expect(handle.model.baseUrl).toBe( - "https://gateway.ai.cloudflare.com/v1/gateway-account-id/workers-ai-gateway/workers-ai/v1"); - expect(handle.aiGatewayLogRoute).toEqual({ - gateway: "workers-ai-gateway", - accountId: "gateway-account-id", - apiToken: "gateway-token", + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/workers-ai/v1"); + expect(handle.aiGatewayLogRoute).toEqual({ gateway: "platform-gateway" }); + + const entry = await captureEntry(handle); + expect(entry.url).toBe( + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/workers-ai/" + + "v1/chat/completions"); + expect((JSON.parse(entry.body) as { model: string }).model) + .toBe("@cf/meta/llama-3.3-70b-instruct-fp8-fast"); + // openai-completions adapters inject `Authorization: Bearer unused` under header-owned + // auth; the gatewayAuthHeaders nulls must delete it before dispatch, else the gateway + // would treat it as a request-supplied provider key overriding stored keys. + const headerNames = Object.keys(entry.headers).map((name) => name.toLowerCase()); + expect(headerNames).not.toContain("authorization"); + expect(headerNames).not.toContain("x-api-key"); + }, 15000); + + it("lets a per-call fetch override the binding transport", async () => { + // Tests and diagnostics inject options.fetch; it must win over the handle's binding fetch. + // The URL is the model's, so it still names the binding route -- only the transport swaps. + const handle = getModel(bindingEnv(), ANTHROPIC_CONFIG, INITIATOR); + + const request = await captureRequest(handle); + expect(capturedEntries).toHaveLength(0); + expect(request.url).toBe( + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/anthropic/v1/messages"); + expect(request.headers.get("cf-aig-authorization")) + .toBe("Bearer cloudflare-gateway-binding"); + }, 15000); + + it("keeps Google on HTTPS with the token while other providers use the binding", async () => { + // Hybrid mode: binding and token both present. pi's Google adapter rejects a custom fetch, + // so Google inference rides HTTPS with the gateway token -- but same-account log reads + // still use the binding. + const hybridEnv = env({ + CF_AI_GATEWAY_PROVIDERS: "anthropic,openai,google,cloudflare", + WORKERS_AI: fakeBinding, }); + + const googleHandle = getModel(hybridEnv, { + provider: "google", + model: "gemini-2.5-flash", + apiToken: "ignored-in-gateway-mode", + }, INITIATOR); + expect(googleHandle.model.baseUrl).toBe( + "https://gateway.ai.cloudflare.com/v1/gateway-account-id/platform-gateway/" + + "google-ai-studio/v1beta"); + expect(googleHandle.aiGatewayLogRoute).toEqual({ gateway: "platform-gateway" }); + + const anthropicHandle = getModel(hybridEnv, ANTHROPIC_CONFIG, INITIATOR); + const entry = await captureEntry(anthropicHandle); + expect(entry.url).toBe( + "https://workers-binding.ai/ai-gateway/gateways/platform-gateway/anthropic/v1/messages"); + // The binding arm carries the sentinel, never the real gateway token. + expect(entry.headers["cf-aig-authorization"]).toBe("Bearer cloudflare-gateway-binding"); + }, 15000); + + it("requires the token when google is an enabled provider", () => { + expect(() => getModel( + bindingEnv({ CF_AI_GATEWAY_PROVIDERS: "anthropic,google" }), + ANTHROPIC_CONFIG, INITIATOR)).toThrow( + "enabling the google provider requires CF_AI_GATEWAY_API_TOKEN"); + }); + + it("rejects a stored google config when the deployment has no token", () => { + expect(() => getModel(bindingEnv(), { + provider: "google", + model: "gemini-2.5-flash", + apiToken: "ignored-in-gateway-mode", + }, INITIATOR)).toThrow( + 'Provider "google" cannot use the Workers AI binding transport'); }); + }); describe("getModel direct routing (no gateway)", () => { diff --git a/packages/workshop-backend/__tests__/web-fetch.test.ts b/packages/workshop-backend/__tests__/web-fetch.test.ts index 275ef6a5..322c25a6 100644 --- a/packages/workshop-backend/__tests__/web-fetch.test.ts +++ b/packages/workshop-backend/__tests__/web-fetch.test.ts @@ -120,10 +120,14 @@ describe("webFetch document conversion", () => { tokens: 1, data: "# Title", })); + // WORKERS_AI is what makes the gateway same-account: it is both the binding toMarkdown runs on + // and the gateway transport, and the overseer takes `ai` and this config from the one env, so a + // config built without it describes a deployment where the conversion could not happen at all. const gateway = new AiGatewayConfig({ CF_AI_GATEWAY: "workers-ai-gateway", CF_AI_GATEWAY_ACCOUNT_ID: "gateway-account-id", CF_AI_GATEWAY_API_TOKEN: "gateway-token", + WORKERS_AI: {} as Ai, } as Cloudflare.Env); await webFetch(makeEnv(toMarkdown, gateway), { url: "https://example.com/page" }); @@ -134,7 +138,7 @@ describe("webFetch document conversion", () => { }); }); - it("does not pass a gateway to toMarkdown when direct Workers AI is configured", async () => { + it("does not pass a gateway to toMarkdown when the gateway is cross-account", async () => { mockResponse("

Title

", "text/html"); const toMarkdown = vi.fn(async (doc: { name: string; blob: Blob }) => ({ @@ -145,11 +149,16 @@ describe("webFetch document conversion", () => { tokens: 1, data: "# Title", })); + // CF_AI_GATEWAY_USE_BINDING=false marks the platform gateway as living in a different + // account; the binding-based toMarkdown call can't log through it. WORKERS_AI stays bound -- + // that is the whole shape of the opt-out: the binding is still there for the conversion, it + // just can't reach this gateway. const gateway = new AiGatewayConfig({ CF_AI_GATEWAY: "platform-gateway", CF_AI_GATEWAY_ACCOUNT_ID: "gateway-account-id", CF_AI_GATEWAY_API_TOKEN: "gateway-token", - CF_AI_GATEWAY_WAI_DIRECT: "true", + CF_AI_GATEWAY_USE_BINDING: "false", + WORKERS_AI: {} as Ai, } as Cloudflare.Env); await webFetch(makeEnv(toMarkdown, gateway), { url: "https://example.com/page" }); diff --git a/packages/workshop-backend/src/ai-gateway.ts b/packages/workshop-backend/src/ai-gateway.ts index b0656408..def09308 100644 --- a/packages/workshop-backend/src/ai-gateway.ts +++ b/packages/workshop-backend/src/ai-gateway.ts @@ -7,35 +7,91 @@ import { UserAiModelRecord } from "./user.js"; // compared to the actual coding model so there's not much reason to use a smaller model. const QUICK_MODEL_ID = "@cf/meta/llama-3.3-70b-instruct-fp8-fast"; +/** + * Providers whose pi API adapter refuses a custom fetch, so their inference cannot ride the + * Workers AI binding and needs CF_AI_GATEWAY_API_TOKEN over HTTPS. pi's Google adapter throws + * "Custom fetch is not supported by the Google Generative AI adapter" whenever the fetch it is + * given is not globalThis.fetch, and the client it builds on offers no hook to route around that: + * @google/genai's `GoogleGenAI` takes only `httpOptions`, whose knobs are + * baseUrl/apiVersion/headers/timeout/extraBody/retryOptions. + * https://github.com/earendil-works/pi/blob/v0.84.2/packages/ai/src/api/google-generative-ai.ts#L80 + * + * pi's Vertex adapter throws the same way, so a google-vertex provider would belong here too; it + * is absent only because this deployment has no such provider. + * https://github.com/earendil-works/pi/blob/v0.84.2/packages/ai/src/api/google-vertex.ts#L98 + */ +const HTTPS_ONLY_PROVIDERS = new Set(["google"]); + export class AiGatewayConfig { readonly gateway: string; - readonly workersAiGateway?: string; + /** + * The gateway name for Workers-AI-binding calls (webFetch's toMarkdown): binding calls only + * reach gateways in the Worker's own account, so this is the platform gateway whenever the + * binding transport is active, and unset when it isn't (see {@link binding}). + */ + readonly sameAccountGateway?: string; readonly accountId: string; - readonly apiToken: string; + readonly apiToken?: string; + /** + * Workers AI binding, used as the gateway transport whenever present unless + * CF_AI_GATEWAY_USE_BINDING=false opts out: binding requests are pre-authenticated in-account, + * so inference and cost-log reads need no API token. Binding requests only reach gateways in + * the Worker's own account, and the Worker can't verify that itself (it can't discover its own + * account ID), so deployments whose gateway lives in a DIFFERENT account must set the opt-out + * and use CF_AI_GATEWAY_API_TOKEN over HTTPS. Absent in local dev unless run-dev-server is + * started with --use-workers-ai-binding. + * + * Such a deployment opts out with the flag rather than by unbinding WORKERS_AI, because the + * binding is not only the gateway transport: webFetch's document-to-Markdown conversion calls + * `env.ai.toMarkdown()` through it (see web-fetch.ts), so unbinding would break that too. + */ + readonly binding?: Ai; readonly providers: Set; constructor(env: Cloudflare.Env) { this.gateway = env.CF_AI_GATEWAY!; - // Inference now goes over HTTPS with tokens (pi has no Workers-binding transport), so the - // account/token pair is required whenever gateway mode is enabled. The token-less - // same-account mode existed only because of the Workers binding. - if (!env.CF_AI_GATEWAY_ACCOUNT_ID || !env.CF_AI_GATEWAY_API_TOKEN) { - throw new Error( - "CF_AI_GATEWAY_ACCOUNT_ID and CF_AI_GATEWAY_API_TOKEN (a Run + Read token) are " + - "required when CF_AI_GATEWAY is set."); + if (!env.CF_AI_GATEWAY_ACCOUNT_ID) { + throw new Error("CF_AI_GATEWAY_ACCOUNT_ID is required when CF_AI_GATEWAY is set."); } this.accountId = env.CF_AI_GATEWAY_ACCOUNT_ID; - this.apiToken = env.CF_AI_GATEWAY_API_TOKEN; - if (env.CF_AI_GATEWAY_WAI_DIRECT === "true" && env.CF_AI_GATEWAY_WAI) { + this.apiToken = env.CF_AI_GATEWAY_API_TOKEN || undefined; + // Normalized once, so a stray " False " opts out rather than reading as unset and silently + // picking the other transport. + const useBinding = env.CF_AI_GATEWAY_USE_BINDING?.trim().toLowerCase(); + this.binding = useBinding === "false" + ? undefined + : (env as { WORKERS_AI?: Ai }).WORKERS_AI; + if (useBinding === "true" && !this.binding) { throw new Error( - "CF_AI_GATEWAY_WAI and CF_AI_GATEWAY_WAI_DIRECT cannot be configured together."); + "CF_AI_GATEWAY_USE_BINDING requires the WORKERS_AI binding; without it the config " + + "would silently fall back to the HTTPS transport."); } - this.workersAiGateway = env.CF_AI_GATEWAY_WAI_DIRECT === "true" - ? undefined - : env.CF_AI_GATEWAY_WAI || this.gateway; + if (!this.apiToken && !this.binding) { + throw new Error( + "AI Gateway mode needs a transport: bind Workers AI (WORKERS_AI; in local dev start " + + "with --use-workers-ai-binding) or set CF_AI_GATEWAY_API_TOKEN (a Run + Read token)."); + } + this.sameAccountGateway = this.binding ? this.gateway : undefined; this.providers = new Set( (env.CF_AI_GATEWAY_PROVIDERS || "").split(",").map(s => s.trim()).filter(s => s !== "") ); + const httpsOnly = [...this.providers].filter(p => HTTPS_ONLY_PROVIDERS.has(p)); + if (httpsOnly.length > 0 && !this.apiToken) { + const names = httpsOnly.join(", "); + throw new Error( + `${names} inference cannot ride the Workers AI binding transport, so enabling the ` + + `${names} provider${httpsOnly.length > 1 ? "s" : ""} requires ` + + "CF_AI_GATEWAY_API_TOKEN."); + } + } + + /** + * Transport for a provider's gateway inference: the Workers AI binding when present, except for + * the providers in {@link HTTPS_ONLY_PROVIDERS}, which ride HTTPS with the token (the + * constructor guarantees a token whenever one of them is an enabled provider). + */ + bindingFor(provider: string): Ai | undefined { + return HTTPS_ONLY_PROVIDERS.has(provider) ? undefined : this.binding; } /** diff --git a/packages/workshop-backend/src/ai-models.ts b/packages/workshop-backend/src/ai-models.ts index 7f7ea8de..b1bb2c3e 100644 --- a/packages/workshop-backend/src/ai-models.ts +++ b/packages/workshop-backend/src/ai-models.ts @@ -1,8 +1,8 @@ import { DurableObject, RpcStub, RpcTarget } from "cloudflare:workers"; import { validateRpc } from "capnweb-validate"; import type { - AnthropicMessagesCompat, Api, AssistantMessageEventStream, Context, Model, ModelCost, - OpenAICompletionsCompat, ProviderHeaders, SimpleStreamOptions, StreamFunction, + AnthropicMessagesCompat, Api, AssistantMessageEventStream, Context, FetchFunction, Model, + ModelCost, OpenAICompletionsCompat, ProviderHeaders, SimpleStreamOptions, StreamFunction, } from "@earendil-works/pi-ai"; import { stream as anthropicMessagesStream } from "@earendil-works/pi-ai/api/anthropic-messages"; import { stream as googleGenerativeAiStream } from "@earendil-works/pi-ai/api/google-generative-ai"; @@ -165,9 +165,11 @@ function workersAiCompat(catalog: Model | undefined): OpenAICompletionsComp } // Build the pi model descriptor for reaching a provider's own native API through an AI Gateway -// (the platform's or a user's). `gatewayUrl` is a gateway root -// (https://gateway.ai.cloudflare.com/v1/{accountId}/{gateway}); each provider's native API is -// exposed under a per-provider path on it. AI Gateway also offers a unified OpenAI-compat +// (the platform's or a user's). `gatewayUrl` is a gateway root -- over HTTPS +// (https://gateway.ai.cloudflare.com/v1/{accountId}/{gateway}) or, for binding-routed requests, +// over the AI binding (https://workers-binding.ai/ai-gateway/gateways/{gateway}); each +// provider's native API is exposed under the same per-provider path on either. AI Gateway also +// offers a unified OpenAI-compat // translation layer (/compat), which we deliberately never use: we already speak every // provider's native API, and the translation drops provider features pi relies on (extended // thinking, Anthropic cache_control prompt caching, the OpenAI Responses API). Billing -- @@ -268,6 +270,10 @@ type HandleArgs = { gatewayMetadata?: GatewayMetadata; sessionAffinity?: string; aiGatewayLogRoute?: AiGatewayLogRoute; + // Transport override for every request on this handle: how a binding-routed model reaches the + // gateway over env.WORKERS_AI.fetch() instead of the global fetch (see bindingFetch). + // A per-call options.fetch still wins, which tests rely on to capture requests. + fetch?: FetchFunction; }; function makeHandle(args: HandleArgs): ModelHandle { @@ -315,6 +321,7 @@ function makeHandle(args: HandleArgs): ModelHandle { ...(thinking ? apiExtras : args.model.api === "anthropic-messages" ? { thinkingEnabled: false } : {}), + ...(args.fetch !== undefined ? { fetch: args.fetch } : {}), ...options, ...(args.apiKey !== undefined ? { apiKey: args.apiKey } : {}), ...(Object.keys(headers).length > 0 ? { headers } : {}), @@ -333,9 +340,6 @@ function makeHandle(args: HandleArgs): ModelHandle { const replaced = await options.onPayload?.(payload, payloadModel); return bridgePdfAttachments(args.model.api, replaced ?? payload) ?? replaced; }, - // NOTE(binding-transport): pi passes `options.fetch` into its SDK clients on all paths. - // If Workers-binding-backed inference returns (upstream ask filed), inject a - // fetch-to-binding shim here and relax the token requirements in ai-gateway.ts. }; return streamFn(model, context, merged); }, @@ -411,6 +415,28 @@ function getModelViaUserGateway( }); } +/** + * Placeholder auth value for binding-routed requests. pi's API impls require an API key or a + * recognized auth header (authorization, x-api-key, cf-aig-authorization) before dispatch; + * binding calls are pre-authenticated in-account, so this satisfies the check and the gateway + * recognizes and strips it rather than treating it as a BYOK provider key. + */ +const CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL = "cloudflare-gateway-binding"; + +/** + * `Ai#fetch` exists at runtime but @cloudflare/workers-types' `Ai` doesn't declare it, so the + * binding is cast structurally to reach the passthrough. + */ +type AiFetchBinding = { + fetch(input: Request | string | URL, init?: RequestInit): Promise; +}; + +// pi drives the model's baseUrl, which already names the gateway route on the binding's host, +// so the binding's fetch passes through unchanged -- no URL rewriting needed. +function bindingFetch(binding: Ai): FetchFunction { + return (input, init) => (binding as unknown as AiFetchBinding).fetch(input, init); +} + // Platform free-tier path: route through the deployment's configured AI Gateway (platform-funded). // Used only for requests that are NOT billed to a connected user's account. function getModelViaGateway( @@ -420,48 +446,40 @@ function getModelViaGateway( options: ModelRoutingOptions, ): ModelHandle { const metadata = buildMetadata(initiator, options.metadata); + const binding = gwConfig.bindingFor(config.provider); + // No binding means either the provider can't ride one or the deployment has none; the second + // case already required a token in the constructor, so this only fires for the first + if (!binding && !gwConfig.apiToken) { + throw new Error(`Provider "${config.provider}" cannot use the Workers AI binding transport, ` + + "and no CF_AI_GATEWAY_API_TOKEN is configured for the HTTPS one."); + } const gatewayAuthHeaders: ProviderHeaders = { // pi's API impls explicitly recognize cf-aig-authorization and skip SDK auth; the null // values suppress the SDKs' own auth headers so the gateway's server-managed provider keys // apply. - "cf-aig-authorization": `Bearer ${gwConfig.apiToken}`, + "cf-aig-authorization": + `Bearer ${binding ? CLOUDFLARE_GATEWAY_BINDING_AUTH_SENTINEL : gwConfig.apiToken}`, Authorization: null, "x-api-key": null, }; const gatewayBase = `https://gateway.ai.cloudflare.com/v1/${gwConfig.accountId}`; - const logRoute = (gateway: string): AiGatewayLogRoute => - ({ gateway, accountId: gwConfig.accountId, apiToken: gwConfig.apiToken }); - - if (config.provider === "cloudflare" && !gwConfig.workersAiGateway) { - // CF_AI_GATEWAY_WAI_DIRECT: the plain Workers AI REST endpoint -- no gateway, no log route, - // no gateway metadata (mirroring the old direct-binding path, which had no - // aiGatewayLogRoute). Reuses the CF_AI_GATEWAY_* account/token pair. - const catalog = catalogModel(config.provider, config.model); - const model: Model = { - id: config.model, - name: catalog?.name ?? config.model, - api: "openai-completions", - provider: "cloudflare-workers-ai", - baseUrl: `https://api.cloudflare.com/client/v4/accounts/${gwConfig.accountId}/ai/v1`, - reasoning: catalog?.reasoning ?? false, - input: catalog?.input ?? ["text"], - cost: catalog?.cost ?? ZERO_COST, - ...modelTokenWindow(config, catalog), - compat: workersAiCompat(catalog), - }; - return makeHandle({ - model, - apiKey: gwConfig.apiToken, - sessionAffinity: options.sessionAffinity, - }); - } - - // Workers AI may be routed through a different gateway than the other providers - // (CF_AI_GATEWAY_WAI); either way, gateway log route and attribution metadata apply. - const gateway = config.provider === "cloudflare" - ? gwConfig.workersAiGateway! : gwConfig.gateway; - const model = gatewayNativeModel(config, `${gatewayBase}/${gateway}`); + // Cost-log reads are same-account, so the binding arm applies whenever the binding transport + // is active (gwConfig.binding is unset when CF_AI_GATEWAY_USE_BINDING=false opts out) -- + // even for Google inference, which itself rides HTTPS (see AiGatewayConfig.bindingFor). + const logRoute = (gateway: string): AiGatewayLogRoute => gwConfig.binding + ? { gateway } + : { gateway, accountId: gwConfig.accountId, apiToken: gwConfig.apiToken! }; + + // Every provider -- Workers AI included -- rides the same gateway, with the same log route + // and attribution metadata. Binding-routed providers address it on the binding's host, which + // takes no account id (the binding channel carries identity); the paths are otherwise the + // same, so the model descriptors are built identically from either root. + const gateway = gwConfig.gateway; + const gatewayUrl = binding + ? `https://workers-binding.ai/ai-gateway/gateways/${gateway}` + : `${gatewayBase}/${gateway}`; + const model = gatewayNativeModel(config, gatewayUrl); if (!model) { throw new Error( `Provider "${config.provider}" is not supported through AI Gateway. ` + @@ -479,6 +497,7 @@ function getModelViaGateway( // the gateway recognizes its own token there and applies the stored Google key instead. ...(config.provider === "google" ? { apiKey: gwConfig.apiToken } : {}), headers: gatewayAuthHeaders, + ...(binding ? { fetch: bindingFetch(binding) } : {}), gatewayMetadata: metadata, sessionAffinity: options.sessionAffinity, aiGatewayLogRoute: logRoute(gateway), diff --git a/packages/workshop-backend/src/env.d.ts b/packages/workshop-backend/src/env.d.ts index e0443836..9ad6c001 100644 --- a/packages/workshop-backend/src/env.d.ts +++ b/packages/workshop-backend/src/env.d.ts @@ -15,15 +15,12 @@ declare global { // AI Gateway mode: when CF_AI_GATEWAY is set, supported providers are routed through // Cloudflare AI Gateway with server-managed keys. Users don't need their own keys. - // Inference goes over HTTPS with tokens (there is no Workers-binding transport), so the - // ACCOUNT_ID/API_TOKEN pair is REQUIRED whenever CF_AI_GATEWAY is set. CF_AI_GATEWAY?: string; // Gateway name (enables gateway mode) CF_AI_GATEWAY_PROVIDERS?: string; // Comma-separated list: "anthropic,openai,google,cloudflare" CF_AI_GATEWAY_ACCOUNT_ID?: string; // Gateway owner account ID (required with CF_AI_GATEWAY) - CF_AI_GATEWAY_API_TOKEN?: string; // Run + Read token for inference and cost-log reads - CF_AI_GATEWAY_WAI?: string; // Optional Workers AI gateway override - CF_AI_GATEWAY_WAI_DIRECT?: string; // "true" to route Workers AI to its plain REST endpoint - // (no gateway, no cost logs) instead of a named Gateway + CF_AI_GATEWAY_API_TOKEN?: string; // Run + Read token; optional when the binding transport + // applies (still required for google) + CF_AI_GATEWAY_USE_BINDING?: string; // Note: outside gateway mode, Workers AI (provider "cloudflare") is BYOK like every other // provider -- the account ID and API token live in the user's model config, not in env. diff --git a/packages/workshop-backend/src/web-fetch.ts b/packages/workshop-backend/src/web-fetch.ts index 3ad43b63..252c0550 100644 --- a/packages/workshop-backend/src/web-fetch.ts +++ b/packages/workshop-backend/src/web-fetch.ts @@ -177,14 +177,15 @@ const TO_MARKDOWN_MIME_TYPES = new Set([ "application/vnd.apple.numbers", // .numbers ]); -// `toMarkdown()` uses the Workers AI binding, so only apply the same-account Workers AI gateway -// resolved by AiGatewayConfig. A cross-account platform gateway cannot be used by this binding. +// `toMarkdown()` uses the Workers AI binding, and binding calls only reach gateways in the +// Worker's own account -- so apply the platform gateway only when AiGatewayConfig resolves it +// as same-account (CF_AI_GATEWAY_USE_BINDING=false marks it cross-account). function buildGatewayOptions( gateway: AiGatewayConfig | null, ): GatewayOptions | undefined { if (!gateway) return undefined; - if (!gateway.workersAiGateway) return undefined; - return { id: gateway.workersAiGateway, metadata: { tool: "webFetch", automated: true } }; + if (!gateway.sameAccountGateway) return undefined; + return { id: gateway.sameAccountGateway, metadata: { tool: "webFetch", automated: true } }; } // Attempt to convert a document to Markdown using the Workers AI binding. Returns the diff --git a/scripts/env-passthrough.test.ts b/scripts/env-passthrough.test.ts index 3fd1989c..88203d51 100644 --- a/scripts/env-passthrough.test.ts +++ b/scripts/env-passthrough.test.ts @@ -71,7 +71,7 @@ const EXPECTED: Record = { forwarded: ["VITE_FRONTEND_ERROR_REPORTING"], external: [ "CF_ACCESS_AUD", "CF_ACCESS_ISS", "CF_AI_GATEWAY", "CF_AI_GATEWAY_ACCOUNT_ID", - "CF_AI_GATEWAY_API_TOKEN", "CF_AI_GATEWAY_PROVIDERS", "CF_AI_GATEWAY_WAI_DIRECT", + "CF_AI_GATEWAY_API_TOKEN", "CF_AI_GATEWAY_PROVIDERS", "CF_AI_GATEWAY_USE_BINDING", "CI_COMMIT_SHA", "CI_PIPELINE_IID", "CLOUDFLARE_ACCOUNT_ID", "CLOUDFLARE_API_TOKEN", "GITHUB_REPOSITORY", "GITHUB_TOKEN", "PREVIEW_ADMINS", "PREVIEW_NAME", "PREVIEW_PR_NUMBER", "PREVIEW_WORKERS_DEV_HOST", "PREVIEW_WRANGLER", "VITE_BACKEND_HOST", diff --git a/scripts/preview/staging-config.test.ts b/scripts/preview/staging-config.test.ts index 58f22e26..84776e0a 100644 --- a/scripts/preview/staging-config.test.ts +++ b/scripts/preview/staging-config.test.ts @@ -41,7 +41,7 @@ const AI_GATEWAY = { accountId: "1".repeat(32), apiToken: "example-run-and-read-token", providers: "cloudflare", - waiDirect: "true", + useBinding: "false", }; // Every input is passed explicitly: each resolver defaults to reading the environment, so a machine // with any of these set would otherwise change what the tests assert. @@ -269,7 +269,7 @@ test("the backend's secrets are the admin list, the Access pair and the AI gatew CF_AI_GATEWAY_ACCOUNT_ID: AI_GATEWAY.accountId, CF_AI_GATEWAY_API_TOKEN: AI_GATEWAY.apiToken, CF_AI_GATEWAY_PROVIDERS: AI_GATEWAY.providers, - CF_AI_GATEWAY_WAI_DIRECT: AI_GATEWAY.waiDirect, + CF_AI_GATEWAY_USE_BINDING: AI_GATEWAY.useBinding, }); }); @@ -278,20 +278,56 @@ test("the AI gateway is optional as a group, but not half-configured", () => { assert.deepEqual(resolveAiGateway({}), {}); assert.deepEqual(resolveAiGateway({ accountId: AI_GATEWAY.accountId }), {}, "orphans are ignored"); - // With one, the account and token are what AiGatewayConfig demands: without them it throws on the + // With one, the account is what AiGatewayConfig demands: without it the backend throws on the // first chat, so the deploy has to be the thing that fails instead. assert.throws(() => resolveAiGateway({ gateway: "g" }), - /CF_AI_GATEWAY_ACCOUNT_ID and CF_AI_GATEWAY_API_TOKEN must be set when CF_AI_GATEWAY is/); - assert.throws(() => resolveAiGateway({ gateway: "g", apiToken: "t" }), - /CF_AI_GATEWAY_ACCOUNT_ID must be set/); - assert.throws(() => resolveAiGateway({ gateway: "g", accountId: "a" }), - /CF_AI_GATEWAY_API_TOKEN must be set/); + /CF_AI_GATEWAY_ACCOUNT_ID must be set when CF_AI_GATEWAY is/); - // The two knobs below the required trio are each independently optional. + // The gateway name and its account are the whole requirement: a preview binds Workers AI, and + // the binding transport is pre-authenticated in-account, so a tokenless gateway is a complete + // configuration. Everything below them is independently optional. + assert.deepEqual(resolveAiGateway({ gateway: "g", accountId: "a" }), + { CF_AI_GATEWAY: "g", CF_AI_GATEWAY_ACCOUNT_ID: "a" }); assert.deepEqual(resolveAiGateway({ gateway: "g", accountId: "a", apiToken: "t" }), { CF_AI_GATEWAY: "g", CF_AI_GATEWAY_ACCOUNT_ID: "a", CF_AI_GATEWAY_API_TOKEN: "t" }); }); +test("a preview that cannot use the binding transport needs the gateway token", () => { + // Both mirror an AiGatewayConfig throw: opting out of the binding leaves only HTTPS, and the + // google SDK cannot take the binding's fetch. Each is a deploy failure rather than a chat one. + assert.throws( + () => resolveAiGateway({ gateway: "g", accountId: "a", useBinding: "false" }), + /CF_AI_GATEWAY_API_TOKEN must be set when CF_AI_GATEWAY_USE_BINDING is false/); + assert.throws( + () => resolveAiGateway({ gateway: "g", accountId: "a", providers: "cloudflare,google" }), + /CF_AI_GATEWAY_API_TOKEN must be set when the google provider is enabled/); + + // With the token, both are configurations rather than errors. + assert.deepEqual( + resolveAiGateway({ gateway: "g", accountId: "a", apiToken: "t", useBinding: "false" }), + { CF_AI_GATEWAY: "g", CF_AI_GATEWAY_ACCOUNT_ID: "a", CF_AI_GATEWAY_API_TOKEN: "t", + CF_AI_GATEWAY_USE_BINDING: "false" }); + // Requiring the binding is the other half of the same knob, and needs no token at all. + assert.deepEqual(resolveAiGateway({ gateway: "g", accountId: "a", useBinding: "true" }), + { CF_AI_GATEWAY: "g", CF_AI_GATEWAY_ACCOUNT_ID: "a", CF_AI_GATEWAY_USE_BINDING: "true" }); + + // Case and padding are normalized before the check, and the canonical form is what is emitted, + // so the backend never receives a value it would have to normalize itself. + assert.deepEqual( + resolveAiGateway({ gateway: "g", accountId: "a", apiToken: "t", useBinding: " FALSE " }), + { CF_AI_GATEWAY: "g", CF_AI_GATEWAY_ACCOUNT_ID: "a", CF_AI_GATEWAY_API_TOKEN: "t", + CF_AI_GATEWAY_USE_BINDING: "false" }); + // A normalized " FALSE " is still the opt-out, so it needs the token like any other. + assert.throws( + () => resolveAiGateway({ gateway: "g", accountId: "a", useBinding: " FALSE " }), + /CF_AI_GATEWAY_API_TOKEN must be set when CF_AI_GATEWAY_USE_BINDING is false/); + + // The backend compares against the two strings and reads anything else as unset, so a value it + // would silently ignore fails here instead + assert.throws(() => resolveAiGateway({ gateway: "g", accountId: "a", useBinding: "yes please" }), + /CF_AI_GATEWAY_USE_BINDING must be "true" or "false", not "yes please"/); +}); + test("no generated config declares a secret's variable", () => { for (const [name, config] of buildAll().configs) { const halves: [string, Record | undefined][] = [ @@ -317,7 +353,7 @@ test("no generated config carries a secret's value anywhere", () => { // Every value that identifies this deployment — the two whose values are ordinary words are left // to the name check above. The bare email as well as its JSON form, so that seeding the admins as // anything other than the secret's exact encoding — a comma-joined var, say — is caught too. - const generic = new Set(["CF_AI_GATEWAY_PROVIDERS", "CF_AI_GATEWAY_WAI_DIRECT"]); + const generic = new Set(["CF_AI_GATEWAY_PROVIDERS", "CF_AI_GATEWAY_USE_BINDING"]); const sensitive = [ ...Object.entries(SECRETS).filter(([key]) => !generic.has(key)), ["an admin's email", ADMIN], diff --git a/scripts/preview/staging-config.ts b/scripts/preview/staging-config.ts index 8d65cad6..04a609c0 100644 --- a/scripts/preview/staging-config.ts +++ b/scripts/preview/staging-config.ts @@ -558,11 +558,16 @@ export function resolveAccess({ * Optional as a group, unlike {@link resolveAccess}: with CF_AI_GATEWAY unset a preview is BYOK, * exactly like a deployment that never configured a gateway, and the agent still works. Set, it * routes inference through Cloudflare AI Gateway with server-managed keys — and then the gateway - * account and its Run + Read token are required, because `AiGatewayConfig` (ai-gateway.ts) throws - * without them. That throw would otherwise land in a chat rather than in this deploy. + * account is required, because `AiGatewayConfig` (ai-gateway.ts) throws without it. That throw + * would otherwise land in a chat rather than in this deploy. * - * CF_AI_GATEWAY_WAI is deliberately not offered: it cannot be combined with CF_AI_GATEWAY_WAI_DIRECT - * (the backend rejects the pair), and one knob for where Workers AI inference goes is enough. + * The Run + Read token is not required with it. Every preview backend binds Workers AI (see + * {@link applyBackend}), and the binding is the gateway transport whenever it is present: those + * requests are pre-authenticated in-account, so neither inference nor a cost-log read needs a + * token. Two configurations still do, and each is the deploy-time mirror of a constructor throw: + * CF_AI_GATEWAY_USE_BINDING=false, which marks the gateway as living in a *different* account and + * so forces the HTTPS transport, and the google provider, whose pi adapter refuses the binding's + * fetch. * * The environment is read in the parameter defaults rather than the body so that * env-passthrough.test.js, whose discovery is textual, can see every name. @@ -572,16 +577,19 @@ export function resolveAiGateway({ accountId = process.env.CF_AI_GATEWAY_ACCOUNT_ID, apiToken = process.env.CF_AI_GATEWAY_API_TOKEN, providers = process.env.CF_AI_GATEWAY_PROVIDERS, - waiDirect = process.env.CF_AI_GATEWAY_WAI_DIRECT, + useBinding: rawUseBinding = process.env.CF_AI_GATEWAY_USE_BINDING, }: { gateway?: string; accountId?: string; apiToken?: string; providers?: string; - waiDirect?: string; + useBinding?: string; } = {}): Record { + // Normalized here rather than in the parameter default, which only runs when the caller omits + // the value -- an explicitly passed " FALSE " would skip it and fail validation below. + const useBinding = rawUseBinding?.trim().toLowerCase(); const rest = { CF_AI_GATEWAY_ACCOUNT_ID: accountId, CF_AI_GATEWAY_API_TOKEN: apiToken, - CF_AI_GATEWAY_PROVIDERS: providers, CF_AI_GATEWAY_WAI_DIRECT: waiDirect }; + CF_AI_GATEWAY_PROVIDERS: providers, CF_AI_GATEWAY_USE_BINDING: useBinding }; if (!gateway) { // Every one of these does nothing without a gateway name, so a set of them without it is a // half-finished configuration rather than a deliberate BYOK preview. @@ -592,22 +600,38 @@ export function resolveAiGateway({ } return {}; } - if (!accountId || !apiToken) { - const missing = [ - ...(accountId ? [] : ["CF_AI_GATEWAY_ACCOUNT_ID"]), - ...(apiToken ? [] : ["CF_AI_GATEWAY_API_TOKEN"]), - ]; - throw new Error(`${missing.join(" and ")} must be set when CF_AI_GATEWAY is: inference goes ` + - "over HTTPS with a Run + Read token, so the backend refuses to start a chat without it"); + if (!accountId) { + throw new Error("CF_AI_GATEWAY_ACCOUNT_ID must be set when CF_AI_GATEWAY is: the backend " + + "cannot discover its own account, and refuses to start a chat without it"); + } + if (useBinding !== undefined && useBinding !== "true" && useBinding !== "false") { + // The backend compares against those two strings and treats anything else as unset, which for + // an intended "false" is the opposite of what was asked for -- silently, and in a preview + // nobody is reading the logs of. + throw new Error(`CF_AI_GATEWAY_USE_BINDING must be "true" or "false", not ` + + `"${rawUseBinding}": the backend reads any other value as unset.`); + } + // The two AiGatewayConfig throws the Workers AI binding does not cover. Raised here so a + // half-configured preview fails its deploy rather than its first chat. + if (!apiToken && useBinding === "false") { + throw new Error("CF_AI_GATEWAY_API_TOKEN must be set when CF_AI_GATEWAY_USE_BINDING is " + + "false: opting out of the binding leaves the HTTPS transport, which needs a Run + Read " + + "token. Drop the opt-out unless the gateway is in another account."); + } + if (!apiToken && providers?.split(",").some(p => p.trim() === "google")) { + throw new Error("CF_AI_GATEWAY_API_TOKEN must be set when the google provider is enabled: " + + "pi's Google adapter refuses a custom fetch, so Google inference cannot ride the Workers " + + "AI binding."); } return { CF_AI_GATEWAY: gateway, CF_AI_GATEWAY_ACCOUNT_ID: accountId, - CF_AI_GATEWAY_API_TOKEN: apiToken, - // Both are optional on their own: no providers means the gateway offers no server-keyed model, - // and no WAI_DIRECT routes Workers AI through the gateway itself. + // Each of the three is optional on its own: no token rides the binding transport, no providers + // means the gateway offers no server-keyed model, and no USE_BINDING takes the binding + // whenever it is bound -- which, for a preview, is always. + ...(apiToken ? { CF_AI_GATEWAY_API_TOKEN: apiToken } : {}), ...(providers ? { CF_AI_GATEWAY_PROVIDERS: providers } : {}), - ...(waiDirect ? { CF_AI_GATEWAY_WAI_DIRECT: waiDirect } : {}), + ...(useBinding ? { CF_AI_GATEWAY_USE_BINDING: useBinding } : {}), }; } diff --git a/scripts/release/manifest-lib.ts b/scripts/release/manifest-lib.ts index be83b6dd..42cdae94 100644 --- a/scripts/release/manifest-lib.ts +++ b/scripts/release/manifest-lib.ts @@ -400,9 +400,12 @@ export function buildWorkerEntry( // the deploy service's backendExtraVars at PUT time, never manifest-templated. vars.PUBLIC_BASE_URL = "$PUBLIC_BASE_URL"; // Every deployed backend gets the Workers AI binding (hardcoded like PUBLIC_BASE_URL, not - // read from wrangler.jsonc): webFetch's toMarkdown conversion depends on it, and it costs - // nothing when unused. (Inference does not — Workers AI models are reached over HTTPS like - // every other provider.) No placeholders — the deploy renderer passes it through. + // read from wrangler.jsonc): webFetch's toMarkdown conversion depends on it, and it is also + // the backend's default AI Gateway transport (the deploy service creates the gateway in the + // user's own account, so the in-account requirement holds; CF_AI_GATEWAY_USE_BINDING=false + // is the cross-account opt-out) — binding requests are pre-authenticated, so inference and + // cost-log reads need no CF_AI_GATEWAY_API_TOKEN (google provider excepted). + // No placeholders — the deploy renderer passes it through. bindings.push({ type: "ai", name: "WORKERS_AI" }); // Installed gatekeepers are called through GATEKEEPER_* service bindings with the // GatekeeperVendor entrypoint (same shape run-dev-server.ts generates for dev). diff --git a/scripts/run-dev-server.ts b/scripts/run-dev-server.ts index 71cde0ae..86c81afb 100644 --- a/scripts/run-dev-server.ts +++ b/scripts/run-dev-server.ts @@ -479,11 +479,14 @@ for (const gk of gatekeepers) { const OPTIONAL_FEATURE_VARS = [ "DISABLE_PASSWORD_AUTH", "AUTH_GATEKEEPERS", "ENABLE_CLOUDFLARE_LIMITS", "PUBLIC_BASE_URL", "DAILY_LLM_CALL_LIMIT", "MINIMUM_CLOUDFLARE_BALANCE", - // Platform AI Gateway — makes the cross-provider model catalog available. The - // ACCOUNT_ID/API_TOKEN pair is required whenever CF_AI_GATEWAY is set (all inference goes - // over HTTPS with tokens). + // Platform AI Gateway — makes the cross-provider model catalog available. CF_AI_GATEWAY + // always needs CF_AI_GATEWAY_ACCOUNT_ID plus one transport: the WORKERS_AI binding + // (start with --use-workers-ai-binding; CF_AI_GATEWAY_USE_BINDING=false opts out, e.g. + // when the gateway lives in a different account than the dev binding) or + // CF_AI_GATEWAY_API_TOKEN over HTTPS. The google provider can't ride the binding and + // needs the token even when the binding is present. "CF_AI_GATEWAY", "CF_AI_GATEWAY_PROVIDERS", "CF_AI_GATEWAY_ACCOUNT_ID", - "CF_AI_GATEWAY_API_TOKEN", "CF_AI_GATEWAY_WAI", "CF_AI_GATEWAY_WAI_DIRECT", + "CF_AI_GATEWAY_API_TOKEN", "CF_AI_GATEWAY_USE_BINDING", ]; // OAuth app credentials (GOOGLE_/GITHUB_/CLOUDFLARE_OAUTH_*) are NOT passed to the backend anymore; // they are injected into the gatekeeper Workers (see SHARED_GATEKEEPER_CREDS below).