diff --git a/AGENTS.md b/AGENTS.md index 0f01b459c..33b932c2c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -235,7 +235,7 @@ The `error-context.md` file contains a YAML accessibility tree of the entire pag - Has a different `name`/`data-testid` than expected - Is behind a lazy-load boundary that hasn't resolved -**Stale state from earlier serial tests** — Mock-LLM tests run serially (`workers: 1`) against a real agent-server. Earlier tests (conversation, automation) persist settings on the server. If your test depends on "clean" state but a prior test configured `llm_base_url`, `llm_model`, etc., the form may render in a different view mode. Use Playwright `page.route()` to intercept and normalize the settings response. Example: `routeOnboardingLlmCatalog` in `tests/e2e/support/onboarding-helpers.ts` intercepts `GET /api/settings` to clear `llm_base_url` so the LLM form always opens in "Basic" view. +**Stale state from earlier serial tests** — Mock-LLM tests run serially (`workers: 1`) against a real agent-server. Earlier tests (conversation, automation) persist settings on the server. If your test needs "first-run" state (e.g. onboarding), use `interceptSettingsForFirstRun(page)` from `mock-llm-helpers.ts` in the `beforeGoto` callback — it intercepts `GET /api/settings` to strip stale `agent_settings.llm` values so the frontend sees schema defaults. This is necessary because (a) the agent-server PATCH API rejects null/empty values for validated fields like `model` and `api_key` (422), and (b) the `inferInitialView` function checks ALL schema fields against their defaults to decide between Basic/Advanced view. The interception must be registered AFTER `routeSessionApiKey` (Playwright LIFO order). **View mode mismatch (Basic vs Advanced)** — `LlmSettingsScreen` switches between "Basic" (renders `ModelSelector` with provider/model dropdowns) and "Advanced" (renders plain text inputs). The view is determined by `getInitialView()` which checks `currentSettings.llm_base_url` — a non-default base URL triggers "Advanced" view. If your test expects `input[name="llm-provider-input"]` but sees text inputs instead, the settings have a stale `base_url`. diff --git a/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts b/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts index fcfe90c69..17382b3ff 100644 --- a/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts +++ b/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts @@ -5,7 +5,11 @@ import { showOnboarding, waitForOnboardingStep, } from "../support/onboarding-helpers"; -import { routeSessionApiKey, SESSION_API_KEY } from "./utils/mock-llm-helpers"; +import { + interceptSettingsForFirstRun, + routeSessionApiKey, + SESSION_API_KEY, +} from "./utils/mock-llm-helpers"; test.describe.configure({ mode: "serial" }); @@ -70,22 +74,8 @@ test.describe("onboarding recent regressions", () => { apiKey: SESSION_API_KEY, beforeGoto: async () => { await routeSessionApiKey(page); - // Intercept GET /api/settings so the LLM form sees a clean - // base_url, forcing basic view mode regardless of what earlier - // specs configured. Registered AFTER routeSessionApiKey so - // Playwright's LIFO matching picks this up first for settings. - await page.route("**/api/settings", async (route, req) => { - if (req.method() !== "GET") { - await route.fallback(); - return; - } - const response = await route.fetch(); - const body = await response.json(); - if (body?.agent_settings?.llm) { - body.agent_settings.llm.base_url = null; - } - await route.fulfill({ response, json: body }); - }); + // Strip stale LLM config so the form sees a first-run state. + await interceptSettingsForFirstRun(page); }, }); await advanceOnboardingToLlmStep(page); diff --git a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts index b557a6aff..3d713b841 100644 --- a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts +++ b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts @@ -422,6 +422,36 @@ export async function resetMockLLM(request: APIRequestContext) { expect(resp.ok(), `Reset mock LLM: ${resp.status()}`).toBe(true); } +/** + * Intercept `GET /api/settings` to strip all LLM configuration, simulating + * a first-run state where no model/api_key/base_url has been configured. + * + * Earlier specs (conversation, automation, etc.) persist custom LLM settings + * on the real agent-server. The settings schema's `inferInitialView` checks + * ALL non-critical fields against their defaults — if any differ (model, + * api_key, base_url), the LLM form renders in "Advanced" view instead of + * "Basic". The PATCH API rejects null/empty values for most fields (422), + * so we can't reset them server-side. Instead we strip the stale values from + * the response so the frontend sees schema defaults. + * + * Must be registered AFTER `routeSessionApiKey` so Playwright's LIFO order + * processes the settings interception first for matching requests. + */ +export async function interceptSettingsForFirstRun(page: Page) { + await page.route("**/api/settings", async (route, req) => { + if (req.method() !== "GET") { + await route.fallback(); + return; + } + const response = await route.fetch(); + const body = await response.json(); + if (body?.agent_settings?.llm) { + body.agent_settings.llm = {}; + } + await route.fulfill({ response, json: body }); + }); +} + /** * Fetch all chat-completion request bodies captured by the mock LLM server * since the last /admin/reset.