From bbb2da7f2c61ca72a362c6458b08d4c990167216 Mon Sep 17 00:00:00 2001 From: Michael Yong Date: Tue, 8 Sep 2026 13:22:55 -0700 Subject: [PATCH 01/10] Add Modal machine and project image plugin --- .../src/services/plugins/builtin-registry.ts | 5 + .../services/plugins/builtin-plugins.test.ts | 1 + docs/cli-guide-and-skill.md | 2 + docs/configuration.md | 22 + .../src/builtin-server-artifacts.test.ts | 2 + plugins/bb-official.json | 4 + plugins/environment-modal-sandbox/README.md | 149 ++++ plugins/environment-modal-sandbox/app.tsx | 233 +++++ .../catalogue/artifact.ts | 40 + .../catalogue/backend.ts | 161 ++++ .../catalogue/base.ts | 29 + .../catalogue/catalogue.test.ts | 389 ++++++++ .../catalogue/cli.ts | 285 ++++++ .../catalogue/context.ts | 111 +++ .../catalogue/contract.ts | 184 ++++ .../catalogue/dockerfile.ts | 163 ++++ .../catalogue/logs.ts | 32 + .../catalogue/migrations.ts | 24 + .../catalogue/model.ts | 176 ++++ .../catalogue/service.test.ts | 560 ++++++++++++ .../catalogue/service.ts | 610 +++++++++++++ .../catalogue/source-contract.ts | 61 ++ .../catalogue/store.ts | 626 +++++++++++++ .../catalogue/verification.ts | 479 ++++++++++ .../catalogue/worker.ts | 155 ++++ .../configuration.test.ts | 36 + .../configuration.ts | 149 ++++ plugins/environment-modal-sandbox/host.ts | 260 ++++++ .../environment-modal-sandbox/lifecycle.ts | 34 + .../environment-modal-sandbox/modal-logo.svg | 10 + .../environment-modal-sandbox/package.json | 48 + .../recipe-editor.test.tsx | 57 ++ .../recipe-editor.tsx | 196 ++++ .../sandbox-backend.test.ts | 161 ++++ .../sandbox-backend.ts | 199 +++++ .../environment-modal-sandbox/server.test.ts | 841 ++++++++++++++++++ plugins/environment-modal-sandbox/server.ts | 645 ++++++++++++++ .../environment-modal-sandbox/settings.tsx | 774 ++++++++++++++++ .../skills/modal-sandboxes/SKILL.md | 114 +++ .../environment-modal-sandbox/tsconfig.json | 37 + plugins/environment-modal-sandbox/ui-state.ts | 31 + .../vitest.config.ts | 16 + pnpm-lock.yaml | 210 ++++- turbo.json | 3 + 44 files changed, 8318 insertions(+), 6 deletions(-) create mode 100644 plugins/environment-modal-sandbox/README.md create mode 100644 plugins/environment-modal-sandbox/app.tsx create mode 100644 plugins/environment-modal-sandbox/catalogue/artifact.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/backend.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/base.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/catalogue.test.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/cli.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/context.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/contract.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/dockerfile.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/logs.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/migrations.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/model.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/service.test.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/service.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/source-contract.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/store.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/verification.ts create mode 100644 plugins/environment-modal-sandbox/catalogue/worker.ts create mode 100644 plugins/environment-modal-sandbox/configuration.test.ts create mode 100644 plugins/environment-modal-sandbox/configuration.ts create mode 100644 plugins/environment-modal-sandbox/host.ts create mode 100644 plugins/environment-modal-sandbox/lifecycle.ts create mode 100644 plugins/environment-modal-sandbox/modal-logo.svg create mode 100644 plugins/environment-modal-sandbox/package.json create mode 100644 plugins/environment-modal-sandbox/recipe-editor.test.tsx create mode 100644 plugins/environment-modal-sandbox/recipe-editor.tsx create mode 100644 plugins/environment-modal-sandbox/sandbox-backend.test.ts create mode 100644 plugins/environment-modal-sandbox/sandbox-backend.ts create mode 100644 plugins/environment-modal-sandbox/server.test.ts create mode 100644 plugins/environment-modal-sandbox/server.ts create mode 100644 plugins/environment-modal-sandbox/settings.tsx create mode 100644 plugins/environment-modal-sandbox/skills/modal-sandboxes/SKILL.md create mode 100644 plugins/environment-modal-sandbox/tsconfig.json create mode 100644 plugins/environment-modal-sandbox/ui-state.ts create mode 100644 plugins/environment-modal-sandbox/vitest.config.ts diff --git a/apps/server/src/services/plugins/builtin-registry.ts b/apps/server/src/services/plugins/builtin-registry.ts index 3907a93cc35..6a1f50160b4 100644 --- a/apps/server/src/services/plugins/builtin-registry.ts +++ b/apps/server/src/services/plugins/builtin-registry.ts @@ -165,6 +165,11 @@ export const BUILTIN_PLUGINS = [ })); export const OFFICIAL_PLUGINS = [ + { + name: "environment-modal-sandbox", + pluginId: "environment-modal-sandbox", + defaultEnabled: true, + }, { name: "browser-automation", pluginId: "browser-automation", diff --git a/apps/server/test/services/plugins/builtin-plugins.test.ts b/apps/server/test/services/plugins/builtin-plugins.test.ts index 4f741b5f8ec..6001b4ac7d9 100644 --- a/apps/server/test/services/plugins/builtin-plugins.test.ts +++ b/apps/server/test/services/plugins/builtin-plugins.test.ts @@ -209,6 +209,7 @@ describe("builtin plugin reconciliation", () => { it("keeps official plugins bundled but out of the auto-install builtins", () => { const optionalNames = OFFICIAL_PLUGINS.map((plugin) => plugin.name); expect(optionalNames).toEqual([ + "environment-modal-sandbox", "browser-automation", "github", "docs", diff --git a/docs/cli-guide-and-skill.md b/docs/cli-guide-and-skill.md index 9ca2eb42e6d..abaf81019dc 100644 --- a/docs/cli-guide-and-skill.md +++ b/docs/cli-guide-and-skill.md @@ -16,3 +16,5 @@ personal-workspace paths skip both hooks. These semantics apply equally to CLI, SDK, and app launches; see [worktrees.md](worktrees.md). The Machines settings creation drawer prepares an existing-machine command when access is ready, otherwise shows setup guidance. After access is ready, Choose a machine provider lets users select a project before reviewing provider inputs; the selected project is passed to the same `hosts.submit`/`bb machine create --project` launch surface. + +Modal project catalogue Settings and commands are documented in [modal-sandboxes](../plugins/environment-modal-sandbox/skills/modal-sandboxes/SKILL.md). `bb modal account inspect`, `project sources`, and `project preflight` support `--json` and typed plugin RPC. Settings edits bb-owned Dockerfile text with revision checks, follows explicit builds, and verifies/promotes images for future launches. `bb machine lifecycle --remove --yes` complements keep/automatic-retention controls. diff --git a/docs/configuration.md b/docs/configuration.md index 413da708f45..b27caf49c99 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1279,3 +1279,25 @@ server gh credentials to machines; `true` enables them again. In Machines → Advanced settings, the automatic GH_TOKEN switch controls the same setting. This does not log the server out or suppress an explicit custom GH_TOKEN. Changes apply to new turns, setup commands and terminals. + +## Modal project image configuration + +`bb modal project configure --project X --expected-revision N --json-input JSON` +stores project CPU cores, memory MiB, and idle/lifetime/retention policy with CAS. +Defaults are 1 core, 4096 MiB, 15 idle minutes, 1440 lifetime minutes, and 30 +retention days. The resource records preserve these values for later lifecycle +policy enforcement. Recipes and uploaded build inputs belong to plugin storage, +not repository files. See the Modal catalogue skill for the command contract. + +Modal `project.configure` also accepts `usableBuildId: null` to clear the project's +selected image with the same revision guard before explicit image cleanup. + +Modal machines default to a 15-minute idle pause, 24-hour compute lifetime and +30-day retention after the last thread. Project policy changes apply to existing +machines on the next lifecycle observation. Open terminals prevent idle pause, +but deadline maintenance closes them. Inspect retention and keep a machine with +`bb machine lifecycle MACHINE --keep --json`; explicit removal is still available. +Preservation covers planned rotation. A server outage spanning the vendor deadline +can lose changes since the last snapshot; lifecycle status reports this risk. + +Modal project catalogue Settings and commands are documented in [modal-sandboxes](../plugins/environment-modal-sandbox/skills/modal-sandboxes/SKILL.md). `bb modal account inspect`, `project sources`, and `project preflight` support `--json` and typed plugin RPC. Settings edits bb-owned Dockerfile text with revision checks, follows explicit builds, and verifies/promotes images for future launches. `bb machine lifecycle --remove --yes` complements keep/automatic-retention controls. diff --git a/packages/plugin-build/src/builtin-server-artifacts.test.ts b/packages/plugin-build/src/builtin-server-artifacts.test.ts index 2409ee78fca..0037b6bba93 100644 --- a/packages/plugin-build/src/builtin-server-artifacts.test.ts +++ b/packages/plugin-build/src/builtin-server-artifacts.test.ts @@ -73,12 +73,14 @@ describe("builtin server artifacts", () => { { pluginDir: "environment-project-checkout" }, { pluginDir: "environment-git-worktree" }, { pluginDir: "environment-personal-workspace" }, + { pluginDir: "environment-modal-sandbox" }, ])( "inlines the environment-provider runtime into the $pluginDir server entry", async ({ pluginDir }) => { const root = await mkdtemp(join(repositoryRoot, ".builtin-server-test-")); tempDirs.push(root); const source = join(repositoryRoot, "plugins", pluginDir); + if (pluginDir === "environment-modal-sandbox") await cp(join(source, "catalogue"), join(root, "catalogue"), { recursive: true }); const fileNames = (await readdir(source)).filter( (fileName) => fileName === "package.json" || diff --git a/plugins/bb-official.json b/plugins/bb-official.json index 31dc16ea87d..867105c24f9 100644 --- a/plugins/bb-official.json +++ b/plugins/bb-official.json @@ -141,5 +141,9 @@ "machine-manual": { "category": "environments", "screenshots": [] + }, + "environment-modal-sandbox": { + "category": "environments", + "screenshots": [] } } diff --git a/plugins/environment-modal-sandbox/README.md b/plugins/environment-modal-sandbox/README.md new file mode 100644 index 00000000000..eed6052360d --- /dev/null +++ b/plugins/environment-modal-sandbox/README.md @@ -0,0 +1,149 @@ +# Modal sandbox + +Creates resumable bb machines in [Modal](https://modal.com) Sandboxes. It is +an official catalog plugin, not installed by default. Installing it adds the +`modal-sandbox` machine provider; it does not add an environment provider. + +Choose **New sandbox** under **New machine** in a project's environment picker to create a machine, +have core clone and register that project's checkout, and run the thread through +the Project checkout provider. The machine remains a normal bb execution +target, so later threads can create Git worktrees or use other environment +providers on the same sandbox. New machines require a project and an explicitly +built catalogue image. Pass its `buildId` to `bb machine create`, or promote a +verified image to select it for future Modal machines in the project. + +Creation prepares enrollment before vendor allocation and awaits a core resource +checkpoint as soon as the allocation ID is known. The checkpoint contains no +bootstrap credentials. Core can remove a cancelled allocation directly from +that checkpoint without rerunning creation, enrollment, or bootstrap. +Core owns project checkout setup and source registration after the machine +connects, using the `project-checkout` environment row. Agent-provider +readiness runs before dispatch: compatible installed CLIs are reused, missing +CLIs use their registered installer, and credential routes are checked from +the machine. Account Pooler injects credentials at runtime. + +## Project images + +Use `bb modal project inspect`, `recipe put --stdin --expected-revision 0`, +`context upload`, and `image build --key` to create a project image. All commands +support `--json`; `image logs BUILD --follow` streams bounded cursor pages. +Recipes live in plugin SQLite storage. Context uploads contain tracked files at +an identified commit plus an explicitly reviewed dirty overlay. See the bundled +[command reference](skills/modal-sandboxes/SKILL.md) for flags and typed RPC names. + +The TypeScript builder prepends a versioned Debian bookworm / Node 22.19 base, +installs pinned Codex and Claude Code CLIs, and embeds the server's credential-free +bb CLI/daemon package with a verified SHA-256 and protocol version. Build provenance +records that package digest. Project Dockerfiles support RUN, COPY, ENV, WORKDIR, +and ARG; FROM and other instructions fail with a source-line error. +Images contain no enrollment or agent login state. Credentials enter only during +runtime bootstrap. Rebuilding requires an explicit request; inspection reports +staleness. GC marks owned images, waits 60 seconds, and rechecks references before +deletion. Machine allocations, project promotion pointers, and verification records +protect builds. + +Run `bb modal image verify BUILD --provider codex --key KEY --json` to start a +durable verification. Repeating its key returns the current result. Verification +runs a real agent turn, independently checks the recorded smoke commands and +commit, then suspends and restores the same machine with a filesystem sentinel. +Readiness and smoke checks run again after restore. Successful verification +retains a suspended machine; failed machines remain available for inspection. + +`bb modal image use BUILD --project PROJECT --provider codex --expected-revision N` +requires successful verification for that agent. It changes only the project's +Modal image pointer, leaving environment preferences and existing machines alone. +`bb machine ready MACHINE --provider codex --project PROJECT --json` exposes the +same generic readiness checks used before agent dispatch. Core runs the repo's `.bb-env-setup.sh` after an environment provider creates a +checkout it owns, including a fresh clone on a new machine. The script owns cache +validation and its unchanged-input no-op path. Readiness requires a successful +core hook outcome for this checkout's commit and lockfile inputs; it does not run +another script. Attaching a user-maintained checkout runs neither setup nor +teardown. Dockerfile recipes remain in plugin storage; setup hooks live in Git. + +Core runs `.bb-env-teardown.sh` before removing an owned environment with a separate +15-minute timeout. Teardown failure is reported and does not block removal. +`.worktreeinclude` does not apply to fresh machine clones. Use core Machine +environment settings for local files and secrets on machines. +Settings → Plugins → Modal sandbox includes a per-project Dockerfile editor, reviewed context upload, explicit builds with log follow/cancel, staleness, image verification/promotion, and resources/lifecycle policy. The recipe lives in bb storage; import/export does not write it to the repository. Account connection checks return no secrets. + +## Lifecycle + +Core observes Modal's running state and vendor deadline. Defaults are a 15-minute +idle pause, 24-hour compute lifetime and 30-day retention after the last thread. +Zero-thread machines also pause while retained. Open terminals prevent idle pause. +Project policy changes apply to existing machines on the next observation. + +Maintenance starts 15 minutes before expiry, or halfway through shorter configured +lifetimes. Core excludes new work, interrupts active turns, closes terminals and +bounds drain to five minutes. The daemon stops its managed runtimes before the +plugin saves a private filesystem snapshot with no expiry. A durable checkpoint +precedes compute termination. Dispatch then restores the same host identity; +provider credentials are supplied again by core on the new continuation turn. Core reruns the repo’s `.bb-env-setup.sh` through its durable hook path to restart services. The hook must be idempotent and may start services; a failed restore hook blocks readiness. +An interrupted turn is never reported as a successful completion or replayed. + +`bb machine lifecycle MACHINE --json` shows expiry, maintenance, the last successful +save, recovery state and the separate automatic removal deadline. Use `--keep` to +retain a machine, or `--no-keep` to restore automatic retention removal. Removal +cascades through owned environments and deletes private snapshots. It remains +available explicitly even when keep is enabled. + +Failed saves keep old compute and retry inside the remaining margin. Failed restore +or account changes remain visible and never substitute a fresh empty checkout. +Preservation covers planned rotation only: a server outage spanning vendor expiry +can lose changes since the last snapshot. A missing running sandbox is marked +lost-since-last-snapshot and blocks automatic dispatch. `bb machine resume MACHINE` +is an explicit request to recover that last snapshot with the disclosed loss risk. + +## What it needs + +- A Modal API token. Set its two halves in the plugin's `tokenId` and + `tokenSecret` settings. +- A git remote when creating through the project picker, because core clones + and registers that project. Standalone machine creation still selects a + project build. +- A URL the sandbox can reach this bb at. + +## How the sandbox reaches this bb + +Configure the instance's default server-access provider so the sandbox can +reach bb. Core's public machine bootstrap helper owns access grants, +enrollment, durable identity, daemon startup, and waiting for a connection. +The plugin supplies Modal exec as the transport, including stdin for secret +bootstrap data. It does not store enrollment credentials in machine resources. + +Creation calls bootstrap with the durable creation key and the preinstalled daemon. +Resume calls the same helper with the original key and a preinstalled daemon, +including when a previous attempt left the sandbox running. Core reuses the +identity and restarts the daemon when needed. The plugin has no `serverUrl` +setting; configure access centrally. + +The exec adapter stops waiting when cancellation is requested. Modal does not +expose per-exec cancellation, so a command already submitted may continue until +its process timeout. Retries reuse the named sandbox and the bootstrap key. + +## Settings + +| Setting | Required | What it is | +| ---------------- | -------- | ----------------------------------------------------------------------------- | +| `tokenId` | yes | The token id half of a Modal API token. | +| `tokenSecret` | yes | The token secret half of the same token. | +| `appName` | no | The Modal app for sandboxes. Defaults to `bb-sandboxes`. | +| `timeoutMinutes` | no | Modal sandbox timeout, 1–1440 minutes. | +| `idleMinutes` | no | Snapshot after this many idle minutes. Defaults to 15; 0 disables suspension. | +| `cpu` | no | Reserved cores. Blank uses Modal's default. | +| `memoryMiB` | no | Reserved memory in MiB. Blank uses Modal's default. | + +## Logo and trademark + +The bundled `modal-logo.svg` is an unmodified copy of +[`Modal-IconMark-Dark-OneColor.svg`](https://drive.google.com/file/d/1JvQGLrZsQvnpZu5DmUafxXPGHXDk6TsI/view), +the web one-color icon mark in [Modal's current official brand +assets](https://modal.com/brand). The light one-color file published beside it +uses the same geometry; bb supplies the visible color through its icon mask. + +Modal's brand-asset folder publishes no separate license or attribution file. +Modal and its logo are trademarks of Modal Labs, Inc., and Modal's +[terms](https://modal.com/legal/terms) reserve its intellectual-property +rights. The mark remains Modal's property and is bundled only to identify the +service this plugin integrates with; no license to reuse it separately is +granted or implied. diff --git a/plugins/environment-modal-sandbox/app.tsx b/plugins/environment-modal-sandbox/app.tsx new file mode 100644 index 00000000000..bca71e69851 --- /dev/null +++ b/plugins/environment-modal-sandbox/app.tsx @@ -0,0 +1,233 @@ +import { z } from "zod"; +import { useEffect, useRef, useState } from "react"; +import { + definePluginApp, + useRpc, + type PluginMachineProviderInputsProps, +} from "@get-bb/plugin-sdk/app"; +import { Input } from "@bb/shared-ui/input"; +import type { modalRpcContract } from "./catalogue/contract.js"; +import type { Build, Project } from "./catalogue/model.js"; +import { ModalSettings } from "./settings.js"; +import { + errorText, + runningHourlyEstimate, + stalenessLabel, + type Staleness, +} from "./ui-state.js"; + +function ModalInputs({ + projectId, + value, + onChange, + experimental_agentProviderId, +}: PluginMachineProviderInputsProps) { + const rpc = useRpc(); + const callback = useRef(onChange); + useEffect(() => { + callback.current = onChange; + }, [onChange]); + const [project, setProject] = useState< + (Project & { staleness: Staleness }) | null + >(null); + const [build, setBuild] = useState(null); + const [message, setMessage] = useState("Checking project image…"); + const initial = z + .object({ + buildId: z.string().optional(), + resources: z + .object({ cpuCores: z.number(), memoryMiB: z.number() }) + .optional(), + }) + .safeParse(value); + const [cpu, setCpu] = useState(() => + initial.success ? (initial.data.resources?.cpuCores ?? null) : null, + ); + const [memory, setMemory] = useState(() => + initial.success ? (initial.data.resources?.memoryMiB ?? null) : null, + ); + const [imageOverride, setImageOverride] = useState(() => + initial.success ? (initial.data.buildId ?? "") : "", + ); + const [images, setImages] = useState([]); + const provider = experimental_agentProviderId ?? "codex"; + useEffect(() => { + let live = true; + setBuild(null); + setProject(null); + callback.current({ + status: "blocked", + reason: "Checking the selected project image", + }); + if (!projectId) { + setMessage("Select a project to use its Modal image"); + return; + } + void (async () => { + const sources = await rpc.call("project.sources", { projectId }); + const source = sources.find((source) => source.primaryHost); + if (source) + await rpc + .call("project.inspect", { projectId, environmentId: source.id }) + .catch(() => undefined); + const [configured, preflight, available] = await Promise.all([ + rpc.call("project.show", { projectId }), + rpc.call("project.preflight", { + projectId, + agentProviderId: provider, + buildId: imageOverride || null, + }), + rpc.call("image.list", { projectId, cursor: null, limit: 100 }), + ]); + if (!live) return; + setProject(configured); + setImages(available.images); + setMessage(preflight.message); + setBuild(preflight.ready ? preflight.build : null); + })().catch((error) => { + if (live) { + setMessage(errorText(error)); + callback.current({ + status: "blocked", + reason: "Modal image preflight failed; review settings", + }); + } + }); + return () => { + live = false; + }; + }, [rpc, projectId, provider, imageOverride]); + useEffect(() => { + if (!build || !project) { + callback.current({ status: "blocked", reason: message }); + return; + } + const cpuCores = cpu ?? project.resources.cpuCores; + const memoryMiB = memory ?? project.resources.memoryMiB; + if ( + !Number.isFinite(cpuCores) || + cpuCores < 0.125 || + cpuCores > 64 || + !Number.isInteger(memoryMiB) || + memoryMiB < 128 || + memoryMiB > 262144 + ) { + callback.current({ + status: "blocked", + reason: "Enter 0.125–64 physical cores and 128–262144 MiB memory", + }); + return; + } + callback.current({ + status: "ready", + value: { + buildId: build.buildId, + accountRef: "default", + appName: build.appName, + resources: { cpuCores, memoryMiB }, + policy: project.policy, + }, + }); + }, [build, project, cpu, memory, message]); + return ( +
+

New Modal sandbox

+

+ {message} +

+ {project && ( + <> +

{stalenessLabel(project.staleness)}

+ +
+ + +
+

+ Pauses after {project.policy.idleMinutes || "no"} idle minutes. + Lifetime {project.policy.lifetimeMinutes} minutes; planned + preservation interrupts active work. Retained{" "} + {project.policy.retentionDays} days after the last thread unless + kept. +

+

+ Estimated running compute $ + {runningHourlyEstimate( + cpu ?? project.resources.cpuCores, + memory ?? project.resources.memoryMiB, + ).toFixed(4)} + /hour; additional charges and unknown snapshot storage apply.{" "} + + Rates: September 8, 2026 + + . +

+ + )} + + Configure Modal + +

+ Creates a new machine for this thread. Choose an existing machine’s + section in the picker to reuse or wake it. Builds are always explicit. +

+
+ ); +} + +export default definePluginApp((app) => { + app.slots.settingsSection({ + id: "catalogue", + title: "Project sandboxes", + component: ModalSettings, + }); + app.slots.experimental_machineProviderInputs({ + machineProviderId: "modal-sandbox", + component: (props) => , + }); +}); diff --git a/plugins/environment-modal-sandbox/catalogue/artifact.ts b/plugins/environment-modal-sandbox/catalogue/artifact.ts new file mode 100644 index 00000000000..805fcd3b4a4 --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/artifact.ts @@ -0,0 +1,40 @@ +import { z } from "zod"; +import { createHash } from "node:crypto"; +const hash = (data: Buffer) => createHash("sha256").update(data).digest("hex"); +const hashSchema = z.string().regex(/^[a-f0-9]{64}$/); + +export const artifactMetadataSchema = z + .object({ + sha256: hashSchema, + version: z.string().min(1), + protocolVersion: z.number().int().positive(), + }) + .strict(); +export type ArtifactMetadata = z.infer; +export interface BaseArtifact { + metadata: ArtifactMetadata; + data: Buffer; +} +export async function fetchBaseArtifact( + serverUrl: string, +): Promise { + const versionResponse = await fetch(`${serverUrl}/install/version`); + if (!versionResponse.ok) + throw new Error("Cannot resolve the server's bb package version"); + const version = z + .object({ + version: z.string(), + protocolVersion: z.number().int().positive(), + }) + .parse(await versionResponse.json()); + const response = await fetch(`${serverUrl}/install/bb-app.tgz`); + if (!response.ok) + throw new Error("Cannot fetch the server's credential-free bb package"); + const digest = hashSchema.parse(response.headers.get("x-bb-artifact-sha256")); + const data = Buffer.from(await response.arrayBuffer()); + if (data.length > 64 * 1024 * 1024) + throw new Error("The bb base package exceeds 64 MiB"); + if (hash(data) !== digest) + throw new Error("The bb base package failed SHA-256 verification"); + return { metadata: { ...version, sha256: digest }, data }; +} diff --git a/plugins/environment-modal-sandbox/catalogue/backend.ts b/plugins/environment-modal-sandbox/catalogue/backend.ts new file mode 100644 index 00000000000..b77b1dc805f --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/backend.ts @@ -0,0 +1,161 @@ +import type { BaseArtifact } from "./artifact.js"; +import { ModalClient, NotFoundError } from "modal"; +import { z } from "zod"; +import { baseCommands, baseManifest } from "./base.js"; +import { quote, translateDockerfile } from "./dockerfile.js"; +import { hash } from "./model.js"; +import type { ModalCredentials } from "../sandbox-backend.js"; + +export interface ImageBuildRequest { + name: string; + appName: string; + dockerfileText: string; + baseArtifact: BaseArtifact | null; + files: ReadonlyMap; +} +export interface ImageBuildHooks { + signal?: AbortSignal; + log(text: string): void; + allocated(imageId: string): void; +} +export interface ImageBackend { + accountIdentity(): Promise; + build(request: ImageBuildRequest, hooks: ImageBuildHooks): Promise; + reconcile(name: string): Promise; + delete(imageId: string): Promise; + resolve(imageId: string): Promise; +} +export type ImageBackendFactory = ( + credentials: ModalCredentials, +) => ImageBackend; +export const createImageBackend: ImageBackendFactory = (credentials) => { + async function lookup(operation: (client: ModalClient) => Promise) { + const client = new ModalClient(credentials); + try { + return await operation(client); + } catch (error) { + if (error instanceof NotFoundError) return null; + throw error; + } finally { + client.close(); + } + } + return { + async accountIdentity() { + const identity = await lookup(async (client) => { + const token = await client.cpClient.tokenInfoGet({}); + if (!token.workspaceId) + throw new Error("Modal did not return an account identity"); + return hash(token.workspaceId); + }); + if (!identity) throw new Error("Modal account was not found"); + return identity; + }, + async build(request, hooks) { + type Middleware = NonNullable< + NonNullable< + ConstructorParameters[0] + >["grpcMiddleware"] + >[number]; + const middleware: Middleware = async function* (call, options) { + const iterator = call.next(call.request, options); + for (;;) { + const item = await iterator.next(); + if (call.method.path.endsWith("/ImageJoinStreaming")) { + const logs = z + .object({ taskLogs: z.array(z.object({ data: z.string() })) }) + .safeParse(item.value); + if (logs.success) + for (const log of logs.data.taskLogs) + if (log.data) hooks.log(log.data); + } + if (item.done) { + if (call.method.path.endsWith("/ImageGetOrCreate")) { + const image = z + .object({ imageId: z.string().min(1) }) + .safeParse(item.value); + const request = z + .object({ + image: z.object({ dockerfileCommands: z.array(z.string()) }), + }) + .safeParse(call.request); + if ( + image.success && + request.success && + request.data.image.dockerfileCommands.some((command) => + command.includes("/opt/bb-project/image-manifest.json"), + ) + ) + hooks.allocated(image.data.imageId); + } + return item.value; + } + yield item.value; + } + }; + const client = new ModalClient({ + ...credentials, + grpcMiddleware: [middleware], + }); + try { + const app = await client.apps.fromName(request.appName, { + createIfMissing: true, + }); + const artifact = request.baseArtifact; + if (!artifact) + throw new Error( + "Build requires the server’s exact credential-free bb package; explicitly rebuild", + ); + const commands = [ + ...baseCommands, + ...translateDockerfile( + "COPY bb-app.tgz /tmp/bb-app.tgz", + new Map([ + ["bb-app.tgz", { data: artifact.data, executable: false }], + ]), + ), + `RUN echo "${artifact.metadata.sha256} /tmp/bb-app.tgz" | sha256sum -c - && npm install -g /tmp/bb-app.tgz && rm /tmp/bb-app.tgz`, + "RUN node --version && npm --version && git --version && python3 --version && bb --version && bb machine enroll --help && codex --version && claude --version", + ...translateDockerfile(request.dockerfileText, request.files), + "RUN mkdir -p /opt/bb-project && printf '%s' " + + quote( + JSON.stringify({ + version: 1, + name: request.name, + base: baseManifest.version, + bbPackage: artifact.metadata, + }), + ) + + " > /opt/bb-project/image-manifest.json", + ]; + hooks.log(`Building ${request.name} with ${baseManifest.builder}`); + const image = await client.images + .fromRegistry(baseManifest.registry) + .dockerfileCommands(commands) + .build(app); + hooks.allocated(image.imageId); + await image.publish(request.name); + hooks.log(`Published ${request.name}`); + return image.imageId; + } finally { + client.close(); + } + }, + reconcile(name) { + return lookup( + async (client) => (await client.images.fromName(name)).imageId, + ); + }, + resolve(imageId) { + return lookup( + async (client) => (await client.images.fromId(imageId)).imageId, + ); + }, + async delete(imageId) { + await lookup(async (client) => { + await client.images.delete(imageId); + return imageId; + }); + }, + }; +}; diff --git a/plugins/environment-modal-sandbox/catalogue/base.ts b/plugins/environment-modal-sandbox/catalogue/base.ts new file mode 100644 index 00000000000..c64de2a15ef --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/base.ts @@ -0,0 +1,29 @@ +import { hash } from "./model.js"; + +export const baseManifest = { + version: "bb-modal-v1.0.1", + platform: "linux/amd64", + registry: + "node:22.19.0-bookworm@sha256:afff6d8c97964a438d2e6a9c96509367e45d8bf93f790ad561a1eaea926303d9", + node: "22.19.0", + distribution: "Debian bookworm", + packages: ["git", "curl", "ca-certificates", "build-essential", "python3"], + npmPackages: { + "@openai/codex": "0.153.4", + "@anthropic-ai/claude-code": "2.1.263", + }, + bbPackage: "server-host-artifact-with-recorded-sha256", + builder: "modal@0.10.0", + credentials: "none", +} as const; +export const baseCommands = [ + "RUN apt-get update && apt-get install -y --no-install-recommends git curl ca-certificates build-essential python3 && rm -rf /var/lib/apt/lists/*", + `RUN npm install -g ${Object.entries(baseManifest.npmPackages) + .map(([name, version]) => `${name}@${version}`) + .join(" ")} && npm cache clean --force`, + + `RUN mkdir -p /opt/bb-project && printf '%s' '${Buffer.from(JSON.stringify(baseManifest)).toString("base64")}' | base64 -d > /opt/bb-project/base-manifest.json`, +]; +export const baseDigest = hash( + JSON.stringify({ manifest: baseManifest, commands: baseCommands }), +); diff --git a/plugins/environment-modal-sandbox/catalogue/catalogue.test.ts b/plugins/environment-modal-sandbox/catalogue/catalogue.test.ts new file mode 100644 index 00000000000..674620f02aa --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/catalogue.test.ts @@ -0,0 +1,389 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { createFakePluginHost } from "@get-bb/plugin-sdk/testing"; +import { Catalogue } from "./store.js"; +import { hash, recipeInputSchema } from "./model.js"; +import { acceptChunk, contextFiles } from "./context.js"; +import { parseDockerfile, translateDockerfile } from "./dockerfile.js"; + +const disposals: Array<() => Promise> = []; +afterEach(async () => { + await Promise.all(disposals.splice(0).map((dispose) => dispose())); +}); +function setup() { + const host = createFakePluginHost({ pluginId: "environment-modal-sandbox" }); + disposals.push(() => host.harness.lifecycle.dispose()); + let now = 1000; + const store = new Catalogue( + host.bb.storage.database(), + host.bb.storage, + () => now, + ); + const recipe = store.putRecipe( + recipeInputSchema.parse({ + projectId: "p", + expectedRevision: 0, + dockerfileText: "COPY package-lock.json ./\nRUN node --version", + contextRules: { include: ["package-lock.json"] }, + }), + ); + const data = Buffer.from("{}"); + const context = store.putContext("p", { + recipeId: recipe.recipeId, + revision: recipe.revision, + source: { + hostId: "local", + path: "/project", + commit: "a".repeat(40), + dirty: [], + submodules: [], + lfs: [], + }, + reviewedDirty: [], + files: [ + { + path: "package-lock.json", + bytes: data.length, + sha256: hash(data), + mode: "100644", + }, + ], + }); + store.db + .prepare("INSERT INTO context_uploads VALUES (?,?,?)") + .run(context.contextId, hash("upload-token"), "local"); + acceptChunk(store, "upload-token", { + contextId: context.contextId, + path: "package-lock.json", + offset: 0, + data: data.toString("base64"), + }); + contextFiles(store, context.contextId); + store.completeContext(context.contextId); + const input = { + projectId: "p", + recipeId: recipe.recipeId, + revision: 1, + contextId: context.contextId, + key: "request", + }; + return { + ...host, + store, + recipe, + context, + input, + advance: (amount: number) => { + now += amount; + }, + }; +} +describe("real migrated image catalogue", () => { + it("claims a hash once across request keys and connection handles; account claims serialize builds", async () => { + const test = setup(); + const second = new Catalogue( + test.store.db, + test.bb.storage, + test.store.now, + ); + const results = await Promise.all( + Array.from({ length: 20 }, (_, index) => + Promise.resolve().then(() => + (index % 2 ? second : test.store).start( + { ...test.input, key: `request-${index}` }, + hash("account"), + "app", + ), + ), + ), + ); + expect(new Set(results.map((result) => result.build.buildId)).size).toBe(1); + expect(results.filter((result) => !result.reused)).toHaveLength(1); + expect(test.store.claim(hash("account"))?.state).toBe("building"); + expect(second.claim(hash("account"))).toBeNull(); + expect(() => + second.start( + { ...test.input, key: "request-0", contextId: "missing" }, + hash("account"), + "app", + ), + ).toThrow(); + expect(() => + second.start( + { ...test.input, key: "request-0" }, + hash("account"), + "other-app", + ), + ).toThrow(/different payload/); + }); + it("uses CAS for immutable recipes and project configuration", () => { + const { store, recipe } = setup(); + expect(() => + store.putRecipe( + recipeInputSchema.parse({ + ...recipe, + recipeId: undefined, + revision: undefined, + recipeHash: undefined, + baseDigest: undefined, + createdAt: undefined, + expectedRevision: 0, + }), + ), + ).toThrow(); + expect(() => + store.putRecipe( + recipeInputSchema.parse({ + projectId: "p", + expectedRevision: 0, + dockerfileText: "RUN true", + }), + ), + ).toThrow(/latest revision 1/); + const next = store.putRecipe( + recipeInputSchema.parse({ + projectId: "p", + expectedRevision: 1, + dockerfileText: "RUN true", + }), + ); + expect(store.recipe("p", 1).dockerfileText).toBe(recipe.dockerfileText); + expect(next.revision).toBe(2); + const project = store.project("p"); + store.configure({ ...project, usableBuildId: null, expectedRevision: 0 }); + expect(() => store.configure({ ...project, usableBuildId: null, expectedRevision: 0 })).toThrow( + /revision conflict/, + ); + }); + it("reconciles restart mid-build without resubmitting or releasing its account claim", () => { + const test = setup(); + const { build } = test.store.start(test.input, hash("account"), "app"); + test.store.claim(hash("account")); + const restarted = new Catalogue( + test.store.db, + test.bb.storage, + test.store.now, + ); + restarted.restart(); + expect(restarted.build(build.buildId).state).toBe("reconciling"); + expect(restarted.claim(hash("account"))).toBeNull(); + expect(restarted.reconciling(hash("account"))[0]?.name).toBe(build.name); + restarted.ready(build.buildId, "im-reconciled", {}); + expect(restarted.build(build.buildId).imageId).toBe("im-reconciled"); + }); + it("retains monotonic paged redacted logs under the byte cap", () => { + const test = setup(); + const { build } = test.store.start(test.input, hash("account"), "app"); + test.store.event( + build.buildId, + "log", + "private-value Bearer bearer-value api_key=key-value", + ["private-value"], + ); + const first = test.store.events(build.buildId, 0, 10); + expect(first.events[0]?.text).not.toMatch( + /private-value|bearer-value|key-value/, + ); + for (let i = 0; i < 165; i++) + test.store.event(build.buildId, "log", "x".repeat(65536)); + const retained = test.store.db + .prepare("SELECT sum(bytes) AS bytes FROM build_events WHERE build_id=?") + .get(build.buildId); + expect(retained).toMatchObject({ bytes: expect.any(Number) }); + let cursor = first.nextCursor; + let truncated = false; + for (;;) { + const page = test.store.events(build.buildId, cursor, 10); + if (!page.events.length) break; + expect(page.nextCursor).toBeGreaterThan(cursor); + truncated ||= page.events.some((event) => event.kind === "truncated"); + cursor = page.nextCursor; + } + expect(truncated).toBe(true); + expect( + test.store.db + .prepare( + "SELECT sum(bytes)<=? AS bounded FROM build_events WHERE build_id=?", + ) + .get(10 * 1024 * 1024, build.buildId), + ).toEqual({ bounded: 1 }); + }); + it("cancelled watchers have no effect and vendor cancellation is only a request", () => { + const test = setup(); + const { build } = test.store.start(test.input, hash("account"), "app"); + test.store.start( + { ...test.input, key: "another-watcher" }, + hash("account"), + "app", + ); + test.store.claim(hash("account")); + test.store.events(build.buildId, 0, 10); + const controller = new AbortController(); + controller.abort(); + expect(test.store.build(build.buildId).state).toBe("building"); + expect(test.store.cancel(build.buildId)).toMatchObject({ + state: "building", + cancelRequested: true, + }); + }); + it("rejects malformed, escaping, unauthorized, corrupt and incomplete archives", () => { + const test = setup(); + const fresh = test.store.putContext("p", test.context.manifest); + test.store.db + .prepare("INSERT INTO context_uploads VALUES (?,?,?)") + .run(fresh.contextId, hash("token"), "local"); + const chunk = { + contextId: fresh.contextId, + path: "package-lock.json", + offset: 0, + data: "e30=", + }; + expect(() => acceptChunk(test.store, "wrong", chunk)).toThrow(/token/); + expect(() => + acceptChunk(test.store, "token", { ...chunk, path: "../outside" }), + ).toThrow(/Unsafe/); + expect(() => + acceptChunk(test.store, "token", { ...chunk, path: "unreviewed" }), + ).toThrow(/allowlisted/); + expect(() => + acceptChunk(test.store, "token", { ...chunk, offset: 1 }), + ).toThrow(/bounds/); + expect(() => contextFiles(test.store, fresh.contextId)).toThrow(/mismatch/); + acceptChunk(test.store, "token", { ...chunk, data: "e3s=" }); + expect(() => contextFiles(test.store, fresh.contextId)).toThrow(/mismatch/); + expect(() => acceptChunk(test.store, "token", chunk)).toThrow( + /different content/, + ); + }); + it("rechecks references at deletion and rejects references after a deletion claim", () => { + const test = setup(); + const { build } = test.store.start(test.input, hash("account"), "app"); + test.store.ready(build.buildId, "im-owned", {}); + expect(test.store.claimDeletion(build.buildId, 1000)).toBe(false); + test.store.reference(build.buildId, "machine", "host"); + test.advance(2000); + expect(test.store.claimDeletion(build.buildId, 1000)).toBe(false); + test.store.release("machine", "host"); + expect(test.store.claimDeletion(build.buildId, 1000)).toBe(false); + test.advance(1000); + expect(test.store.claimDeletion(build.buildId, 1000)).toBe(true); + expect(() => + test.store.reference(build.buildId, "machine", "late-host"), + ).toThrow(/being deleted/); + test.store.deleted(build.buildId); + expect(test.store.images("p", null, 10)).toEqual([]); + }); + it("isolates recipes, contexts, build events and GC by authenticated owner", () => { + const test = setup(); + const { build } = test.store.start(test.input, hash("account"), "app"); + test.store.ready(build.buildId, "im-owner", {}); + const other = new Catalogue( + test.store.db, + test.bb.storage, + test.store.now, + "different-owner", + ); + expect(other.recipes(null, 10)).toEqual([]); + expect(other.gcCandidates()).toEqual([]); + expect(() => other.recipe("p")).toThrow(/not found/); + expect(() => other.context(test.context.contextId)).toThrow(/not found/); + expect(() => other.events(build.buildId, 0, 10)).toThrow(/not found/); + expect(() => other.cancel(build.buildId)).toThrow(/not found/); + }); +}); +describe("Dockerfile extension subset", () => { + it.each([ + "FROM ubuntu", + "ADD file /", + "CMD true", + "ENTRYPOINT true", + "USER root", + "RUN --mount=type=secret true", + ])("reports the exact rejected source line for %s", (instruction) => { + expect(() => + parseDockerfile(`# project\nRUN true\n${instruction}`), + ).toThrow(/line 3/); + }); + it("materializes exact binary COPY contents without depending on a server path or URL", () => { + const data = Buffer.from([0, 255, 1, 128]); + const commands = translateDockerfile( + 'WORKDIR /opt/cache\nCOPY ["lock file", "./"]\nENV HELLO=world\nARG BUILD_MODE=release\nRUN echo ok', + new Map([["lock file", { data, executable: false }]]), + ); + expect(commands.join("\n")).toContain(data.toString("base64")); + expect(commands.join("\n")).toContain("'/opt/cache/lock file'"); + expect(commands.join("\n")).not.toContain("COPY"); + expect(() => translateDockerfile("COPY absent ./", new Map())).toThrow( + /line 1.*absent/, + ); + }); +}); +it("bounds log pages even when JSON escaping expands large chunks", () => { + const { store, input } = setup(); + const { build } = store.start(input, hash("account"), "app"); + for (let i = 0; i < 20; i++) + store.event(build.buildId, "log", "\u0001".repeat(65536)); + const page = store.events(build.buildId, 0, 200); + expect(Buffer.byteLength(JSON.stringify(page))).toBeLessThan(1024 * 1024); + expect(page.events).toHaveLength(2); + expect( + store.events(build.buildId, page.nextCursor, 200).events[0]?.sequence, + ).toBe(3); +}); +it("replays a completed request after staging expires and resets GC state after an explicit rebuild", () => { + const { store, input, advance } = setup(); + const { build } = store.start(input, hash("account"), "app"); + store.ready(build.buildId, "im-first", {}); + advance(24 * 60 * 60 * 1000 + 1); + expect(store.start(input, hash("account"), "app").build.buildId).toBe( + build.buildId, + ); + store.claimDeletion(build.buildId, 0); + store.deleted(build.buildId); + expect(store.images("p", null, 50)).toHaveLength(0); + store.ready(build.buildId, "im-rebuilt", {}); + store.reference(build.buildId, "allocation", "new-machine"); + expect(store.images("p", null, 50)).toHaveLength(1); + expect(store.protected(build.buildId)).toBe(true); +}); + +it("migrates stored recipe setup text away and rejects the removed input", async () => { + const { migrations } = await import("./migrations.js"); + const host = createFakePluginHost({ pluginId: "environment-modal-sandbox" }); + disposals.push(() => host.harness.lifecycle.dispose()); + const db = host.bb.storage.database(); + host.bb.storage.migrate(db, migrations.slice(0, -1)); + db.prepare("INSERT INTO catalogue_owner VALUES ('owner', 'user')").run(); + const legacy = { + projectId: "project", + recipeId: "recipe", + revision: 1, + dockerfileText: "RUN true", + setupScriptText: "legacy-private-setup", + contextRules: { include: [], exclude: [] }, + smoke: { commands: [], timeoutSeconds: 120 }, + recipeHash: hash("legacy"), + baseDigest: hash("base"), + createdAt: 1, + }; + db.prepare("INSERT INTO recipes VALUES (?,?,?,?,?,?)").run( + "user", + "project", + "recipe", + 1, + legacy.recipeHash, + JSON.stringify(legacy), + ); + const store = new Catalogue(db, host.bb.storage); + expect(store.recipe("project")).not.toHaveProperty("setupScriptText"); + expect( + JSON.stringify(db.prepare("SELECT data FROM recipes").all()), + ).not.toContain("legacy-private-setup"); + expect(() => + recipeInputSchema.parse({ + projectId: "project", + expectedRevision: 1, + dockerfileText: "RUN true", + setupScriptText: "ignored", + }), + ).toThrow(); +}); diff --git a/plugins/environment-modal-sandbox/catalogue/cli.ts b/plugins/environment-modal-sandbox/catalogue/cli.ts new file mode 100644 index 00000000000..b53af2f2c72 --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/cli.ts @@ -0,0 +1,285 @@ +import type { BbPluginApi, PluginCliResult } from "@get-bb/plugin-sdk"; +import { z } from "zod"; +import { modalRpcContract } from "./contract.js"; +import { CatalogueError } from "./model.js"; +import type { CatalogueService } from "./service.js"; + +const commandFlags: Record = { + "account inspect": [], + "project sources": ["project"], + "project preflight": ["project", "provider"], + "project inspect": ["project", "environment"], + "recipe put": ["project", "expected-revision", "input-text", "json-input"], + "recipe show": ["project"], + "recipe list": ["cursor", "limit"], + "context upload": [ + "project", + "environment", + "recipe", + "revision", + "reviewed-dirty-json", + ], + "image verify": ["provider", "key"], + "image use": ["project", "provider", "expected-revision"], + "image build": ["project", "recipe", "revision", "context", "key"], + "image logs": ["follow", "cursor", "limit"], + "image status": [], + "image cancel": [], + "image list": ["project", "cursor", "limit"], + "image gc": ["apply", "dry-run"], + "project configure": ["project", "expected-revision", "json-input"], + "project show": ["project"], +}; +export const commandNames = Object.keys(commandFlags); +export function registerCatalogueCli( + bb: BbPluginApi, + service: CatalogueService, +) { + bb.cli.register({ + name: "modal", + summary: + "Inspect projects and explicitly build credential-free Modal images", + commands: commandNames.map((name) => ({ + name: name.replaceAll(" ", "-"), + summary: name, + usage: `bb modal ${name} --json`, + })), + async run(argv) { + try { + const [group, action, ...rest] = argv; + const flags = new Map(); + const positional = []; + for (let i = 0; i < rest.length; i++) { + const value = rest[i]!; + if (!value.startsWith("--")) { + positional.push(value); + continue; + } + if (flags.has(value)) + throw new CatalogueError(400, `Repeated flag ${value}`); + flags.set( + value, + rest[i + 1] && !rest[i + 1]!.startsWith("--") ? rest[++i]! : "true", + ); + } + const command = `${group} ${action}`; + const allowed = commandFlags[command]; + if (!allowed) + throw new CatalogueError(400, `Commands: ${commandNames.join(", ")}`); + for (const flag of flags.keys()) { + if (flag !== "--json" && !allowed.includes(flag.slice(2))) + throw new CatalogueError( + 400, + `Unknown flag ${flag} for ${command}`, + ); + } + const expectsBuild = [ + "image verify", + "image use", + "image logs", + "image status", + "image cancel", + ].includes(command); + if (positional.length !== (expectsBuild ? 1 : 0)) + throw new CatalogueError( + 400, + expectsBuild + ? "Provide exactly one build ID" + : "Unexpected positional argument", + ); + const required = (name: string) => { + const value = flags.get(`--${name}`); + if (!value || value === "true") + throw new CatalogueError(400, `--${name} is required`); + return value; + }; + const number = (name: string) => + z.coerce.number().int().nonnegative().parse(required(name)); + async function projectId() { + const value = required("project"); + const projects = await bb.sdk.projects.list(); + const matches = projects.filter( + (project) => project.id === value || project.name === value, + ); + if (matches.length !== 1) + throw new CatalogueError( + 400, + "Project must resolve to exactly one stable ID", + ); + return matches[0]!.id; + } + async function invoke( + method: Method, + input: unknown, + ) { + const parsed = modalRpcContract[method].input.parse(input); + const handler = service.handlers[method] as ( + input: typeof parsed, + ) => unknown; + return await handler(parsed); + } + const page = { + cursor: flags.get("--cursor") ?? null, + limit: flags.has("--limit") ? number("limit") : 50, + }; + let result: unknown; + let continuation: { argv: string[]; delayMs: number } | undefined; + switch (`${group} ${action}`) { + case "account inspect": result = await invoke("account.inspect", {}); break; + case "project sources": result = await invoke("project.sources", { projectId: await projectId() }); break; + case "project preflight": result = await invoke("project.preflight", { projectId: await projectId(), agentProviderId: flags.get("--provider") ?? "codex", buildId: null }); break; + case "project inspect": + result = await invoke("project.inspect", { + projectId: await projectId(), + environmentId: required("environment"), + }); + break; + case "recipe put": { + const text = flags.get("--input-text") ?? flags.get("--json-input"); + if (text === undefined) + throw new CatalogueError( + 400, + "Pipe Dockerfile text or a JSON recipe envelope using --stdin", + ); + const content = text.trim().startsWith("{") + ? z.record(z.string(), z.unknown()).parse(JSON.parse(text)) + : { dockerfileText: text }; + result = await invoke("recipe.put", { + ...content, + projectId: await projectId(), + expectedRevision: number("expected-revision"), + }); + break; + } + case "recipe show": + result = await invoke("recipe.get", { + projectId: await projectId(), + }); + break; + case "recipe list": + result = await invoke("recipe.list", page); + break; + case "context upload": { + const prepared = await service.handlers["context.prepare"]( + modalRpcContract["context.prepare"].input.parse({ + projectId: await projectId(), + environmentId: required("environment"), + recipeId: required("recipe"), + revision: number("revision"), + reviewedDirty: JSON.parse( + flags.get("--reviewed-dirty-json") ?? "[]", + ), + }), + ); + result = await service.handlers["context.upload"]({ + contextId: prepared.contextId, + uploadToken: prepared.uploadToken, + }); + break; + } + case "image build": + result = await invoke("build.start", { + projectId: await projectId(), + recipeId: required("recipe"), + revision: number("revision"), + contextId: required("context"), + key: required("key"), + }); + break; + case "image logs": { + const input = modalRpcContract["build.events"].input.parse({ + buildId: positional[0], + cursor: Number(flags.get("--cursor") ?? 0), + limit: flags.has("--limit") ? number("limit") : 200, + }); + const events = await service.handlers["build.events"](input); + result = events; + if (flags.has("--follow") && !events.terminal) + continuation = { + argv: [ + "image", + "logs", + input.buildId, + "--follow", + "--cursor", + String(events.nextCursor), + "--json", + ], + delayMs: events.events.length ? 0 : 1000, + }; + break; + } + case "image verify": + result = await invoke("verification.start", { + buildId: positional[0], + agentProviderId: required("provider"), + key: required("key"), + }); + break; + case "image use": + result = await invoke("project.useImage", { + buildId: positional[0], + projectId: await projectId(), + agentProviderId: flags.get("--provider") ?? "codex", + expectedRevision: number("expected-revision"), + }); + break; + case "image status": + result = await invoke("build.get", { buildId: positional[0] }); + break; + case "image cancel": + result = await invoke("build.cancel", { buildId: positional[0] }); + break; + case "image list": + result = await invoke("image.list", { + ...page, + projectId: flags.has("--project") ? await projectId() : null, + }); + break; + case "image gc": { + if (flags.has("--apply") === flags.has("--dry-run")) + throw new CatalogueError(400, "Choose --dry-run or --apply"); + result = await invoke("image.gc", { + dryRun: flags.has("--dry-run"), + }); + break; + } + case "project configure": + result = await invoke("project.configure", { + ...z + .record(z.string(), z.unknown()) + .parse(JSON.parse(required("json-input"))), + projectId: await projectId(), + expectedRevision: number("expected-revision"), + }); + break; + case "project show": + result = await invoke("project.show", { + projectId: await projectId(), + }); + break; + default: + throw new CatalogueError( + 400, + `Commands: ${commandNames.join(", ")}`, + ); + } + return { + exitCode: 0, + stdout: JSON.stringify(result), + ...(continuation ? { experimental_continue: continuation } : {}), + } satisfies PluginCliResult; + } catch (error) { + return { + exitCode: 1, + stderr: JSON.stringify({ + error: error instanceof Error ? error.message : String(error), + status: error instanceof CatalogueError ? error.status : 400, + latestRevision: + error instanceof CatalogueError ? error.latestRevision : null, + }), + }; + } + }, + }); +} diff --git a/plugins/environment-modal-sandbox/catalogue/context.ts b/plugins/environment-modal-sandbox/catalogue/context.ts new file mode 100644 index 00000000000..af6a5c14fb3 --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/context.ts @@ -0,0 +1,111 @@ +import { timingSafeEqual } from "node:crypto"; +import { z } from "zod"; +import { CatalogueError, hash } from "./model.js"; +import { safePath } from "./dockerfile.js"; +import type { Catalogue } from "./store.js"; + +export const chunkSchema = z + .object({ + contextId: z.string(), + path: z.string(), + offset: z.number().int().nonnegative(), + data: z + .string() + .max(350000) + .regex( + /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/, + ), + }) + .strict(); +export function acceptChunk( + store: Catalogue, + token: string, + input: z.infer, +) { + store.db + .transaction(() => { + const context = store.context(input.contextId); + const upload = z + .object({ token_hash: z.string() }) + .parse( + store.db + .prepare( + "SELECT token_hash FROM context_uploads WHERE context_id=?", + ) + .get(context.contextId), + ); + if ( + !timingSafeEqual( + Buffer.from(hash(token)), + Buffer.from(upload.token_hash), + ) + ) + throw new CatalogueError(403, "Invalid scoped context upload token"); + if (context.uploaded || context.expiresAt <= store.now()) + throw new CatalogueError(409, "Context is sealed or expired"); + safePath(input.path); + const file = context.manifest.files.find( + (file) => file.path === input.path, + ); + if (!file) + throw new CatalogueError(400, "Archive path is not allowlisted"); + const data = Buffer.from(input.data, "base64"); + if ( + input.offset + data.length > file.bytes || + input.offset % (256 * 1024) !== 0 || + (input.offset + data.length < file.bytes && data.length !== 256 * 1024) + ) + throw new CatalogueError(400, "Invalid archive chunk bounds"); + const previous = store.db + .prepare( + "SELECT data FROM context_chunks WHERE context_id=? AND path=? AND offset=?", + ) + .get(context.contextId, input.path, input.offset); + if ( + previous && + !z + .object({ data: z.instanceof(Buffer) }) + .parse(previous) + .data.equals(data) + ) + throw new CatalogueError( + 409, + "Archive chunk already has different content", + ); + store.db + .prepare("INSERT OR IGNORE INTO context_chunks VALUES (?,?,?,?)") + .run(context.contextId, input.path, input.offset, data); + }) + .immediate(); +} +export function contextFiles(store: Catalogue, contextId: string) { + const context = store.context(contextId); + const files = new Map(); + for (const file of context.manifest.files) { + const chunks = store.db + .prepare( + "SELECT offset,data FROM context_chunks WHERE context_id=? AND path=? ORDER BY offset", + ) + .all(contextId, file.path) + .map((row) => + z.object({ offset: z.number(), data: z.instanceof(Buffer) }).parse(row), + ); + let offset = 0; + for (const chunk of chunks) { + if (chunk.offset !== offset) + throw new CatalogueError( + 400, + `Archive has missing chunks: ${file.path}`, + ); + offset += chunk.data.length; + } + const data = Buffer.concat(chunks.map((chunk) => chunk.data)); + if (offset !== file.bytes || hash(data) !== file.sha256) + throw new CatalogueError(400, `Archive hash/size mismatch: ${file.path}`); + files.set(safePath(file.path), { + data, + executable: file.mode === "100755", + }); + } + return files; +} diff --git a/plugins/environment-modal-sandbox/catalogue/contract.ts b/plugins/environment-modal-sandbox/catalogue/contract.ts new file mode 100644 index 00000000000..8fd1e93baef --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/contract.ts @@ -0,0 +1,184 @@ +import { defineRpcContract } from "@get-bb/plugin-sdk"; +import { z } from "zod"; +import { + verificationSchema, + buildSchema, + contextSchema, + eventSchema, + idSchema, + policySchema, + projectSchema, + recipeInputSchema, + recipeSchema, + resourcesSchema, +} from "./model.js"; +import { inspectionSchema } from "./source-contract.js"; + +const project = z.object({ projectId: idSchema }).strict(); +const build = z.object({ buildId: idSchema }).strict(); +const page = z + .object({ + cursor: idSchema.nullable().default(null), + limit: z.number().int().min(1).max(100).default(50), + }) + .strict(); +export const modalRpcContract = defineRpcContract({ + "catalogue.projects": { + input: z.object({}).strict(), + output: z.array(z.object({ id: idSchema, name: z.string() })), + }, + "project.sources": { + input: project, + output: z.array( + z.object({ + id: idSchema, + hostId: idSchema, + primaryHost: z.boolean(), + path: z.string(), + name: z.string(), + }), + ), + }, + "account.inspect": { + input: z.object({}).strict(), + output: z.object({ + available: z.boolean(), + accountIdentity: z.string().nullable(), + appName: z.string().nullable(), + baseVersion: z.string(), + message: z.string(), + }), + }, + "project.preflight": { + input: project.extend({ + agentProviderId: idSchema.default("codex"), + buildId: idSchema.nullable().default(null), + }), + output: z.object({ + ready: z.boolean(), + message: z.string(), + build: buildSchema.nullable(), + }), + }, + "verification.list": { + input: build, + output: z.array(verificationSchema), + }, + "project.inspect": { + input: project.extend({ environmentId: idSchema }), + output: inspectionSchema, + }, + "recipe.put": { input: recipeInputSchema, output: recipeSchema }, + "recipe.get": { input: project, output: recipeSchema }, + "recipe.list": { + input: page, + output: z.object({ + recipes: z.array(recipeSchema), + nextCursor: idSchema.nullable(), + }), + }, + "context.prepare": { + input: project.extend({ + environmentId: idSchema, + recipeId: idSchema, + revision: z.number().int().positive(), + reviewedDirty: z.array(z.string()).default([]), + }), + output: contextSchema.extend({ + files: z.number(), + uploadToken: z.string(), + uploadUrl: z.string(), + }), + }, + "context.upload": { + input: z.object({ contextId: idSchema, uploadToken: z.string() }).strict(), + output: contextSchema, + }, + "context.complete": { + input: z.object({ contextId: idSchema }).strict(), + output: contextSchema, + }, + "build.start": { + input: project.extend({ + recipeId: idSchema, + revision: z.number().int().positive(), + contextId: idSchema, + key: idSchema, + }), + output: z.object({ + buildId: idSchema, + state: buildSchema.shape.state, + reused: z.boolean(), + }), + }, + "build.events": { + input: build.extend({ + cursor: z.number().int().nonnegative().default(0), + limit: z.number().int().min(1).max(200).default(200), + }), + output: z.object({ + events: z.array(eventSchema), + nextCursor: z.number(), + terminal: z.boolean(), + }), + }, + "build.get": { input: build, output: buildSchema }, + "build.cancel": { input: build, output: buildSchema }, + "verification.start": { + input: build.extend({ agentProviderId: idSchema, key: idSchema }), + output: verificationSchema, + }, + "verification.get": { + input: z.object({ verificationId: idSchema }).strict(), + output: verificationSchema, + }, + "project.useImage": { + input: project.extend({ + buildId: idSchema, + agentProviderId: idSchema.default("codex"), + expectedRevision: z.number().int().nonnegative(), + }), + output: projectSchema.extend({ available: z.literal(true) }), + }, + "image.list": { + input: page.extend({ projectId: idSchema.nullable().default(null) }), + output: z.object({ + images: z.array(buildSchema), + nextCursor: idSchema.nullable(), + }), + }, + "image.gc": { + input: z.object({ dryRun: z.boolean() }).strict(), + output: z.object({ + candidates: z.array( + z.object({ + buildId: idSchema, + imageId: idSchema, + markedAt: z.number().nullable(), + deleted: z.boolean(), + }), + ), + blockedReferences: z.array(idSchema), + }), + }, + "project.configure": { + input: project.extend({ + usableBuildId: z.null().optional(), + resources: resourcesSchema, + policy: policySchema, + expectedRevision: z.number().int().nonnegative(), + }), + output: projectSchema, + }, + "project.show": { + input: project, + output: projectSchema.extend({ + staleness: z.object({ + dockerfileChanged: z.boolean(), + lockfilesChanged: z.boolean().nullable(), + reason: z.string().nullable(), + lastCheckedAt: z.number().nullable(), + }), + }), + }, +}); diff --git a/plugins/environment-modal-sandbox/catalogue/dockerfile.ts b/plugins/environment-modal-sandbox/catalogue/dockerfile.ts new file mode 100644 index 00000000000..994f67794c0 --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/dockerfile.ts @@ -0,0 +1,163 @@ +import { posix } from "node:path"; +import { CatalogueError } from "./model.js"; + +export interface Instruction { + line: number; + instruction: "RUN" | "COPY" | "ENV" | "WORKDIR" | "ARG"; + argument: string; +} +export const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; +export function safePath(path: string): string { + if ( + !path || + path.includes("\\") || + path.includes("\0") || + path.startsWith("/") || + path.split("/").some((part) => part === ".." || part === ".git") + ) + throw new CatalogueError(400, `Unsafe context path: ${path}`); + return path; +} +export function parseDockerfile(text: string): Instruction[] { + const result: Instruction[] = []; + const lines = text.replaceAll("\r\n", "\n").split("\n"); + for (let index = 0; index < lines.length; index++) { + const line = index + 1; + let value = lines[index]!.trim(); + if (!value || value.startsWith("#")) continue; + while (value.endsWith("\\")) { + if (++index >= lines.length) + throw new CatalogueError( + 400, + `Dockerfile line ${line}: unfinished continuation`, + ); + value = `${value.slice(0, -1)} ${lines[index]!.trim()}`; + } + const match = /^([A-Za-z]+)\s+(.+)$/s.exec(value); + const instruction = match?.[1]?.toUpperCase(); + const argument = match?.[2] ?? ""; + if ( + instruction !== "RUN" && + instruction !== "COPY" && + instruction !== "ENV" && + instruction !== "WORKDIR" && + instruction !== "ARG" + ) + throw new CatalogueError( + 400, + `Dockerfile line ${line}: unsupported instruction ${instruction ?? value}; bb supplies FROM and the runtime command`, + ); + if (argument.startsWith("--") || argument.includes("<<")) + throw new CatalogueError( + 400, + `Dockerfile line ${line}: flags and heredocs are unsupported`, + ); + if ( + /\b(?:BB_SERVER_URL|BB_ENROLLMENT|BB_APP_URL|MODAL_TOKEN_SECRET|OPENAI_API_KEY|ANTHROPIC_API_KEY|POOL_TOKEN)\b|-----BEGIN .*PRIVATE KEY-----/i.test( + argument, + ) + ) + throw new CatalogueError( + 400, + `Dockerfile line ${line}: runtime credentials and server configuration cannot be baked into images`, + ); + result.push({ line, instruction, argument }); + } + return result; +} +export function copyArguments(step: Instruction): string[] { + let words: string[]; + if (step.argument.startsWith("[")) { + let value: unknown; + try { + value = JSON.parse(step.argument); + } catch { + throw new CatalogueError( + 400, + `Dockerfile line ${step.line}: invalid COPY JSON`, + ); + } + if ( + !Array.isArray(value) || + !value.every((word): word is string => typeof word === "string") + ) + throw new CatalogueError( + 400, + `Dockerfile line ${step.line}: COPY requires string paths`, + ); + words = value; + } else { + if (/["'\\]/.test(step.argument)) + throw new CatalogueError( + 400, + `Dockerfile line ${step.line}: use JSON COPY for quoted paths`, + ); + words = step.argument.split(/\s+/); + } + if (words.length < 2) + throw new CatalogueError( + 400, + `Dockerfile line ${step.line}: COPY needs source and destination`, + ); + for (const source of words.slice(0, -1)) safePath(source); + return words; +} +export function translateDockerfile( + text: string, + files: ReadonlyMap, +): string[] { + let workdir = "/"; + const result: string[] = []; + for (const step of parseDockerfile(text)) { + if (step.instruction === "WORKDIR") { + if (/[\s$]/.test(step.argument)) + throw new CatalogueError( + 400, + `Dockerfile line ${step.line}: WORKDIR must be a literal path`, + ); + workdir = posix.resolve(workdir, step.argument); + } + if (step.instruction !== "COPY") { + result.push(`${step.instruction} ${step.argument}`); + continue; + } + const args = copyArguments(step); + const destination = args.at(-1)!; + if (destination.includes("$") || destination.includes("\0")) + throw new CatalogueError( + 400, + `Dockerfile line ${step.line}: COPY destination must be literal`, + ); + const sources = args.slice(0, -1); + if (sources.length > 1 && !destination.endsWith("/")) + throw new CatalogueError( + 400, + `Dockerfile line ${step.line}: multiple sources require a directory destination`, + ); + for (const source of sources) { + const file = files.get(source); + if (!file) + throw new CatalogueError( + 400, + `Dockerfile line ${step.line}: COPY source ${source} is absent; v1 requires explicit regular-file paths`, + ); + const target = posix.resolve( + workdir, + destination, + ...(destination.endsWith("/") ? [posix.basename(source)] : []), + ); + const encoded = file.data.toString("base64"); + result.push( + `RUN mkdir -p ${quote(posix.dirname(target))} && : > ${quote(target)}`, + ); + for (let offset = 0; offset < encoded.length; offset += 32768) + result.push( + `RUN printf '%s' ${quote(encoded.slice(offset, offset + 32768))} | base64 -d >> ${quote(target)}`, + ); + result.push( + `RUN chmod ${file.executable ? "755" : "644"} ${quote(target)}`, + ); + } + } + return result; +} diff --git a/plugins/environment-modal-sandbox/catalogue/logs.ts b/plugins/environment-modal-sandbox/catalogue/logs.ts new file mode 100644 index 00000000000..fb357b3bd57 --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/logs.ts @@ -0,0 +1,32 @@ +export function createBuildLogWriter( + write: (kind: "log" | "truncated", text: string) => void, +) { + let pending = ""; + let dropping = false; + return { + append(text: string) { + for (const part of text.split(/(?<=\n)/)) { + const complete = part.endsWith("\n"); + if (!dropping) { + pending += part; + if (Buffer.byteLength(pending) > 65536) { + write( + "truncated", + "An oversized log line was omitted at the 64 KiB limit", + ); + pending = ""; + dropping = true; + } else if (complete) { + write("log", pending); + pending = ""; + } + } + if (complete) dropping = false; + } + }, + flush() { + if (pending) write("log", pending); + pending = ""; + }, + }; +} diff --git a/plugins/environment-modal-sandbox/catalogue/migrations.ts b/plugins/environment-modal-sandbox/catalogue/migrations.ts new file mode 100644 index 00000000000..5f80295d4da --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/migrations.ts @@ -0,0 +1,24 @@ +export const migrations = [ + `CREATE TABLE catalogue_owner (id TEXT PRIMARY KEY CHECK(id = 'owner'), identity TEXT NOT NULL UNIQUE)`, + `CREATE TABLE recipes (user_id TEXT NOT NULL, project_id TEXT NOT NULL, id TEXT NOT NULL, revision INTEGER NOT NULL, hash TEXT NOT NULL, data TEXT NOT NULL CHECK(json_valid(data)), PRIMARY KEY(user_id, project_id, revision), UNIQUE(user_id, id, revision))`, + `CREATE TABLE contexts (id TEXT PRIMARY KEY, user_id TEXT NOT NULL, project_id TEXT NOT NULL, hash TEXT NOT NULL, expiry INTEGER NOT NULL, data TEXT NOT NULL CHECK(json_valid(data)))`, + `CREATE TABLE builds (id TEXT PRIMARY KEY, user_id TEXT NOT NULL, project_id TEXT NOT NULL, account_id TEXT NOT NULL, app TEXT NOT NULL, hash TEXT NOT NULL, state TEXT NOT NULL CHECK(state IN ('queued','building','ready','failed','cancelled','reconciling')), vendor_ref TEXT NOT NULL, lease TEXT, data TEXT NOT NULL CHECK(json_valid(data)), UNIQUE(user_id,account_id,app,hash))`, + `CREATE INDEX builds_queue ON builds(state,account_id)`, + `CREATE INDEX builds_project ON builds(user_id,project_id,id)`, + `CREATE UNIQUE INDEX builds_account_claim ON builds(account_id) WHERE state IN ('building','reconciling')`, + `CREATE TABLE build_requests (user_id TEXT NOT NULL, key TEXT NOT NULL, payload_hash TEXT NOT NULL, build_id TEXT NOT NULL REFERENCES builds(id), PRIMARY KEY(user_id,key))`, + `CREATE TABLE build_events (build_id TEXT NOT NULL REFERENCES builds(id), sequence INTEGER NOT NULL, kind TEXT NOT NULL, text TEXT NOT NULL, time INTEGER NOT NULL, bytes INTEGER NOT NULL, PRIMARY KEY(build_id,sequence))`, + `CREATE TABLE images (build_id TEXT PRIMARY KEY REFERENCES builds(id), user_id TEXT NOT NULL, image_id TEXT NOT NULL, manifest_json TEXT NOT NULL CHECK(json_valid(manifest_json)), marked_at INTEGER, deleting INTEGER NOT NULL DEFAULT 0, deleted_at INTEGER)`, + `CREATE TABLE project_images (user_id TEXT NOT NULL, project_id TEXT NOT NULL, usable_build_id TEXT REFERENCES builds(id), revision INTEGER NOT NULL, data TEXT NOT NULL CHECK(json_valid(data)), PRIMARY KEY(user_id,project_id))`, + `CREATE TABLE verifications (id TEXT PRIMARY KEY, user_id TEXT NOT NULL, build_id TEXT NOT NULL REFERENCES builds(id), state TEXT NOT NULL, data TEXT NOT NULL CHECK(json_valid(data)))`, + `CREATE INDEX verifications_build ON verifications(user_id,build_id,state)`, + `CREATE TABLE image_references (user_id TEXT NOT NULL, owner_kind TEXT NOT NULL, owner_id TEXT NOT NULL, build_id TEXT NOT NULL REFERENCES builds(id), PRIMARY KEY(user_id,owner_kind,owner_id,build_id))`, + `CREATE INDEX image_references_build ON image_references(user_id,build_id)`, + `CREATE TABLE context_uploads (context_id TEXT PRIMARY KEY REFERENCES contexts(id), token_hash TEXT NOT NULL, host_id TEXT NOT NULL)`, + `CREATE TABLE context_chunks (context_id TEXT NOT NULL REFERENCES contexts(id), path TEXT NOT NULL, offset INTEGER NOT NULL, data BLOB NOT NULL, PRIMARY KEY(context_id,path,offset))`, + `CREATE TABLE source_observations (user_id TEXT NOT NULL, project_id TEXT NOT NULL, data TEXT NOT NULL CHECK(json_valid(data)), checked_at INTEGER NOT NULL, PRIMARY KEY(user_id,project_id))`, + `CREATE TABLE base_artifacts (hash TEXT PRIMARY KEY, metadata_json TEXT NOT NULL CHECK(json_valid(metadata_json)), data BLOB NOT NULL)`, + `CREATE INDEX contexts_expiry ON contexts(user_id,expiry)`, + `CREATE TABLE verification_requests (user_id TEXT NOT NULL, key TEXT NOT NULL, payload_hash TEXT NOT NULL, verification_id TEXT NOT NULL REFERENCES verifications(id), PRIMARY KEY(user_id,key))`, + `UPDATE recipes SET data=json_remove(data, '$.setupScriptText')`, +]; diff --git a/plugins/environment-modal-sandbox/catalogue/model.ts b/plugins/environment-modal-sandbox/catalogue/model.ts new file mode 100644 index 00000000000..309582f0196 --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/model.ts @@ -0,0 +1,176 @@ +import { artifactMetadataSchema } from "./artifact.js"; +import { experimental_PluginRpcConflict } from "@get-bb/plugin-sdk"; +import { createHash } from "node:crypto"; +import { z } from "zod"; + +export const hash = (text: string | Buffer) => + createHash("sha256").update(text).digest("hex"); +export const idSchema = z.string().min(1).max(200); +export const hashSchema = z.string().regex(/^[a-f0-9]{64}$/); +export const resourcesSchema = z + .object({ + cpuCores: z.number().min(0.125).max(64).default(1), + memoryMiB: z.number().int().min(128).max(262144).default(4096), + }) + .strict(); +export const policySchema = z + .object({ + idleMinutes: z.number().int().min(0).max(1440).default(15), + lifetimeMinutes: z.number().int().min(1).max(1440).default(1440), + retentionDays: z.number().int().min(1).max(365).default(30), + }) + .strict(); +export const recipeInputSchema = z + .object({ + projectId: idSchema, + expectedRevision: z.number().int().nonnegative(), + dockerfileText: z.string().max(256 * 1024), + contextRules: z + .object({ + include: z.array(z.string()).max(1000), + exclude: z.array(z.string()).max(1000).default([]), + }) + .strict() + .default({ include: [], exclude: [] }), + smoke: z + .object({ + commands: z.array(z.string()).max(50).default([]), + timeoutSeconds: z.number().int().min(1).max(600).default(120), + }) + .strict() + .default({ commands: [], timeoutSeconds: 120 }), + }) + .strict(); +export const recipeSchema = recipeInputSchema + .omit({ expectedRevision: true }) + .extend({ + recipeId: idSchema, + revision: z.number().int().positive(), + recipeHash: hashSchema, + baseDigest: hashSchema, + createdAt: z.number(), + }); +export const fileSchema = z + .object({ + path: z.string().min(1).max(4096), + sha256: hashSchema, + bytes: z.number().int().nonnegative(), + mode: z.enum(["100644", "100755"]), + }) + .strict(); +export const sourceSchema = z + .object({ + hostId: idSchema, + path: z.string(), + commit: z.string().regex(/^[a-f0-9]{40,64}$/), + dirty: z.array(z.string()), + submodules: z.array(z.string()), + lfs: z.array(z.string()), + }) + .strict(); +export const manifestSchema = z + .object({ + source: sourceSchema, + files: z.array(fileSchema).max(20000), + reviewedDirty: z.array(z.string()), + recipeId: idSchema, + revision: z.number().int().positive(), + }) + .strict(); +export const contextSchema = z.object({ + contextId: idSchema, + projectId: idSchema, + manifestHash: hashSchema, + manifest: manifestSchema, + bytes: z.number(), + expiresAt: z.number(), + uploaded: z.boolean(), +}); +export const buildStateSchema = z.enum([ + "queued", + "building", + "ready", + "failed", + "cancelled", + "reconciling", +]); +export const buildSchema = z.object({ + baseArtifact: artifactMetadataSchema.nullable().default(null), + buildId: idSchema, + projectId: idSchema, + recipeId: idSchema, + revision: z.number().int(), + contextId: idSchema, + accountIdentity: hashSchema, + appName: idSchema, + hash: hashSchema, + name: idSchema, + state: buildStateSchema, + imageId: idSchema.nullable(), + failure: z.string().nullable(), + cancelRequested: z.boolean(), + lastEventSequence: z.number().int(), + createdAt: z.number(), + updatedAt: z.number(), +}); +export const eventSchema = z.object({ + sequence: z.number().int(), + kind: z.enum(["log", "state", "truncated"]), + text: z.string(), + time: z.number(), +}); +export const projectSchema = z.object({ + projectId: idSchema, + revision: z.number().int(), + usableBuildId: idSchema.nullable(), + resources: resourcesSchema, + policy: policySchema, +}); +export type Recipe = z.infer; +export type Context = z.infer; +export type Build = z.infer; +export type Manifest = z.infer; +export type Project = z.infer; +export class CatalogueError extends experimental_PluginRpcConflict { + constructor( + public readonly status: number, + message: string, + latestRevision: number | null = null, + ) { + super(message, latestRevision); + if (status !== 409) this.name = "CatalogueError"; + } +} + +export const verificationSchema = z.object({ + verificationId: idSchema, + buildId: idSchema, + agentProviderId: idSchema, + key: idSchema, + state: z.enum([ + "queued", + "allocating", + "preparing", + "starting", + "running", + "checking", + "suspending", + "resuming", + "restoring", + "retaining", + "passed", + "failed", + ]), + hostId: idSchema.nullable(), + environmentId: idSchema.nullable(), + threadId: idSchema.nullable(), + completedTurnSeq: z.number().int().nullable(), + restored: z.boolean().default(false), + checks: z.array( + z.object({ command: z.string(), exitCode: z.number().int() }), + ), + failure: z.string().nullable(), + createdAt: z.number(), + updatedAt: z.number(), +}); +export type Verification = z.infer; diff --git a/plugins/environment-modal-sandbox/catalogue/service.test.ts b/plugins/environment-modal-sandbox/catalogue/service.test.ts new file mode 100644 index 00000000000..2074d50c749 --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/service.test.ts @@ -0,0 +1,560 @@ +import { acceptChunk, contextFiles } from "./context.js"; +import { afterEach, expect, it, vi } from "vitest"; +import { + createFakePluginHost, + makeThreadResponse, +} from "@get-bb/plugin-sdk/testing"; +import { createCatalogueService } from "./service.js"; +import { registerCatalogueCli } from "./cli.js"; +import { hash } from "./model.js"; +import { createVerificationService } from "./verification.js"; +import type { ImageBackend } from "./backend.js"; + +const disposals: Array<() => Promise> = []; +afterEach(async () => { + await Promise.all(disposals.splice(0).map((dispose) => dispose())); +}); +async function fixture(now = Date.now) { + const project = { + id: "project", + kind: "standard" as const, + name: "fixture", + gitRemoteUrl: null, + createdAt: 1, + updatedAt: 1, + sources: [], + }; + const host = createFakePluginHost({ + pluginId: "environment-modal-sandbox", + sdk: { + projects: { get: async () => project, list: async () => [project] }, + }, + }); + disposals.push(() => host.harness.lifecycle.dispose()); + const backend: ImageBackend = { + accountIdentity: async () => hash("account"), + build: vi.fn(async (_request, hooks) => { + hooks.allocated("im-fixture"); + hooks.log("token=secret private-value"); + return "im-fixture"; + }), + reconcile: vi.fn(async () => null), + resolve: vi.fn(async (id) => id), + delete: vi.fn(async () => {}), + }; + const service = createCatalogueService( + host.bb, + async () => ({ + tokenId: "token-id", + tokenSecret: "private-value", + appName: "test-app", + environmentVariables: {}, + timeoutMs: 60000, + idleMs: 60000, + cpu: 1, + memoryMiB: 4096, + }), + () => backend, + now, + async () => ({ + metadata: { + sha256: hash("fixture"), + version: "test", + protocolVersion: 1, + }, + data: Buffer.from("fixture"), + }), + ); + registerCatalogueCli(host.bb, service); + const recipe = await service.handlers["recipe.put"]({ + projectId: project.id, + expectedRevision: 0, + dockerfileText: "RUN true", + contextRules: { include: [], exclude: [] }, + smoke: { commands: ["node --version"], timeoutSeconds: 120 }, + }); + const context = service.store.putContext(project.id, { + recipeId: recipe.recipeId, + revision: 1, + source: { + hostId: "local", + path: "/fixture", + commit: "a".repeat(40), + dirty: [], + submodules: [], + lfs: [], + }, + reviewedDirty: [], + files: [], + }); + service.store.completeContext(context.contextId); + const input = { + projectId: project.id, + recipeId: recipe.recipeId, + revision: 1, + contextId: context.contextId, + key: "request", + }; + return { ...host, service, backend, input, recipe }; +} +it("uses the same CLI/RPC handlers and keeps a shared build alive after a follower disconnects", async () => { + const test = await fixture(); + let finish: (imageId: string) => void = () => {}; + const pending = new Promise((resolve) => { + finish = resolve; + }); + test.backend.build = vi.fn(async (_request, hooks) => { + hooks.allocated("im-shared"); + hooks.log("Downloading dependencies"); + return pending; + }); + const first = await test.service.handlers["build.start"](test.input); + const cli = await test.harness.behavior.runCli([ + "image", + "build", + "--project", + "fixture", + "--recipe", + test.input.recipeId, + "--revision", + "1", + "--context", + test.input.contextId, + "--key", + "second", + "--json", + ]); + expect(JSON.parse(cli.stdout)).toEqual({ ...first, reused: true }); + const running = test.service.sweep(); + await vi.waitFor(() => expect(test.backend.build).toHaveBeenCalledTimes(1)); + const controller = new AbortController(); + const logs = await test.harness.behavior.runCli( + ["image", "logs", first.buildId, "--follow", "--json"], + { signal: controller.signal }, + ); + expect(logs.experimental_continue?.argv).toContain(first.buildId); + controller.abort(); + expect(test.service.store.build(first.buildId)).toMatchObject({ + state: "building", + cancelRequested: false, + }); + finish("im-shared"); + await running; + expect( + await test.harness.behavior.callRpc("build.get", { + buildId: first.buildId, + }), + ).toMatchObject({ state: "ready", imageId: "im-shared" }); + expect(test.backend.build).toHaveBeenCalledTimes(1); +}); +it("reports missing vendor images and permits an explicit retry with a new key", async () => { + const test = await fixture(); + const first = await test.service.handlers["build.start"](test.input); + await test.service.sweep(); + test.backend.resolve = async () => null; + await expect( + test.service.handlers["build.start"]({ ...test.input, key: "check" }), + ).rejects.toThrow(/image is missing/); + expect(test.service.store.build(first.buildId).state).toBe("failed"); + const retried = await test.service.handlers["build.start"]({ + ...test.input, + key: "explicit-retry", + }); + expect(retried.state).toBe("queued"); +}); +it("reconciles deterministic names after transient errors instead of silently submitting again", async () => { + const test = await fixture(); + test.backend.build = vi.fn(async () => { + throw new Error("UNAVAILABLE: socket disconnected"); + }); + const first = await test.service.handlers["build.start"](test.input); + await test.service.sweep(); + expect(test.service.store.build(first.buildId).state).toBe("reconciling"); + await test.service.sweep(); + expect(test.backend.build).toHaveBeenCalledTimes(1); + test.backend.reconcile = vi.fn(async () => "im-reconciled"); + await test.service.sweep(); + expect(test.service.store.build(first.buildId)).toMatchObject({ + state: "ready", + imageId: "im-reconciled", + }); + expect(test.backend.reconcile).toHaveBeenCalledWith( + test.service.store.build(first.buildId).name, + ); +}); +it("returns structured CAS conflict errors through the real RPC boundary", async () => { + const test = await fixture(); + await expect( + test.harness.behavior.callRpc("recipe.put", { + projectId: "project", + expectedRevision: 0, + dockerfileText: "RUN changed", + }), + ).rejects.toMatchObject({ + code: "conflict", + message: expect.stringContaining("latest revision 1"), + }); +}); +it("fences a late worker result after restart and reconciliation", async () => { + const test = await fixture(); + let finish: (id: string) => void = () => {}; + test.backend.build = async () => + new Promise((resolve) => { + finish = resolve; + }); + const first = await test.service.handlers["build.start"](test.input); + const running = test.service.sweep(); + await vi.waitFor(() => + expect(test.service.store.build(first.buildId).state).toBe("building"), + ); + test.service.store.restart(); + test.backend.reconcile = async () => "im-reconciled"; + await test.service.sweep(); + finish("im-late-worker"); + await running; + expect(test.service.store.build(first.buildId)).toMatchObject({ + state: "ready", + imageId: "im-reconciled", + }); +}); +it("redacts credentials split across vendor log chunks", async () => { + const test = await fixture(); + test.backend.build = async (_request, hooks) => { + hooks.log("private-"); + hooks.log("value\n"); + return "im-safe"; + }; + const first = await test.service.handlers["build.start"](test.input); + await test.service.sweep(); + const page = test.service.store.events(first.buildId, 0, 200); + expect(JSON.stringify(page)).not.toContain("private-"); + expect(JSON.stringify(page)).not.toContain("value"); + expect(JSON.stringify(page)).toContain("[REDACTED]"); +}); + +it("rejects failed smoke and incomplete proof, then promotes the selected agent with CAS", async () => { + const test = await fixture(); + const build = await test.service.handlers["build.start"](test.input); + await test.service.sweep(); + const job = await test.service.verifications.start({ + buildId: build.buildId, + agentProviderId: "codex", + key: "verify", + }); + expect( + await test.service.verifications.start({ + buildId: build.buildId, + agentProviderId: "codex", + key: "verify", + }), + ).toEqual(job); + await expect( + test.service.verifications.start({ + buildId: build.buildId, + agentProviderId: "claude-code", + key: "verify", + }), + ).rejects.toThrow(/different payload/); + const verifiedCli = await test.harness.behavior.runCli([ + "image", + "verify", + build.buildId, + "--provider", + "codex", + "--key", + "verify", + "--json", + ]); + expect(verifiedCli.exitCode).toBe(0); + expect(JSON.parse(verifiedCli.stdout)).toEqual(job); + const use = { + projectId: "project", + buildId: build.buildId, + agentProviderId: "codex", + expectedRevision: 0, + }; + const save = ( + state: "failed" | "passed", + exitCode: number, + completedTurnSeq: number | null, + ) => { + const data = { + ...job, + restored: true, + state, + hostId: "machine", + environmentId: "checkout", + threadId: "smoke-thread", + completedTurnSeq, + checks: [{ command: "node --version", exitCode }], + }; + test.service.store.db + .prepare("UPDATE verifications SET state=?,data=? WHERE id=?") + .run(state, JSON.stringify(data), job.verificationId); + }; + save("failed", 1, 42); + expect(() => test.service.verifications.promote(use)).toThrow( + /successful verification/, + ); + save("passed", 1, 42); + expect(() => test.service.verifications.promote(use)).toThrow( + /independent command/, + ); + save("passed", 0, null); + expect(() => test.service.verifications.promote(use)).toThrow(/agent/); + save("passed", 0, 42); + expect(() => + test.service.verifications.promote({ + ...use, + agentProviderId: "claude-code", + }), + ).toThrow(/successful verification/); + expect(test.service.verifications.promote(use)).toMatchObject({ + usableBuildId: build.buildId, + available: true, + revision: 1, + }); + expect(() => test.service.verifications.promote(use)).toThrow(/revision/); + expect(test.service.store.protected(build.buildId)).toBe(true); + const project = test.service.store.project("project"); + expect(() => + test.service.store.configure({ + ...project, + usableBuildId: null, + expectedRevision: 0, + }), + ).toThrow(/revision/); + expect(test.service.store.project("project").usableBuildId).toBe( + build.buildId, + ); + expect( + test.service.store.configure({ + ...project, + usableBuildId: null, + expectedRevision: 1, + }).usableBuildId, + ).toBeNull(); +}); + +it("keeps failed allocation reconcilable and retains the machine when preparation blocks", async () => { + const test = await fixture(); + const build = await test.service.handlers["build.start"](test.input); + await test.service.sweep(); + test.harness.sdk.stub("system.providerStates", async () => ({ + providers: [ + { + providerId: "codex", + displayName: "Codex", + status: "ready", + statusMessage: null, + accountEmail: null, + planLabel: null, + installedVersion: "1.0.0", + minimumSupportedVersion: "1.0.0", + canInstall: true, + canUpdate: false, + loginCommand: null, + }, + ], + })); + const submit = vi.fn().mockResolvedValue({ + id: "launch", + phase: "failed", + hostId: null, + step: "bootstrap", + log: "", + message: "Retryable transport failure", + cancelPending: false, + terminal: true, + }); + test.harness.sdk.stub("hosts.submit", submit); + const job = await test.service.verifications.start({ + buildId: build.buildId, + agentProviderId: "codex", + key: "durable", + }); + await test.service.verifications.sweep(); + expect(test.service.verifications.get(job.verificationId)).toMatchObject({ + state: "allocating", + }); + expect(test.service.store.protected(build.buildId)).toBe(true); + submit.mockResolvedValue({ + id: "launch", + phase: "ready", + hostId: "machine", + step: "ready", + log: "", + message: null, + cancelPending: false, + terminal: true, + }); + test.harness.sdk.stub("projects.sources.add", async () => { + throw new Error("Repository access denied"); + }); + await test.service.verifications.sweep(); + expect(test.service.verifications.get(job.verificationId)).toMatchObject({ + state: "failed", + hostId: "machine", + threadId: null, + failure: expect.stringContaining("preparing"), + }); + expect(submit.mock.calls.map(([input]) => input.key)).toEqual([ + `modal-${job.verificationId}`, + `modal-${job.verificationId}`, + ]); +}); + +it("reconciles a failed smoke turn after restart and refuses promotion", async () => { + const test = await fixture(); + const build = await test.service.handlers["build.start"](test.input); + await test.service.sweep(); + const job = await test.service.verifications.start({ + buildId: build.buildId, + agentProviderId: "codex", + key: "failed-turn", + }); + test.service.store.db + .prepare("UPDATE verifications SET state=?,data=? WHERE id=?") + .run( + "running", + JSON.stringify({ + ...job, + state: "running", + hostId: "machine", + threadId: "smoke-thread", + }), + job.verificationId, + ); + test.harness.sdk.stub("threads.get", async () => + makeThreadResponse({ + id: "smoke-thread", + projectId: "project", + environmentId: "checkout", + status: "error", + }), + ); + test.harness.sdk.stub("threads.events.list", async () => []); + const restarted = createVerificationService(test.bb, test.service.store); + await restarted.sweep(); + expect(restarted.get(job.verificationId)).toMatchObject({ + state: "failed", + hostId: "machine", + threadId: "smoke-thread", + environmentId: "checkout", + completedTurnSeq: null, + failure: "Smoke thread failed before a completed agent turn", + }); + expect(() => + restarted.promote({ + projectId: "project", + buildId: build.buildId, + agentProviderId: "codex", + expectedRevision: 0, + }), + ).toThrow(/successful verification/); +}); + +it("preflights the selected agent without allocation and never returns account secrets", async () => { + const test = await fixture(); + const account = await test.service.handlers["account.inspect"]({}); + expect(account).toMatchObject({ available: true, appName: "test-app" }); + expect(JSON.stringify(account)).not.toContain("private-value"); + const empty = await test.service.handlers["project.preflight"]({ + projectId: "project", + agentProviderId: "codex", + buildId: null, + }); + expect(empty.ready).toBe(false); + const build = await test.service.handlers["build.start"](test.input); + await test.service.sweep(); + const unverified = await test.service.handlers["project.preflight"]({ + projectId: "project", + agentProviderId: "claude-code", + buildId: build.buildId, + }); + expect(unverified).toMatchObject({ ready: false, build: null }); + expect(unverified.message).toContain("successful verification"); + test.backend.accountIdentity = async () => { + throw new Error("private-value"); + }; + const failed = await test.service.handlers["account.inspect"]({}); + expect(failed.available).toBe(false); + expect(JSON.stringify(failed)).not.toContain("private-value"); +}); + +it("retries identical build content using the newly uploaded context after expired chunks are collected", async () => { + let now = 1000; + const test = await fixture(() => now); + const recipe = await test.service.handlers["recipe.put"]({ + projectId: "project", + expectedRevision: 1, + dockerfileText: "COPY package-lock.json /tmp/package-lock.json", + contextRules: { include: ["package-lock.json"], exclude: [] }, + smoke: { commands: ["true"], timeoutSeconds: 120 }, + }); + const manifest = { + recipeId: recipe.recipeId, + revision: recipe.revision, + source: { + hostId: "local", + path: "/fixture", + commit: "a".repeat(40), + dirty: [], + submodules: [], + lfs: [], + }, + reviewedDirty: [], + files: [ + { + path: "package-lock.json", + bytes: 2, + sha256: hash("{}"), + mode: "100644" as const, + }, + ], + }; + const upload = () => { + const context = test.service.store.putContext("project", manifest); + test.service.store.db + .prepare("INSERT INTO context_uploads VALUES (?,?,?)") + .run(context.contextId, hash("fixture-token"), "local"); + acceptChunk(test.service.store, "fixture-token", { + contextId: context.contextId, + path: "package-lock.json", + offset: 0, + data: "e30=", + }); + test.service.store.completeContext(context.contextId); + return context; + }; + const old = upload(); + vi.mocked(test.backend.build).mockRejectedValueOnce( + new Error("Build syntax failure"), + ); + const input = { + ...test.input, + revision: recipe.revision, + contextId: old.contextId, + }; + const first = await test.service.handlers["build.start"](input); + await test.service.sweep(); + expect(test.service.store.build(first.buildId).state).toBe("failed"); + now += 86400001; + await test.service.sweep(); + expect(() => contextFiles(test.service.store, old.contextId)).toThrow( + "Archive hash/size mismatch", + ); + const fresh = upload(); + const retry = await test.service.handlers["build.start"]({ + ...input, + key: "retry-fresh-context", + contextId: fresh.contextId, + }); + expect(retry.buildId).toBe(first.buildId); + expect(test.service.store.build(retry.buildId).contextId).toBe( + fresh.contextId, + ); + expect(contextFiles(test.service.store, fresh.contextId).size).toBe(1); + await test.service.sweep(); + expect(test.service.store.build(retry.buildId).state).toBe("ready"); +}); diff --git a/plugins/environment-modal-sandbox/catalogue/service.ts b/plugins/environment-modal-sandbox/catalogue/service.ts new file mode 100644 index 00000000000..9f63fc7dc24 --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/service.ts @@ -0,0 +1,610 @@ +import { createVerificationService } from "./verification.js"; +import { randomBytes } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import type { BbPluginApi, PluginRpcHandlers } from "@get-bb/plugin-sdk"; +import { z } from "zod"; +import type { ResolvedSettings } from "../configuration.js"; +import { + fetchBaseArtifact, + artifactMetadataSchema, + type BaseArtifact, +} from "./artifact.js"; +import { baseManifest } from "./base.js"; +import { createImageBackend, type ImageBackendFactory } from "./backend.js"; +import { acceptChunk, chunkSchema, contextFiles } from "./context.js"; +import { modalRpcContract } from "./contract.js"; +import { + CatalogueError, + hash, + verificationSchema, + type Build, +} from "./model.js"; +import { + inspectionSchema, + sourceContract, + isLockfile, +} from "./source-contract.js"; +import { createBuildLogWriter } from "./logs.js"; +import { Catalogue } from "./store.js"; + +export const accountIdentity = ( + settings: ResolvedSettings, + factory: ImageBackendFactory = createImageBackend, +) => factory(settings).accountIdentity(); +export const GC_GRACE_MS = 60 * 1000; +export function createCatalogueService( + bb: BbPluginApi, + settings: () => Promise, + backendFactory: ImageBackendFactory = createImageBackend, + now = Date.now, + loadArtifact: () => Promise = () => + fetchBaseArtifact(bb.server.loopbackBaseUrl), +) { + const store = new Catalogue(bb.storage.database(), bb.storage, now); + const verifications = createVerificationService(bb, store); + const hosts = bb.hosts.experimental_client({ contract: sourceContract }); + const uploadPath = "/context-chunk"; + const uploadUrl = () => + `${bb.server.experimental_appUrl ?? bb.server.loopbackBaseUrl}/api/v1/plugins/${bb.pluginId}/http${uploadPath}`; + async function source(projectId: string, environmentId: string) { + await bb.sdk.projects.get({ projectId }); + const environment = await bb.sdk.environments.get({ environmentId }); + if (environment.projectId !== projectId || !environment.path) + throw new CatalogueError( + 400, + "Choose a source environment in this project with a checkout", + ); + return environment; + } + async function inspect(projectId: string, environmentId: string) { + await assertProject(projectId); + try { + const environment = await source(projectId, environmentId); + const result = await hosts.call( + "inspect", + { path: environment.path!, hostId: environment.hostId }, + { hostId: environment.hostId }, + ); + store.db + .prepare( + "INSERT INTO source_observations VALUES (?,?,?,?) ON CONFLICT(user_id,project_id) DO UPDATE SET data=excluded.data,checked_at=excluded.checked_at", + ) + .run(store.owner, projectId, JSON.stringify(result), now()); + return result; + } catch (error) { + store.db + .prepare( + "DELETE FROM source_observations WHERE user_id=? AND project_id=?", + ) + .run(store.owner, projectId); + throw error; + } + } + async function assertProject(projectId: string) { + await bb.sdk.projects.get({ projectId }); + } + const handlers: PluginRpcHandlers = { + "catalogue.projects": async () => + (await bb.sdk.projects.list()).map(({ id, name }) => ({ id, name })), + "project.sources": async ({ projectId }) => { + await assertProject(projectId); + const sources = await bb.sdk.environments.list({ projectId }); + const primary = (await bb.sdk.hosts.list()).find( + (host) => host.machineProviderId === null, + ); + return sources + .filter((row) => row.status === "ready" && row.path !== null) + .map((row) => ({ + id: row.id, + hostId: row.hostId, + primaryHost: row.hostId === primary?.id, + path: row.path!, + name: row.path!, + })); + }, + "account.inspect": async () => { + try { + const resolved = await settings(); + return { + available: true, + accountIdentity: await backendFactory(resolved).accountIdentity(), + appName: resolved.appName, + baseVersion: baseManifest.version, + message: "Modal account is reachable", + }; + } catch { + return { + available: false, + accountIdentity: null, + appName: null, + baseVersion: baseManifest.version, + message: + "Configure the Modal token and app above, then test the connection", + }; + } + }, + "verification.list": ({ buildId }) => + store.db + .prepare( + "SELECT data FROM verifications WHERE user_id=? AND build_id=? ORDER BY rowid DESC LIMIT 50", + ) + .all(store.owner, buildId) + .map((row) => + verificationSchema.parse( + JSON.parse(z.object({ data: z.string() }).parse(row).data), + ), + ), + "project.preflight": async ({ projectId, agentProviderId, buildId }) => { + await assertProject(projectId); + const selected = buildId ?? store.project(projectId).usableBuildId; + if (!selected) + return { + ready: false, + message: + "Build, verify and use an image for this project in Modal settings", + build: null, + }; + try { + const build = store.build(selected); + if ( + build.projectId !== projectId || + build.state !== "ready" || + build.imageId === null + ) + throw new CatalogueError( + 409, + "Choose a ready image for this project", + ); + verifications.assertPassed(selected, agentProviderId); + const resolved = await settings(); + if ( + build.accountIdentity !== + (await accountIdentity(resolved, backendFactory)) + ) + throw new CatalogueError( + 409, + "The image belongs to a different account; restore its account configuration", + ); + if (!(await backendFactory(resolved).resolve(build.imageId))) + throw new CatalogueError( + 409, + "The image is missing; explicitly build a replacement", + ); + return { + ready: true, + message: + "Verified image available; checkout and credentials are checked before the first turn", + build, + }; + } catch (error) { + return { + ready: false, + message: + error instanceof CatalogueError + ? error.message + : "Image preflight failed; check account configuration and vendor availability", + build: null, + }; + } + }, + "project.inspect": ({ projectId, environmentId }) => + inspect(projectId, environmentId), + "recipe.put": async (input) => { + await assertProject(input.projectId); + return store.putRecipe(input); + }, + "recipe.get": async ({ projectId }) => { + await assertProject(projectId); + return store.recipe(projectId); + }, + "recipe.list": ({ cursor, limit }) => { + const recipes = store.recipes(cursor, limit); + return { + recipes, + nextCursor: recipes.length === limit ? recipes.at(-1)!.projectId : null, + }; + }, + "context.prepare": async (input) => { + const environment = await source(input.projectId, input.environmentId); + const recipe = store.recipe(input.projectId, input.revision); + if (recipe.recipeId !== input.recipeId) + throw new CatalogueError(409, "Recipe does not belong to this project"); + const manifest = await hosts.call( + "manifest", + { + path: environment.path!, + hostId: environment.hostId, + recipeId: recipe.recipeId, + revision: recipe.revision, + ...recipe.contextRules, + reviewedDirty: input.reviewedDirty, + }, + { hostId: environment.hostId }, + ); + const context = store.putContext(input.projectId, manifest); + const uploadToken = randomBytes(32).toString("hex"); + store.db + .prepare("INSERT INTO context_uploads VALUES (?,?,?)") + .run(context.contextId, hash(uploadToken), environment.hostId); + return { + ...context, + files: manifest.files.length, + uploadToken, + uploadUrl: uploadUrl(), + }; + }, + "context.upload": async ({ contextId, uploadToken }) => { + const context = store.context(contextId); + await hosts.call( + "upload", + { + contextId, + path: context.manifest.source.path, + manifest: context.manifest, + url: uploadUrl(), + token: uploadToken, + }, + { hostId: context.manifest.source.hostId }, + ); + return handlers["context.complete"]({ contextId }); + }, + "context.complete": ({ contextId }) => { + contextFiles(store, contextId); + store.completeContext(contextId); + return store.context(contextId); + }, + "build.start": async (input) => { + await assertProject(input.projectId); + const resolved = await settings(); + const artifact = await loadArtifact(); + store.db + .prepare( + "INSERT INTO base_artifacts VALUES (?,?,?) ON CONFLICT(hash) DO NOTHING", + ) + .run( + artifact.metadata.sha256, + JSON.stringify(artifact.metadata), + artifact.data, + ); + const { build, reused } = store.start( + input, + await accountIdentity(resolved, backendFactory), + resolved.appName, + artifact.metadata, + ); + if ( + reused && + build.state === "ready" && + build.imageId && + !(await backendFactory(resolved).resolve(build.imageId)) + ) { + store.saveBuild({ + ...build, + state: "failed", + failure: "Published image is missing", + }); + throw new CatalogueError( + 409, + "Published image is missing; explicitly build again with a new request key", + ); + } + return { buildId: build.buildId, state: build.state, reused }; + }, + "verification.start": (input) => verifications.start(input), + "verification.get": ({ verificationId }) => + verifications.get(verificationId), + "project.useImage": async (input) => { + await assertProject(input.projectId); + const build = store.build(input.buildId); + const resolved = await settings(); + if ( + build.accountIdentity !== + (await accountIdentity(resolved, backendFactory)) || + !build.imageId || + !(await backendFactory(resolved).resolve(build.imageId)) + ) + throw new CatalogueError( + 409, + "Published image is missing or belongs to a different account", + ); + return verifications.promote(input); + }, + "build.events": ({ buildId, cursor, limit }) => + store.events(buildId, cursor, limit), + "build.get": ({ buildId }) => store.build(buildId), + "build.cancel": ({ buildId }) => store.cancel(buildId), + "image.list": ({ projectId, cursor, limit }) => { + const images = store.images(projectId, cursor, limit); + return { + images, + nextCursor: images.length === limit ? images.at(-1)!.buildId : null, + }; + }, + "image.gc": async ({ dryRun }) => { + const resolved = await settings(); + const backend = backendFactory(resolved); + const candidates = []; + const blockedReferences = []; + for (const row of store.gcCandidates()) { + const build = store.build(row.build_id); + if ( + build.accountIdentity !== + (await accountIdentity(resolved, backendFactory)) || + store.protected(build.buildId) + ) { + blockedReferences.push(build.buildId); + continue; + } + let deleted = false; + if (!dryRun && store.claimDeletion(build.buildId, GC_GRACE_MS)) { + await backend.delete(row.image_id); + store.deleted(build.buildId); + deleted = true; + } + candidates.push({ + buildId: build.buildId, + imageId: row.image_id, + markedAt: dryRun + ? row.marked_at + : z + .object({ marked_at: z.number().nullable() }) + .parse( + store.db + .prepare( + "SELECT marked_at FROM images WHERE build_id=? AND user_id=?", + ) + .get(build.buildId, store.owner), + ).marked_at, + deleted, + }); + } + return { candidates, blockedReferences }; + }, + "project.configure": async (input) => { + await assertProject(input.projectId); + return store.configure(input); + }, + "project.show": async ({ projectId }) => { + await assertProject(projectId); + const project = store.project(projectId); + const latest = store.db + .prepare( + "SELECT data FROM builds WHERE user_id=? AND project_id=? AND state='ready' ORDER BY rowid DESC LIMIT 1", + ) + .get(store.owner, projectId); + const build = project.usableBuildId + ? store.build(project.usableBuildId) + : latest + ? store.build( + z + .object({ data: z.string() }) + .transform( + (row) => + z + .object({ buildId: z.string() }) + .parse(JSON.parse(row.data)).buildId, + ) + .parse(latest), + ) + : null; + const observed = store.db + .prepare( + "SELECT data,checked_at FROM source_observations WHERE user_id=? AND project_id=?", + ) + .get(store.owner, projectId); + const observation = observed + ? z.object({ data: z.string(), checked_at: z.number() }).parse(observed) + : null; + const facts = observation + ? inspectionSchema.parse(JSON.parse(observation.data)) + : null; + const previous = build + ? store.context(build.contextId).manifest.files + : []; + const lockfilesChanged = + facts && build + ? facts.evidence + .filter((file) => file.kind === "lockfile") + .some( + (file) => + previous.find((before) => before.path === file.path) + ?.sha256 !== file.sha256, + ) || + previous.some( + (before) => + isLockfile(before.path) && + !facts.evidence.some((file) => file.path === before.path), + ) + : null; + return { + ...project, + staleness: { + dockerfileChanged: build + ? store.recipe(projectId).recipeHash !== + store.recipe(projectId, build.revision).recipeHash + : false, + lockfilesChanged, + reason: !build + ? "No built image" + : facts + ? null + : "Source not checked", + lastCheckedAt: observation?.checked_at ?? null, + }, + }; + }, + }; + bb.http.route( + "POST", + uploadPath, + async (context) => { + try { + const body = await context.req.text(); + if (Buffer.byteLength(body) > 400000) + return context.json({ error: "Archive chunk exceeds limit" }, 413); + const token = + context.req.header("authorization")?.replace(/^Bearer /, "") ?? ""; + acceptChunk(store, token, chunkSchema.parse(JSON.parse(body))); + return context.json({ ok: true }); + } catch (error) { + return context.json( + { error: error instanceof Error ? error.message : "Invalid archive" }, + error instanceof CatalogueError && error.status === 403 ? 403 : 400, + ); + } + }, + { auth: "none" }, + ); + async function runBuild( + build: Build, + resolved: ResolvedSettings, + signal: AbortSignal, + ) { + const backend = backendFactory(resolved); + const lease = store.lease(build.buildId); + const currentLease = () => + store.lease(build.buildId) === lease && + store.build(build.buildId).state === "building"; + const recipe = store.recipe(build.projectId, build.revision); + const logs = createBuildLogWriter((kind, text) => { + if (currentLease()) + store.event(build.buildId, kind, text, [ + resolved.tokenId, + resolved.tokenSecret, + ]); + }); + try { + const row = build.baseArtifact + ? store.db + .prepare( + "SELECT metadata_json,data FROM base_artifacts WHERE hash=?", + ) + .get(build.baseArtifact.sha256) + : null; + const artifact = row + ? z + .object({ metadata_json: z.string(), data: z.instanceof(Buffer) }) + .parse(row) + : null; + const baseArtifact = artifact + ? { + metadata: artifactMetadataSchema.parse( + JSON.parse(artifact.metadata_json), + ), + data: artifact.data, + } + : null; + const imageId = await backend.build( + { + baseArtifact, + name: build.name, + appName: build.appName, + dockerfileText: recipe.dockerfileText, + files: contextFiles(store, build.contextId), + }, + { + signal, + log: (text) => logs.append(text), + allocated: (imageId) => { + if (!currentLease()) return; + store.db + .transaction(() => { + const current = store.build(build.buildId); + store.saveBuild({ ...current, imageId }); + store.db + .prepare( + "INSERT INTO images(build_id,user_id,image_id,manifest_json) VALUES (?,?,?,?) ON CONFLICT(build_id) DO NOTHING", + ) + .run( + build.buildId, + store.owner, + imageId, + JSON.stringify({ + version: 1, + state: "building", + base: baseManifest, + }), + ); + }) + .immediate(); + }, + }, + ); + logs.flush(); + if (!currentLease()) return; + store.ready(build.buildId, imageId, { + version: 1, + base: baseManifest, + bbPackage: build.baseArtifact, + recipe, + context: store.context(build.contextId).manifest, + }); + } catch (error) { + logs.flush(); + if (!currentLease()) return; + const text = error instanceof Error ? error.message : String(error); + store.event(build.buildId, "log", text, [ + resolved.tokenId, + resolved.tokenSecret, + ]); + const transient = + /unavailable|deadline|timeout|ECONN|socket|429|rate.limit|cancel|aborted|worker stopped/i.test( + text, + ); + store.saveBuild({ + ...store.build(build.buildId), + state: transient ? "reconciling" : "failed", + failure: transient + ? "Vendor outcome unknown; reconciling published name" + : "Build failed; inspect redacted logs", + }); + } + } + async function sweep(signal = new AbortController().signal) { + const resolved = await settings(); + const backend = backendFactory(resolved); + for (const build of store.reconciling( + await accountIdentity(resolved, backendFactory), + )) { + const imageId = await backend.reconcile(build.name); + if (imageId) + store.ready(build.buildId, imageId, { + version: 1, + base: baseManifest, + recipe: store.recipe(build.projectId, build.revision), + context: store.context(build.contextId).manifest, + }); + } + const build = store.claim(await accountIdentity(resolved, backendFactory)); + if (build) await runBuild(build, resolved, signal); + store.db + .prepare( + "DELETE FROM context_chunks WHERE context_id IN (SELECT id FROM contexts WHERE user_id=? AND expiry {}); + } + }, + }); + bb.rpc.register(modalRpcContract, handlers); + return { store, handlers, sweep, backendFactory, settings, verifications }; +} +export type CatalogueService = ReturnType; diff --git a/plugins/environment-modal-sandbox/catalogue/source-contract.ts b/plugins/environment-modal-sandbox/catalogue/source-contract.ts new file mode 100644 index 00000000000..22092a555a2 --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/source-contract.ts @@ -0,0 +1,61 @@ +import { defineRpcContract } from "@get-bb/plugin-sdk"; +import { z } from "zod"; +import { fileSchema, manifestSchema, sourceSchema } from "./model.js"; + +export const isLockfile = (path: string) => + /(^|\/)(?:.*lock.*|go\.sum|Cargo\.lock)$/i.test(path); + +export const inspectionSchema = z.object({ + source: sourceSchema, + evidence: z.array(fileSchema.extend({ kind: z.string() })), + setupHooks: z.array(z.string()), + missing: z.array(z.string()), +}); +export const sourceContract = defineRpcContract({ + smoke: { + input: z + .object({ + path: z.string(), + commands: z.array(z.string()).min(1).max(32), + timeoutMs: z.number().int().positive().max(600000), + expectedCommit: z.string(), + }) + .strict(), + output: z.object({ + commit: z.string(), + results: z.array( + z.object({ command: z.string(), exitCode: z.number().int() }), + ), + }), + }, + inspect: { + input: z.object({ path: z.string(), hostId: z.string() }).strict(), + output: inspectionSchema, + }, + manifest: { + input: z + .object({ + path: z.string(), + hostId: z.string(), + recipeId: z.string(), + revision: z.number().int(), + include: z.array(z.string()), + exclude: z.array(z.string()), + reviewedDirty: z.array(z.string()), + }) + .strict(), + output: manifestSchema, + }, + upload: { + input: z + .object({ + path: z.string(), + manifest: manifestSchema, + url: z.string().url(), + token: z.string(), + contextId: z.string(), + }) + .strict(), + output: z.object({ uploaded: z.boolean() }), + }, +}); diff --git a/plugins/environment-modal-sandbox/catalogue/store.ts b/plugins/environment-modal-sandbox/catalogue/store.ts new file mode 100644 index 00000000000..82d84c31acd --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/store.ts @@ -0,0 +1,626 @@ +import type { ArtifactMetadata } from "./artifact.js"; +import { randomUUID } from "node:crypto"; +import type { BbPluginApi } from "@get-bb/plugin-sdk"; +import { z } from "zod"; +import { baseDigest, baseManifest } from "./base.js"; +import { parseDockerfile } from "./dockerfile.js"; +import { migrations } from "./migrations.js"; +import { + buildSchema, + CatalogueError, + contextSchema, + eventSchema, + hash, + projectSchema, + recipeInputSchema, + recipeSchema, + type Build, + type Context, + type Manifest, + type Project, +} from "./model.js"; + +const rowSchema = z.object({ data: z.string() }); +const keySchema = z.object({ build_id: z.string(), payload_hash: z.string() }); +const idRow = z.object({ id: z.string() }); +const MAX_LOG_BYTES = 10 * 1024 * 1024; +export type CatalogueDb = ReturnType; +export class Catalogue { + readonly owner: string; + constructor( + readonly db: CatalogueDb, + storage: BbPluginApi["storage"], + readonly now = Date.now, + owner?: string, + ) { + storage.migrate(db, migrations); + db.prepare("INSERT OR IGNORE INTO catalogue_owner VALUES ('owner', ?)").run( + randomUUID(), + ); + this.owner = + owner ?? + z + .object({ identity: z.string() }) + .parse( + db + .prepare("SELECT identity FROM catalogue_owner WHERE id='owner'") + .get(), + ).identity; + } + recipe(projectId: string, revision?: number) { + const row = + revision === undefined + ? this.db + .prepare( + "SELECT data FROM recipes WHERE user_id=? AND project_id=? ORDER BY revision DESC LIMIT 1", + ) + .get(this.owner, projectId) + : this.db + .prepare( + "SELECT data FROM recipes WHERE user_id=? AND project_id=? AND revision=?", + ) + .get(this.owner, projectId, revision); + if (!row) throw new CatalogueError(404, "Recipe not found"); + return recipeSchema.parse(JSON.parse(rowSchema.parse(row).data)); + } + putRecipe(input: z.infer) { + parseDockerfile(input.dockerfileText); + return this.db + .transaction(() => { + const previous = this.db + .prepare( + "SELECT data FROM recipes WHERE user_id=? AND project_id=? ORDER BY revision DESC LIMIT 1", + ) + .get(this.owner, input.projectId); + const current = previous + ? recipeSchema.parse(JSON.parse(rowSchema.parse(previous).data)) + : null; + if ((current?.revision ?? 0) !== input.expectedRevision) + throw new CatalogueError( + 409, + `Recipe revision conflict; latest revision ${current?.revision ?? 0}`, + current?.revision ?? 0, + ); + const { expectedRevision, ...content } = input; + const recipe = recipeSchema.parse({ + ...content, + recipeId: current?.recipeId ?? `r_${randomUUID()}`, + revision: expectedRevision + 1, + recipeHash: hash(JSON.stringify({ ...content, baseDigest })), + baseDigest, + createdAt: this.now(), + }); + this.db + .prepare("INSERT INTO recipes VALUES (?,?,?,?,?,?)") + .run( + this.owner, + input.projectId, + recipe.recipeId, + recipe.revision, + recipe.recipeHash, + JSON.stringify(recipe), + ); + return recipe; + }) + .immediate(); + } + recipes(cursor: string | null, limit: number) { + return this.db + .prepare( + `SELECT r.data FROM recipes r WHERE user_id=? AND project_id>? AND revision=(SELECT max(revision) FROM recipes latest WHERE latest.user_id=r.user_id AND latest.project_id=r.project_id) ORDER BY project_id LIMIT ?`, + ) + .all(this.owner, cursor ?? "", limit) + .map((row) => recipeSchema.parse(JSON.parse(rowSchema.parse(row).data))); + } + putContext(projectId: string, manifest: Manifest) { + const recipe = this.recipe(projectId, manifest.revision); + if (recipe.recipeId !== manifest.recipeId) + throw new CatalogueError( + 409, + "Context recipe does not match project revision", + ); + const context: Context = { + contextId: `c_${randomUUID()}`, + projectId, + manifest, + manifestHash: hash(JSON.stringify(manifest)), + bytes: manifest.files.reduce((sum, file) => sum + file.bytes, 0), + expiresAt: this.now() + 86400000, + uploaded: false, + }; + if (context.bytes > 256 * 1024 * 1024) + throw new CatalogueError(413, "Context exceeds 256 MiB"); + this.db + .prepare("INSERT INTO contexts VALUES (?,?,?,?,?,?)") + .run( + context.contextId, + this.owner, + projectId, + context.manifestHash, + context.expiresAt, + JSON.stringify(context), + ); + return context; + } + context(id: string) { + const row = this.db + .prepare("SELECT data FROM contexts WHERE id=? AND user_id=?") + .get(id, this.owner); + if (!row) throw new CatalogueError(404, "Context not found"); + return contextSchema.parse(JSON.parse(rowSchema.parse(row).data)); + } + completeContext(id: string) { + const context = this.context(id); + context.uploaded = true; + this.db + .prepare("UPDATE contexts SET data=? WHERE id=? AND user_id=?") + .run(JSON.stringify(context), id, this.owner); + } + build(id: string): Build { + const row = this.db + .prepare("SELECT data FROM builds WHERE id=? AND user_id=?") + .get(id, this.owner); + if (!row) throw new CatalogueError(404, "Build not found"); + return buildSchema.parse(JSON.parse(rowSchema.parse(row).data)); + } + saveBuild(build: Build) { + this.db + .prepare( + "UPDATE builds SET state=?,data=json_set(?, '$.lastEventSequence', max(json_extract(data, '$.lastEventSequence'), ?)) WHERE id=? AND user_id=?", + ) + .run( + build.state, + JSON.stringify({ ...build, updatedAt: this.now() }), + build.lastEventSequence, + build.buildId, + this.owner, + ); + } + start( + input: { + projectId: string; + recipeId: string; + revision: number; + contextId: string; + key: string; + }, + accountIdentity: string, + appName: string, + baseArtifact: ArtifactMetadata | null = null, + ) { + return this.db + .transaction(() => { + const recipe = this.recipe(input.projectId, input.revision); + const context = this.context(input.contextId); + if ( + recipe.recipeId !== input.recipeId || + context.projectId !== input.projectId || + context.manifest.recipeId !== input.recipeId || + context.manifest.revision !== input.revision + ) + throw new CatalogueError(409, "Build recipe/context mismatch"); + const payloadHash = hash( + JSON.stringify({ input, accountIdentity, appName, baseArtifact }), + ); + const request = this.db + .prepare( + "SELECT build_id,payload_hash FROM build_requests WHERE user_id=? AND key=?", + ) + .get(this.owner, input.key); + if (request) { + const previous = keySchema.parse(request); + if (previous.payload_hash !== payloadHash) + throw new CatalogueError( + 409, + "Request key already has a different payload", + ); + return { build: this.build(previous.build_id), reused: true }; + } + if (!context.uploaded || context.expiresAt <= this.now()) + throw new CatalogueError( + 409, + "Context is incomplete or expired; upload it again", + ); + const buildHash = hash( + JSON.stringify({ + recipeHash: recipe.recipeHash, + baseDigest: recipe.baseDigest, + files: context.manifest.files, + platform: baseManifest.platform, + builder: baseManifest.builder, + baseArtifact, + }), + ); + const match = this.db + .prepare( + "SELECT data FROM builds WHERE user_id=? AND account_id=? AND app=? AND hash=?", + ) + .get(this.owner, accountIdentity, appName, buildHash); + let build: Build; + if (match) { + build = buildSchema.parse(JSON.parse(rowSchema.parse(match).data)); + if (["failed", "cancelled", "reconciling"].includes(build.state)) { + build = { + ...build, + contextId: context.contextId, + state: "queued", + failure: null, + cancelRequested: false, + }; + this.saveBuild(build); + this.event( + build.buildId, + "state", + "Explicit retry requested; an unresolved vendor build may still incur cost", + ); + } + } else { + build = { + buildId: `b_${randomUUID()}`, + baseArtifact, + projectId: input.projectId, + recipeId: input.recipeId, + revision: input.revision, + contextId: input.contextId, + accountIdentity, + appName, + hash: buildHash, + name: `bb-${hash(this.owner).slice(0, 12)}-${hash(appName).slice(0, 12)}:${buildHash}`, + state: "queued", + imageId: null, + failure: null, + cancelRequested: false, + lastEventSequence: 0, + createdAt: this.now(), + updatedAt: this.now(), + }; + this.db + .prepare("INSERT INTO builds VALUES (?,?,?,?,?,?,?,?,?,?)") + .run( + build.buildId, + this.owner, + build.projectId, + accountIdentity, + appName, + buildHash, + build.state, + build.name, + null, + JSON.stringify(build), + ); + } + this.db + .prepare("INSERT INTO build_requests VALUES (?,?,?,?)") + .run(this.owner, input.key, payloadHash, build.buildId); + return { build, reused: Boolean(match) }; + }) + .immediate(); + } + claim(accountIdentity: string): Build | null { + return this.db + .transaction(() => { + const active = this.db + .prepare( + "SELECT id FROM builds WHERE account_id=? AND state IN ('building','reconciling') LIMIT 1", + ) + .get(accountIdentity); + if (active) return null; + const row = this.db + .prepare( + "SELECT data FROM builds WHERE user_id=? AND account_id=? AND state='queued' ORDER BY rowid LIMIT 1", + ) + .get(this.owner, accountIdentity); + if (!row) return null; + const build = buildSchema.parse(JSON.parse(rowSchema.parse(row).data)); + build.state = "building"; + this.saveBuild(build); + this.db + .prepare("UPDATE builds SET lease=? WHERE id=?") + .run(randomUUID(), build.buildId); + this.event(build.buildId, "state", "building"); + return this.build(build.buildId); + }) + .immediate(); + } + lease(id: string) { + return z + .object({ lease: z.string().nullable() }) + .parse( + this.db + .prepare("SELECT lease FROM builds WHERE id=? AND user_id=?") + .get(id, this.owner), + ).lease; + } + restart() { + this.db + .transaction(() => { + const rows = this.db + .prepare("SELECT id FROM builds WHERE user_id=? AND state='building'") + .all(this.owner); + for (const row of rows) { + const build = this.build(idRow.parse(row).id); + this.db + .prepare("UPDATE builds SET lease=NULL WHERE id=? AND user_id=?") + .run(build.buildId, this.owner); + this.saveBuild({ + ...build, + state: "reconciling", + failure: + "Worker stopped; reconciling immutable published name before any retry", + }); + } + }) + .immediate(); + } + reconciling(accountIdentity: string) { + return this.db + .prepare( + "SELECT data FROM builds WHERE user_id=? AND account_id=? AND state='reconciling'", + ) + .all(this.owner, accountIdentity) + .map((row) => buildSchema.parse(JSON.parse(rowSchema.parse(row).data))); + } + cancel(id: string) { + return this.db + .transaction(() => { + const build = this.build(id); + if (build.state === "queued") build.state = "cancelled"; + else if (build.state === "building" || build.state === "reconciling") + build.cancelRequested = true; + this.saveBuild(build); + this.event( + id, + "state", + build.state === "cancelled" + ? "Cancelled before vendor submission" + : "Cancellation requested; vendor termination is not confirmed", + ); + return this.build(id); + }) + .immediate(); + } + event( + id: string, + kind: "state" | "log" | "truncated", + text: string, + secrets: readonly string[] = [], + ) { + this.db + .transaction(() => { + const build = this.build(id); + let safe = text; + for (const secret of secrets) + if (secret) safe = safe.replaceAll(secret, "[REDACTED]"); + safe = safe.replace( + /(?:Bearer\s+\S+|(?:token|secret|password|api[_-]?key)\s*[=:]\s*\S+|https?:\/\/[^\s/]+:[^\s@]+@[^\s]+)/gi, + "[REDACTED]", + ); + safe = safe.replace( + /'[A-Za-z0-9+/=]{512,}'/g, + "'[encoded file contents]'", + ); + const clipped = Buffer.byteLength(safe) > 65536; + safe = Buffer.from(safe).subarray(0, 65536).toString("utf8"); + const insert = (eventKind: string, value: string) => { + build.lastEventSequence++; + this.db + .prepare("INSERT INTO build_events VALUES (?,?,?,?,?,?)") + .run( + id, + build.lastEventSequence, + eventKind, + value, + this.now(), + Buffer.byteLength(value), + ); + }; + insert(kind, safe); + if (clipped) + insert("truncated", "One log chunk exceeded the 64 KiB chunk limit"); + const total = z + .object({ bytes: z.number() }) + .parse( + this.db + .prepare( + "SELECT coalesce(sum(bytes),0) AS bytes FROM build_events WHERE build_id=?", + ) + .get(id), + ).bytes; + if (total > MAX_LOG_BYTES) { + this.db + .prepare( + `DELETE FROM build_events WHERE build_id=? AND sequence IN (SELECT sequence FROM (SELECT sequence,sum(bytes) OVER (ORDER BY sequence DESC) AS retained FROM build_events WHERE build_id=?) WHERE retained>?)`, + ) + .run(id, id, MAX_LOG_BYTES - 1024); + insert( + "truncated", + "Earlier build logs were truncated at the 10 MiB retention limit", + ); + } + this.saveBuild(build); + }) + .immediate(); + } + events(id: string, cursor: number, limit: number) { + const build = this.build(id); + const events = this.db + .prepare( + "SELECT sequence,kind,text,time FROM (SELECT sequence,kind,text,time,sum(bytes) OVER (ORDER BY sequence) AS page_bytes FROM build_events WHERE build_id=? AND sequence>?) WHERE page_bytes<=131072 ORDER BY sequence LIMIT ?", + ) + .all(id, cursor, limit) + .map((row) => eventSchema.parse(row)); + return { + events, + nextCursor: events.at(-1)?.sequence ?? cursor, + terminal: + ["ready", "failed", "cancelled"].includes(build.state) && + (events.at(-1)?.sequence ?? cursor) >= build.lastEventSequence, + }; + } + ready(id: string, imageId: string, imageManifest: object) { + this.db + .transaction(() => { + const build = this.build(id); + this.db + .prepare( + "INSERT INTO images(build_id,user_id,image_id,manifest_json) VALUES (?,?,?,?) ON CONFLICT(build_id) DO UPDATE SET image_id=excluded.image_id,manifest_json=excluded.manifest_json,marked_at=NULL,deleting=0,deleted_at=NULL", + ) + .run(id, this.owner, imageId, JSON.stringify(imageManifest)); + this.saveBuild({ ...build, state: "ready", imageId, failure: null }); + this.event(id, "state", "ready"); + }) + .immediate(); + } + project(projectId: string): Project { + const row = this.db + .prepare( + "SELECT data FROM project_images WHERE user_id=? AND project_id=?", + ) + .get(this.owner, projectId); + return row + ? projectSchema.parse(JSON.parse(rowSchema.parse(row).data)) + : projectSchema.parse({ + projectId, + revision: 0, + usableBuildId: null, + resources: {}, + policy: {}, + }); + } + configure(input: { + projectId: string; + usableBuildId?: null; + expectedRevision: number; + resources: Project["resources"]; + policy: Project["policy"]; + }) { + return this.db + .transaction(() => { + const previous = this.project(input.projectId); + if (previous.revision !== input.expectedRevision) + throw new CatalogueError( + 409, + `Project revision conflict; latest revision ${previous.revision}`, + previous.revision, + ); + const next = { + ...previous, + usableBuildId: + input.usableBuildId === null ? null : previous.usableBuildId, + resources: input.resources, + policy: input.policy, + revision: previous.revision + 1, + }; + this.db + .prepare( + "INSERT INTO project_images VALUES (?,?,?,?,?) ON CONFLICT(user_id,project_id) DO UPDATE SET usable_build_id=excluded.usable_build_id,revision=excluded.revision,data=excluded.data", + ) + .run( + this.owner, + input.projectId, + next.usableBuildId, + next.revision, + JSON.stringify(next), + ); + return next; + }) + .immediate(); + } + reference(buildId: string, kind: string, ownerId: string) { + this.db + .transaction(() => { + this.build(buildId); + const image = this.db + .prepare( + "SELECT image_id FROM images WHERE build_id=? AND user_id=? AND deleting=0 AND deleted_at IS NULL", + ) + .get(buildId, this.owner); + if (!image) + throw new CatalogueError(409, "Image is missing or being deleted"); + this.db + .prepare("INSERT OR IGNORE INTO image_references VALUES (?,?,?,?)") + .run(this.owner, kind, ownerId, buildId); + this.db + .prepare( + "UPDATE images SET marked_at=NULL WHERE build_id=? AND user_id=?", + ) + .run(buildId, this.owner); + }) + .immediate(); + } + release(kind: string, ownerId: string) { + this.db + .prepare( + "DELETE FROM image_references WHERE user_id=? AND owner_kind=? AND owner_id=?", + ) + .run(this.owner, kind, ownerId); + } + protected(buildId: string) { + return Boolean( + this.db + .prepare( + `SELECT 1 FROM image_references WHERE user_id=? AND build_id=? UNION ALL SELECT 1 FROM project_images WHERE user_id=? AND usable_build_id=? UNION ALL SELECT 1 FROM verifications WHERE user_id=? AND build_id=? AND state IN ('queued','running') UNION ALL SELECT 1 FROM builds WHERE user_id=? AND id=? AND state IN ('queued','building','reconciling') LIMIT 1`, + ) + .get( + this.owner, + buildId, + this.owner, + buildId, + this.owner, + buildId, + this.owner, + buildId, + ), + ); + } + images(projectId: string | null, cursor: string | null, limit: number) { + return this.db + .prepare( + `SELECT b.data FROM builds b JOIN images i ON i.build_id=b.id WHERE b.user_id=? AND (? IS NULL OR b.project_id=?) AND b.id>? AND i.deleted_at IS NULL ORDER BY b.id LIMIT ?`, + ) + .all(this.owner, projectId, projectId, cursor ?? "", limit) + .map((row) => buildSchema.parse(JSON.parse(rowSchema.parse(row).data))); + } + gcCandidates() { + return this.db + .prepare( + "SELECT build_id,image_id,marked_at FROM images WHERE user_id=? AND deleted_at IS NULL ORDER BY build_id", + ) + .all(this.owner) + .map((row) => + z + .object({ + build_id: z.string(), + image_id: z.string(), + marked_at: z.number().nullable(), + }) + .parse(row), + ); + } + claimDeletion(buildId: string, graceMs: number) { + return this.db + .transaction(() => { + if (this.protected(buildId)) return false; + this.db + .prepare( + "UPDATE images SET marked_at=? WHERE user_id=? AND build_id=? AND marked_at IS NULL", + ) + .run(this.now(), this.owner, buildId); + return ( + this.db + .prepare( + "UPDATE images SET deleting=1 WHERE user_id=? AND build_id=? AND marked_at<=? AND deleted_at IS NULL", + ) + .run(this.owner, buildId, this.now() - graceMs).changes > 0 + ); + }) + .immediate(); + } + deleted(buildId: string) { + this.db + .prepare( + "UPDATE images SET deleted_at=? WHERE user_id=? AND build_id=? AND deleting=1", + ) + .run(this.now(), this.owner, buildId); + } +} diff --git a/plugins/environment-modal-sandbox/catalogue/verification.ts b/plugins/environment-modal-sandbox/catalogue/verification.ts new file mode 100644 index 00000000000..e46e2ebc29a --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/verification.ts @@ -0,0 +1,479 @@ +import { randomUUID } from "node:crypto"; +import { z } from "zod"; +import type { BbPluginApi } from "@get-bb/plugin-sdk"; +import { + CatalogueError, + hash, + verificationSchema, + type Verification, +} from "./model.js"; +import type { Catalogue } from "./store.js"; +import { sourceContract } from "./source-contract.js"; + +const rowSchema = z.object({ data: z.string() }); +export function createVerificationService(bb: BbPluginApi, store: Catalogue) { + const hosts = bb.hosts.experimental_client({ contract: sourceContract }); + const get = (id: string) => { + const row = store.db + .prepare("SELECT data FROM verifications WHERE user_id=? AND id=?") + .get(store.owner, id); + if (!row) throw new CatalogueError(404, "Verification not found"); + return verificationSchema.parse(JSON.parse(rowSchema.parse(row).data)); + }; + const save = (record: Verification) => { + record.updatedAt = store.now(); + store.db + .prepare( + "UPDATE verifications SET state=?,data=? WHERE user_id=? AND id=?", + ) + .run( + record.state, + JSON.stringify(record), + store.owner, + record.verificationId, + ); + }; + async function start(input: { + buildId: string; + agentProviderId: string; + key: string; + }) { + const build = store.build(input.buildId); + if (build.state !== "ready") + throw new CatalogueError(409, "Verification requires a ready image"); + const recipe = store.recipe(build.projectId, build.revision); + if (!recipe.smoke.commands.length) + throw new CatalogueError( + 400, + "Record at least one smoke command before verification", + ); + return store.db + .transaction(() => { + const payloadHash = hash(JSON.stringify(input)); + const previous = store.db + .prepare( + "SELECT payload_hash,verification_id FROM verification_requests WHERE user_id=? AND key=?", + ) + .get(store.owner, input.key); + if (previous) { + const row = z + .object({ payload_hash: z.string(), verification_id: z.string() }) + .parse(previous); + if (row.payload_hash !== payloadHash) + throw new CatalogueError( + 409, + "Verification key has a different payload", + ); + return get(row.verification_id); + } + const record: Verification = { + ...input, + verificationId: `v_${randomUUID()}`, + state: "queued", + hostId: null, + environmentId: null, + threadId: null, + completedTurnSeq: null, + restored: false, + checks: [], + failure: null, + createdAt: store.now(), + updatedAt: store.now(), + }; + store.reference(build.buildId, "verification", record.verificationId); + store.db + .prepare("INSERT INTO verifications VALUES (?,?,?,?,?)") + .run( + record.verificationId, + store.owner, + build.buildId, + record.state, + JSON.stringify(record), + ); + store.db + .prepare("INSERT INTO verification_requests VALUES (?,?,?,?)") + .run(store.owner, input.key, payloadHash, record.verificationId); + return record; + }) + .immediate(); + } + function assertPassed(buildId: string, agentProviderId: string) { + const build = store.build(buildId); + const passed = store.db + .prepare( + "SELECT data FROM verifications WHERE user_id=? AND build_id=? AND state='passed' AND json_extract(data,'$.agentProviderId')=? ORDER BY rowid DESC LIMIT 1", + ) + .get(store.owner, buildId, agentProviderId); + if (!passed) + throw new CatalogueError( + 409, + "A successful verification for the selected agent is required", + ); + const proof = verificationSchema.parse( + JSON.parse(rowSchema.parse(passed).data), + ); + const commands = store.recipe(build.projectId, build.revision).smoke + .commands; + if ( + !proof.restored || + proof.completedTurnSeq === null || + proof.threadId === null || + proof.hostId === null || + proof.environmentId === null || + !commands.length || + proof.checks.length !== commands.length || + proof.checks.some( + (check, index) => + check.command !== commands[index] || check.exitCode !== 0, + ) + ) + throw new CatalogueError( + 409, + "Verification lacks successful agent and independent command evidence", + ); + return proof; + } + function promote(input: { + projectId: string; + buildId: string; + agentProviderId: string; + expectedRevision: number; + }) { + return store.db + .transaction(() => { + const build = store.build(input.buildId); + if (build.projectId !== input.projectId || build.state !== "ready") + throw new CatalogueError( + 409, + "Choose a ready image for this project", + ); + assertPassed(input.buildId, input.agentProviderId); + const previous = store.project(input.projectId); + if (previous.revision !== input.expectedRevision) + throw new CatalogueError( + 409, + "Project revision changed", + previous.revision, + ); + store.reference(input.buildId, "promotion", input.projectId); + const next = { + ...previous, + usableBuildId: input.buildId, + revision: previous.revision + 1, + }; + store.db + .prepare( + "INSERT INTO project_images VALUES (?,?,?,?,?) ON CONFLICT(user_id,project_id) DO UPDATE SET usable_build_id=excluded.usable_build_id,revision=excluded.revision,data=excluded.data", + ) + .run( + store.owner, + input.projectId, + input.buildId, + next.revision, + JSON.stringify(next), + ); + store.release("promotion", input.projectId); + return { ...next, available: true as const }; + }) + .immediate(); + } + async function advance(record: Verification) { + const build = store.build(record.buildId); + const title = `Modal verification ${record.verificationId}`; + if (record.state === "queued") { + const health = await bb.sdk.system.providerStates(); + if ( + !health.providers.some( + (provider) => + provider.providerId === record.agentProviderId && + provider.status === "ready", + ) + ) + throw new CatalogueError( + 409, + "Configure a usable provider credential route before verification", + ); + record.state = "allocating"; + save(record); + } + if (record.state === "allocating") { + const launch = await bb.sdk.hosts.submit({ + machineProviderId: "modal-sandbox", + projectId: build.projectId, + inputs: { buildId: build.buildId, policy: { idleMinutes: 0 } }, + key: `modal-${record.verificationId}`, + }); + if (launch.phase === "cancelled") + throw new CatalogueError( + 409, + "Verification machine allocation was cancelled", + ); + if (launch.phase === "failed") { + record.failure = + "Machine allocation has failed; awaiting core reconciliation or explicit cancellation"; + save(record); + return; + } + if (launch.phase !== "ready" || !launch.hostId) return; + record.failure = null; + record.hostId = launch.hostId; + record.state = "preparing"; + save(record); + } + if (record.state === "preparing" && record.hostId) { + const project = await bb.sdk.projects.get({ projectId: build.projectId }); + if (!project.sources.some((source) => source.hostId === record.hostId)) { + await bb.sdk.projects.sources.add({ + projectId: build.projectId, + type: "clone", + hostId: record.hostId, + }); + } + record.state = "starting"; + save(record); + const recipe = store.recipe(build.projectId, build.revision); + const thread = await bb.sdk.threads.spawn({ + projectId: build.projectId, + providerId: record.agentProviderId, + origin: "plugin", + originPluginId: bb.pluginId, + title, + prompt: `Run these recorded smoke commands, report each exit code and cwd, then stop. Do not change tracked source files.\n${recipe.smoke.commands.join("\n")}`, + environment: { + type: "provider", + environmentProviderId: "project-checkout", + machine: { type: "existing", hostId: record.hostId }, + inputs: { + branch: { + kind: "new", + baseBranch: store.context(build.contextId).manifest.source.commit, + }, + }, + }, + }); + record.threadId = thread.id; + record.state = "running"; + save(record); + } + if (record.state === "starting") { + const matches = await bb.sdk.threads.search({ + query: record.verificationId, + limitPerGroup: "10", + }); + const threads = [...matches.active.results, ...matches.archived.results] + .map((result) => result.thread) + .filter( + (thread) => + thread.title === title && thread.projectId === build.projectId, + ); + if (threads.length !== 1) return; + record.threadId = threads[0]!.id; + record.state = "running"; + save(record); + } + if (record.state === "running" && record.threadId) { + const thread = await bb.sdk.threads.get({ threadId: record.threadId }); + record.environmentId = thread.environmentId; + const events = await bb.sdk.threads.events.list({ + threadId: record.threadId, + types: ["turn/completed"], + limit: "1", + order: "desc", + }); + const completed = events[0]; + if (completed?.type !== "turn/completed") { + if (thread.status === "error") + throw new CatalogueError( + 409, + "Smoke thread failed before a completed agent turn", + ); + save(record); + return; + } + if (completed.data.status !== "completed") + throw new CatalogueError( + 409, + "The real agent smoke turn did not complete successfully", + ); + record.completedTurnSeq = completed.seq; + record.state = "checking"; + save(record); + } + if (record.state === "checking" && record.hostId && record.environmentId) { + const environment = await bb.sdk.environments.get({ + environmentId: record.environmentId, + }); + if (!environment.path || environment.hostId !== record.hostId) + throw new CatalogueError(409, "Verification checkout is unavailable"); + const recipe = store.recipe(build.projectId, build.revision); + const result = await hosts.call( + "smoke", + { + path: environment.path, + commands: recipe.smoke.commands, + timeoutMs: recipe.smoke.timeoutSeconds * 1000, + expectedCommit: store.context(build.contextId).manifest.source.commit, + }, + { hostId: record.hostId }, + ); + record.checks = result.results; + if (record.checks.some((check) => check.exitCode !== 0)) + throw new CatalogueError(409, "Independent smoke command failed"); + const sentinel = await hosts.call( + "smoke", + { + path: environment.path, + commands: [ + `printf '%s' '${record.verificationId}' > .bb-modal-verification-sentinel`, + ], + timeoutMs: 30000, + expectedCommit: store.context(build.contextId).manifest.source.commit, + }, + { hostId: record.hostId }, + ); + if (sentinel.results.some((check) => check.exitCode !== 0)) + throw new CatalogueError( + 409, + "Cannot persist the verification sentinel", + ); + record.state = "suspending"; + save(record); + } + if (record.state === "suspending" && record.hostId) { + const host = await bb.sdk.hosts.get({ hostId: record.hostId }); + if (host.lifecycle.phase !== "suspended") { + await bb.sdk.hosts.suspend({ hostId: record.hostId }); + return; + } + record.state = "resuming"; + save(record); + } + if (record.state === "resuming" && record.hostId) { + const host = await bb.sdk.hosts.get({ hostId: record.hostId }); + if (host.lifecycle.phase !== "active") { + await bb.sdk.hosts.resume({ hostId: record.hostId }); + return; + } + record.state = "restoring"; + save(record); + } + if (record.state === "restoring" && record.hostId && record.environmentId) { + const ready = await bb.sdk.hosts.experimental_ensureReady({ + hostId: record.hostId, + projectId: build.projectId, + providerId: record.agentProviderId, + }); + if (ready.status !== "ready") + throw new CatalogueError(409, "Restored machine readiness failed"); + const environment = await bb.sdk.environments.get({ + environmentId: record.environmentId, + }); + if (!environment.path || environment.hostId !== record.hostId) + throw new CatalogueError( + 409, + "Restored verification checkout is unavailable", + ); + const recipe = store.recipe(build.projectId, build.revision); + const result = await hosts.call( + "smoke", + { + path: environment.path, + commands: [ + `test "$(cat .bb-modal-verification-sentinel)" = '${record.verificationId}'`, + ...recipe.smoke.commands, + ], + timeoutMs: recipe.smoke.timeoutSeconds * 1000, + expectedCommit: store.context(build.contextId).manifest.source.commit, + }, + { hostId: record.hostId }, + ); + if (result.results.some((check) => check.exitCode !== 0)) + throw new CatalogueError( + 409, + "Restored sentinel or independent smoke command failed", + ); + record.checks = result.results.slice(1); + record.restored = true; + record.state = "retaining"; + save(record); + } + if (record.state === "retaining" && record.hostId) { + const host = await bb.sdk.hosts.get({ hostId: record.hostId }); + if (host.lifecycle.phase !== "suspended") { + await bb.sdk.hosts.suspend({ hostId: record.hostId }); + return; + } + record.state = "passed"; + save(record); + store.release("verification", record.verificationId); + } + } + async function sweep() { + const rows = store.db + .prepare( + "SELECT data FROM verifications WHERE user_id=? AND state NOT IN ('passed','failed') ORDER BY rowid LIMIT 10", + ) + .all(store.owner); + for (const row of rows) { + const record = verificationSchema.parse( + JSON.parse(rowSchema.parse(row).data), + ); + try { + if (store.now() - record.createdAt > 30 * 60 * 1000) + throw new CatalogueError( + 409, + "Verification timed out; inspect the retained machine and thread", + ); + await advance(record); + } catch (error) { + const expired = store.now() - record.createdAt > 30 * 60 * 1000; + if (record.state === "allocating" && expired) { + try { + const cancelled = await bb.sdk.hosts.cancel({ + id: `modal-${record.verificationId}`, + }); + if (cancelled.cancelPending) continue; + } catch { + record.failure = + "Verification timed out; allocation cancellation is pending"; + save(record); + continue; + } + } + if ( + !expired && + [ + "queued", + "allocating", + "suspending", + "resuming", + "retaining", + ].includes(record.state) && + !(error instanceof CatalogueError) + ) { + record.failure = `Verification ${record.state} is awaiting reconciliation`; + save(record); + continue; + } + if ( + record.state === "starting" && + !(error instanceof CatalogueError) && + store.now() - record.createdAt <= 30 * 60 * 1000 + ) { + record.failure = + "Smoke submission outcome is uncertain; reconciling its durable title"; + save(record); + continue; + } + record.failure = + error instanceof CatalogueError + ? error.message + : `Verification failed during ${record.state}; inspect machine launch modal-${record.verificationId} and the retained smoke thread`; + record.state = "failed"; + save(record); + store.release("verification", record.verificationId); + } + } + } + return { start, get, promote, sweep, assertPassed }; +} diff --git a/plugins/environment-modal-sandbox/catalogue/worker.ts b/plugins/environment-modal-sandbox/catalogue/worker.ts new file mode 100644 index 00000000000..09151f750e9 --- /dev/null +++ b/plugins/environment-modal-sandbox/catalogue/worker.ts @@ -0,0 +1,155 @@ +import { artifactMetadataSchema } from "./artifact.js"; +import { spawn } from "node:child_process"; +import { z } from "zod"; +import { createImageBackend, type ImageBackendFactory } from "./backend.js"; + +const requestSchema = z + .object({ + credentials: z.object({ tokenId: z.string(), tokenSecret: z.string() }), + baseArtifact: z + .object({ metadata: artifactMetadataSchema, data: z.string() }) + .nullable(), + name: z.string(), + appName: z.string(), + dockerfileText: z.string(), + files: z.array( + z.object({ path: z.string(), data: z.string(), executable: z.boolean() }), + ), + }) + .strict(); +const messageSchema = z.discriminatedUnion("type", [ + z.object({ type: z.literal("log"), text: z.string() }), + z.object({ type: z.literal("allocated"), imageId: z.string() }), + z.object({ type: z.literal("result"), imageId: z.string() }), + z.object({ type: z.literal("error"), text: z.string() }), +]); +export async function runImageBuildWorker() { + const input = requestSchema.parse( + await new Promise((resolve) => process.once("message", resolve)), + ); + const send = (message: z.infer) => + process.send?.(message); + try { + const backend = createImageBackend(input.credentials); + const imageId = await backend.build( + { + name: input.name, + appName: input.appName, + dockerfileText: input.dockerfileText, + baseArtifact: input.baseArtifact + ? { + metadata: input.baseArtifact.metadata, + data: Buffer.from(input.baseArtifact.data, "base64"), + } + : null, + files: new Map( + input.files.map((file) => [ + file.path, + { + data: Buffer.from(file.data, "base64"), + executable: file.executable, + }, + ]), + ), + }, + { + log: (text) => send({ type: "log", text }), + allocated: (imageId) => send({ type: "allocated", imageId }), + }, + ); + send({ type: "result", imageId }); + } catch (error) { + send({ + type: "error", + text: error instanceof Error ? error.message : String(error), + }); + } finally { + process.disconnect?.(); + } +} +export function createWorkerImageBackend( + entryUrl: string, +): ImageBackendFactory { + return (credentials) => ({ + ...createImageBackend(credentials), + async build(request, hooks) { + const child = spawn( + process.execPath, + [ + ...(entryUrl.endsWith(".ts") + ? ["--conditions=source", "--import", "tsx"] + : []), + "--input-type=module", + "-e", + `import { runImageBuildWorker } from ${JSON.stringify(entryUrl)}; await runImageBuildWorker();`, + ], + { + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + TMPDIR: process.env.TMPDIR, + LANG: "C.UTF-8", + ELECTRON_RUN_AS_NODE: "1", + }, + stdio: ["ignore", "pipe", "pipe", "ipc"], + }, + ); + const log = (data: Buffer) => hooks.log(data.toString("utf8")); + child.stdout?.on("data", log); + child.stderr?.on("data", log); + const onAbort = () => child.kill(); + hooks.signal?.addEventListener("abort", onAbort, { once: true }); + if (hooks.signal?.aborted) child.kill(); + try { + return await new Promise((resolve, reject) => { + child.on("error", reject); + child.on("exit", (code) => + reject( + new Error( + `Build worker stopped (${code}); vendor outcome unknown`, + ), + ), + ); + child.on("message", (value) => { + const parsed = messageSchema.safeParse(value); + if (!parsed.success) { + reject(new Error("Malformed build worker response")); + return; + } + const message = parsed.data; + if (message.type === "log") hooks.log(message.text); + else if (message.type === "allocated") + hooks.allocated(message.imageId); + else if (message.type === "result") resolve(message.imageId); + else reject(new Error(message.text)); + }); + child.send( + { + credentials, + name: request.name, + appName: request.appName, + dockerfileText: request.dockerfileText, + baseArtifact: request.baseArtifact + ? { + metadata: request.baseArtifact.metadata, + data: request.baseArtifact.data.toString("base64"), + } + : null, + files: [...request.files].map(([path, file]) => ({ + path, + data: file.data.toString("base64"), + executable: file.executable, + })), + }, + (error) => { + if (error) reject(error); + }, + ); + }); + } finally { + hooks.signal?.removeEventListener("abort", onAbort); + child.kill(); + } + }, + }); +} diff --git a/plugins/environment-modal-sandbox/configuration.test.ts b/plugins/environment-modal-sandbox/configuration.test.ts new file mode 100644 index 00000000000..6d774efbfb4 --- /dev/null +++ b/plugins/environment-modal-sandbox/configuration.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { resolveSettings, type RawSettings } from "./configuration.js"; + +function settings(overrides: Partial = {}): RawSettings { + return { + tokenId: "token-id", + tokenSecret: "token-secret", + appName: "bb-sandboxes", + timeoutMinutes: "60", + idleMinutes: "15", + cpu: "", + memoryMiB: "", + ...overrides, + }; +} + +describe("idle hibernation", () => { + it("defaults to a concrete delay and allows disabling it", () => { + expect(resolveSettings(settings())).toMatchObject({ + ok: true, + settings: { idleMs: 15 * 60_000 }, + }); + expect(resolveSettings(settings({ idleMinutes: "0" }))).toMatchObject({ + ok: true, + settings: { idleMs: null }, + }); + }); + + it("rejects a delay outside the supported range", () => { + expect(resolveSettings(settings({ idleMinutes: "1.5" }))).toEqual({ + ok: false, + message: + "Modal sandbox idleMinutes must be a whole number between 0 and 1440, not 1.5.", + }); + }); +}); diff --git a/plugins/environment-modal-sandbox/configuration.ts b/plugins/environment-modal-sandbox/configuration.ts new file mode 100644 index 00000000000..cc75e40905b --- /dev/null +++ b/plugins/environment-modal-sandbox/configuration.ts @@ -0,0 +1,149 @@ +export const SETTING_DESCRIPTORS = { + tokenId: { + type: "string", + secret: true, + label: "Modal token id", + description: + "The token id half of a Modal API token (modal token new writes one to ~/.modal.toml).", + }, + tokenSecret: { + type: "string", + label: "Modal token secret", + secret: true, + description: "The token secret half of the same Modal API token.", + }, + appName: { + type: "string", + label: "Modal app name", + description: "The Modal app the sandboxes are created in.", + default: "bb-sandboxes", + }, + timeoutMinutes: { + type: "string", + label: "Sandbox lifetime (minutes)", + description: + "Modal terminates the sandbox after this long. Between 1 and 1440.", + default: "1440", + }, + idleMinutes: { + type: "string", + label: "Hibernate after idle (minutes)", + description: + "Snapshot and stop an idle sandbox after this long. Use 0 to keep it running until Modal's lifetime limit.", + default: "15", + }, + cpu: { + type: "string", + label: "CPU cores", + description: + "Reserved physical cores, fractional allowed. Blank for Modal's default.", + default: "", + }, + memoryMiB: { + type: "string", + label: "Memory (MiB)", + description: "Reserved memory in MiB. Blank for Modal's default.", + default: "", + }, +} as const; + +export interface ResolvedSettings { + tokenId: string; + tokenSecret: string; + appName: string; + environmentVariables: Readonly>; + timeoutMs: number; + idleMs: number | null; + cpu: number | null; + memoryMiB: number | null; +} + +export type SettingsResolution = + | { ok: true; settings: ResolvedSettings } + | { ok: false; message: string }; + +export interface RawSettings { + tokenId: string | undefined; + tokenSecret: string | undefined; + appName: string; + timeoutMinutes: string; + idleMinutes: string; + cpu: string; + memoryMiB: string; +} + +const MAX_TIMEOUT_MINUTES = 24 * 60; +const MAX_IDLE_MINUTES = 24 * 60; +function parseNumber(raw: string): number | null { + const trimmed = raw.trim(); + if (trimmed.length === 0) return null; + const value = Number(trimmed); + return Number.isFinite(value) && value > 0 ? value : Number.NaN; +} + +export function resolveSettings(raw: RawSettings): SettingsResolution { + const tokenId = (raw.tokenId ?? "").trim(); + const tokenSecret = (raw.tokenSecret ?? "").trim(); + const missing: string[] = []; + if (tokenId.length === 0) missing.push("tokenId"); + if (tokenSecret.length === 0) missing.push("tokenSecret"); + if (missing.length > 0) { + return { + ok: false, + message: `Modal sandbox is not configured: set ${missing.join(", ")} in the plugin's settings.`, + }; + } + const appName = raw.appName.trim(); + if (appName.length === 0) { + return { ok: false, message: "Modal sandbox appName must not be blank." }; + } + const timeoutMinutes = Number(raw.timeoutMinutes.trim()); + if ( + !Number.isInteger(timeoutMinutes) || + timeoutMinutes < 1 || + timeoutMinutes > MAX_TIMEOUT_MINUTES + ) { + return { + ok: false, + message: `Modal sandbox timeoutMinutes must be a whole number between 1 and ${MAX_TIMEOUT_MINUTES}, not ${raw.timeoutMinutes}.`, + }; + } + const cpu = parseNumber(raw.cpu); + if (cpu !== null && Number.isNaN(cpu)) { + return { + ok: false, + message: `Modal sandbox cpu must be a positive number or blank, not ${raw.cpu}.`, + }; + } + const memoryMiB = parseNumber(raw.memoryMiB); + if (memoryMiB !== null && Number.isNaN(memoryMiB)) { + return { + ok: false, + message: `Modal sandbox memoryMiB must be a positive number or blank, not ${raw.memoryMiB}.`, + }; + } + const idleMinutes = Number(raw.idleMinutes.trim()); + if ( + !Number.isInteger(idleMinutes) || + idleMinutes < 0 || + idleMinutes > MAX_IDLE_MINUTES + ) { + return { + ok: false, + message: `Modal sandbox idleMinutes must be a whole number between 0 and ${MAX_IDLE_MINUTES}, not ${raw.idleMinutes}.`, + }; + } + return { + ok: true, + settings: { + tokenId, + tokenSecret, + appName, + environmentVariables: {}, + timeoutMs: timeoutMinutes * 60_000, + idleMs: idleMinutes === 0 ? null : idleMinutes * 60_000, + cpu, + memoryMiB, + }, + }; +} diff --git a/plugins/environment-modal-sandbox/host.ts b/plugins/environment-modal-sandbox/host.ts new file mode 100644 index 00000000000..13a259dc0ab --- /dev/null +++ b/plugins/environment-modal-sandbox/host.ts @@ -0,0 +1,260 @@ +import { spawn } from "node:child_process"; +import { isLockfile } from "./catalogue/source-contract.js"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { readFile, lstat, realpath } from "node:fs/promises"; +import { resolve, relative, matchesGlob } from "node:path"; +import { experimental_defineHostEntry } from "@get-bb/plugin-sdk/host"; +import { sourceContract } from "./catalogue/source-contract.js"; +import { CatalogueError, hash, type Manifest } from "./catalogue/model.js"; +import { safePath } from "./catalogue/dockerfile.js"; + +const exec = promisify(execFile); +async function git(path: string, args: string[]) { + return ( + await exec("git", ["-C", path, ...args], { + encoding: "buffer", + maxBuffer: 256 * 1024 * 1024, + }) + ).stdout; +} +const split = (buffer: Buffer) => + buffer.toString("utf8").split("\0").filter(Boolean); +const forbidden = + /(^|\/)(?:\.git|\.env[^/]*|node_modules|\.cache|\.ssh|\.aws|\.config|\.npmrc|\.pypirc|credentials[^/]*)(\/|$)|\.(?:pem|key|p12)$/i; + +const evidence = + /(^|\/)(?:AGENTS\.md|package\.json|.*lock.*|.*\.toml|.*\.mod|.*\.sum|\.nvmrc|\.node-version|Dockerfile[^/]*|\.bb-env-setup\.sh|\.bb-env-teardown\.sh|\.worktreeinclude|.*\.ya?ml)$/; +export async function inspectSource(path: string, hostId: string) { + const commit = (await git(path, ["rev-parse", "HEAD"])).toString().trim(); + const dirty = split(await git(path, ["diff", "HEAD", "--name-only", "-z"])); + const untracked = split( + await git(path, ["ls-files", "--others", "--exclude-standard", "-z"]), + ); + const tracked = split(await git(path, ["ls-files", "-z"])); + const stage = (await git(path, ["ls-files", "--stage"])).toString(); + const submodules = stage + .split("\n") + .filter((line) => line.startsWith("160000 ")) + .map((line) => line.split("\t")[1]!); + const facts = []; + const lfs = []; + for (const name of tracked.filter( + (name) => evidence.test(name) && !forbidden.test(name), + )) { + const data = await workingFile(path, name).catch(() => null); + if (!data) continue; + if ( + data.data + .subarray(0, 100) + .toString() + .startsWith("version https://git-lfs.github.com/spec/") + ) + lfs.push(name); + facts.push({ + path: name, + sha256: hash(data.data), + bytes: data.data.length, + mode: data.executable ? ("100755" as const) : ("100644" as const), + kind: isLockfile(name) ? "lockfile" : "setup", + }); + } + return { + source: { + hostId, + path, + commit, + dirty: [...new Set([...dirty, ...untracked])].sort(), + submodules, + lfs, + }, + evidence: facts, + setupHooks: tracked.filter((name) => + [".bb-env-setup.sh", ".bb-env-teardown.sh", ".worktreeinclude"].includes( + name, + ), + ), + missing: facts.some((file) => file.kind === "lockfile") + ? [] + : ["No lockfile found"], + }; +} +async function workingFile(root: string, name: string) { + safePath(name); + const path = resolve(root, name); + const stat = await lstat(path); + const rel = relative(await realpath(root), await realpath(path)); + if (rel.startsWith("..") || !stat.isFile() || stat.isSymbolicLink()) + throw new CatalogueError( + 400, + `Context accepts regular files inside the checkout only: ${name}`, + ); + return { data: await readFile(path), executable: Boolean(stat.mode & 0o111) }; +} +async function content(path: string, manifest: Manifest, name: string) { + const file = manifest.files.find((entry) => entry.path === name); + if (!file) throw new CatalogueError(400, "File is not in reviewed manifest"); + const data = manifest.reviewedDirty.includes(name) + ? (await workingFile(path, name)).data + : await git(path, ["show", `${manifest.source.commit}:${name}`]); + if (data.length !== file.bytes || hash(data) !== file.sha256) + throw new CatalogueError(409, `Context changed since review: ${name}`); + return data; +} +export default experimental_defineHostEntry({ + contract: sourceContract, + handlers: { + inspect({ path, hostId }) { + return inspectSource(path, hostId); + }, + async manifest(input) { + const inspection = await inspectSource(input.path, input.hostId); + const dirty = inspection.source.dirty; + if ( + JSON.stringify([...input.reviewedDirty].sort()) !== + JSON.stringify(dirty) + ) + throw new CatalogueError( + 409, + `Review the dirty overlay explicitly: ${JSON.stringify(dirty)}`, + ); + if (inspection.source.submodules.length || inspection.source.lfs.length) + throw new CatalogueError( + 400, + "Submodules and LFS require explicit exported regular-file contexts; unsupported in v1", + ); + const tracked = split( + await git(input.path, [ + "ls-tree", + "-r", + "--name-only", + "-z", + inspection.source.commit, + ]), + ); + const secretRules = ( + await readFile(resolve(input.path, ".worktreeinclude"), "utf8").catch( + () => "", + ) + ) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith("#")); + const names = [...new Set([...tracked, ...dirty])] + .filter( + (name) => + input.include.some((rule) => matchesGlob(name, rule)) && + ![...input.exclude, ...secretRules].some((rule) => + matchesGlob(name, rule), + ) && + !forbidden.test(name), + ) + .sort(); + const manifest: Manifest = { + source: inspection.source, + files: [], + reviewedDirty: input.reviewedDirty, + recipeId: input.recipeId, + revision: input.revision, + }; + for (const name of names) { + safePath(name); + const staged = ( + await git(input.path, [ + "ls-tree", + inspection.source.commit, + "--", + name, + ]) + ).toString(); + if (staged && !/^100(?:644|755) blob /.test(staged)) + throw new CatalogueError(400, `Unsupported archive entry ${name}`); + const value = dirty.includes(name) + ? await workingFile(input.path, name) + : { + data: await git(input.path, [ + "show", + `${inspection.source.commit}:${name}`, + ]), + executable: staged.startsWith("100755"), + }; + if ( + /-----BEGIN .*PRIVATE KEY-----|(?:sk-[A-Za-z0-9_-]{20,}|AKIA[A-Z0-9]{16})/.test( + value.data.toString("utf8"), + ) + ) + throw new CatalogueError( + 400, + `Credential-like content excluded: ${name}`, + ); + manifest.files.push({ + path: name, + sha256: hash(value.data), + bytes: value.data.length, + mode: value.executable ? "100755" : "100644", + }); + } + if ( + manifest.files.reduce((sum, file) => sum + file.bytes, 0) > + 256 * 1024 * 1024 + ) + throw new CatalogueError(413, "Context exceeds 256 MiB"); + return manifest; + }, + async smoke({ path, commands, timeoutMs, expectedCommit }) { + const commit = (await git(path, ["rev-parse", "HEAD"])) + .toString("utf8") + .trim(); + if (commit !== expectedCommit) + throw new Error( + "Verification checkout does not match the recorded build commit", + ); + const results = []; + for (const command of commands) { + const exitCode = await new Promise((resolve, reject) => { + const child = spawn("sh", ["-eu", "-c", command], { + cwd: path, + stdio: "ignore", + timeout: timeoutMs, + }); + child.once("error", reject); + child.once("exit", (code) => resolve(code ?? 1)); + }); + results.push({ command, exitCode }); + } + return { commit, results }; + }, + async upload({ path, manifest, url, token, contextId }) { + for (const file of manifest.files) { + const data = await content(path, manifest, file.path); + for ( + let offset = 0; + offset < Math.max(data.length, 1); + offset += 256 * 1024 + ) { + const response = await fetch(url, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ + contextId, + path: file.path, + offset, + data: data + .subarray(offset, offset + 256 * 1024) + .toString("base64"), + }), + }); + if (!response.ok) + throw new CatalogueError( + response.status, + `Context upload failed: ${await response.text()}`, + ); + } + } + return { uploaded: true }; + }, + }, +}); diff --git a/plugins/environment-modal-sandbox/lifecycle.ts b/plugins/environment-modal-sandbox/lifecycle.ts new file mode 100644 index 00000000000..5b730ec5fd8 --- /dev/null +++ b/plugins/environment-modal-sandbox/lifecycle.ts @@ -0,0 +1,34 @@ +import { z } from "zod"; +import { policySchema, resourcesSchema } from "./catalogue/model.js"; + +const legacyResourceSchema = z + .object({ + version: z.literal(3), + key: z.string().min(1), + sandboxId: z.string().min(1).nullable(), + snapshotImageId: z.string().min(1).nullable(), + pendingSnapshotImageIds: z.array(z.string().min(1)), + }) + .strict(); +export const pinnedResourceSchema = legacyResourceSchema + .extend({ + version: z.literal(4), + buildId: z.string().min(1), + imageId: z.string().min(1), + accountRef: z.literal("default"), + accountIdentity: z.string().min(1), + appName: z.string().min(1), + resources: resourcesSchema, + policy: policySchema, + expiresAt: z.number().nullable(), + policyRevision: z.number().int().nonnegative().default(0), + }) + .strict(); +export const modalMachineResourceSchema = z.union([ + pinnedResourceSchema, + legacyResourceSchema, +]); +export type ModalMachineResource = z.infer; +export function readModalMachineResource(value: unknown): ModalMachineResource { + return modalMachineResourceSchema.parse(value); +} diff --git a/plugins/environment-modal-sandbox/modal-logo.svg b/plugins/environment-modal-sandbox/modal-logo.svg new file mode 100644 index 00000000000..94c4fc0e9f9 --- /dev/null +++ b/plugins/environment-modal-sandbox/modal-logo.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/plugins/environment-modal-sandbox/package.json b/plugins/environment-modal-sandbox/package.json new file mode 100644 index 00000000000..58da2494ef4 --- /dev/null +++ b/plugins/environment-modal-sandbox/package.json @@ -0,0 +1,48 @@ +{ + "name": "bb-plugin-environment-modal-sandbox", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Run resumable bb machines in Modal Sandboxes", + "engines": { + "bb": ">=0.0" + }, + "bb": { + "name": "Modal sandbox", + "description": "Run resumable bb machines in Modal Sandboxes", + "branding": { + "icon": "./modal-logo.svg" + }, + "server": "./server.ts", + "host": "./host.ts", + "skills": [ + "skills" + ], + "app": "./app.tsx" + }, + "keywords": [ + "bb-plugin" + ], + "scripts": { + "lint": "oxlint .", + "test": "vitest run --config vitest.config.ts", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "modal": "0.10.0", + "zod": "^4.3.6", + "@bb/shared-ui": "workspace:*" + }, + "devDependencies": { + "@get-bb/plugin-sdk": "workspace:*", + "@types/node": "^22.0.0", + "typescript": "npm:@typescript/typescript6@^6.0.2", + "vitest": "^4.1.1", + "better-sqlite3": "^12.6.2", + "@testing-library/react": "^16.3.2", + "@types/react": "^19.0.0", + "jsdom": "^29.0.1", + "react": "^19.0.0", + "react-dom": "^19.0.0" + } +} diff --git a/plugins/environment-modal-sandbox/recipe-editor.test.tsx b/plugins/environment-modal-sandbox/recipe-editor.test.tsx new file mode 100644 index 00000000000..44e17b0b03e --- /dev/null +++ b/plugins/environment-modal-sandbox/recipe-editor.test.tsx @@ -0,0 +1,57 @@ +// @vitest-environment jsdom +import { cleanup, fireEvent, render, waitFor } from "@testing-library/react"; +import { afterEach, expect, it, vi } from "vitest"; +import { RecipeEditor } from "./recipe-editor.js"; +import { stalenessLabel } from "./ui-state.js"; + +afterEach(cleanup); + +it("preserves a conflicting draft and its revision until explicitly reloaded", async () => { + const save = vi + .fn() + .mockRejectedValue(new Error("Revision conflict: latest revision 2")); + const view = render( + , + ); + fireEvent.change(view.getByLabelText("Dockerfile recipe"), { + target: { value: "RUN npm --version\n" }, + }); + fireEvent.click(view.getByText("Save recipe")); + await view.findByRole("alert"); + expect(view.getByRole("alert").textContent).toContain("Your draft is kept"); + expect(view.container.querySelector("textarea")?.value).toBe( + "RUN npm --version\n", + ); + fireEvent.click(view.getByText("Save recipe")); + await waitFor(() => expect(save).toHaveBeenCalledTimes(2)); + expect(save.mock.calls.map((call) => call[0].expectedRevision)).toEqual([ + 0, 0, + ]); +}); + +it("never presents an unchecked source as fresh and combines changed inputs", () => { + const state = { + dockerfileChanged: false, + lockfilesChanged: null, + reason: null, + lastCheckedAt: null, + }; + expect(stalenessLabel(state)).toBe("Source not checked"); + expect( + stalenessLabel({ + ...state, + dockerfileChanged: true, + lockfilesChanged: true, + }), + ).toContain("Dockerfile and lockfiles changed"); + expect(stalenessLabel({ ...state, lockfilesChanged: false })).toContain( + "match the recorded build", + ); +}); diff --git a/plugins/environment-modal-sandbox/recipe-editor.tsx b/plugins/environment-modal-sandbox/recipe-editor.tsx new file mode 100644 index 00000000000..8375a5ef41a --- /dev/null +++ b/plugins/environment-modal-sandbox/recipe-editor.tsx @@ -0,0 +1,196 @@ +import { useEffect, useState } from "react"; +import { Button } from "@bb/shared-ui/button"; +import type { Recipe } from "./catalogue/model.js"; +import { errorText, lines } from "./ui-state.js"; + +type Input = Omit< + Recipe, + "recipeId" | "revision" | "recipeHash" | "baseDigest" | "createdAt" +> & { expectedRevision: number }; + +export function RecipeEditor({ + projectId, + recipe, + save, + reload, + saved, + dirtyChanged, +}: { + dirtyChanged: (dirty: boolean) => void; + projectId: string; + recipe: Recipe | null; + save: (input: Input) => Promise; + reload: () => Promise; + saved: (recipe: Recipe) => void; +}) { + const [revision, setRevision] = useState(recipe?.revision ?? 0); + const [text, setText] = useState( + recipe?.dockerfileText ?? "RUN node --version\n", + ); + const [include, setInclude] = useState( + recipe?.contextRules.include.join("\n") ?? "", + ); + const [exclude, setExclude] = useState( + recipe?.contextRules.exclude.join("\n") ?? "", + ); + const [smoke, setSmoke] = useState(recipe?.smoke.commands.join("\n") ?? ""); + useEffect(() => { + dirtyChanged( + text !== recipe?.dockerfileText || + include !== recipe.contextRules.include.join("\n") || + exclude !== recipe.contextRules.exclude.join("\n") || + smoke !== recipe.smoke.commands.join("\n"), + ); + }, [text, include, exclude, smoke, recipe, dirtyChanged]); + const [busy, setBusy] = useState(false); + const [message, setMessage] = useState(""); + const [failed, setFailed] = useState(false); + const inputClass = + "min-h-24 w-full rounded-md border border-border bg-background p-3 font-mono text-sm text-foreground"; + function accept(value: Recipe) { + setRevision(value.revision); + saved(value); + setFailed(false); + } + async function submit() { + setBusy(true); + try { + const value = await save({ + projectId, + expectedRevision: revision, + dockerfileText: text, + contextRules: { include: lines(include), exclude: lines(exclude) }, + smoke: { + commands: lines(smoke), + timeoutSeconds: recipe?.smoke.timeoutSeconds ?? 120, + }, + }); + accept(value); + setMessage(`Saved revision ${value.revision}`); + } catch (error) { + setFailed(true); + setMessage( + `${errorText(error)}. Your draft is kept; reload the stored recipe to review changes before saving again.`, + ); + } finally { + setBusy(false); + } + } + return ( +
+
+

Dockerfile recipe

+ + Stored in bb · revision {revision} + +
+

+ bb supplies the credential-free base. Use RUN, COPY, ENV, WORKDIR and + ARG. This recipe is stored per project in bb, outside the repository. +

+