Skip to content

Commit 1b55206

Browse files
authored
Merge pull request #38 from browser-use/feat/harness-data-dir-relocation-v2
feat(harness): relocate to data dir, split runtime/scratch, archive on upgrade
2 parents 2ac59c8 + 28e0c44 commit 1b55206

6 files changed

Lines changed: 206 additions & 89 deletions

File tree

‎packages/bcode-browser/script/embed-harness.ts‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,19 @@
33
// The build script (`packages/opencode/script/build.ts`) calls
44
// `createEmbeddedHarnessBundle()` and plumbs the result into
55
// `Bun.build({ files: { "bcode-harness.gen.ts": <result> } })`. The generated
6-
// virtual module exports `{ "<rel>": "<bunfs path>" }` for every harness file.
7-
// `harness.ts` reads it lazily in compiled mode and extracts the files to a
8-
// per-version cache dir on first use (decisions.md §4.6).
6+
// virtual module exports `{ "<rel>": "<bunfs path>" }` for every harness file
7+
// plus a content-hash `buildHash` used as the on-disk extraction sentinel.
8+
// `harness.ts` reads it in compiled mode and extracts the files to
9+
// `<dataDir>/harness/` on session start, skipping when the sentinel matches.
910
//
1011
// The walk is glob-driven (not hand-enumerated): when skill files leave the
1112
// repo for the cloud-fetch architecture (decisions.md §4.7) the embed shrinks
1213
// automatically with no script change. Excludes mirror `harness/.gitignore`
1314
// so local artifacts (`.venv/`, `__pycache__/`, `*.egg-info/`, etc.) never
1415
// land in the binary.
1516

17+
import crypto from "crypto"
18+
import fs from "fs/promises"
1619
import path from "path"
1720
import { fileURLToPath } from "url"
1821

@@ -29,6 +32,18 @@ const ignored = [
2932
new Bun.Glob("**/uv.lock"),
3033
]
3134

35+
// SHA-256 over (rel + NUL + content) for each file in sorted order. Stable
36+
// across builds when content is identical, so warm launches skip extraction.
37+
const computeBuildHash = async (files: string[]) => {
38+
const hash = crypto.createHash("sha256")
39+
for (const rel of files) {
40+
hash.update(rel)
41+
hash.update("\0")
42+
hash.update(await fs.readFile(path.join(HARNESS_DIR, rel)))
43+
}
44+
return hash.digest("hex")
45+
}
46+
3247
export const createEmbeddedHarnessBundle = async (buildCwd: string) => {
3348
console.log("Embedding harness files into the binary")
3449
const files = (await Array.fromAsync(new Bun.Glob("**/*").scan({ cwd: HARNESS_DIR, dot: true })))
@@ -37,6 +52,7 @@ export const createEmbeddedHarnessBundle = async (buildCwd: string) => {
3752
.sort()
3853

3954
console.log(`Embedding ${files.length} harness files`)
55+
const buildHash = await computeBuildHash(files)
4056

4157
const imports = files.map((file, i) => {
4258
const spec = path.relative(buildCwd, path.join(HARNESS_DIR, file)).replaceAll("\\", "/")
@@ -47,6 +63,7 @@ export const createEmbeddedHarnessBundle = async (buildCwd: string) => {
4763
`// Auto-generated by packages/bcode-browser/script/embed-harness.ts`,
4864
`// Maps "<rel>" -> bunfs path for every embedded harness file.`,
4965
...imports,
66+
`export const buildHash = ${JSON.stringify(buildHash)}`,
5067
`export default {`,
5168
...entries,
5269
`} as Record<string, string>`,

‎packages/bcode-browser/src/browser-execute.ts‎

Lines changed: 39 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -13,29 +13,39 @@
1313
// pipe stdout+stderr back. BU_NAME is namespaced by sessionID so parallel
1414
// sub-agents (each with their own session) get isolated daemons + browsers.
1515
//
16-
// BH_TMP_DIR points at a per-session scratch dir so sock/port/pid/log + screenshot
17-
// output land somewhere predictable per session, instead of all sessions sharing
18-
// /tmp. The Level-2 wrapper supplies the cache root; we own the layout convention.
16+
// Two per-session dirs, separated by lifetime + path-length sensitivity:
17+
// BH_TMP_DIR — screenshots, debug overlays, daemon log. Persistent under
18+
// <dataDir>/sessions/<sid>/. Long path is fine; the cloud
19+
// UI / read tool finds artifacts here.
20+
// BH_RUNTIME_DIR — sock, port, pid. Volatile under <runtimeRoot>/bcode/<sid>/.
21+
// Path-length budgeted on macOS (AF_UNIX sun_path = 104).
1922
//
2023
// Level 1 per decisions.md §1c — substantial implementation lives here. The
2124
// Level-2 hook in packages/opencode is a one-line wrapper.
2225

2326
import fs from "fs/promises"
27+
import os from "os"
2428
import path from "path"
2529
import { Effect, Schema, Stream } from "effect"
2630
import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"
27-
import { resolveHarnessDir } from "./harness"
31+
import { harnessArchiveDir, resolveHarnessDir } from "./harness"
2832
import { uvLocate } from "./uv-locate"
2933

30-
// Canonical per-session scratch dir layout. Caller supplies dataDir
31-
// (e.g. opencode's Global.Path.data); we own the `sessions/<id>` shape.
32-
// AF_UNIX sun_path is 104 bytes on macOS — `<dataDir>/sessions/<sessionID>/bu-<sessionID>.sock`
33-
// must fit. SessionID is `ses_` + 26 chars (30 chars). The literal suffix is
34-
// `/sessions/` (10) + 30 + `/bu-` (4) + 30 + `.sock` (5) = 79 chars, leaving
35-
// 25 chars of headroom for dataDir. Typical XDG dataDir is well under that.
34+
// Per-session persistent scratch under <dataDir>/sessions/<sid>/. Holds
35+
// screenshots, debug overlays, daemon log. Caller supplies dataDir
36+
// (e.g. opencode's Global.Path.data).
3637
export const sessionScratchDir = (dataDir: string, sessionID: string) =>
3738
path.join(dataDir, "sessions", sessionID)
3839

40+
// Per-session volatile runtime dir under <runtimeRoot>/bcode/<sid>/. Holds
41+
// AF_UNIX sock + port file + pid. macOS sun_path is 104 bytes:
42+
// `/tmp/bcode/ses_<26ch>/bu.sock` is 50 chars — well within budget.
43+
// On Windows the daemon listens on TCP so the path doesn't need to be short,
44+
// but using os.tmpdir() keeps the layout consistent.
45+
const RUNTIME_ROOT = process.platform === "win32" ? os.tmpdir() : "/tmp"
46+
export const sessionRuntimeDir = (sessionID: string) =>
47+
path.join(RUNTIME_ROOT, "bcode", sessionID)
48+
3949
const DEFAULT_TIMEOUT_MS = 60 * 1000
4050
const MAX_TIMEOUT_MS = 10 * 60 * 1000
4151

@@ -50,11 +60,12 @@ export type Parameters = Schema.Schema.Type<typeof parameters>
5060

5161
export interface ExecuteContext {
5262
readonly sessionID: string
53-
// Per-session scratch dir, passed to the harness as BH_TMP_DIR. The harness
54-
// mkdirs it on import, but we mkdir-p here too so failures surface as a
55-
// direct effect error rather than a child-process exit. Pre-compute via
56-
// sessionScratchDir(dataDir, sessionID).
57-
readonly bhTmpDir: string
63+
// BH_TMP_DIR. Persistent per-session dir for screenshots/log. Pre-compute
64+
// via sessionScratchDir(dataDir, sessionID).
65+
readonly bhScratchDir: string
66+
// BH_RUNTIME_DIR. Volatile short-path per-session dir for sock/port/pid.
67+
// Pre-compute via sessionRuntimeDir(sessionID).
68+
readonly bhRuntimeDir: string
5869
// Optional progress callback invoked per output chunk (combined stdout+stderr).
5970
// Level-2 supplies this to drive TUI streaming via opencode's `ctx.metadata`.
6071
// The callback receives the fully accumulated output so far, not just the
@@ -83,29 +94,37 @@ const isUvMissing = (err: unknown): boolean => {
8394
return false
8495
}
8596

86-
export const make = Effect.fn("BrowserExecute.make")(function* () {
97+
// dataDir is opencode's XDG_DATA_HOME for bcode (~/.local/share/bcode/). The
98+
// harness lives at <dataDir>/harness/. We resolve eagerly at make-time so the
99+
// extraction (compiled mode) happens before the agent reads SKILL.md.
100+
export const make = Effect.fn("BrowserExecute.make")(function* (dataDir: string) {
87101
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
88102
const locate = yield* uvLocate
103+
const harnessDir = yield* Effect.promise(() => resolveHarnessDir(dataDir))
89104

90105
const execute = (args: Parameters, ctx: ExecuteContext) =>
91106
Effect.gen(function* () {
92-
const harnessDir = yield* Effect.promise(() => resolveHarnessDir())
93107
// Pre-flight check on harnessDir: spawn ENOENT on a missing cwd surfaces
94108
// with `path: "uv"` on Bun/Windows, which is indistinguishable from a
95109
// truly-missing uv. Catch it here so the user gets the real cause
96110
// instead of a misleading "uv not on PATH" hint.
97111
if (!(yield* Effect.promise(() => fs.access(harnessDir).then(() => true, () => false)))) {
98112
return yield* Effect.fail(new Error(`harness directory not found at ${harnessDir} — bcode build is broken; please reinstall`))
99113
}
100-
yield* Effect.promise(() => fs.mkdir(ctx.bhTmpDir, { recursive: true }))
114+
yield* Effect.promise(() => fs.mkdir(ctx.bhScratchDir, { recursive: true }))
115+
yield* Effect.promise(() => fs.mkdir(ctx.bhRuntimeDir, { recursive: true }))
101116
const uv = yield* locate
102117
const proc = ChildProcess.make(
103118
uv,
104119
["run", "--project", harnessDir, "browser-harness", "-c", args.python],
105120
{
106121
cwd: harnessDir,
107122
extendEnv: true,
108-
env: { BU_NAME: ctx.sessionID, BH_TMP_DIR: ctx.bhTmpDir },
123+
env: {
124+
BU_NAME: ctx.sessionID,
125+
BH_TMP_DIR: ctx.bhScratchDir,
126+
BH_RUNTIME_DIR: ctx.bhRuntimeDir,
127+
},
109128
stdin: "ignore",
110129
},
111130
)
@@ -138,7 +157,7 @@ export const make = Effect.fn("BrowserExecute.make")(function* () {
138157
}),
139158
)
140159

141-
return { parameters, execute }
160+
return { parameters, execute, harnessDir, harnessArchiveDir: harnessArchiveDir(dataDir) }
142161
})
143162

144163
export * as BrowserExecute from "./browser-execute"

‎packages/bcode-browser/src/harness.ts‎

Lines changed: 100 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -11,25 +11,31 @@
1111
// `import.meta.url` lives under `/$bunfs/` (or `B:/~BUN/` on Windows), a
1212
// read-only virtual filesystem. uv cannot write `.venv/` there. We extract
1313
// the embedded harness (built into the binary by `script/embed-harness.ts`)
14-
// to a single un-versioned directory at `~/.cache/bcode/harness/`.
14+
// to `<dataDir>/harness/`, where dataDir is opencode's XDG_DATA_HOME for
15+
// bcode (~/.local/share/bcode/ on Linux/Mac). The harness is data, not
16+
// cache: it accumulates agent edits to `agent-workspace/agent_helpers.py`
17+
// that must outlive a `~/.cache` wipe.
1518
//
16-
// Per decisions §4.8, the cache is **un-versioned** so agent edits to
17-
// `agent-workspace/agent_helpers.py` survive binary upgrades. Extraction
18-
// policy on every launch: walk the embed map and write each file out, with
19-
// one exception — `agent-workspace/agent_helpers.py` is preserved if
20-
// already present. Everything else (`src/browser_harness/*.py`,
21-
// `pyproject.toml`, skills, etc.) is overwritten unconditionally; the
22-
// binary is the source of truth for those, and we want curated skill /
23-
// daemon / setup updates to land on upgrade.
24-
// `agent-workspace/agent_helpers.py` is the one Green-zone file (decisions
25-
// §3.7, §4.5) where agent learnings accumulate and must outlive upgrades.
26-
// Upstream moved the agent-editable surface from root `helpers.py` to
27-
// `agent-workspace/agent_helpers.py` in PR #229; the core `helpers.py`
28-
// inside `src/browser_harness/` is now baseline-overwrite.
19+
// A content-hash sentinel at `<harness>/.bcode-build` records the embed
20+
// bundle that produced the on-disk tree. On session start we compare it to
21+
// the bundle hash and skip extraction when they match — warm launches cost
22+
// one stat. Mismatch (binary upgrade) snapshots the active tree to
23+
// `<dataDir>/harness-archive/<old-buildHash>/` (excluding `.venv/` and
24+
// `__pycache__/`) so the agent can read the old skills + helpers when
25+
// migrating its own customizations, then re-extracts every embed file
26+
// except anything under `agent-workspace/` (the Green-zone subtree —
27+
// decisions §3.7, §4.5: agent_helpers.py and any agent-authored files
28+
// like domain-skills/<host>/*.md persist across upgrades). The core
29+
// `src/browser_harness/` package and shipped skill files are
30+
// baseline-overwrite.
2931
//
3032
// Concurrent first-callers are deduplicated via an in-process promise.
3133
// Bun.write is atomic per file; cross-process races just result in the
3234
// same bytes being written, which is fine.
35+
//
36+
// On first launch after the relocation, any pre-existing harness at the
37+
// legacy `~/.cache/bcode/harness/` is moved to the new location so agent
38+
// edits under `agent-workspace/` survive the upgrade.
3339

3440
import fs from "fs/promises"
3541
import os from "os"
@@ -47,41 +53,105 @@ const isCompiled = (() => {
4753
return d.startsWith("/$bunfs/") || d.startsWith("B:/~BUN/")
4854
})()
4955
const DEV_HARNESS_DIR = path.resolve(__dirname, "..", "harness")
50-
const cachedHarnessDir = path.join(os.homedir(), ".cache", "bcode", "harness")
56+
const LEGACY_CACHE_DIR = path.join(os.homedir(), ".cache", "bcode", "harness")
57+
const SENTINEL_NAME = ".bcode-build"
58+
59+
// Embed paths that are agent-editable and must be preserved across binary
60+
// upgrades. Per decisions §3.7 / §4.5 the entire `agent-workspace/` subtree
61+
// is the Green zone (agent_helpers.py plus any agent-authored files such as
62+
// domain-skills/<host>/*.md). The core `src/browser_harness/` package and
63+
// shipped skill files are baseline-overwrite.
64+
const PRESERVED_PREFIX = "agent-workspace/"
65+
66+
// Compute the harness directory for a given dataDir without touching the
67+
// filesystem. The agent permission whitelist uses this; runtime extraction
68+
// uses `resolveHarnessDir`.
69+
export const harnessDir = (dataDir: string) => path.join(dataDir, "harness")
5170

52-
// Files that are agent-editable and must be preserved across binary upgrades.
53-
// Everything in the embed map that isn't in this set is baseline-overwrite.
54-
// Per decisions §3.7 / §4.5: only `agent-workspace/agent_helpers.py` is
55-
// Green-zone editable inside the harness. The core `src/browser_harness/`
56-
// package (daemon, admin, helpers, run, _ipc) is baseline-only.
57-
const PRESERVED_PATHS = new Set(["agent-workspace/agent_helpers.py"])
71+
// Where past-version snapshots live. Each subdir is named for the buildHash
72+
// of the harness it was extracted from. Read-only after creation.
73+
export const harnessArchiveDir = (dataDir: string) => path.join(dataDir, "harness-archive")
74+
75+
// Skipped during archive copies — regenerable (.venv) or junk (__pycache__).
76+
// Match by basename at any depth so nested __pycache__/ inside src/ is also
77+
// excluded.
78+
const ARCHIVE_EXCLUDE = new Set([".venv", "__pycache__"])
5879

5980
const exists = (p: string) => fs.access(p).then(() => true, () => false)
6081

61-
const extractEmbeddedHarness = async (): Promise<string> => {
82+
const readSentinel = async (dir: string) => {
83+
try { return await fs.readFile(path.join(dir, SENTINEL_NAME), "utf8") }
84+
catch { return null }
85+
}
86+
87+
const migrateLegacyIfPresent = async (target: string) => {
88+
if (!(await exists(LEGACY_CACHE_DIR))) return
89+
if (await exists(target)) return
90+
await fs.mkdir(path.dirname(target), { recursive: true })
91+
try { await fs.rename(LEGACY_CACHE_DIR, target) }
92+
catch (err) {
93+
if ((err as { code?: string }).code !== "EXDEV") throw err
94+
await fs.cp(LEGACY_CACHE_DIR, target, { recursive: true })
95+
await fs.rm(LEGACY_CACHE_DIR, { recursive: true, force: true })
96+
}
97+
}
98+
99+
const archiveExistingHarness = async (dataDir: string, target: string, oldHash: string) => {
100+
const archiveTarget = path.join(harnessArchiveDir(dataDir), oldHash)
101+
if (await exists(archiveTarget)) return // already archived (re-entry); nothing to do
102+
await fs.mkdir(harnessArchiveDir(dataDir), { recursive: true })
103+
await fs.cp(target, archiveTarget, {
104+
recursive: true,
105+
filter: (src) => !ARCHIVE_EXCLUDE.has(path.basename(src)),
106+
})
107+
}
108+
109+
const extractEmbeddedHarness = async (dataDir: string): Promise<string> => {
110+
const target = harnessDir(dataDir)
111+
await migrateLegacyIfPresent(target)
112+
62113
// @ts-expect-error generated at build time
63114
const mod = await import("bcode-harness.gen.ts").catch(() => null)
64115
if (!mod) throw new Error("bcode-harness.gen.ts not found in compiled binary — was the build script updated?")
65116
const fileMap = mod.default as Record<string, string>
117+
const buildHash = mod.buildHash as string
118+
119+
const existing = await readSentinel(target)
120+
if (existing === buildHash) return target
121+
if (existing) await archiveExistingHarness(dataDir, target, existing)
66122

67-
await fs.mkdir(cachedHarnessDir, { recursive: true })
123+
await fs.mkdir(target, { recursive: true })
68124
await Promise.all(
69125
Object.entries(fileMap).map(async ([rel, bunfsPath]) => {
70-
const dest = path.join(cachedHarnessDir, rel)
71-
if (PRESERVED_PATHS.has(rel) && (await exists(dest))) return
126+
const dest = path.join(target, rel)
127+
if (rel.startsWith(PRESERVED_PREFIX) && (await exists(dest))) return
72128
await fs.mkdir(path.dirname(dest), { recursive: true })
73129
await Bun.write(dest, Bun.file(bunfsPath))
74130
}),
75131
)
76-
return cachedHarnessDir
132+
await fs.writeFile(path.join(target, SENTINEL_NAME), buildHash, "utf8")
133+
return target
77134
}
78135

79-
let extractPromise: Promise<string> | null = null
136+
// Per-dataDir cache. In production opencode passes the same Global.Path.data
137+
// every call, so this is effectively a singleton; tests and any future
138+
// multi-instance setup that resolves against multiple dataDirs each get their
139+
// own deduplicated extraction without cross-directory contamination.
140+
const extractCache = new Map<string, Promise<string>>()
80141

81-
export const resolveHarnessDir = (): Promise<string> => {
142+
export const resolveHarnessDir = (dataDir: string): Promise<string> => {
82143
if (!isCompiled) return Promise.resolve(DEV_HARNESS_DIR)
83-
if (!extractPromise) extractPromise = extractEmbeddedHarness()
84-
return extractPromise
144+
const cached = extractCache.get(dataDir)
145+
if (cached) return cached
146+
const fresh = extractEmbeddedHarness(dataDir)
147+
extractCache.set(dataDir, fresh)
148+
// Evict on rejection so a transient failure (FS hiccup, partial write) doesn't
149+
// permanently brick subsequent calls. The `===` guard avoids clobbering a
150+
// retry that started after the failure but before this handler fired.
151+
fresh.catch(() => {
152+
if (extractCache.get(dataDir) === fresh) extractCache.delete(dataDir)
153+
})
154+
return fresh
85155
}
86156

87157
export * as Harness from "./harness"

0 commit comments

Comments
 (0)