diff --git a/src/cli/bg.ts b/src/cli/bg.ts index c2a29c213e..02321b27bf 100644 --- a/src/cli/bg.ts +++ b/src/cli/bg.ts @@ -1043,6 +1043,7 @@ export async function attachHandler( export async function killHandler( args: string[] | string | undefined, + options: { retentionSettingsReady?: boolean } = {}, ): Promise { const target = normalizeArgs(args)[0] if (!target) fail('Usage: openclaude kill ') @@ -1054,6 +1055,19 @@ export async function killHandler( `Failed to kill background session ${session.id}: ${errorMessage(error)}`, ) }) + if (options.retentionSettingsReady !== false) { + try { + const { runBackgroundSessionRetention } = await import( + '../utils/cleanup.js' + ) + await runBackgroundSessionRetention({ + trigger: 'explicit-kill', + sessionId: killed.id, + }) + } catch { + // A retention failure must not replace the successful kill outcome. + } + } console.log(`Killed background session ${killed.id}.`) } diff --git a/src/cli/bgFinalizer.fixture.ts b/src/cli/bgFinalizer.fixture.ts index 72072a950d..d2a158b77d 100644 --- a/src/cli/bgFinalizer.fixture.ts +++ b/src/cli/bgFinalizer.fixture.ts @@ -1,14 +1,43 @@ import { writeFile } from 'node:fs/promises' import { handleBgFlag } from './bg.js' -import { prepareBackgroundSessionFinalizer } from './bgFinalizer.js' +import * as backgroundFinalizer from './bgFinalizer.js' import { noteBackgroundSessionTerminationSignal } from '../utils/backgroundSessionTermination.js' const invocation = process.argv.slice(2) -if (invocation[0] === 'launcher') { - await handleBgFlag(['--bg', invocation[1] ?? 'success']) +const cleanupWorkerEnv = 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_WORKER' +if (process.env[cleanupWorkerEnv] === '1') { + const runCleanupWorker = ( + backgroundFinalizer as typeof backgroundFinalizer & { + runBackgroundSessionCleanupWorker?: () => Promise + } + ).runBackgroundSessionCleanupWorker + if (!runCleanupWorker) { + throw new Error('background cleanup worker is unavailable') + } + await runCleanupWorker() +} else if (invocation[0] === 'launcher') { + const name = invocation[2] + const originalLog = console.log + if ( + process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_OUTPUT === 'omit-logs' + ) { + console.log = (...args: unknown[]) => { + if (String(args[0]).startsWith('Logs:')) return + originalLog(...args) + } + } + try { + await handleBgFlag([ + '--bg', + ...(name ? ['--name', name] : []), + invocation[1] ?? 'success', + ]) + } finally { + console.log = originalLog + } } else { const mode = invocation.at(-1) - await prepareBackgroundSessionFinalizer() + await backgroundFinalizer.prepareBackgroundSessionFinalizer() if (mode === 'throw') { throw new Error('intentional background finalizer fixture failure') @@ -29,7 +58,45 @@ if (invocation[0] === 'launcher') { }) } const readyPath = process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_READY - if (readyPath) await writeFile(readyPath, 'ready') + if (mode === 'handled-throw') { + const releasePath = process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_RELEASE + if (!readyPath || !releasePath) { + throw new Error('handled-throw fixture routing is missing') + } + process.once('uncaughtException', async () => { + await writeFile(readyPath, 'ready') + const deadline = Date.now() + 5_000 + while (!(await Bun.file(releasePath).exists())) { + if (Date.now() >= deadline) { + process.exit(24) + } + await new Promise(resolve => setTimeout(resolve, 10)) + } + process.exit(23) + }) + queueMicrotask(() => { + throw new Error('intentional handled fixture failure') + }) + } else if ( + readyPath && + process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_SKIP_READY !== '1' + ) { + await writeFile(readyPath, 'ready') + } + if (mode === 'controlled' || mode === 'controlled-exit') { + const releasePath = process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_RELEASE + if (!releasePath) { + throw new Error('controlled fixture release path is missing') + } + const deadline = Date.now() + 5_000 + while (!(await Bun.file(releasePath).exists())) { + if (Date.now() >= deadline) { + throw new Error('controlled fixture release timed out') + } + await new Promise(resolve => setTimeout(resolve, 10)) + } + if (mode === 'controlled-exit') process.exit(0) + } if (mode === 'wait' || mode === 'sigint' || mode === 'sigterm') { setInterval(() => {}, 1_000) } diff --git a/src/cli/bgFinalizer.test.ts b/src/cli/bgFinalizer.test.ts index 7af3edae76..a8f46538d4 100644 --- a/src/cli/bgFinalizer.test.ts +++ b/src/cli/bgFinalizer.test.ts @@ -1,13 +1,16 @@ import { once } from 'node:events' import { spawn } from 'node:child_process' -import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises' +import { createHash } from 'node:crypto' +import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { basename, join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { isProcessRunning } from '../utils/genericProcessUtils.js' import { BACKGROUND_SESSION_ID_ENV, BACKGROUND_SESSION_LAUNCHER_PID_ENV, prepareBackgroundSessionFinalizer, + startBackgroundSessionCleanupWorker, } from './bgFinalizer.js' import { buildBackgroundChildProcessConfig } from './bg.js' import { @@ -25,6 +28,11 @@ import { const fixturePath = join(import.meta.dir, 'bgFinalizer.fixture.ts') const installedLauncherPath = join(import.meta.dir, '../../bin/openclaude') +const BACKGROUND_SESSION_CLEANUP_WORKER_ENV = + 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_WORKER' +const BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV = + 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_OWNER_PID' +const FIXTURE_OUTPUT_LIMIT_BYTES = 64 * 1024 describe('background session finalizer', () => { let configDir: string @@ -182,12 +190,14 @@ describe('background session finalizer', () => { let exitListener: ((code: number) => void) | undefined const finalized: number[] = [] const finalizedSync: number[] = [] + const registeredSession = ownedSession('bg-owned', 500) + let finalizationOwner: BackgroundSession | undefined const preparation = await prepareBackgroundSessionFinalizer({ env, pid: 500, readSession: async () => - ++reads < 3 ? null : ownedSession('bg-owned', 500), + ++reads < 3 ? null : registeredSession, isLauncherAlive: () => true, sleep: async () => {}, registrationWaitMs: 10, @@ -202,13 +212,15 @@ describe('background session finalizer', () => { onExit: listener => { exitListener = listener }, - finalize: async (_id, termination) => { + finalize: async (_id, termination, options) => { finalized.push(termination.exitCode ?? -1) + finalizationOwner = options?.expectedSession return ownedSession('bg-owned', 500) }, finalizeSync: (_id, termination) => { finalizedSync.push(termination.exitCode ?? -1) }, + startCleanupWorker: () => {}, }) expect(preparation).toBe('installed') @@ -225,6 +237,244 @@ describe('background session finalizer', () => { } expect(finalized).toEqual([7]) expect(finalizedSync).toEqual([]) + expect(finalizationOwner).toEqual(registeredSession) + }) + + it('starts a detached cleanup owner after awaited finalization', async () => { + let beforeExitListener: (() => void | Promise) | undefined + const order: string[] = [] + let spawnedArgs: readonly string[] | undefined + let spawnedEnv: NodeJS.ProcessEnv | undefined + let unrefCalls = 0 + + await prepareBackgroundSessionFinalizer({ + env: { + [BACKGROUND_SESSION_ID_ENV]: 'bg-launcher-handoff', + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: '123', + }, + pid: 500, + readSession: async () => ownedSession('bg-launcher-handoff', 500), + registerCleanup: () => () => {}, + onBeforeExit: listener => { + beforeExitListener = listener + }, + onExit: () => {}, + finalize: async () => { + order.push('finalize') + return ownedSession('bg-launcher-handoff', 500) + }, + settingsArgs: [ + '--settings', + JSON.stringify({ + cleanupPeriodDays: 0, + permissions: { defaultMode: 'bogus' }, + }), + ], + settingsCwd: configDir, + spawnCleanupWorkerProcess: ((_command, args, options) => { + order.push('worker') + spawnedArgs = args + spawnedEnv = options.env + return { + once: () => {}, + unref: () => { + unrefCalls++ + }, + } + }) as unknown as typeof spawn, + }) + + await beforeExitListener?.() + expect(order).toEqual(['finalize', 'worker']) + expect(spawnedArgs).not.toContain('--settings') + expect( + spawnedEnv?.[ + 'OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE' + ], + ).toBe('invalid') + expect(unrefCalls).toBe(1) + }) + + it('passes only settings-source inputs to the detached cleanup worker', () => { + let spawnedArgs: readonly string[] | undefined + let spawnedEnv: NodeJS.ProcessEnv | undefined + let unrefCalls = 0 + startBackgroundSessionCleanupWorker({ + sessionId: 'bg-worker-settings', + ownerPid: 500, + launcherPid: 123, + execPath: 'node', + execArgv: ['--expose-gc'], + entrypoint: '/openclaude/cli.mjs', + settingsArgs: [ + '--provider', + 'openai', + '--settings={"cleanupPeriodDays":0,"apiKeyHelper":"private"}', + '--setting-sources', + 'project,user', + ], + spawnProcess: ((_command, args, options) => { + spawnedArgs = args + spawnedEnv = options.env + return { + once: () => {}, + unref: () => { + unrefCalls++ + }, + } + }) as unknown as typeof spawn, + }) + + expect(spawnedArgs).toEqual([ + '--expose-gc', + '/openclaude/cli.mjs', + '--settings', + '{"cleanupPeriodDays":0}', + '--setting-sources', + 'project,user', + ]) + expect(spawnedEnv?.[BACKGROUND_SESSION_ID_ENV]).toBe( + 'bg-worker-settings', + ) + expect( + spawnedEnv?.[ + 'OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE' + ], + ).toBe('preserved') + expect(unrefCalls).toBe(1) + }) + + it('marks a projected inline settings handoff invalid when a sibling setting fails validation', () => { + let spawnedArgs: readonly string[] | undefined + let spawnedEnv: NodeJS.ProcessEnv | undefined + startBackgroundSessionCleanupWorker({ + sessionId: 'bg-worker-invalid-settings', + ownerPid: 500, + launcherPid: 123, + execPath: 'node', + entrypoint: '/openclaude/cli.mjs', + settingsArgs: [ + '--settings', + JSON.stringify({ + cleanupPeriodDays: 0, + permissions: { defaultMode: 'bogus' }, + }), + ], + spawnProcess: ((_command, args, options) => { + spawnedArgs = args + spawnedEnv = options.env + return { once: () => {}, unref: () => {} } + }) as unknown as typeof spawn, + }) + + expect(spawnedArgs).toEqual(['/openclaude/cli.mjs']) + expect( + spawnedEnv?.[ + 'OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE' + ], + ).toBe('invalid') + }) + + it('retains cleanup when the detached worker cannot observe process exit', async () => { + const { runBackgroundSessionCleanupWorker } = await import( + './bgFinalizer.js' + ) + const env = { + [BACKGROUND_SESSION_CLEANUP_WORKER_ENV]: '1', + [BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV]: '500', + [BACKGROUND_SESSION_ID_ENV]: 'bg-worker-timeout', + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: '123', + } + let observedExit: boolean | undefined + + await runBackgroundSessionCleanupWorker({ + env, + isProcessAlive: () => true, + sleep: async () => {}, + waitMs: 2, + pollMs: 1, + cleanup: async (_sessionId, waitForProcessesToExit) => { + observedExit = await waitForProcessesToExit() + }, + }) + + expect(observedExit).toBe(false) + expect(env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV]).toBeUndefined() + expect(env[BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV]).toBeUndefined() + expect(env[BACKGROUND_SESSION_LAUNCHER_PID_ENV]).toBeUndefined() + }) + + it('does not run detached cleanup for an invalid originating policy', async () => { + const { runBackgroundSessionCleanupWorker } = await import( + './bgFinalizer.js' + ) + const env = { + [BACKGROUND_SESSION_CLEANUP_WORKER_ENV]: '1', + [BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV]: '500', + [BACKGROUND_SESSION_ID_ENV]: 'bg-worker-invalid-policy', + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: '123', + OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE: 'invalid', + } + let cleanupRuns = 0 + + await runBackgroundSessionCleanupWorker({ + env, + cleanup: async () => { + cleanupRuns++ + }, + }) + + expect(cleanupRuns).toBe(0) + expect( + env.OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE, + ).toBeUndefined() + }) + + it('passes the registered generation to the synchronous exit fallback', async () => { + let cleanup: (() => void | Promise) | undefined + let exitListener: ((code: number) => void) | undefined + const registeredSession: BackgroundSession = { + ...ownedSession('bg-sync-generation', 500), + processMarker: 'a'.repeat(64), + terminalFactGeneration: 'a'.repeat(64), + } + let syncOwner: BackgroundSession | undefined + const cleanupWorkers: Array<[string, number, number]> = [] + + await prepareBackgroundSessionFinalizer({ + env: { + [BACKGROUND_SESSION_ID_ENV]: registeredSession.id, + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: '123', + }, + pid: registeredSession.pid, + readSession: async () => registeredSession, + isLauncherAlive: () => true, + registerCleanup: fn => { + cleanup = fn + return () => {} + }, + onBeforeExit: () => {}, + onExit: listener => { + exitListener = listener + }, + finalize: async () => { + throw Object.assign(new Error('contended'), { code: 'ELOCKED' }) + }, + debug: () => {}, + finalizeSync: (_id, _termination, options) => { + syncOwner = options?.expectedSession + }, + startCleanupWorker: (sessionId, ownerPid, launcherPid) => { + cleanupWorkers.push([sessionId, ownerPid, launcherPid]) + }, + }) + + await cleanup?.() + exitListener?.(23) + expect(syncOwner).toEqual(registeredSession) + expect(cleanupWorkers).toEqual([ + [registeredSession.id, registeredSession.pid, 123], + ]) }) it('keeps the original exit code when both persistence paths fail', async () => { @@ -274,6 +524,40 @@ describe('background session finalizer', () => { expect(diagnostics.join('\n')).not.toContain('private sync details') }) + it('reports post-finalization cleanup failure without changing the outcome', async () => { + let cleanup: (() => void | Promise) | undefined + const diagnostics: string[] = [] + + await prepareBackgroundSessionFinalizer({ + env: { + [BACKGROUND_SESSION_ID_ENV]: 'bg-cleanup-failure', + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: '123', + }, + pid: 500, + readSession: async () => ownedSession('bg-cleanup-failure', 500), + isLauncherAlive: () => false, + registerCleanup: fn => { + cleanup = fn + return () => {} + }, + onBeforeExit: () => {}, + onExit: () => {}, + finalize: async () => ownedSession('bg-cleanup-failure', 500), + startCleanupWorker: () => { + throw Object.assign(new Error('private cleanup path'), { code: 'EIO' }) + }, + debug: message => { + diagnostics.push(message) + }, + }) + + await cleanup?.() + expect(diagnostics).toEqual([ + 'Background session post-finalization cleanup failed (EIO)', + ]) + expect(diagnostics[0]).not.toContain('private cleanup path') + }) + it('records an observed shutdown signal instead of a successful exit code', async () => { let cleanup: (() => void | Promise) | undefined let termination: { exitCode?: number; signal?: string } | undefined @@ -299,6 +583,7 @@ describe('background session finalizer', () => { termination = observed return ownedSession('bg-observed-sigint', 500) }, + startCleanupWorker: () => {}, }) await cleanup?.() @@ -309,7 +594,15 @@ describe('background session finalizer', () => { }) async function runFixture( - mode: 'success' | 'fail' | 'throw' | 'wait' | 'sigint' | 'sigterm', + mode: + | 'success' + | 'fail' + | 'throw' + | 'handled-throw' + | 'wait' + | 'sigint' + | 'sigterm', + releasePath?: string, ): Promise<{ id: string child: ReturnType @@ -324,6 +617,9 @@ describe('background session finalizer', () => { [BACKGROUND_SESSION_ID_ENV]: id, [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: String(process.pid), OPENCLAUDE_BG_FINALIZER_FIXTURE_READY: readyPath, + ...(releasePath + ? { OPENCLAUDE_BG_FINALIZER_FIXTURE_RELEASE: releasePath } + : {}), }, stdio: ['ignore', 'ignore', 'ignore'], }) @@ -338,13 +634,246 @@ describe('background session finalizer', () => { } async function waitForFile(path: string): Promise { - for (let attempt = 0; attempt < 200; attempt += 1) { + for (let attempt = 0; attempt < 500; attempt += 1) { if (await Bun.file(path).exists()) return - await new Promise(resolve => setTimeout(resolve, 5)) + await new Promise(resolve => setTimeout(resolve, 10)) } throw new Error('fixture readiness file was not created') } + async function waitForProcessStop(pid: number): Promise { + for (let attempt = 0; attempt < 500; attempt += 1) { + if (!isProcessRunning(pid)) return true + await new Promise(resolve => setTimeout(resolve, 10)) + } + return !isProcessRunning(pid) + } + + async function stopDetachedFixtureChild( + childPid: number | undefined, + ): Promise { + if ( + childPid === undefined || + !Number.isSafeInteger(childPid) || + !isProcessRunning(childPid) + ) { + return + } + try { + process.kill(childPid, 'SIGTERM') + } catch {} + if (await waitForProcessStop(childPid)) return + try { + process.kill(childPid, 'SIGKILL') + } catch {} + await waitForProcessStop(childPid) + } + + function sessionArtifactPaths(id: string, name: string): string[] { + const reservationDigest = createHash('sha256').update(name).digest('hex') + return [ + join(sessionsRoot, 'sessions', `${id}.json`), + join(sessionsRoot, 'logs', `${id}.out.log`), + join(sessionsRoot, 'logs', `${id}.err.log`), + join(sessionsRoot, 'names', `${reservationDigest}.json`), + ] + } + + async function expectSessionArtifactsReclaimed( + id: string, + name: string, + ): Promise { + const artifacts = sessionArtifactPaths(id, name) + for (let attempt = 0; attempt < 500; attempt += 1) { + const present = await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), + ) + if (present.every(exists => !exists)) break + await new Promise(resolve => setTimeout(resolve, 10)) + } + expect( + await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), + ), + ).toEqual([false, false, false, false]) + expect( + (await readdir(join(sessionsRoot, 'terminal'))).filter(file => + file.startsWith(`${id}~`), + ), + ).toEqual([]) + } + + async function expectSessionArtifactsRetained( + id: string, + name: string, + stabilityMs: number = 1_000, + ): Promise { + const artifacts = sessionArtifactPaths(id, name) + const deadline = Date.now() + stabilityMs + do { + expect( + await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), + ), + ).toEqual([true, true, true, false]) + await new Promise(resolve => setTimeout(resolve, 20)) + } while (Date.now() < deadline) + } + + async function waitForChildClose( + child: ReturnType, + label: string, + timeoutMs: number = 15_000, + ): Promise { + const close = once(child, 'close') as Promise<[number | null]> + let timedOut = false + let timer: ReturnType | undefined + try { + return await Promise.race([ + close.then(([code]) => code), + new Promise((_, reject) => { + timer = setTimeout(() => { + timedOut = true + child.kill('SIGKILL') + reject(new Error(`${label} exceeded ${timeoutMs}ms`)) + }, timeoutMs) + timer.unref?.() + }), + ]) + } finally { + if (timer !== undefined) clearTimeout(timer) + if (timedOut) { + await Promise.race([ + close.catch(() => [null] as [null]), + new Promise(resolve => setTimeout(resolve, 2_000)), + ]) + } + } + } + + function captureBoundedOutput( + child: ReturnType, + stream: NodeJS.ReadableStream | null, + label: string, + ): { read: () => string; assertWithinLimit: () => void } { + let text = '' + let bytes = 0 + let exceeded = false + stream?.setEncoding('utf8') + stream?.on('data', chunk => { + if (exceeded) return + const value = String(chunk) + bytes += Buffer.byteLength(value) + if (bytes > FIXTURE_OUTPUT_LIMIT_BYTES) { + exceeded = true + child.kill('SIGKILL') + return + } + text += value + }) + return { + read: () => text, + assertWithinLimit: () => { + if (exceeded) { + throw new Error( + `${label} exceeded ${FIXTURE_OUTPUT_LIMIT_BYTES} bytes`, + ) + } + }, + } + } + + async function runBuiltBackgroundProviderExit( + name: string, + retentionArgs: string[] = [], + ): Promise<{ id: string; childPid: number }> { + const homeDir = join(configDir, 'home') + const cacheDir = join(configDir, 'cache') + await mkdir(homeDir, { recursive: true }) + await mkdir(cacheDir, { recursive: true }) + const childEnv: NodeJS.ProcessEnv = { + ...process.env, + HOME: homeDir, + XDG_CACHE_HOME: cacheDir, + OPENCLAUDE_CONFIG_DIR: configDir, + } + for (const key of [ + 'ANTHROPIC_API_KEY', + 'OPENAI_API_KEY', + 'OPENAI_API_KEYS', + 'GEMINI_API_KEY', + 'NODE_ENV', + ]) { + delete childEnv[key] + } + delete childEnv.OPENCLAUDE_DISABLE_CLI_ENTRYPOINT_AUTO_RUN + const launcher = spawn( + 'node', + [ + installedLauncherPath, + '--bg', + '--name', + name, + ...retentionArgs, + '--provider', + 'openai', + '--print', + 'noop', + ], + { + cwd: configDir, + env: childEnv, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + const stdoutCapture = captureBoundedOutput( + launcher, + launcher.stdout, + 'built background launcher stdout', + ) + const stderrCapture = captureBoundedOutput( + launcher, + launcher.stderr, + 'built background launcher stderr', + ) + let childPid: number | undefined + let ownershipTransferred = false + try { + const launcherCode = await waitForChildClose( + launcher, + 'built background launcher', + ) + stdoutCapture.assertWithinLimit() + stderrCapture.assertWithinLimit() + const stdout = stdoutCapture.read() + const parsedChildPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + if (Number.isSafeInteger(parsedChildPid)) childPid = parsedChildPid + const stderr = stderrCapture.read() + expect(launcherCode).toBe(0) + expect(stderr).toBe('') + const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] + const id = stdoutLogPath + ? basename(stdoutLogPath, '.out.log') + : undefined + expect(id).toBeDefined() + expect(childPid).toBeDefined() + if (!id || childPid === undefined) { + throw new Error('built launch output omitted its session identity') + } + ownershipTransferred = true + return { id, childPid } + } finally { + if (!ownershipTransferred) { + if (childPid === undefined) { + const stdout = stdoutCapture.read() + const parsedChildPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + if (Number.isSafeInteger(parsedChildPid)) childPid = parsedChildPid + } + await stopDetachedFixtureChild(childPid) + } + } + } + async function runBuiltCliSession( id: string, args: string[], @@ -386,7 +915,7 @@ describe('background session finalizer', () => { } async function waitForTerminalSession(): Promise { - for (let attempt = 0; attempt < 200; attempt += 1) { + for (let attempt = 0; attempt < 500; attempt += 1) { const [session] = await listBackgroundSessions() if ( session && @@ -394,7 +923,7 @@ describe('background session finalizer', () => { ) { return session } - await new Promise(resolve => setTimeout(resolve, 5)) + await new Promise(resolve => setTimeout(resolve, 10)) } throw new Error('detached fixture did not persist a terminal outcome') } @@ -409,22 +938,253 @@ describe('background session finalizer', () => { }, stdio: ['ignore', 'pipe', 'pipe'], }) - let stdout = '' - let stderr = '' - launcher.stdout?.setEncoding('utf8') - launcher.stderr?.setEncoding('utf8') - launcher.stdout?.on('data', chunk => { - stdout += chunk - }) - launcher.stderr?.on('data', chunk => { - stderr += chunk - }) + const stdoutCapture = captureBoundedOutput( + launcher, + launcher.stdout, + 'detached launcher stdout', + ) + const stderrCapture = captureBoundedOutput( + launcher, + launcher.stderr, + 'detached launcher stderr', + ) const [code] = (await once(launcher, 'exit')) as [number] + stdoutCapture.assertWithinLimit() + stderrCapture.assertWithinLimit() + const stdout = stdoutCapture.read() + const stderr = stderrCapture.read() expect(code).toBe(0) expect(stderr).toBe('') return { stdout, session: await waitForTerminalSession() } } + async function runWaitingDetachedLaunch( + name: string, + fixtureEnv: NodeJS.ProcessEnv = {}, + ): Promise<{ id: string; childPid: number; readyPath: string }> { + const readyPath = join(configDir, `${name}.ready`) + const launcher = spawn( + process.execPath, + [fixturePath, 'launcher', 'wait', name], + { + cwd: configDir, + env: { + ...process.env, + OPENCLAUDE_CONFIG_DIR: configDir, + OPENCLAUDE_BG_FINALIZER_FIXTURE_READY: readyPath, + ...fixtureEnv, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + const stdoutCapture = captureBoundedOutput( + launcher, + launcher.stdout, + 'detached wait launcher stdout', + ) + const stderrCapture = captureBoundedOutput( + launcher, + launcher.stderr, + 'detached wait launcher stderr', + ) + let childPid: number | undefined + let ownershipTransferred = false + try { + const launcherCode = await waitForChildClose( + launcher, + 'detached wait launcher', + ) + stdoutCapture.assertWithinLimit() + stderrCapture.assertWithinLimit() + const stdout = stdoutCapture.read() + const parsedChildPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + if (Number.isSafeInteger(parsedChildPid)) childPid = parsedChildPid + const stderr = stderrCapture.read() + expect(launcherCode).toBe(0) + expect(stderr).toBe('') + const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] + const id = stdoutLogPath + ? basename(stdoutLogPath, '.out.log') + : undefined + expect(id).toBeDefined() + expect(childPid).toBeDefined() + if (!id || childPid === undefined) { + throw new Error('detached wait launch omitted its session identity') + } + await waitForFile(readyPath) + ownershipTransferred = true + return { id, childPid, readyPath } + } finally { + if (!ownershipTransferred) { + if (childPid === undefined) { + const stdout = stdoutCapture.read() + const parsedChildPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + if (Number.isSafeInteger(parsedChildPid)) childPid = parsedChildPid + } + await stopDetachedFixtureChild(childPid) + } + } + } + + for (const scenario of [ + { + label: 'launcher output omits Logs', + env: { + OPENCLAUDE_BG_FINALIZER_FIXTURE_OUTPUT: 'omit-logs', + }, + }, + { + label: 'readiness is never published', + env: { + OPENCLAUDE_BG_FINALIZER_FIXTURE_SKIP_READY: '1', + }, + }, + ]) { + it( + `reaps the detached child when ${scenario.label}`, + async () => { + const name = `failed-handoff-${scenario.label.replaceAll(' ', '-')}` + await expect( + runWaitingDetachedLaunch(name, scenario.env), + ).rejects.toThrow() + const session = (await listBackgroundSessions()).find( + candidate => candidate.name === name, + ) + expect(session).toBeDefined() + expect( + session ? isProcessRunning(session.pid) : true, + ).toBe(false) + }, + 30_000, + ) + } + + async function runBuiltKill( + id: string, + settingsArgs: string[] = [], + ): Promise { + const killEnv: NodeJS.ProcessEnv = { + ...process.env, + OPENCLAUDE_CONFIG_DIR: configDir, + } + delete killEnv.NODE_ENV + delete killEnv.OPENCLAUDE_DISABLE_CLI_ENTRYPOINT_AUTO_RUN + const kill = spawn( + 'node', + [installedLauncherPath, 'kill', id, ...settingsArgs], + { + cwd: configDir, + env: killEnv, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + const stdoutCapture = captureBoundedOutput( + kill, + kill.stdout, + 'built kill stdout', + ) + const stderrCapture = captureBoundedOutput( + kill, + kill.stderr, + 'built kill stderr', + ) + const code = await waitForChildClose(kill, 'built kill command') + stdoutCapture.assertWithinLimit() + stderrCapture.assertWithinLimit() + const stdout = stdoutCapture.read() + const stderr = stderrCapture.read() + expect(code, stderr).toBe(0) + expect(stderr).toBe('') + return stdout + } + + for (const scenario of [ + { label: 'null input', input: 'process.exit(null)', expected: 0 }, + { + label: 'cleared exitCode', + input: 'process.exitCode = null; process.exit()', + expected: 0, + }, + { label: 'omitted input', input: 'process.exit()', expected: 23 }, + { + label: 'undefined input', + input: 'process.exit(undefined)', + expected: 0, + }, + ]) { + it(`records Node's actual explicit exit with ${scenario.label}`, async () => { + const id = 'bg-node-explicit-exit' + const preload = join(configDir, 'explicit-exit.cjs') + // Run the input as soon as the built CLI installs its exit wrapper. + // Node owns conversion of the input and the observed child status. + await writeFile( + preload, + ` + let exit = process.exit + Object.defineProperty(process, 'exit', { + configurable: true, + get: () => exit, + set: value => { + exit = value + queueMicrotask(() => { + require('node:fs').writeFileSync(${JSON.stringify(join(configDir, 'exit-input-ran'))}, 'ran') + process.exitCode = 23 + ${scenario.input} + }) + }, + }) + `, + ) + const child = spawn( + 'node', + ['--require', preload, installedLauncherPath, '--print', 'fixture'], + { + cwd: configDir, + env: { + ...process.env, + NODE_ENV: '', + OPENCLAUDE_DISABLE_HEAP_RELAUNCH: '1', + OPENCLAUDE_DISABLE_CLI_ENTRYPOINT_AUTO_RUN: '0', + OPENCLAUDE_CONFIG_DIR: configDir, + [BACKGROUND_SESSION_ID_ENV]: id, + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: String(process.pid), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + const closed = waitForChildClose(child, 'Node explicit exit') + const stderr = captureBoundedOutput( + child, + child.stderr, + 'Node explicit exit stderr', + ) + try { + if (!child.pid) + throw new Error('Node explicit-exit fixture did not start') + await mkdir(join(sessionsRoot, 'sessions'), { recursive: true }) + await writeFile( + join(sessionsRoot, 'sessions', `${id}.json`), + JSON.stringify(ownedSession(id, child.pid)), + ) + const code = await closed + expect( + await Bun.file(join(configDir, 'exit-input-ran')).exists(), + ).toBe(true) + expect(code, stderr.read()).toBe(scenario.expected) + stderr.assertWithinLimit() + expect((await listBackgroundSessions())[0]).toMatchObject({ + id, + status: scenario.expected === 0 ? 'exited' : 'failed', + exitCode: scenario.expected, + }) + } finally { + if (child.exitCode === null && child.signalCode === null) + child.kill('SIGKILL') + await closed.catch(() => {}) + } + }, 30_000) + } + for (const expectation of [ { mode: 'success' as const, status: 'exited', exitCode: 0 }, { mode: 'fail' as const, status: 'failed', exitCode: 23 }, @@ -444,6 +1204,122 @@ describe('background session finalizer', () => { }) } + for (const scenario of [ + { label: 'zero-day', days: 0, reclaimed: true, settingsArgs: [] }, + { label: 'positive', days: 30, reclaimed: false, settingsArgs: [] }, + { + label: 'failed retention bootstrap', + days: 0, + reclaimed: false, + settingsArgs: ['--settings', 'missing-retention-settings.json'], + }, + ]) { + it( + `keeps built explicit kill successful with ${scenario.label} retention`, + async () => { + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: scenario.days }), + ) + const name = `built-kill-${scenario.label.replaceAll(' ', '-')}` + const { id, childPid } = await runWaitingDetachedLaunch(name) + try { + expect(isProcessRunning(childPid)).toBe(true) + expect(await runBuiltKill(id, scenario.settingsArgs)).toContain( + `Killed background session ${id}.`, + ) + expect(await waitForProcessStop(childPid)).toBe(true) + if (scenario.reclaimed) { + await expectSessionArtifactsReclaimed(id, name) + } else { + await expectSessionArtifactsRetained(id, name) + } + } finally { + if (isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + } + }, + 30_000, + ) + } + + it( + 'keeps built kill retention independent of malformed project MCP config', + async () => { + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + await writeFile(join(configDir, '.mcp.json'), '{invalid') + const name = 'built-kill-malformed-mcp' + const { id, childPid } = await runWaitingDetachedLaunch(name) + try { + expect(await runBuiltKill(id)).toContain( + `Killed background session ${id}.`, + ) + expect(await waitForProcessStop(childPid)).toBe(true) + await expectSessionArtifactsReclaimed(id, name) + } finally { + if (isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + } + }, + 30_000, + ) + + it( + 'keeps a handled uncaught exception live until the actual exit', + async () => { + const releasePath = join(configDir, 'handled-throw.release') + const { id, child, readyPath } = await runFixture( + 'handled-throw', + releasePath, + ) + const childExit = once(child, 'exit') as Promise< + [number | null, NodeJS.Signals | null] + > + const forceTimer = setTimeout(() => child.kill('SIGKILL'), 10_000) + try { + await waitForFile(readyPath) + expect(child.pid).toBeDefined() + expect(isProcessRunning(child.pid!)).toBe(true) + expect( + (await listBackgroundSessions()).find(item => item.id === id), + ).toMatchObject({ + id, + status: 'running', + }) + + await writeFile(releasePath, 'release') + const [code, signal] = await childExit + expect(signal).toBeNull() + expect(code).toBe(23) + expect( + (await listBackgroundSessions()).find(item => item.id === id), + ).toMatchObject({ + id, + status: 'failed', + exitCode: 23, + terminalReason: 'exit_code', + }) + } finally { + clearTimeout(forceTimer) + await writeFile(releasePath, 'release').catch(() => {}) + if (child.exitCode === null && child.signalCode === null) { + child.kill('SIGKILL') + await childExit.catch(() => {}) + } + } + }, + 30_000, + ) + for (const expectation of [ { mode: 'sigint' as const, @@ -483,27 +1359,247 @@ describe('background session finalizer', () => { { mode: 'success' as const, status: 'exited', exitCode: 0 }, { mode: 'fail' as const, status: 'failed', exitCode: 23 }, ]) { - it(`finalizes a detached ${expectation.mode} launch through handleBgFlag`, async () => { - const { stdout, session } = await runDetachedLaunch(expectation.mode) + it( + `finalizes a detached ${expectation.mode} launch through handleBgFlag`, + async () => { + const { stdout, session } = await runDetachedLaunch(expectation.mode) + + expect(stdout).toMatch( + new RegExp( + `(?:Started background session ${session.id}\\.|Background session ${session.id} finished with status ${expectation.status}\\.)`, + ), + ) + expect(session).toMatchObject({ + status: expectation.status, + exitCode: expectation.exitCode, + terminalReason: 'exit_code', + }) + expect(isValidBackgroundProcessMarker(session.processMarker)).toBe( + true, + ) + expect(session.command).toContain( + backgroundProcessMarkerToken(session.processMarker!), + ) + expect(stdout).not.toContain(BACKGROUND_PROCESS_MARKER_FLAG) + expect(await Bun.file(session.stdoutLogPath).exists()).toBe(true) + expect(await Bun.file(session.stderrLogPath).exists()).toBe(true) + }, + 30_000, + ) + } - expect(stdout).toMatch( - new RegExp( - `(?:Started background session ${session.id}\\.|Background session ${session.id} finished with status ${expectation.status}\\.)`, + async function runZeroRetentionFinalizerScenario( + mode: 'controlled' | 'controlled-exit', + ): Promise { + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const homeDir = join(configDir, 'home') + const cacheDir = join(configDir, 'cache') + await mkdir(homeDir, { recursive: true }) + await mkdir(cacheDir, { recursive: true }) + const name = `zero-retention-${mode}` + const readyPath = join(configDir, `${mode}.ready`) + const releasePath = join(configDir, `${mode}.release`) + const launcher = spawn( + process.execPath, + [fixturePath, 'launcher', mode, name], + { + cwd: configDir, + env: { + ...process.env, + HOME: homeDir, + XDG_CACHE_HOME: cacheDir, + OPENCLAUDE_CONFIG_DIR: configDir, + OPENCLAUDE_BG_FINALIZER_FIXTURE_READY: readyPath, + OPENCLAUDE_BG_FINALIZER_FIXTURE_RELEASE: releasePath, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + const stdoutCapture = captureBoundedOutput( + launcher, + launcher.stdout, + 'zero-retention launcher stdout', + ) + const stderrCapture = captureBoundedOutput( + launcher, + launcher.stderr, + 'zero-retention launcher stderr', + ) + const launcherClose = waitForChildClose(launcher, 'zero-retention launcher') + let childPid: number | undefined + try { + const launcherCode = await launcherClose + stdoutCapture.assertWithinLimit() + stderrCapture.assertWithinLimit() + const stdout = stdoutCapture.read() + const stderr = stderrCapture.read() + expect(launcherCode).toBe(0) + expect(stderr).toBe('') + const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] + const id = stdoutLogPath + ? basename(stdoutLogPath, '.out.log') + : undefined + const parsedChildPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + expect(id).toBeDefined() + expect(Number.isSafeInteger(parsedChildPid)).toBe(true) + if (!id || !Number.isSafeInteger(parsedChildPid)) { + throw new Error('detached launch output omitted its session identity') + } + childPid = parsedChildPid + const artifacts = sessionArtifactPaths(id, name) + await waitForFile(readyPath) + expect(isProcessRunning(childPid)).toBe(true) + expect( + await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), ), + ).toEqual([true, true, true, true]) + await writeFile(releasePath, 'release') + + expect(await waitForProcessStop(childPid)).toBe(true) + await expectSessionArtifactsReclaimed(id, name) + } finally { + await writeFile(releasePath, 'release').catch(() => {}) + if (launcher.exitCode === null && launcher.signalCode === null) { + launcher.kill('SIGKILL') + await launcherClose.catch(() => {}) + } + if (childPid !== undefined && isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGTERM') + } catch {} + if (!(await waitForProcessStop(childPid))) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + } + } + } + + for (const mode of ['controlled', 'controlled-exit'] as const) { + it( + `reclaims a short-lived zero-retention ${mode} launch without a recurring pass`, + async () => runZeroRetentionFinalizerScenario(mode), + 30_000, + ) + } + + it( + 'reclaims zero-retention artifacts after a built provider-validation exit', + async () => { + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), ) - expect(session).toMatchObject({ - status: expectation.status, - exitCode: expectation.exitCode, - terminalReason: 'exit_code', - }) - expect(isValidBackgroundProcessMarker(session.processMarker)).toBe(true) - expect(session.command).toContain( - backgroundProcessMarkerToken(session.processMarker!), - ) - expect(stdout).not.toContain(BACKGROUND_PROCESS_MARKER_FLAG) - expect(await Bun.file(session.stdoutLogPath).exists()).toBe(true) - expect(await Bun.file(session.stderrLogPath).exists()).toBe(true) - }) + const name = 'node-provider-exit' + const { id, childPid } = await runBuiltBackgroundProviderExit(name) + try { + expect(await waitForProcessStop(childPid)).toBe(true) + + await expectSessionArtifactsReclaimed(id, name) + } finally { + if (isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + } + }, + 30_000, + ) + + for (const scenario of [ + { + label: 'settings file retains against global zero-day policy', + globalDays: 0, + prepare: async () => { + const path = join(configDir, 'retain-settings.json') + await writeFile(path, JSON.stringify({ cleanupPeriodDays: 30 })) + return ['--settings', path] + }, + reclaimed: false, + }, + { + label: 'inline settings enable zero-day against global retention', + globalDays: 30, + prepare: async () => [ + '--settings', + JSON.stringify({ cleanupPeriodDays: 0 }), + ], + reclaimed: true, + }, + { + label: 'invalid inline sibling retains against global zero-day policy', + globalDays: 0, + prepare: async () => [ + '--settings', + JSON.stringify({ + cleanupPeriodDays: 0, + permissions: { defaultMode: 'bogus' }, + }), + ], + reclaimed: false, + }, + { + label: 'project-only sources retain against global zero-day policy', + globalDays: 0, + prepare: async () => { + await mkdir(join(configDir, '.openclaude'), { recursive: true }) + await writeFile( + join(configDir, '.openclaude', 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + return ['--setting-sources', 'project'] + }, + reclaimed: false, + }, + { + label: 'project-only sources enable zero-day against global retention', + globalDays: 30, + prepare: async () => { + await mkdir(join(configDir, '.openclaude'), { recursive: true }) + await writeFile( + join(configDir, '.openclaude', 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + return ['--setting-sources', 'project'] + }, + reclaimed: true, + }, + ]) { + it( + `preserves ${scenario.label} across the cleanup-worker handoff`, + async () => { + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: scenario.globalDays }), + ) + const name = `worker-policy-${scenario.globalDays}-${scenario.reclaimed ? 'clean' : 'retain'}` + const { id, childPid } = await runBuiltBackgroundProviderExit( + name, + await scenario.prepare(), + ) + try { + expect(await waitForProcessStop(childPid)).toBe(true) + if (scenario.reclaimed) { + await expectSessionArtifactsReclaimed(id, name) + } else { + await expectSessionArtifactsRetained(id, name) + } + } finally { + if (isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + } + }, + 30_000, + ) } it.skipIf(process.platform === 'win32')( diff --git a/src/cli/bgFinalizer.ts b/src/cli/bgFinalizer.ts index a6afc63360..875e166829 100644 --- a/src/cli/bgFinalizer.ts +++ b/src/cli/bgFinalizer.ts @@ -1,6 +1,13 @@ +import { spawn } from 'node:child_process' +import { resolve } from 'node:path' import { isProcessRunning } from '../utils/genericProcessUtils.js' import { logForDebugging } from '../utils/debug.js' import { registerCleanup } from '../utils/cleanupRegistry.js' +import { SettingsSchema } from '../utils/settings/types.js' +import { + filterInvalidModelPricing, + filterInvalidPermissionRules, +} from '../utils/settings/validation.js' import { beginBackgroundSessionSignalTracking, type ObservedBackgroundSessionSignal, @@ -12,11 +19,15 @@ import { type BackgroundSession, } from './bgRegistry.js' import { + BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV, + BACKGROUND_SESSION_CLEANUP_WORKER_ENV, BACKGROUND_SESSION_ID_ENV, BACKGROUND_SESSION_LAUNCHER_PID_ENV, } from './bgRouting.js' export { + BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV, + BACKGROUND_SESSION_CLEANUP_WORKER_ENV, BACKGROUND_SESSION_ID_ENV, BACKGROUND_SESSION_LAUNCHER_PID_ENV, } from './bgRouting.js' @@ -24,6 +35,17 @@ export { const SAFE_ID_RE = /^[A-Za-z0-9._-]+$/ const DEFAULT_REGISTRATION_WAIT_MS = 5_000 const DEFAULT_REGISTRATION_POLL_MS = 10 +const DEFAULT_CLEANUP_WORKER_WAIT_MS = 10_000 +const DEFAULT_CLEANUP_WORKER_POLL_MS = 25 +const BACKGROUND_SESSION_RETENTION_POLICY_STATE_ENV = + 'OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE' + +type BackgroundRetentionPolicyState = 'preserved' | 'invalid' + +type BackgroundRetentionSettingsHandoff = { + args: string[] + policyState: BackgroundRetentionPolicyState +} type PrepareBackgroundSessionFinalizerOptions = { env?: NodeJS.ProcessEnv @@ -38,6 +60,16 @@ type PrepareBackgroundSessionFinalizerOptions = { onExit?: (listener: (code: number) => void) => void finalize?: typeof recordBackgroundSessionNaturalTermination finalizeSync?: typeof recordBackgroundSessionNaturalTerminationSync + startCleanupWorker?: ( + sessionId: string, + ownerPid: number, + launcherPid: number, + settingsArgs: string[], + policyState: BackgroundRetentionPolicyState, + ) => void + spawnCleanupWorkerProcess?: typeof spawn + settingsArgs?: string[] + settingsCwd?: string getObservedSignal?: () => ObservedBackgroundSessionSignal | undefined debug?: (message: string) => void } @@ -63,9 +95,10 @@ function defaultDebug(message: string): void { logForDebugging(message, { level: 'error' }) } -function currentProcessExitCode(): number { - const value = process.exitCode - if (value === undefined) return 0 +function normalizeProcessExitCode( + value: string | number | null | undefined, +): number { + if (value === undefined || value === null) return 0 const parsed = typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : value return typeof parsed === 'number' && @@ -75,6 +108,10 @@ function currentProcessExitCode(): number { : 1 } +function currentProcessExitCode(): number { + return normalizeProcessExitCode(process.exitCode) +} + function reportFinalizationFailure( debug: (message: string) => void, error: unknown, @@ -88,6 +125,19 @@ function reportFinalizationFailure( } } +function reportPostFinalizationCleanupFailure( + debug: (message: string) => void, + error: unknown, +): void { + try { + debug( + `Background session post-finalization cleanup failed (${boundedFailureKind(error)})`, + ) + } catch { + // Diagnostics must never replace the child process's original outcome. + } +} + function parsePositivePid(value: string | undefined): number | undefined { if (!value || !/^\d+$/.test(value)) return undefined const parsed = Number(value) @@ -120,28 +170,243 @@ async function waitForOwnedSession( | 'registrationPollMs' > >, -): Promise<'owned' | 'mismatch' | 'timeout'> { +): Promise { const attempts = Math.max( 1, Math.ceil(options.registrationWaitMs / options.registrationPollMs), ) for (let attempt = 0; attempt < attempts; attempt += 1) { const session = await options.readSession(id) - if (session) return session.pid === ownerPid ? 'owned' : 'mismatch' + if (session) return session.pid === ownerPid ? session : 'mismatch' if (!options.isLauncherAlive(launcherPid)) { const finalSession = await options.readSession(id) if (finalSession) { - return finalSession.pid === ownerPid ? 'owned' : 'mismatch' + return finalSession.pid === ownerPid ? finalSession : 'mismatch' } return 'timeout' } await options.sleep(options.registrationPollMs) } const session = await options.readSession(id) - if (session) return session.pid === ownerPid ? 'owned' : 'mismatch' + if (session) return session.pid === ownerPid ? session : 'mismatch' return 'timeout' } +type BackgroundSessionCleanupWorkerStartOptions = { + sessionId: string + ownerPid: number + launcherPid: number + env?: NodeJS.ProcessEnv + execPath?: string + execArgv?: string[] + entrypoint?: string + settingsArgs?: string[] + settingsCwd?: string + policyState?: BackgroundRetentionPolicyState + spawnProcess?: typeof spawn +} + +function backgroundRetentionSettingsHandoff( + args: string[], + settingsCwd: string, +): BackgroundRetentionSettingsHandoff { + const selected: string[] = [] + let policyState: BackgroundRetentionPolicyState = 'preserved' + for (const flag of ['--settings', '--setting-sources']) { + const inlinePrefix = `${flag}=` + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] + if (arg?.startsWith(inlinePrefix)) { + const handoff = backgroundRetentionSettingsValue( + flag, + arg.slice(inlinePrefix.length), + settingsCwd, + ) + if (handoff.policyState === 'invalid') policyState = 'invalid' + if (handoff.value !== undefined) selected.push(flag, handoff.value) + break + } + if (arg === flag && index + 1 < args.length) { + const handoff = backgroundRetentionSettingsValue( + flag, + args[index + 1]!, + settingsCwd, + ) + if (handoff.policyState === 'invalid') policyState = 'invalid' + if (handoff.value !== undefined) selected.push(flag, handoff.value) + break + } + } + } + return { args: selected, policyState } +} + +function backgroundRetentionSettingsValue( + flag: string, + value: string, + settingsCwd: string, +): { value?: string; policyState: BackgroundRetentionPolicyState } { + if (flag !== '--settings') { + return { value, policyState: 'preserved' } + } + const trimmed = value.trim() + if (!trimmed.startsWith('{') || !trimmed.endsWith('}')) { + return { + value: resolve(settingsCwd, value), + policyState: 'preserved', + } + } + try { + const parsed = JSON.parse(trimmed) as unknown + const inlineErrors = [ + ...filterInvalidPermissionRules(parsed, '--settings'), + ...filterInvalidModelPricing(parsed, '--settings'), + ] + if ( + !parsed || + typeof parsed !== 'object' || + Array.isArray(parsed) || + inlineErrors.length > 0 || + !SettingsSchema().safeParse(parsed).success + ) { + return { policyState: 'invalid' } + } + if (!Object.hasOwn(parsed, 'cleanupPeriodDays')) { + return { policyState: 'preserved' } + } + return { + value: JSON.stringify({ + cleanupPeriodDays: (parsed as { cleanupPeriodDays?: unknown }) + .cleanupPeriodDays, + }), + policyState: 'preserved', + } + } catch { + return { policyState: 'invalid' } + } +} + +export function startBackgroundSessionCleanupWorker( + options: BackgroundSessionCleanupWorkerStartOptions, +): void { + const entrypoint = options.entrypoint ?? process.argv[1] + if (!entrypoint) { + throw new Error('Background cleanup worker entrypoint is unavailable') + } + const workerEnv = { ...(options.env ?? process.env) } + if (!SAFE_ID_RE.test(options.sessionId)) { + throw new Error('Invalid background cleanup session ID') + } + workerEnv[BACKGROUND_SESSION_ID_ENV] = options.sessionId + workerEnv[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] = '1' + workerEnv[BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV] = String( + options.ownerPid, + ) + workerEnv[BACKGROUND_SESSION_LAUNCHER_PID_ENV] = String(options.launcherPid) + const settingsHandoff = backgroundRetentionSettingsHandoff( + options.settingsArgs ?? process.argv.slice(2), + options.settingsCwd ?? process.cwd(), + ) + workerEnv[BACKGROUND_SESSION_RETENTION_POLICY_STATE_ENV] = + options.policyState === 'invalid' + ? 'invalid' + : settingsHandoff.policyState + const child = (options.spawnProcess ?? spawn)( + options.execPath ?? process.execPath, + [ + ...(options.execArgv ?? process.execArgv), + entrypoint, + ...settingsHandoff.args, + ], + { + detached: true, + env: workerEnv, + stdio: 'ignore', + }, + ) + child.once('error', () => {}) + child.unref() +} + +type RunBackgroundSessionCleanupWorkerOptions = { + env?: NodeJS.ProcessEnv + isProcessAlive?: (pid: number) => boolean + sleep?: (ms: number) => Promise + waitMs?: number + pollMs?: number + cleanup?: ( + sessionId: string, + waitForProcessesToExit: () => Promise, + reloadSettings?: () => boolean, + ) => Promise + reloadSettings?: () => boolean +} + +async function runDefaultPostFinalizationCleanup( + sessionId: string, + waitForProcessesToExit: () => Promise, + reloadSettings?: () => boolean, +): Promise { + const { cleanupBackgroundSessionsAfterFinalization } = await import( + '../utils/cleanup.js' + ) + await cleanupBackgroundSessionsAfterFinalization( + sessionId, + waitForProcessesToExit, + reloadSettings, + ) +} + +export async function runBackgroundSessionCleanupWorker( + options: RunBackgroundSessionCleanupWorkerOptions = {}, +): Promise { + const env = options.env ?? process.env + if (env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] !== '1') return + const ownerPid = parsePositivePid( + env[BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV], + ) + const launcherPid = parsePositivePid( + env[BACKGROUND_SESSION_LAUNCHER_PID_ENV], + ) + const sessionId = env[BACKGROUND_SESSION_ID_ENV] + const policyState = env[BACKGROUND_SESSION_RETENTION_POLICY_STATE_ENV] + delete env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] + delete env[BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV] + delete env[BACKGROUND_SESSION_LAUNCHER_PID_ENV] + delete env[BACKGROUND_SESSION_ID_ENV] + delete env[BACKGROUND_SESSION_RETENTION_POLICY_STATE_ENV] + if ( + ownerPid === undefined || + launcherPid === undefined || + !sessionId || + !SAFE_ID_RE.test(sessionId) + ) { + return + } + if (policyState !== undefined && policyState !== 'preserved') return + + const isProcessAlive = options.isProcessAlive ?? isBackgroundLauncherAlive + const sleep = + options.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))) + const waitMs = options.waitMs ?? DEFAULT_CLEANUP_WORKER_WAIT_MS + const pollMs = options.pollMs ?? DEFAULT_CLEANUP_WORKER_POLL_MS + const pids = [...new Set([ownerPid, launcherPid])] + const waitForProcessesToExit = async (): Promise => { + const attempts = Math.max(1, Math.ceil(waitMs / pollMs)) + for (let attempt = 0; attempt < attempts; attempt += 1) { + if (pids.every(pid => !isProcessAlive(pid))) return true + await sleep(pollMs) + } + return pids.every(pid => !isProcessAlive(pid)) + } + + await (options.cleanup ?? runDefaultPostFinalizationCleanup)( + sessionId, + waitForProcessesToExit, + options.reloadSettings, + ) +} + export async function prepareBackgroundSessionFinalizer( options: PrepareBackgroundSessionFinalizerOptions = {}, ): Promise { @@ -159,7 +424,7 @@ export async function prepareBackgroundSessionFinalizer( } const ownerPid = options.pid ?? process.pid - const ownership = await waitForOwnedSession(id, ownerPid, launcherPid, { + const ownedSession = await waitForOwnedSession(id, ownerPid, launcherPid, { readSession: options.readSession ?? readBackgroundSessionForOwner, isLauncherAlive: options.isLauncherAlive ?? isBackgroundLauncherAlive, sleep: @@ -170,11 +435,11 @@ export async function prepareBackgroundSessionFinalizer( registrationPollMs: options.registrationPollMs ?? DEFAULT_REGISTRATION_POLL_MS, }) - if (ownership === 'mismatch') { + if (ownedSession === 'mismatch') { scrubRoutingEnvironment(env) return 'invalid-routing' } - if (ownership === 'timeout') { + if (ownedSession === 'timeout') { scrubRoutingEnvironment(env) throw new Error('Background session registration was not established') } @@ -187,8 +452,43 @@ export async function prepareBackgroundSessionFinalizer( const getObservedSignal = options.getObservedSignal ?? beginBackgroundSessionSignalTracking() const debug = options.debug ?? defaultDebug + const retentionSettingsHandoff = backgroundRetentionSettingsHandoff( + options.settingsArgs ?? process.argv.slice(2), + options.settingsCwd ?? process.cwd(), + ) + const startCleanupWorker = + options.startCleanupWorker ?? + (( + cleanupSessionId, + cleanupOwnerPid, + cleanupLauncherPid, + cleanupSettingsArgs, + cleanupPolicyState, + ) => + startBackgroundSessionCleanupWorker({ + sessionId: cleanupSessionId, + ownerPid: cleanupOwnerPid, + launcherPid: cleanupLauncherPid, + settingsArgs: cleanupSettingsArgs, + policyState: cleanupPolicyState, + spawnProcess: options.spawnCleanupWorkerProcess, + })) let finalized = false + const startPostFinalizationCleanup = () => { + try { + startCleanupWorker( + id, + ownerPid, + launcherPid, + retentionSettingsHandoff.args, + retentionSettingsHandoff.policyState, + ) + } catch (error) { + reportPostFinalizationCleanupFailure(debug, error) + } + } + const currentTermination = () => { const signal = getObservedSignal() return signal === undefined @@ -199,11 +499,16 @@ export async function prepareBackgroundSessionFinalizer( const finalizeAwaited = async () => { if (finalized) return try { - await finalize(id, currentTermination(), { ownerPid }) + await finalize(id, currentTermination(), { + ownerPid, + expectedSession: ownedSession, + }) finalized = true } catch (error) { reportFinalizationFailure(debug, error) + return } + startPostFinalizationCleanup() } const registerFinalizerCleanup = options.registerCleanup ?? registerCleanup registerFinalizerCleanup(finalizeAwaited) @@ -221,16 +526,25 @@ export async function prepareBackgroundSessionFinalizer( finalizeSync( id, signal === undefined ? { exitCode: code } : { signal }, - { ownerPid }, + { ownerPid, expectedSession: ownedSession }, ) finalized = true } catch (error) { reportFinalizationFailure(debug, error) + return } + startPostFinalizationCleanup() } if (options.onExit) { options.onExit(onExit) } else { + const originalExit = process.exit + process.exit = ((...args: Parameters) => { + onExit( + normalizeProcessExitCode(args.length === 0 ? process.exitCode : args[0]), + ) + return originalExit(...args) + }) as typeof process.exit process.once('exit', onExit) } diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts new file mode 100644 index 0000000000..6ea0b1be08 --- /dev/null +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -0,0 +1,1862 @@ +import { afterEach, beforeEach, describe, expect, it, spyOn } from 'bun:test' +import { createHash } from 'node:crypto' +import * as fsPromises from 'node:fs/promises' +import { + lstat, + chmod, + rename, + mkdir, + mkdtemp, + readdir, + readFile, + rm, + symlink, + unlink, + writeFile, +} from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import * as nameReservationLock from '../utils/lockfile.js' +import { backgroundProcessMarkerToken } from './bgRouting.js' +import { + _setBackgroundSessionsRootForTesting, + cleanupBackgroundSessionsBefore, + createBackgroundSession, + markBackgroundSessionKilled, + recordBackgroundSessionNaturalTermination, + reconcileBackgroundSessionTerminalFacts, + refreshBackgroundSessionStatuses, + resolveBackgroundSession, + type BackgroundSession, + type BackgroundSessionNaturalTermination, +} from './bgRegistry.js' + +const CUTOFF = new Date('2026-07-01T00:00:00.000Z') +const OLD_FINISH = new Date('2026-06-01T00:00:00.000Z') +const RECENT_FINISH = new Date('2026-07-01T00:00:01.000Z') +const OLD_PROCESS_MARKER = 'a'.repeat(64) +const REPLACEMENT_PROCESS_MARKER = 'b'.repeat(64) +const NONCANONICAL_FINISHED_AT_CASES = [ + ['normalized-date', '2026-02-30'], + ['numeric-date', '0'], +] as const + +describe('background session retention cleanup', () => { + let configDir: string + let root: string + let nextPid: number + + function paths(id: string): { + metadata: string + stdout: string + stderr: string + natural: string + killed: string + } { + return { + metadata: join(root, 'sessions', `${id}.json`), + stdout: join(root, 'logs', `${id}.out.log`), + stderr: join(root, 'logs', `${id}.err.log`), + natural: join(root, 'terminal', `${id}.natural.json`), + killed: join(root, 'terminal', `${id}.killed.json`), + } + } + + function reservationPath(name: string): string { + const digest = createHash('sha256').update(name).digest('hex') + return join(root, 'names', `${digest}.json`) + } + + function markedTerminalFactPath( + id: string, + kind: 'natural' | 'killed', + processMarker: string, + ): string { + return join(root, 'terminal', `${id}~${processMarker}.${kind}.json`) + } + + async function exists(path: string): Promise { + try { + await lstat(path) + return true + } catch { + return false + } + } + + async function createRunning( + id: string, + options: { name?: string; startedAt?: Date } = {}, + ): Promise { + return await createBackgroundSession({ + id, + ...(options.name ? { name: options.name } : {}), + pid: nextPid++, + cwd: configDir, + command: ['openclaude', '--print', id], + sessionId: `${id}-conversation`, + now: options.startedAt ?? new Date(OLD_FINISH.getTime() - 60_000), + }) + } + + async function finishNaturally( + session: BackgroundSession, + termination: BackgroundSessionNaturalTermination, + finishedAt: Date = OLD_FINISH, + ): Promise { + await recordBackgroundSessionNaturalTermination(session.id, termination, { + ownerPid: session.pid, + now: finishedAt, + }) + } + + async function writeRawSession(options: { + id: string + status: BackgroundSession['status'] + finishedAt?: string + name?: string + stdoutLogPath?: string + stderrLogPath?: string + createLogs?: boolean + processMarker?: string + }): Promise { + const idPaths = paths(options.id) + await Promise.all([ + mkdir(join(root, 'sessions'), { recursive: true }), + mkdir(join(root, 'logs'), { recursive: true }), + mkdir(join(root, 'terminal'), { recursive: true }), + mkdir(join(root, 'names'), { recursive: true }), + ]) + if (options.createLogs !== false) { + await writeFile(idPaths.stdout, 'stdout') + await writeFile(idPaths.stderr, 'stderr') + } + const session: BackgroundSession = { + id: options.id, + ...(options.name ? { name: options.name } : {}), + pid: nextPid++, + cwd: configDir, + status: options.status, + sessionId: `${options.id}-conversation`, + ...(options.processMarker !== undefined + ? { processMarker: options.processMarker } + : {}), + startedAt: new Date(OLD_FINISH.getTime() - 60_000).toISOString(), + updatedAt: options.finishedAt ?? OLD_FINISH.toISOString(), + command: ['openclaude', '--print', options.id], + stdoutLogPath: options.stdoutLogPath ?? idPaths.stdout, + stderrLogPath: options.stderrLogPath ?? idPaths.stderr, + ...(options.finishedAt ? { finishedAt: options.finishedAt } : {}), + } + await writeFile(idPaths.metadata, JSON.stringify(session)) + return session + } + + async function writeReservation(name: string, id: string): Promise { + await mkdir(join(root, 'names'), { recursive: true }) + await writeFile( + reservationPath(name), + JSON.stringify({ name, id, creatorPid: process.pid }), + ) + } + + function deniedError(): NodeJS.ErrnoException { + return Object.assign(new Error('denied'), { code: 'EACCES' }) + } + + beforeEach(async () => { + configDir = await mkdtemp(join(tmpdir(), 'openclaude-bg-cleanup-')) + root = join(configDir, 'bg-sessions') + nextPid = 20_000 + _setBackgroundSessionsRootForTesting(root) + }) + + afterEach(async () => { + _setBackgroundSessionsRootForTesting(undefined) + await rm(configDir, { recursive: true, force: true }) + }) + + for (const scenario of [ + { + name: 'old natural exit zero', + id: 'bg-exit-zero', + termination: { exitCode: 0 } as const, + }, + { + name: 'old natural nonzero failure', + id: 'bg-exit-nonzero', + termination: { exitCode: 23 } as const, + }, + { + name: 'old signal failure', + id: 'bg-signal', + termination: { signal: 'SIGTERM' } as const, + }, + ]) { + it(`removes ${scenario.name} artifacts`, async () => { + const session = await createRunning(scenario.id) + await finishNaturally(session, scenario.termination) + + const result = await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(result).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + for (const path of Object.values(paths(scenario.id))) { + expect(await exists(path)).toBe(false) + } + }) + } + + for (const consumer of ['reconciliation', 'cleanup'] as const) { + for (const fault of [ + 'invalid', + 'unreadable', + 'replaced', + 'replaced-before-read', + ] as const) { + if (consumer === 'reconciliation' && fault === 'replaced-before-read') + continue + it.skipIf( + fault === 'unreadable' && + (process.platform === 'win32' || process.geteuid?.() === 0), + )(`reports ${fault} ${consumer} inventory as unacknowledged responsibility`, async () => { + const session = await writeRawSession({ + id: 'bg-inventory-report', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + const directory = join(root, 'sessions') + const saved = join(root, 'saved-sessions') + const retries: unknown[] = [] + const onRetry = (target?: unknown) => { + retries.push(target) + } + let readSpy: ReturnType | undefined + let replaced = false + let metadataSnapshotRead = false + const lstatForTesting = async (path: string) => { + if (path === directory) { + if (metadataSnapshotRead && !replaced) { + replaced = true + await rename(directory, saved) + await mkdir(directory) + } + metadataSnapshotRead = true + } + return await lstat(path) + } + try { + if (fault === 'invalid') { + await rename(directory, saved) + await writeFile(directory, 'not a directory') + } else if (fault === 'unreadable') { + await chmod(directory, 0o000) + } else if (fault === 'replaced') { + const original = fsPromises.readdir + const replacement = async ( + ...args: Parameters + ) => { + const entries = await original(...args) + if (String(args[0]) === directory && !replaced) { + replaced = true + await rename(directory, saved) + await mkdir(directory) + } + return entries + } + readSpy = spyOn(fsPromises, 'readdir').mockImplementation( + replacement as typeof fsPromises.readdir, + ) + } + const result = + consumer === 'reconciliation' + ? await reconcileBackgroundSessionTerminalFacts({ onRetry }) + : await cleanupBackgroundSessionsBefore(CUTOFF, { + onRetry, + lstatFile: + fault === 'replaced-before-read' + ? lstatForTesting + : undefined, + }) + expect(result.errors).toBeGreaterThan(0) + expect(retries).toHaveLength(1) + expect(retries[0]).toBeUndefined() + if (fault.startsWith('replaced')) expect(replaced).toBe(true) + expect( + await exists( + fault === 'unreadable' + ? paths(session.id).metadata + : join(saved, `${session.id}.json`), + ), + ).toBe(fault !== 'unreadable') + } finally { + readSpy?.mockRestore() + if (fault === 'unreadable') await chmod(directory, 0o755) + } + if (fault === 'unreadable') + expect(await exists(paths(session.id).metadata)).toBe(true) + }) + } + } + + for (const missingDirectory of [false, true]) { + it(`reclaims exact generation orphans without scanning directories, missing metadata directory ${missingDirectory}`, async () => { + const id = 'bg-targeted-orphan' + await writeRawSession({ + id, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + const factPath = markedTerminalFactPath( + id, + 'natural', + OLD_PROCESS_MARKER, + ) + const fact = { + version: 1, + id, + pid: nextPid++, + generation: OLD_PROCESS_MARKER, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + } + await writeFile(factPath, JSON.stringify(fact)) + const unrelated = markedTerminalFactPath( + 'bg-unrelated-orphan', + 'natural', + OLD_PROCESS_MARKER, + ) + await writeFile( + unrelated, + JSON.stringify({ ...fact, id: 'bg-unrelated-orphan' }), + ) + if (missingDirectory) + await rm(join(root, 'sessions'), { recursive: true }) + else await unlink(paths(id).metadata) + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + sessionIds: [id], + orphanedTerminalFacts: [ + { id, generation: OLD_PROCESS_MARKER }, + { id, generation: '../outside' }, + ], + maxDirectoryEntries: 0, + }), + ).toEqual({ sessionsRemoved: 0, artifactsRemoved: 1, errors: 0 }) + expect(await exists(factPath)).toBe(false) + expect(await exists(unrelated)).toBe(true) + }) + } + + it('targets prompt cleanup to the requested session generation', async () => { + const target = await createRunning('bg-targeted-cleanup') + const retained = await createRunning('bg-targeted-retained') + await finishNaturally(target, { exitCode: 0 }) + await finishNaturally(retained, { exitCode: 0 }) + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + sessionIds: [target.id], + }), + ).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + expect(await exists(paths(target.id).metadata)).toBe(false) + expect(await exists(paths(retained.id).metadata)).toBe(true) + }) + + it('bounds prompt cleanup before materializing the metadata directory', async () => { + for (const id of [ + 'bg-bounded-cleanup-a', + 'bg-bounded-cleanup-b', + 'bg-bounded-cleanup-c', + ]) { + const session = await createRunning(id) + await finishNaturally(session, { exitCode: 0 }) + } + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + maxDirectoryEntries: 1, + }), + ).toMatchObject({ sessionsRemoved: 1, errors: 0 }) + expect( + (await readdir(join(root, 'sessions'))).filter(name => + name.endsWith('.json'), + ), + ).toHaveLength(2) + }) + + it('removes old explicit-kill artifacts', async () => { + const session = await createBackgroundSession({ + id: 'bg-killed', + pid: nextPid++, + cwd: configDir, + command: [ + 'openclaude', + backgroundProcessMarkerToken(OLD_PROCESS_MARKER), + '--print', + 'killed generation', + ], + sessionId: 'killed-generation-conversation', + processMarker: OLD_PROCESS_MARKER, + }) + await markBackgroundSessionKilled(session.id, { now: OLD_FINISH }) + const killedFact = markedTerminalFactPath( + session.id, + 'killed', + OLD_PROCESS_MARKER, + ) + + const result = await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(result).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await exists(killedFact)).toBe(false) + }) + + it('retains recent completed sessions', async () => { + const session = await createRunning('bg-recent') + await finishNaturally(session, { exitCode: 0 }, RECENT_FINISH) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).natural)).toBe(true) + }) + + it('retains running, unknown, and stale sessions', async () => { + const running = await createRunning('bg-running') + const unknown = await createRunning('bg-unknown') + const stale = await createRunning('bg-stale') + await refreshBackgroundSessionStatuses({ + isProcessAlive: pid => pid !== stale.pid, + getProcessCommand: pid => + pid === running.pid ? running.command.join(' ') : null, + now: OLD_FINISH, + }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + for (const session of [running, unknown, stale]) { + expect(await exists(paths(session.id).metadata)).toBe(true) + } + }) + + it('retains completed metadata without a valid finishedAt', async () => { + const missing = await writeRawSession({ + id: 'bg-missing-finished', + status: 'exited', + }) + const invalid = await writeRawSession({ + id: 'bg-invalid-finished', + status: 'failed', + finishedAt: 'not-a-date', + }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(missing.id).metadata)).toBe(true) + expect(await exists(paths(invalid.id).metadata)).toBe(true) + }) + + it('retains completed metadata with noncanonical finishedAt values', async () => { + const sessions: BackgroundSession[] = [] + for (const [suffix, finishedAt] of NONCANONICAL_FINISHED_AT_CASES) { + const id = `bg-metadata-${suffix}` + const name = `metadata-${suffix}` + const session = await writeRawSession({ + id, + name, + status: 'running', + finishedAt, + }) + await writeReservation(name, id) + await writeFile( + paths(id).natural, + JSON.stringify({ + version: 1, + id, + pid: session.pid, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + sessions.push(session) + } + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + for (const session of sessions) { + for (const path of [ + paths(session.id).metadata, + paths(session.id).stdout, + paths(session.id).stderr, + paths(session.id).natural, + reservationPath(session.name!), + ]) { + expect(await exists(path)).toBe(true) + } + } + }) + + it('removes an old completed session with a valid process marker', async () => { + const session = await writeRawSession({ + id: 'bg-valid-process-marker', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + processMarker: 'a'.repeat(64), + }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 3, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(false) + }) + + it('retains metadata with an invalid process marker', async () => { + const session = await writeRawSession({ + id: 'bg-invalid-process-marker', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + processMarker: 'not-a-valid-marker', + }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).stdout)).toBe(true) + }) + + it('retains a completion exactly at the cutoff', async () => { + const session = await writeRawSession({ + id: 'bg-at-cutoff', + status: 'exited', + finishedAt: CUTOFF.toISOString(), + }) + + await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(await exists(paths(session.id).metadata)).toBe(true) + }) + + it('uses authoritative facts to clean stale metadata', async () => { + const session = await createRunning('bg-stale-with-fact') + await refreshBackgroundSessionStatuses({ + isProcessAlive: () => false, + now: new Date(OLD_FINISH.getTime() - 1_000), + }) + await finishNaturally(session, { exitCode: 0 }) + + const result = await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(result.sessionsRemoved).toBe(1) + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await exists(paths(session.id).natural)).toBe(false) + }) + + it('ignores malformed metadata without deleting same-looking logs', async () => { + const id = 'bg-malformed' + const idPaths = paths(id) + await mkdir(join(root, 'sessions'), { recursive: true }) + await mkdir(join(root, 'logs'), { recursive: true }) + await writeFile(idPaths.metadata, '{bad json') + await writeFile(idPaths.stdout, 'keep stdout') + await writeFile(idPaths.stderr, 'keep stderr') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(idPaths.stdout)).toBe(true) + expect(await exists(idPaths.stderr)).toBe(true) + }) + + it('does not follow symlinked metadata outside the registry root', async () => { + const id = 'bg-symlink-metadata' + const idPaths = paths(id) + const externalMetadata = join(configDir, 'external-metadata.json') + await mkdir(join(root, 'sessions'), { recursive: true }) + await mkdir(join(root, 'logs'), { recursive: true }) + await writeFile( + externalMetadata, + JSON.stringify({ + id, + pid: 123, + cwd: configDir, + status: 'exited', + sessionId: 'external-conversation', + startedAt: '2026-05-31T23:59:00.000Z', + updatedAt: OLD_FINISH.toISOString(), + finishedAt: OLD_FINISH.toISOString(), + command: ['openclaude'], + stdoutLogPath: idPaths.stdout, + stderrLogPath: idPaths.stderr, + }), + ) + await symlink(externalMetadata, idPaths.metadata) + await writeFile(idPaths.stdout, 'keep stdout') + await writeFile(idPaths.stderr, 'keep stderr') + + await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(await exists(externalMetadata)).toBe(true) + expect(await exists(idPaths.metadata)).toBe(true) + expect(await exists(idPaths.stdout)).toBe(true) + }) + + it('does not follow a symlinked metadata directory outside the registry root', async () => { + const id = 'bg-symlink-sessions-dir' + const idPaths = paths(id) + const externalSessions = join(configDir, 'external-sessions') + await mkdir(externalSessions, { recursive: true }) + await mkdir(join(root, 'logs'), { recursive: true }) + await mkdir(join(root, 'terminal'), { recursive: true }) + await mkdir(join(root, 'names'), { recursive: true }) + await writeFile( + join(externalSessions, `${id}.json`), + JSON.stringify({ + id, + pid: 123, + cwd: configDir, + status: 'exited', + sessionId: 'external-conversation', + startedAt: '2026-05-31T23:59:00.000Z', + updatedAt: OLD_FINISH.toISOString(), + finishedAt: OLD_FINISH.toISOString(), + command: ['openclaude'], + stdoutLogPath: idPaths.stdout, + stderrLogPath: idPaths.stderr, + }), + ) + await symlink(externalSessions, join(root, 'sessions'), 'dir') + await writeFile(idPaths.stdout, 'keep stdout') + await writeFile(idPaths.stderr, 'keep stderr') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 1, + }) + expect(await exists(join(externalSessions, `${id}.json`))).toBe(true) + expect(await exists(idPaths.stdout)).toBe(true) + expect(await exists(idPaths.stderr)).toBe(true) + }) + + it('does not reconcile through symlinked metadata outside the registry root', async () => { + const id = 'bg-symlink-reconciliation' + const idPaths = paths(id) + const externalMetadata = join(configDir, 'external-reconciliation.json') + await mkdir(join(root, 'sessions'), { recursive: true }) + await mkdir(join(root, 'terminal'), { recursive: true }) + const stored = { + id, + pid: 124, + cwd: configDir, + status: 'stale', + sessionId: 'external-reconciliation-conversation', + processMarker: OLD_PROCESS_MARKER, + terminalFactGeneration: OLD_PROCESS_MARKER, + startedAt: '2026-05-31T23:59:00.000Z', + updatedAt: OLD_FINISH.toISOString(), + command: ['openclaude'], + stdoutLogPath: idPaths.stdout, + stderrLogPath: idPaths.stderr, + } + await writeFile(externalMetadata, JSON.stringify(stored)) + await symlink(externalMetadata, idPaths.metadata) + await writeFile( + markedTerminalFactPath(id, 'natural', OLD_PROCESS_MARKER), + JSON.stringify({ + version: 1, + id, + pid: stored.pid, + generation: OLD_PROCESS_MARKER, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 0, + errors: 0, + }) + expect(JSON.parse(await readFile(externalMetadata, 'utf8'))).toEqual( + stored, + ) + expect((await lstat(idPaths.metadata)).isSymbolicLink()).toBe(true) + }) + + it('does not reconcile through a symlinked metadata directory', async () => { + const id = 'bg-symlink-reconciliation-dir' + const externalSessions = join(configDir, 'external-reconciliation-sessions') + const externalMetadata = join(externalSessions, `${id}.json`) + await mkdir(externalSessions, { recursive: true }) + await mkdir(join(root, 'terminal'), { recursive: true }) + const stored = { + id, + pid: 125, + cwd: configDir, + status: 'stale', + sessionId: 'external-reconciliation-dir-conversation', + processMarker: OLD_PROCESS_MARKER, + terminalFactGeneration: OLD_PROCESS_MARKER, + startedAt: '2026-05-31T23:59:00.000Z', + updatedAt: OLD_FINISH.toISOString(), + command: ['openclaude'], + stdoutLogPath: paths(id).stdout, + stderrLogPath: paths(id).stderr, + } + await writeFile(externalMetadata, JSON.stringify(stored)) + await symlink(externalSessions, join(root, 'sessions'), 'dir') + await writeFile( + markedTerminalFactPath(id, 'natural', OLD_PROCESS_MARKER), + JSON.stringify({ + version: 1, + id, + pid: stored.pid, + generation: OLD_PROCESS_MARKER, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 0, + errors: 1, + }) + expect(JSON.parse(await readFile(externalMetadata, 'utf8'))).toEqual( + stored, + ) + }) + + it('does not follow symlinked artifact directories outside the registry root', async () => { + const externalLogs = join(configDir, 'external-logs') + const session = await writeRawSession({ + id: 'bg-symlink-logs-dir', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + createLogs: false, + }) + await rm(join(root, 'logs'), { recursive: true, force: true }) + await mkdir(externalLogs, { recursive: true }) + await writeFile(join(externalLogs, `${session.id}.out.log`), 'outside') + await writeFile(join(externalLogs, `${session.id}.err.log`), 'outside') + await symlink(externalLogs, join(root, 'logs'), 'dir') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 1, + }) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect( + await exists(join(externalLogs, `${session.id}.out.log`)), + ).toBe(true) + expect( + await exists(join(externalLogs, `${session.id}.err.log`)), + ).toBe(true) + }) + + it('stops when the metadata directory identity changes during cleanup', async () => { + const session = await writeRawSession({ + id: 'bg-sessions-dir-swapped', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + const sessionsDir = join(root, 'sessions') + const externalSessions = join(configDir, 'swapped-sessions') + await mkdir(externalSessions, { recursive: true }) + await writeFile( + join(externalSessions, `${session.id}.json`), + await readFile(paths(session.id).metadata, 'utf8'), + ) + let swapped = false + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + _beforeMetadataDirectoryReadForTesting: async () => { + swapped = true + await rm(sessionsDir, { recursive: true, force: true }) + await symlink(externalSessions, sessionsDir, 'dir') + }, + }) + + expect(result).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 1, + }) + expect(swapped).toBe(true) + expect( + await exists(join(externalSessions, `${session.id}.json`)), + ).toBe(true) + expect(await exists(paths(session.id).stdout)).toBe(true) + }) + + it('never trusts persisted external log paths', async () => { + const externalStdout = join(configDir, 'external.out.log') + const externalStderr = join(configDir, 'external.err.log') + await writeFile(externalStdout, 'external stdout') + await writeFile(externalStderr, 'external stderr') + const session = await writeRawSession({ + id: 'bg-external-logs', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + stdoutLogPath: externalStdout, + stderrLogPath: externalStderr, + }) + + await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await Bun.file(externalStdout).text()).toBe('external stdout') + expect(await Bun.file(externalStderr).text()).toBe('external stderr') + }) + + it('unlinks symlinked expected logs without touching their targets', async () => { + const id = 'bg-symlink-log' + const external = join(configDir, 'external-target.log') + await writeFile(external, 'outside') + const session = await writeRawSession({ + id, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + createLogs: false, + }) + await symlink(external, paths(id).stdout) + await writeFile(paths(id).stderr, 'stderr') + + await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(await exists(paths(session.id).stdout)).toBe(false) + expect(await Bun.file(external).text()).toBe('outside') + }) + + it('preserves malformed, wrong-owner, and symlinked terminal facts', async () => { + const malformed = await writeRawSession({ + id: 'bg-malformed-fact', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + await writeFile(paths(malformed.id).killed, '{bad json') + + const wrongOwner = await writeRawSession({ + id: 'bg-wrong-owner-fact', + status: 'failed', + finishedAt: OLD_FINISH.toISOString(), + }) + await writeFile( + paths(wrongOwner.id).natural, + JSON.stringify({ + version: 1, + id: wrongOwner.id, + pid: wrongOwner.pid + 1, + status: 'failed', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 1, + }), + ) + + const symlinked = await writeRawSession({ + id: 'bg-symlink-fact', + status: 'killed', + finishedAt: OLD_FINISH.toISOString(), + }) + const externalFact = join(configDir, 'external-fact.json') + await writeFile( + externalFact, + JSON.stringify({ + version: 1, + id: symlinked.id, + pid: symlinked.pid, + status: 'killed', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'explicit_kill', + }), + ) + await symlink(externalFact, paths(symlinked.id).killed) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + for (const session of [malformed, wrongOwner, symlinked]) { + expect(await exists(paths(session.id).metadata)).toBe(true) + } + expect(await exists(externalFact)).toBe(true) + }) + + it('preserves terminal facts with invalid timestamps', async () => { + const session = await writeRawSession({ + id: 'bg-invalid-fact-time', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + await writeFile( + paths(session.id).natural, + JSON.stringify({ + version: 1, + id: session.id, + pid: session.pid, + status: 'exited', + finishedAt: 'not-a-date', + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).natural)).toBe(true) + }) + + it('preserves terminal facts with noncanonical timestamps', async () => { + const sessions: BackgroundSession[] = [] + for (const [suffix, finishedAt] of NONCANONICAL_FINISHED_AT_CASES) { + for (const malformedKind of ['natural', 'killed'] as const) { + const id = `bg-${malformedKind}-${suffix}` + const name = `${malformedKind}-${suffix}` + const session = await writeRawSession({ id, name, status: 'running' }) + await writeReservation(name, id) + await writeFile( + paths(id).natural, + JSON.stringify({ + version: 1, + id, + pid: session.pid, + status: 'exited', + finishedAt: + malformedKind === 'natural' + ? finishedAt + : OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + await writeFile( + paths(id).killed, + JSON.stringify({ + version: 1, + id, + pid: session.pid, + status: 'killed', + finishedAt: + malformedKind === 'killed' + ? finishedAt + : OLD_FINISH.toISOString(), + terminalReason: 'explicit_kill', + }), + ) + sessions.push(session) + } + } + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + for (const session of sessions) { + for (const path of [ + ...Object.values(paths(session.id)), + reservationPath(session.name!), + ]) { + expect(await exists(path)).toBe(true) + } + } + }) + + it('preserves orphaned terminal facts with noncanonical timestamps', async () => { + await mkdir(join(root, 'terminal'), { recursive: true }) + const factPaths: string[] = [] + for (const [suffix, finishedAt] of NONCANONICAL_FINISHED_AT_CASES) { + for (const kind of ['natural', 'killed'] as const) { + const id = `bg-orphan-${kind}-${suffix}` + const factPath = paths(id)[kind] + await writeFile( + factPath, + JSON.stringify({ + version: 1, + id, + pid: nextPid++, + status: kind === 'natural' ? 'exited' : 'killed', + finishedAt, + terminalReason: + kind === 'natural' ? 'exit_code' : 'explicit_kill', + ...(kind === 'natural' ? { exitCode: 0 } : {}), + }), + ) + factPaths.push(factPath) + } + } + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + for (const factPath of factPaths) { + expect(await exists(factPath)).toBe(true) + } + }) + + it('bounds orphan recovery when the metadata directory is absent', async () => { + await mkdir(join(root, 'terminal'), { recursive: true }) + for (const id of [ + 'bg-bounded-orphan-a', + 'bg-bounded-orphan-b', + 'bg-bounded-orphan-c', + ]) { + await writeFile( + paths(id).natural, + JSON.stringify({ + version: 1, + id, + pid: nextPid++, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + } + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + maxDirectoryEntries: 1, + }), + ).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 1, + errors: 0, + }) + expect( + (await readdir(join(root, 'terminal'))).filter(name => + name.endsWith('.json'), + ), + ).toHaveLength(2) + }) + + it('does not broaden targeted cleanup into orphan recovery', async () => { + const orphanId = 'bg-unrelated-targeted-orphan' + await mkdir(join(root, 'terminal'), { recursive: true }) + await writeFile( + paths(orphanId).natural, + JSON.stringify({ + version: 1, + id: orphanId, + pid: nextPid++, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + sessionIds: ['bg-missing-target'], + }), + ).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(orphanId).natural)).toBe(true) + }) + + it('skips orphan recovery after inspecting an existing target', async () => { + const target = await createRunning('bg-existing-target') + const orphanId = 'bg-unrelated-existing-target-orphan' + await writeFile( + paths(orphanId).natural, + JSON.stringify({ + version: 1, + id: orphanId, + pid: nextPid++, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + sessionIds: [target.id], + }), + ).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(target.id).metadata)).toBe(true) + expect(await exists(paths(orphanId).natural)).toBe(true) + }) + + it('preserves an orphaned fact whose generation does not match its path', async () => { + await mkdir(join(root, 'terminal'), { recursive: true }) + const id = 'bg-orphan-generation-mismatch' + const factPath = markedTerminalFactPath( + id, + 'natural', + OLD_PROCESS_MARKER, + ) + await writeFile( + factPath, + JSON.stringify({ + version: 1, + id, + pid: nextPid++, + generation: REPLACEMENT_PROCESS_MARKER, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(factPath)).toBe(true) + }) + + it('treats missing artifacts as idempotent success', async () => { + const session = await writeRawSession({ + id: 'bg-missing-artifacts', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + createLogs: false, + }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 1, + errors: 0, + }) + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(false) + }) + + it('counts one unlink failure and continues with unrelated sessions', async () => { + const blocked = await createRunning('bg-blocked-unlink') + await finishNaturally(blocked, { exitCode: 0 }) + const removable = await createRunning('bg-other-removable') + await finishNaturally(removable, { exitCode: 0 }) + const blockedStdout = paths(blocked.id).stdout + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === blockedStdout) throw deniedError() + await unlink(path) + }, + }) + + expect(result.errors).toBe(1) + expect(result.sessionsRemoved).toBe(1) + expect(await exists(paths(blocked.id).metadata)).toBe(true) + expect(await exists(paths(blocked.id).natural)).toBe(true) + expect(await exists(paths(removable.id).metadata)).toBe(false) + + expect((await cleanupBackgroundSessionsBefore(CUTOFF)).errors).toBe(0) + expect(await exists(paths(blocked.id).metadata)).toBe(false) + }) + + it('counts a metadata read error and continues with unrelated sessions', async () => { + const blocked = await writeRawSession({ + id: 'bg-metadata-read-error', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + const removable = await writeRawSession({ + id: 'bg-after-metadata-read-error', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + const blockedMetadata = paths(blocked.id).metadata + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + readTextFile: async path => { + if (path === blockedMetadata) throw deniedError() + return await readFile(path, 'utf8') + }, + }) + + expect(result.errors).toBe(1) + expect(result.sessionsRemoved).toBe(1) + expect(await exists(blockedMetadata)).toBe(true) + expect(await exists(paths(blocked.id).stdout)).toBe(true) + expect(await exists(paths(removable.id).metadata)).toBe(false) + }) + + it('counts a terminal-fact read error and retains the session', async () => { + const session = await createRunning('bg-fact-read-error') + await finishNaturally(session, { exitCode: 0 }) + const naturalFact = paths(session.id).natural + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + readTextFile: async path => { + if (path === naturalFact) throw deniedError() + return await readFile(path, 'utf8') + }, + }) + + expect(result.errors).toBe(1) + expect(result.sessionsRemoved).toBe(0) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(naturalFact)).toBe(true) + }) + + it('keeps authoritative facts retryable after metadata unlink fails', async () => { + const session = await createRunning('bg-metadata-retry') + await finishNaturally(session, { exitCode: 0 }) + const metadataPath = paths(session.id).metadata + + const first = await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === metadataPath) throw deniedError() + await unlink(path) + }, + }) + + expect(first.errors).toBe(1) + expect(first.sessionsRemoved).toBe(0) + expect(await exists(metadataPath)).toBe(true) + expect(await exists(paths(session.id).natural)).toBe(true) + + const retry = await cleanupBackgroundSessionsBefore(CUTOFF) + expect(retry.sessionsRemoved).toBe(1) + expect(await exists(metadataPath)).toBe(false) + expect(await exists(paths(session.id).natural)).toBe(false) + }) + + it('reclaims an old terminal fact after metadata-first partial cleanup', async () => { + const session = await createRunning('bg-terminal-fact-retry') + await finishNaturally(session, { exitCode: 0 }) + const naturalFact = paths(session.id).natural + + const first = await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === naturalFact) throw deniedError() + await unlink(path) + }, + }) + + expect(first.errors).toBe(1) + expect(first.sessionsRemoved).toBe(1) + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await exists(naturalFact)).toBe(true) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 1, + errors: 0, + }) + expect(await exists(naturalFact)).toBe(false) + }) + + for (const legacyFact of [false, true]) { + const suffix = legacyFact ? 'legacy' : 'marked' + it(`isolates a retained ${suffix} terminal fact from a same-ID replacement generation`, async () => { + const id = `bg-terminal-fact-generation-${suffix}` + const oldSession = await createBackgroundSession({ + id, + pid: nextPid++, + cwd: configDir, + command: [ + 'openclaude', + backgroundProcessMarkerToken(OLD_PROCESS_MARKER), + '--print', + 'old generation', + ], + sessionId: `old-generation-${suffix}`, + processMarker: OLD_PROCESS_MARKER, + }) + const retainedFact = legacyFact + ? paths(id).natural + : markedTerminalFactPath(id, 'natural', OLD_PROCESS_MARKER) + if (legacyFact) { + const metadata = JSON.parse( + await readFile(paths(id).metadata, 'utf8'), + ) as Record + delete metadata.terminalFactGeneration + await writeFile(paths(id).metadata, JSON.stringify(metadata)) + await writeFile( + retainedFact, + JSON.stringify({ + version: 1, + id, + pid: oldSession.pid, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + } else { + await finishNaturally(oldSession, { exitCode: 0 }) + } + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === retainedFact) throw deniedError() + await unlink(path) + }, + }), + ).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 3, + errors: 1, + }) + expect(await exists(paths(id).metadata)).toBe(false) + expect(await exists(retainedFact)).toBe(true) + + const replacement = await createBackgroundSession({ + id, + pid: nextPid++, + cwd: configDir, + command: [ + 'openclaude', + backgroundProcessMarkerToken(REPLACEMENT_PROCESS_MARKER), + '--print', + 'replacement generation', + ], + sessionId: `replacement-generation-${suffix}`, + processMarker: REPLACEMENT_PROCESS_MARKER, + }) + await recordBackgroundSessionNaturalTermination( + replacement.id, + { exitCode: 23 }, + { + ownerPid: replacement.pid, + now: OLD_FINISH, + }, + ) + + expect(await resolveBackgroundSession(id)).toMatchObject({ + id, + pid: replacement.pid, + status: 'failed', + finishedAt: OLD_FINISH.toISOString(), + exitCode: 23, + }) + const replacementFact = markedTerminalFactPath( + id, + 'natural', + REPLACEMENT_PROCESS_MARKER, + ) + expect(await exists(retainedFact)).toBe(true) + expect(await exists(replacementFact)).toBe(true) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + expect(await exists(paths(id).metadata)).toBe(false) + expect(await exists(replacementFact)).toBe(false) + expect(await exists(retainedFact)).toBe(true) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 1, + errors: 0, + }) + expect(await exists(retainedFact)).toBe(false) + }) + } + + it('reclaims an old killed fact after metadata-first partial cleanup', async () => { + const session = await createRunning('bg-killed-fact-retry') + await markBackgroundSessionKilled(session.id, { now: OLD_FINISH }) + const killedFact = paths(session.id).killed + + const first = await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === killedFact) throw deniedError() + await unlink(path) + }, + }) + + expect(first.errors).toBe(1) + expect(first.sessionsRemoved).toBe(1) + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await exists(killedFact)).toBe(true) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 1, + errors: 0, + }) + expect(await exists(killedFact)).toBe(false) + }) + + it('retains recent and malformed orphaned terminal facts', async () => { + const recent = await createRunning('bg-recent-orphan') + await finishNaturally(recent, { exitCode: 0 }, RECENT_FINISH) + await unlink(paths(recent.id).metadata) + + const malformed = await createRunning('bg-malformed-orphan') + await finishNaturally(malformed, { exitCode: 0 }) + await unlink(paths(malformed.id).metadata) + await writeFile(paths(malformed.id).natural, '{bad json') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(recent.id).natural)).toBe(true) + expect(await exists(paths(malformed.id).natural)).toBe(true) + }) + + it('reclaims eligible orphaned facts beyond a retained prefix in one full sweep', async () => { + const terminalDir = join(root, 'terminal') + await mkdir(terminalDir, { recursive: true }) + const preservedFacts = Array.from({ length: 256 }, (_, index) => { + const id = `bg-preserved-orphan-${index.toString().padStart(3, '0')}` + return writeFile( + paths(id).natural, + JSON.stringify({ + version: 1, + id, + pid: nextPid++, + status: 'exited', + finishedAt: RECENT_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + }) + await Promise.all(preservedFacts) + const oldId = 'zz-bg-old-orphan' + await writeFile( + paths(oldId).natural, + JSON.stringify({ + version: 1, + id: oldId, + pid: nextPid++, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 1, + errors: 0, + }) + expect(await exists(paths(oldId).natural)).toBe(false) + }) + + it('removes only a reservation that still belongs to the session', async () => { + const matching = await createRunning('bg-matching-name', { + name: 'matching-name', + }) + await finishNaturally(matching, { exitCode: 0 }) + await writeReservation(matching.name!, matching.id) + + const reassigned = await createRunning('bg-reassigned-name', { + name: 'reassigned-name', + }) + await finishNaturally(reassigned, { exitCode: 0 }) + await writeReservation(reassigned.name!, 'bg-new-owner') + + await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(await exists(reservationPath(matching.name!))).toBe(false) + expect(await exists(reservationPath(reassigned.name!))).toBe(true) + expect(await exists(paths(reassigned.id).metadata)).toBe(false) + }) + + it('does not remove a reservation replaced during artifact cleanup', async () => { + const session = await createRunning('bg-reservation-replaced', { + name: 'reservation-replaced', + }) + await finishNaturally(session, { exitCode: 0 }) + await writeReservation(session.name!, session.id) + let replaced = false + + await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + await unlink(path) + if (!replaced && path === paths(session.id).stdout) { + replaced = true + await writeReservation(session.name!, 'bg-new-owner') + } + }, + }) + + expect( + JSON.parse(await readFile(reservationPath(session.name!), 'utf8')), + ).toMatchObject({ id: 'bg-new-owner' }) + expect(await exists(paths(session.id).metadata)).toBe(false) + }) + + it('does not remove logs created by a same-ID replacement', async () => { + const session = await createRunning('bg-log-generation-replacement') + await finishNaturally(session, { exitCode: 0 }) + let hookCalls = 0 + let replacement: BackgroundSession | undefined + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + _beforeArtifactRemovalForTesting: async id => { + if (id !== session.id || hookCalls++ > 0) return + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + sessionIds: [session.id], + }), + ).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + replacement = await createBackgroundSession({ + id: session.id, + pid: nextPid++, + cwd: configDir, + command: ['openclaude', '--print', 'replacement'], + sessionId: 'replacement-log-conversation', + }) + await writeFile(paths(session.id).stdout, 'replacement stdout') + await writeFile(paths(session.id).stderr, 'replacement stderr') + }, + }) + + expect(hookCalls).toBe(1) + expect(replacement).toBeDefined() + expect(result).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await readFile(paths(session.id).stdout, 'utf8')).toBe( + 'replacement stdout', + ) + expect(await readFile(paths(session.id).stderr, 'utf8')).toBe( + 'replacement stderr', + ) + expect( + JSON.parse(await readFile(paths(session.id).metadata, 'utf8')), + ).toMatchObject({ + id: replacement?.id, + pid: replacement?.pid, + sessionId: replacement?.sessionId, + }) + }) + + it( + 'serializes cleanup with a replacement using the supported name writer', + async () => { + const session = await createRunning('bg-reservation-generation', { + name: 'reservation-generation', + }) + await finishNaturally(session, { exitCode: 0 }) + await writeReservation(session.name!, session.id) + const target = reservationPath(session.name!) + let replacementPromise: Promise | undefined + let replacementStarted = false + let cleanupLockAcquired = false + let cleanupLockReleased = false + let replacementLockAttempted = false + let targetLockCalls = 0 + let signalCleanupLockRelease!: () => void + const cleanupLockRelease = new Promise(resolve => { + signalCleanupLockRelease = resolve + }) + let signalReplacementLockAttempt!: () => void + const replacementLockAttempt = new Promise(resolve => { + signalReplacementLockAttempt = resolve + }) + const originalLock = nameReservationLock.lock + const lockSpy = spyOn(nameReservationLock, 'lock').mockImplementation( + async (path, options) => { + if (path !== target) return await originalLock(path, options) + targetLockCalls++ + if (!replacementStarted) { + cleanupLockAcquired = true + const release = await originalLock(path, options) + return async () => { + try { + await release() + } finally { + cleanupLockReleased = true + signalCleanupLockRelease() + } + } + } + + replacementLockAttempted = true + signalReplacementLockAttempt() + await cleanupLockRelease + return await originalLock(path, options) + }, + ) + + const result = await (async () => { + try { + return await cleanupBackgroundSessionsBefore(CUTOFF, { + _beforeReservationRemovalForTesting: async path => { + if (path !== target) return + if (!cleanupLockAcquired) { + throw new Error( + 'cleanup did not acquire the name lock before reservation removal', + ) + } + replacementStarted = true + replacementPromise = createBackgroundSession({ + id: 'bg-new-generation', + name: session.name, + pid: nextPid++, + cwd: configDir, + command: ['openclaude', '--print', 'replacement'], + sessionId: 'replacement-conversation', + }) + let timeout: ReturnType | undefined + try { + await Promise.race([ + replacementLockAttempt, + new Promise((_, reject) => { + timeout = setTimeout( + () => + reject( + new Error('replacement lock barrier timed out'), + ), + 5_000, + ) + timeout.unref?.() + }), + ]) + } finally { + if (timeout !== undefined) clearTimeout(timeout) + } + }, + }) + } finally { + lockSpy.mockRestore() + } + })() + + expect({ + cleanupLockAcquired, + cleanupLockReleased, + replacementLockAttempted, + replacementStarted, + targetLockCalls, + }).toEqual({ + cleanupLockAcquired: true, + cleanupLockReleased: true, + replacementLockAttempted: true, + replacementStarted: true, + targetLockCalls: 2, + }) + expect(result).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 5, + errors: 0, + }) + expect(replacementPromise).toBeDefined() + if (!replacementPromise) { + throw new Error('replacement writer did not run') + } + const replacement = await replacementPromise + expect( + JSON.parse(await readFile(target, 'utf8')), + ).toMatchObject({ id: replacement.id }) + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await exists(paths(replacement.id).metadata)).toBe(true) + }, + 10_000, + ) + + it('allows a replacement to claim the name after cleanup', async () => { + const session = await createRunning('bg-reservation-final-window', { + name: 'reservation-final-window', + }) + await finishNaturally(session, { exitCode: 0 }) + await writeReservation(session.name!, session.id) + const target = reservationPath(session.name!) + + await cleanupBackgroundSessionsBefore(CUTOFF) + const replacement = await createBackgroundSession({ + id: 'bg-reservation-final-replacement', + name: session.name, + pid: nextPid++, + cwd: configDir, + command: ['openclaude', '--print', 'replacement'], + sessionId: 'replacement-conversation', + }) + + expect( + JSON.parse(await readFile(target, 'utf8')), + ).toMatchObject({ id: replacement.id }) + }) + + it('counts reservation unlink failure and keeps metadata for retry', async () => { + const session = await createRunning('bg-reservation-error', { + name: 'reservation-error', + }) + await finishNaturally(session, { exitCode: 0 }) + await writeReservation(session.name!, session.id) + const target = reservationPath(session.name!) + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === target) throw deniedError() + await unlink(path) + }, + }) + + expect(result.errors).toBe(1) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).natural)).toBe(true) + expect(await exists(target)).toBe(true) + }) + + it('counts a reservation read error and retains the session', async () => { + const session = await createRunning('bg-reservation-read-error', { + name: 'reservation-read-error', + }) + await finishNaturally(session, { exitCode: 0 }) + await writeReservation(session.name!, session.id) + const target = reservationPath(session.name!) + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + readTextFile: async path => { + if (path === target) throw deniedError() + return await readFile(path, 'utf8') + }, + }) + + expect(result.errors).toBe(1) + expect(result.sessionsRemoved).toBe(0) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).natural)).toBe(true) + expect(await exists(target)).toBe(true) + }) + + it('preserves a malformed reservation while cleaning owned artifacts', async () => { + const session = await createRunning('bg-malformed-reservation', { + name: 'malformed-reservation', + }) + await finishNaturally(session, { exitCode: 0 }) + const target = reservationPath(session.name!) + await writeFile(target, '{bad json') + + const result = await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(result).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + expect(await exists(target)).toBe(true) + expect(await exists(paths(session.id).metadata)).toBe(false) + }) + + it('reports sessions-directory errors without throwing', async () => { + await mkdir(root, { recursive: true }) + await writeFile(join(root, 'sessions'), 'not a directory') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 1, + }) + }) + + it('treats a missing sessions directory as an empty registry', async () => { + await mkdir(root, { recursive: true }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + }) + + it('retains eligible sessions when the terminal directory is invalid', async () => { + const session = await writeRawSession({ + id: 'bg-invalid-terminal-directory', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + await rm(join(root, 'terminal'), { recursive: true, force: true }) + await writeFile(join(root, 'terminal'), 'not a directory') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 1, + }) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).stdout)).toBe(true) + }) + + it('cleans an unnamed session when only the names directory is invalid', async () => { + const session = await writeRawSession({ + id: 'bg-invalid-names-directory', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + await rm(join(root, 'names'), { recursive: true, force: true }) + await writeFile(join(root, 'names'), 'not a directory') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 3, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(false) + }) +}) diff --git a/src/cli/bgRegistry.test.ts b/src/cli/bgRegistry.test.ts index 90983089a7..b9a6069a9b 100644 --- a/src/cli/bgRegistry.test.ts +++ b/src/cli/bgRegistry.test.ts @@ -1,18 +1,30 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import { createHash } from 'node:crypto' -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { + mkdir, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import * as lockfile from '../utils/lockfile.js' import * as backgroundSessionRegistry from './bgRegistry.js' import { _setBackgroundSessionsRootForTesting, + cleanupBackgroundSessionsBefore, createBackgroundSession, isBackgroundSessionProcessAlive, isTerminalBackgroundSession, listBackgroundSessions, markBackgroundSessionKilled, + readBackgroundSessionForOwner, refreshBackgroundSessionStatuses, resolveBackgroundSession, + snapshotBackgroundSessionRecoveryJournal, + takeBackgroundSessionRecoveryBatch, verifyBackgroundSessionProcessIdentity, type BackgroundSession, } from './bgRegistry.js' @@ -27,6 +39,7 @@ const backgroundProcessMarkerToken = (marker: string) => `--openclaude-bg-session-marker=${marker}` const { + reconcileBackgroundSessionTerminalFacts, recordBackgroundSessionNaturalTermination, recordBackgroundSessionNaturalTerminationSync, } = backgroundSessionRegistry @@ -42,8 +55,15 @@ describe('background session registry', () => { function terminalFactPath( id: string, kind: 'natural' | 'killed', + generation?: string, ): string { - return join(configDir, 'bg-sessions', 'terminal', `${id}.${kind}.json`) + const generationSuffix = generation ? `~${generation}` : '' + return join( + configDir, + 'bg-sessions', + 'terminal', + `${id}${generationSuffix}.${kind}.json`, + ) } async function writeTerminalFact( @@ -114,6 +134,7 @@ describe('background session registry', () => { model: 'gpt-5', sessionId: 'conversation-1', processMarker: TEST_PROCESS_MARKER, + terminalFactGeneration: TEST_PROCESS_MARKER, startedAt: '2026-06-15T08:00:00.000Z', updatedAt: '2026-06-15T08:00:00.000Z', command: [ @@ -158,6 +179,7 @@ describe('background session registry', () => { const [session] = await listBackgroundSessions() expect(session?.id).toBe('bg-legacy') expect(session?.processMarker).toBeUndefined() + expect(session?.terminalFactGeneration).toBeUndefined() }) it('rejects malformed process markers on creation', async () => { @@ -211,6 +233,28 @@ describe('background session registry', () => { ) }), ) + await writeFile( + join( + configDir, + 'bg-sessions', + 'sessions', + 'bg-mismatched-terminal-generation.json', + ), + JSON.stringify({ + id: 'bg-mismatched-terminal-generation', + pid: 999, + cwd: '/repo', + status: 'running', + sessionId: 'conversation-mismatched-terminal-generation', + processMarker: TEST_PROCESS_MARKER, + terminalFactGeneration: OTHER_PROCESS_MARKER, + startedAt: '2026-06-15T08:00:00.000Z', + updatedAt: '2026-06-15T08:00:00.000Z', + command: ['openclaude', '--print', 'work'], + stdoutLogPath: '/tmp/stdout.log', + stderrLogPath: '/tmp/stderr.log', + }), + ) expect(await listBackgroundSessions()).toEqual([]) }) @@ -572,6 +616,27 @@ describe('background session registry', () => { expect((await resolveBackgroundSession('bg-collision')).name).toBe('first') }) + it('keeps a markerless id unavailable while a terminal fact remains', async () => { + await writeTerminalFact('bg-retained-fact', 'natural', { + pid: 111, + status: 'exited', + finishedAt: '2026-06-15T08:04:00.000Z', + terminalReason: 'exit_code', + exitCode: 0, + }) + + await expect( + createBackgroundSession({ + id: 'bg-retained-fact', + pid: 222, + cwd: '/repo', + command: ['openclaude', '--print', 'replacement'], + sessionId: 'conversation-replacement', + }), + ).rejects.toThrow('already exists') + expect(await listBackgroundSessions()).toEqual([]) + }) + it('rejects non-positive pids at creation', async () => { await expect( createBackgroundSession({ @@ -975,6 +1040,294 @@ describe('background session registry', () => { }) }) + it('does not follow a symlinked recovery journal after terminal persistence', async () => { + const session = await createBackgroundSession({ + id: 'bg-symlinked-recovery-journal', + pid: 451, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: 'conversation-symlinked-recovery-journal', + }) + const outside = join(configDir, 'outside-recovery-journal') + await writeFile(outside, 'keep') + await symlink( + outside, + join(configDir, 'bg-sessions', '.recovery-journal'), + ) + + expect( + await recordBackgroundSessionNaturalTermination( + session.id, + { exitCode: 0 }, + { ownerPid: session.pid }, + ), + ).toMatchObject({ status: 'exited' }) + expect(await readFile(outside, 'utf8')).toBe('keep') + }) + + it('queues async terminal persistence for bounded recovery', async () => { + const session = await createBackgroundSession({ + id: 'bg-async-recovery-queue', + pid: 452, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: 'conversation-async-recovery-queue', + }) + await recordBackgroundSessionNaturalTermination( + session.id, + { exitCode: 0 }, + { ownerPid: session.pid }, + ) + + const batch = await takeBackgroundSessionRecoveryBatch(1) + expect(batch.sessionIds).toEqual([session.id]) + await batch.commit() + expect( + (await takeBackgroundSessionRecoveryBatch(1)).sessionIds, + ).toEqual([]) + }) + + it('persists only owned retries behind new work before acknowledging a batch', async () => { + const root = join(configDir, 'bg-sessions') + await mkdir(root, { recursive: true }) + const journal = join(root, '.recovery-journal') + await writeFile(journal, 'bg-retry\nbg-done\nbg-next\n') + const batch = await takeBackgroundSessionRecoveryBatch(2) + await batch.commit(['bg-retry', 'bg-foreign', 'bg-retry', '../outside']) + const contents = await readFile(journal, 'utf8') + expect(contents).toBe('bg-retry\nbg-done\nbg-next\nbg-retry\n') + const cursor = JSON.parse( + await readFile(join(root, '.recovery-cursor.json'), 'utf8'), + ) + expect(cursor.offset).toBe(Buffer.byteLength('bg-retry\nbg-done\n')) + await batch.commit(['bg-retry']) + expect(await readFile(journal, 'utf8')).toBe(contents) + + const next = await takeBackgroundSessionRecoveryBatch(2) + expect(next.sessionIds).toEqual(['bg-next', 'bg-retry']) + await next.commit(['bg-retry']) + // Repeated failures rotate the consumed prefix instead of growing forever. + expect(await readFile(journal, 'utf8')).toBe('bg-retry\n') + const retry = await takeBackgroundSessionRecoveryBatch(2) + expect(retry.sessionIds).toEqual(['bg-retry']) + await retry.commit() + expect(await readFile(journal, 'utf8')).toBe('') + }) + + it('retains validated generations for terminal retries after metadata removal', async () => { + const root = join(configDir, 'bg-sessions') + await mkdir(root, { recursive: true }) + const journal = join(root, '.recovery-journal') + const generation = 'a'.repeat(64) + await writeFile(journal, 'bg-fact-retry\n') + const batch = await takeBackgroundSessionRecoveryBatch(1) + await batch.commit( + ['bg-fact-retry'], + [ + { id: 'bg-fact-retry', generation }, + { id: 'bg-foreign', generation }, + { id: 'bg-fact-retry', generation: '../outside' }, + ], + ) + const retry = await takeBackgroundSessionRecoveryBatch(256) + expect(retry.sessionIds).toEqual(['bg-fact-retry']) + expect(retry.terminalFacts).toEqual([ + { id: 'bg-fact-retry', generation: undefined }, + { id: 'bg-fact-retry', generation }, + ]) + await retry.commit(['bg-fact-retry']) + expect( + (await takeBackgroundSessionRecoveryBatch(256)).terminalFacts, + ).toEqual(retry.terminalFacts) + await writeFile( + journal, + `bg-bad~invalid\nbg-bad~${generation}~extra\nbg-good\n`, + ) + expect( + (await takeBackgroundSessionRecoveryBatch(256)).sessionIds, + ).toEqual(['bg-good']) + }) + + for (const synchronous of [false, true]) { + it(`journals marked ownership before interrupted cleanup, sync ${synchronous}`, async () => { + const id = 'bg-marked-interruption' + const session = await createBackgroundSession({ + id, + pid: 452, + cwd: '/repo', + command: ['openclaude', '--print', 'fixture'], + sessionId: 'marked-interruption', + processMarker: TEST_PROCESS_MARKER, + }) + const options = { + ownerPid: session.pid, + expectedSession: session, + now: new Date(0), + } + if (synchronous) + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 0 }, + options, + ) + else + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + options, + ) + const batch = await takeBackgroundSessionRecoveryBatch(256) + expect(batch.terminalFacts).toEqual([ + { id, generation: TEST_PROCESS_MARKER }, + ]) + const root = join(configDir, 'bg-sessions') + // Simulate interruption after logs/metadata were removed, before the fact unlink. + await rm(join(root, 'sessions', `${id}.json`)) + await rm(session.stdoutLogPath, { force: true }) + await rm(session.stderrLogPath, { force: true }) + expect( + await cleanupBackgroundSessionsBefore(new Date(), { + sessionIds: batch.sessionIds, + orphanedTerminalFacts: batch.terminalFacts, + maxDirectoryEntries: 0, + }), + ).toEqual({ sessionsRemoved: 0, artifactsRemoved: 1, errors: 0 }) + await batch.commit() + expect( + (await takeBackgroundSessionRecoveryBatch(256)).sessionIds, + ).toEqual([]) + }) + } + + it('caps recovery batches at 256 journal records', async () => { + const root = join(configDir, 'bg-sessions') + await mkdir(root, { recursive: true }) + await writeFile( + join(root, '.recovery-journal'), + `${Array.from({ length: 300 }, (_, index) => `bg-batch-cap-${index}`).join('\n')}\n`, + ) + + expect( + (await takeBackgroundSessionRecoveryBatch(1_000)).sessionIds, + ).toHaveLength(256) + }) + + it('queues sync terminal persistence for bounded recovery', async () => { + const session = await createBackgroundSession({ + id: 'bg-sync-recovery-queue', + pid: 453, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: 'conversation-sync-recovery-queue', + }) + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { ownerPid: session.pid, expectedSession: session }, + ) + + const batch = await takeBackgroundSessionRecoveryBatch(1) + expect(batch.sessionIds).toEqual([session.id]) + }) + + it('preserves records appended after a full-sweep journal snapshot', async () => { + const first = await createBackgroundSession({ + id: 'bg-recovery-snapshot-first', + pid: 455, + cwd: '/repo', + command: ['openclaude', '--print', 'first'], + sessionId: 'conversation-recovery-snapshot-first', + }) + await recordBackgroundSessionNaturalTermination( + first.id, + { exitCode: 0 }, + { ownerPid: first.pid }, + ) + const snapshot = + await snapshotBackgroundSessionRecoveryJournal() + + const second = await createBackgroundSession({ + id: 'bg-recovery-snapshot-second', + pid: 456, + cwd: '/repo', + command: ['openclaude', '--print', 'second'], + sessionId: 'conversation-recovery-snapshot-second', + }) + await recordBackgroundSessionNaturalTermination( + second.id, + { exitCode: 0 }, + { ownerPid: second.pid }, + ) + + expect(await snapshot.commit()).toBe(true) + expect( + (await takeBackgroundSessionRecoveryBatch(2)).sessionIds, + ).toEqual([second.id]) + }) + + it('ignores a stale batch commit after journal rotation', async () => { + const first = await createBackgroundSession({ + id: 'bg-stale-recovery-batch-first', + pid: 457, + cwd: '/repo', + command: ['openclaude', '--print', 'first'], + sessionId: 'conversation-stale-recovery-batch-first', + }) + await recordBackgroundSessionNaturalTermination( + first.id, + { exitCode: 0 }, + { ownerPid: first.pid }, + ) + const staleBatch = await takeBackgroundSessionRecoveryBatch(1) + const snapshot = + await snapshotBackgroundSessionRecoveryJournal() + expect(await snapshot.commit()).toBe(true) + + const second = await createBackgroundSession({ + id: 'bg-stale-recovery-batch-second', + pid: 458, + cwd: '/repo', + command: ['openclaude', '--print', 'second'], + sessionId: 'conversation-stale-recovery-batch-second', + }) + await recordBackgroundSessionNaturalTermination( + second.id, + { exitCode: 0 }, + { ownerPid: second.pid }, + ) + await staleBatch.commit([first.id]) + + expect( + (await takeBackgroundSessionRecoveryBatch(1)).sessionIds, + ).toEqual([second.id]) + }) + + it('does not follow a symlinked recovery journal during sync persistence', async () => { + const session = await createBackgroundSession({ + id: 'bg-sync-symlinked-recovery-journal', + pid: 454, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: 'conversation-sync-symlinked-recovery-journal', + }) + const outside = join(configDir, 'outside-sync-recovery-journal') + await writeFile(outside, 'keep') + await symlink( + outside, + join(configDir, 'bg-sessions', '.recovery-journal'), + ) + + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { ownerPid: session.pid, expectedSession: session }, + ) + expect((await resolveBackgroundSession(session.id)).status).toBe( + 'exited', + ) + expect(await readFile(outside, 'utf8')).toBe('keep') + }) + it('does not let a late natural finalizer replace the first valid fact', async () => { await createBackgroundSession({ id: 'bg-first-fact', @@ -1012,8 +1365,14 @@ describe('background session registry', () => { id: 'bg-concurrent-natural', pid: 346, cwd: '/repo', - command: ['openclaude', '--print', 'work'], + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], sessionId: 'conversation-concurrent-natural', + processMarker: TEST_PROCESS_MARKER, }) const results = await Promise.all([ @@ -1123,13 +1482,248 @@ describe('background session registry', () => { }) }) + it('does not let a stale refresh overwrite a same-ID replacement generation', async () => { + const id = 'bg-refresh-generation-race' + const name = 'refresh-generation-race' + const oldSession = await createBackgroundSession({ + id, + name, + pid: 360, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'old generation', + ], + sessionId: 'conversation-refresh-generation-old', + processMarker: TEST_PROCESS_MARKER, + }) + let replacement: BackgroundSession | undefined + + const refreshed = await refreshBackgroundSessionStatuses({ + isProcessAlive: () => false, + _beforeStatusWriteForTesting: async () => { + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + { + ownerPid: oldSession.pid, + now: new Date('2026-06-01T00:00:00.000Z'), + }, + ) + await cleanupBackgroundSessionsBefore( + new Date('2026-07-01T00:00:00.000Z'), + ) + replacement = await createBackgroundSession({ + id, + name, + pid: 361, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(OTHER_PROCESS_MARKER), + '--print', + 'replacement generation', + ], + sessionId: 'conversation-refresh-generation-replacement', + processMarker: OTHER_PROCESS_MARKER, + }) + }, + }) + + if (!replacement) throw new Error('replacement session was not created') + expect(refreshed).toHaveLength(1) + expect(refreshed[0]).toMatchObject({ + id, + pid: replacement.pid, + status: 'running', + processMarker: OTHER_PROCESS_MARKER, + terminalFactGeneration: OTHER_PROCESS_MARKER, + }) + expect(await Bun.file(nameReservationPath(name)).json()).toMatchObject({ + id, + }) + + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 23 }, + { + ownerPid: replacement.pid, + now: new Date('2026-07-01T00:00:01.000Z'), + }, + ) + expect(await resolveBackgroundSession(id)).toMatchObject({ + pid: replacement.pid, + status: 'failed', + exitCode: 23, + }) + }) + + it('does not let a stale kill mark a same-ID replacement generation', async () => { + const id = 'bg-kill-generation-race' + const name = 'kill-generation-race' + const reusedPid = 362 + const oldSession = await createBackgroundSession({ + id, + name, + pid: reusedPid, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'old generation', + ], + sessionId: 'conversation-kill-generation-old', + processMarker: TEST_PROCESS_MARKER, + }) + let replacement: BackgroundSession | undefined + + await expect( + markBackgroundSessionKilled(id, { + _beforeMarkWriteForTesting: async () => { + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + { + ownerPid: oldSession.pid, + now: new Date('2026-06-01T00:00:00.000Z'), + }, + ) + await cleanupBackgroundSessionsBefore( + new Date('2026-07-01T00:00:00.000Z'), + ) + replacement = await createBackgroundSession({ + id, + name, + pid: reusedPid, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(OTHER_PROCESS_MARKER), + '--print', + 'replacement generation', + ], + sessionId: 'conversation-kill-generation-replacement', + processMarker: OTHER_PROCESS_MARKER, + }) + }, + }), + ).rejects.toThrow('changed before it could be marked killed') + + if (!replacement) throw new Error('replacement session was not created') + expect(await Bun.file(nameReservationPath(name)).json()).toMatchObject({ + id, + }) + expect( + await Bun.file( + terminalFactPath(id, 'killed', OTHER_PROCESS_MARKER), + ).exists(), + ).toBe(false) + + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 9 }, + { + ownerPid: replacement.pid, + now: new Date('2026-07-01T00:00:01.000Z'), + }, + ) + expect(await resolveBackgroundSession(id)).toMatchObject({ + pid: replacement.pid, + status: 'failed', + exitCode: 9, + }) + }) + + it('does not let a late finalizer write a same-ID replacement generation', async () => { + const id = 'bg-finalizer-generation-race' + const reusedPid = 363 + const oldSession = await createBackgroundSession({ + id, + pid: reusedPid, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'old generation', + ], + sessionId: 'conversation-finalizer-generation-old', + processMarker: TEST_PROCESS_MARKER, + }) + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + { + ownerPid: reusedPid, + expectedSession: oldSession, + now: new Date('2026-06-01T00:00:00.000Z'), + }, + ) + await cleanupBackgroundSessionsBefore( + new Date('2026-07-01T00:00:00.000Z'), + ) + const replacement = await createBackgroundSession({ + id, + pid: reusedPid, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(OTHER_PROCESS_MARKER), + '--print', + 'replacement generation', + ], + sessionId: 'conversation-finalizer-generation-replacement', + processMarker: OTHER_PROCESS_MARKER, + }) + + await expect( + recordBackgroundSessionNaturalTermination( + id, + { exitCode: 17 }, + { ownerPid: reusedPid, expectedSession: oldSession }, + ), + ).rejects.toThrow('does not own this session') + expect(() => + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 17 }, + { ownerPid: reusedPid, expectedSession: oldSession }, + ), + ).toThrow('does not own this session') + expect( + await Bun.file( + terminalFactPath(id, 'natural', OTHER_PROCESS_MARKER), + ).exists(), + ).toBe(false) + + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 7 }, + { ownerPid: reusedPid, expectedSession: replacement }, + ) + expect(await resolveBackgroundSession(id)).toMatchObject({ + processMarker: OTHER_PROCESS_MARKER, + status: 'failed', + exitCode: 7, + }) + }) + it('records a bounded observed signal without inventing an exit code', async () => { await createBackgroundSession({ id: 'bg-observed-signal', pid: 344, cwd: '/repo', - command: ['openclaude', '--print', 'work'], + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], sessionId: 'conversation-observed-signal', + processMarker: TEST_PROCESS_MARKER, }) const failed = await recordBackgroundSessionNaturalTermination( 'bg-observed-signal', @@ -1145,6 +1739,445 @@ describe('background session registry', () => { expect('exitCode' in failed).toBe(false) }) + it('keeps the first marked sync terminal fact immutable', async () => { + await createBackgroundSession({ + id: 'bg-marked-sync-natural', + pid: 348, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], + sessionId: 'conversation-marked-sync-natural', + processMarker: TEST_PROCESS_MARKER, + }) + recordBackgroundSessionNaturalTerminationSync( + 'bg-marked-sync-natural', + { exitCode: 19 }, + { + ownerPid: 348, + now: new Date('2026-06-15T08:09:10.000Z'), + }, + ) + recordBackgroundSessionNaturalTerminationSync( + 'bg-marked-sync-natural', + { exitCode: 0 }, + { + ownerPid: 348, + now: new Date('2026-06-15T08:09:20.000Z'), + }, + ) + + expect(await resolveBackgroundSession('bg-marked-sync-natural')).toMatchObject( + { + status: 'failed', + finishedAt: '2026-06-15T08:09:10.000Z', + exitCode: 19, + }, + ) + expect( + await Bun.file( + terminalFactPath( + 'bg-marked-sync-natural', + 'natural', + TEST_PROCESS_MARKER, + ), + ).json(), + ).toMatchObject({ generation: TEST_PROCESS_MARKER, exitCode: 19 }) + }) + + it('writes a durable marked fact when the metadata lock is contended', async () => { + const id = 'bg-contended-sync-fact' + const name = 'contended-sync-fact' + const session = await createBackgroundSession({ + id, + name, + pid: 350, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], + sessionId: 'conversation-contended-sync-fact', + processMarker: TEST_PROCESS_MARKER, + }) + const metadataPath = join( + configDir, + 'bg-sessions', + 'sessions', + `${id}.json`, + ) + const release = await lockfile.lock(metadataPath, { realpath: false }) + try { + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 19 }, + { + ownerPid: session.pid, + expectedSession: session, + now: new Date('2026-06-15T08:09:24.000Z'), + }, + ) + } finally { + await release() + } + + expect( + await Bun.file( + terminalFactPath(id, 'natural', TEST_PROCESS_MARKER), + ).json(), + ).toMatchObject({ + generation: TEST_PROCESS_MARKER, + status: 'failed', + exitCode: 19, + }) + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(true) + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 1, + errors: 0, + }) + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(false) + }) + + it('bounds recurring reconciliation by terminal directory entries', async () => { + const sessions = await Promise.all( + ['bg-bounded-reconcile-a', 'bg-bounded-reconcile-b'].map( + async (id, index) => + await createBackgroundSession({ + id, + pid: 360 + index, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + id, + ], + sessionId: `conversation-${id}`, + processMarker: TEST_PROCESS_MARKER, + }), + ), + ) + for (const session of sessions) { + const metadataPath = join( + configDir, + 'bg-sessions', + 'sessions', + `${session.id}.json`, + ) + const release = await lockfile.lock(metadataPath, { realpath: false }) + try { + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 17 }, + { ownerPid: session.pid, expectedSession: session }, + ) + } finally { + await release() + } + } + + expect( + await reconcileBackgroundSessionTerminalFacts({ + terminalScanLimit: 1, + }), + ).toEqual({ sessionsUpdated: 1, errors: 0 }) + const statuses = await Promise.all( + sessions.map( + async session => + ( + (await Bun.file( + join( + configDir, + 'bg-sessions', + 'sessions', + `${session.id}.json`, + ), + ).json()) as { status: string } + ).status, + ), + ) + expect(statuses.sort()).toEqual(['failed', 'running']) + }) + + it('does not bypass a contended metadata lock for a markerless session', async () => { + const id = 'bg-contended-sync-legacy' + const session = await createBackgroundSession({ + id, + pid: 351, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: 'conversation-contended-sync-legacy', + }) + const metadataPath = join( + configDir, + 'bg-sessions', + 'sessions', + `${id}.json`, + ) + const release = await lockfile.lock(metadataPath, { realpath: false }) + try { + let error: unknown + try { + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 0 }, + { ownerPid: session.pid, expectedSession: session }, + ) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ code: 'ELOCKED' }) + } finally { + await release() + } + expect(await Bun.file(terminalFactPath(id, 'natural')).exists()).toBe( + false, + ) + }) + + it('uses the isolated legacy fact path for a marker-only sync handoff', async () => { + const id = 'bg-contended-sync-marker-only' + const session = await createBackgroundSession({ + id, + pid: 352, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], + sessionId: 'conversation-contended-sync-marker-only', + processMarker: TEST_PROCESS_MARKER, + }) + const metadataPath = join( + configDir, + 'bg-sessions', + 'sessions', + `${id}.json`, + ) + const metadata = (await Bun.file(metadataPath).json()) as Record< + string, + unknown + > + delete metadata.terminalFactGeneration + await writeFile(metadataPath, JSON.stringify(metadata)) + const markerOnlySession = await readBackgroundSessionForOwner(id) + if (!markerOnlySession) throw new Error('marker-only session was not read') + expect(markerOnlySession.terminalFactGeneration).toBeUndefined() + + const release = await lockfile.lock(metadataPath, { realpath: false }) + try { + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 31 }, + { + ownerPid: session.pid, + expectedSession: markerOnlySession, + now: new Date('2026-06-15T08:09:26.000Z'), + }, + ) + } finally { + await release() + } + + expect( + await Bun.file( + terminalFactPath(id, 'natural'), + ).json(), + ).toMatchObject({ + status: 'failed', + exitCode: 31, + }) + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 1, + errors: 0, + }) + expect(await Bun.file(metadataPath).json()).toMatchObject({ + status: 'failed', + exitCode: 31, + }) + }) + + it('reconciles a sync finalization after refresh holds the metadata lock', async () => { + const id = 'bg-contended-sync-natural' + const name = 'contended-sync-natural' + const session = await createBackgroundSession({ + id, + name, + pid: 349, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], + sessionId: 'conversation-contended-sync-natural', + processMarker: TEST_PROCESS_MARKER, + }) + const metadataPath = join( + configDir, + 'bg-sessions', + 'sessions', + `${id}.json`, + ) + expect( + await refreshBackgroundSessionStatuses({ + isProcessAlive: () => false, + _whileStatusWriteLockedForTesting: () => { + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 23 }, + { + ownerPid: session.pid, + expectedSession: session, + now: new Date('2026-06-15T08:09:25.000Z'), + }, + ) + }, + }), + ).toMatchObject([{ status: 'failed', exitCode: 23 }]) + + expect(await Bun.file(metadataPath).json()).toMatchObject({ + status: 'stale', + }) + expect( + await Bun.file( + terminalFactPath(id, 'natural', TEST_PROCESS_MARKER), + ).json(), + ).toMatchObject({ + generation: TEST_PROCESS_MARKER, + status: 'failed', + exitCode: 23, + }) + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 1, + errors: 0, + }) + expect(await Bun.file(metadataPath).json()).toMatchObject({ + status: 'failed', + exitCode: 23, + finishedAt: '2026-06-15T08:09:25.000Z', + }) + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(false) + + expect( + await cleanupBackgroundSessionsBefore( + new Date('2026-07-01T00:00:00.000Z'), + ), + ).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + expect(await Bun.file(metadataPath).exists()).toBe(false) + }) + + it('reconciles stronger terminal facts and retries reservation release', async () => { + const id = 'bg-terminal-reconciliation-retry' + const name = 'terminal-reconciliation-retry' + const session = await createBackgroundSession({ + id, + name, + pid: 353, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], + sessionId: 'conversation-terminal-reconciliation-retry', + processMarker: TEST_PROCESS_MARKER, + }) + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 5 }, + { ownerPid: session.pid }, + ) + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 1, + errors: 0, + }) + await markBackgroundSessionKilled(id, { + now: new Date('2026-06-15T08:09:27.000Z'), + }) + await writeNameReservation(name, { id }) + + const release = await lockfile.lock(nameReservationPath(name), { + realpath: false, + }) + try { + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 1, + errors: 1, + }) + } finally { + await release() + } + expect( + await Bun.file( + join(configDir, 'bg-sessions', 'sessions', `${id}.json`), + ).json(), + ).toMatchObject({ + status: 'killed', + terminalReason: 'explicit_kill', + }) + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(true) + + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 0, + errors: 0, + }) + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(false) + }) + + for (const sync of [false, true]) { + it(`preserves a mismatched reservation name during ${sync ? 'sync' : 'async'} finalization`, async () => { + const suffix = sync ? 'sync' : 'async' + const id = `bg-mismatched-reservation-${suffix}` + const name = `mismatched-reservation-${suffix}` + const session = await createBackgroundSession({ + id, + name, + pid: sync ? 354 : 355, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: `conversation-mismatched-reservation-${suffix}`, + }) + await writeFile( + nameReservationPath(name), + JSON.stringify({ name: `${name}-other`, id }), + ) + + if (sync) { + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 0 }, + { ownerPid: session.pid }, + ) + } else { + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + { ownerPid: session.pid }, + ) + } + + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(true) + expect(await Bun.file(nameReservationPath(name)).json()).toEqual({ + name: `${name}-other`, + id, + }) + }) + } + it('does not let a late natural finalizer overwrite an explicit kill fact', async () => { await createBackgroundSession({ id: 'bg-killed-absorbing', @@ -1179,8 +2212,14 @@ describe('background session registry', () => { id: 'bg-sync-killed-absorbing', pid: 347, cwd: '/repo', - command: ['openclaude', '--print', 'work'], + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], sessionId: 'conversation-sync-killed-absorbing', + processMarker: TEST_PROCESS_MARKER, }) await markBackgroundSessionKilled('bg-sync-killed-absorbing', { now: new Date('2026-06-15T08:09:30.000Z'), @@ -1201,9 +2240,22 @@ describe('background session registry', () => { }) expect( await Bun.file( - terminalFactPath('bg-sync-killed-absorbing', 'natural'), + terminalFactPath( + 'bg-sync-killed-absorbing', + 'natural', + TEST_PROCESS_MARKER, + ), ).exists(), ).toBe(false) + expect( + await Bun.file( + terminalFactPath( + 'bg-sync-killed-absorbing', + 'killed', + TEST_PROCESS_MARKER, + ), + ).json(), + ).toMatchObject({ generation: TEST_PROCESS_MARKER }) }) it('keeps killed strongest when kill and natural completion race', async () => { diff --git a/src/cli/bgRegistry.ts b/src/cli/bgRegistry.ts index ae9782d0b5..7a5962e208 100644 --- a/src/cli/bgRegistry.ts +++ b/src/cli/bgRegistry.ts @@ -1,7 +1,9 @@ import { link, + lstat, mkdir, open, + opendir, readFile, readdir, rename, @@ -11,13 +13,17 @@ import { } from 'node:fs/promises' import { closeSync, + fstatSync, fsyncSync, linkSync, + lstatSync, mkdirSync, openSync, readFileSync, unlinkSync, writeFileSync, + type Dirent, + type Stats, } from 'node:fs' import { createHash, randomUUID } from 'node:crypto' import { basename, join } from 'node:path' @@ -26,6 +32,7 @@ import { getProcessCommand, isProcessRunning, } from '../utils/genericProcessUtils.js' +import * as lockfile from '../utils/lockfile.js' import { jsonParse, jsonStringify } from '../utils/slowOperations.js' import { backgroundProcessMarkerToken, @@ -50,6 +57,7 @@ export type BackgroundSession = { model?: string sessionId: string processMarker?: string + terminalFactGeneration?: string startedAt: string updatedAt: string command: string[] @@ -70,6 +78,7 @@ type BackgroundSessionTerminalFact = { version: 1 id: string pid: number + generation?: string status: 'exited' | 'failed' | 'killed' finishedAt: string terminalReason: BackgroundSessionTerminalReason @@ -104,6 +113,12 @@ type BackgroundSessionNameReservation = { createdAt?: string } +export type BackgroundCleanupResult = { + sessionsRemoved: number + artifactsRemoved: number + errors: number +} + const TERMINAL_STATUSES = new Set([ 'exited', 'failed', @@ -115,10 +130,61 @@ const ALL_STATUSES = new Set([ 'unknown', ...TERMINAL_STATUSES, ]) +const COMPLETED_STATUSES = new Set([ + 'exited', + 'failed', + 'killed', +]) const SAFE_ID_RE = /^[A-Za-z0-9._-]+$/ const SAFE_SIGNAL_RE = /^SIG[A-Z0-9]{1,24}$/ +const BACKGROUND_RECOVERY_JOURNAL_ENTRY_LIMIT = 256 +const BACKGROUND_RECOVERY_JOURNAL_RECORD_MAX_BYTES = 512 +const BACKGROUND_RECOVERY_JOURNAL_VERSION = 1 +const NAME_RESERVATION_LOCK_OPTIONS = { + realpath: false, + retries: { + retries: 20, + factor: 1, + minTimeout: 5, + maxTimeout: 25, + randomize: true, + }, +} satisfies NonNullable[1]> +const NAME_RESERVATION_SYNC_LOCK_OPTIONS = { + realpath: false, + retries: 0, +} satisfies NonNullable[1]> let backgroundSessionsRootForTesting: string | undefined +type BackgroundSessionRecoveryJournalIdentity = { + dev: number + ino: number +} + +type BackgroundSessionRecoveryCursor = { + version: typeof BACKGROUND_RECOVERY_JOURNAL_VERSION + offset: number +} & BackgroundSessionRecoveryJournalIdentity + +export type BackgroundSessionRecoveryTarget = { + id: string + generation?: string +} + +export type BackgroundSessionRecoveryBatch = { + sessionIds: string[] + terminalFacts: BackgroundSessionRecoveryTarget[] + commit: ( + retrySessionIds?: readonly string[], + retryTerminalFacts?: readonly BackgroundSessionRecoveryTarget[], + ) => Promise +} + +export type BackgroundSessionRecoverySnapshot = { + retry: (targets: readonly BackgroundSessionRecoveryTarget[]) => Promise + commit: () => Promise +} + export function _setBackgroundSessionsRootForTesting( root: string | undefined, ): void { @@ -148,11 +214,519 @@ function getBackgroundSessionTerminalDir(): string { return join(getBackgroundSessionsRoot(), 'terminal') } +function getBackgroundSessionRecoveryJournalPath(): string { + return join(getBackgroundSessionsRoot(), '.recovery-journal') +} + +function getBackgroundSessionRecoveryCursorPath(): string { + return join(getBackgroundSessionsRoot(), '.recovery-cursor.json') +} + +async function readDirectoryEntries( + path: string, + maxEntries?: number, +): Promise[]> { + if (maxEntries === undefined) { + return await readdir(path, { withFileTypes: true }) + } + if (!Number.isSafeInteger(maxEntries) || maxEntries < 1) return [] + + const entries: Dirent[] = [] + const directory = await opendir(path) + try { + for await (const entry of directory) { + entries.push(entry) + if (entries.length >= maxEntries) break + } + } finally { + try { + await directory.close() + } catch { + // The async iterator closes the directory after exhaustion or break. + } + } + return entries +} + +async function withBackgroundRecoveryJournalLock( + operation: () => Promise, +): Promise { + await mkdir(getBackgroundSessionsRoot(), { recursive: true, mode: 0o700 }) + const journalPath = getBackgroundSessionRecoveryJournalPath() + const handle = await openVerifiedBackgroundRecoveryJournal('a') + await handle.close() + const release = await lockfile.lock( + journalPath, + NAME_RESERVATION_LOCK_OPTIONS, + ) + try { + return await operation() + } finally { + await release().catch(() => {}) + } +} + +function withBackgroundRecoveryJournalLockSync(operation: () => T): T { + mkdirSync(getBackgroundSessionsRoot(), { recursive: true, mode: 0o700 }) + const journalPath = getBackgroundSessionRecoveryJournalPath() + const journalFd = openVerifiedBackgroundRecoveryJournalSync('a') + closeSync(journalFd) + const release = lockfile.lockSync( + journalPath, + NAME_RESERVATION_SYNC_LOCK_OPTIONS, + ) + try { + return operation() + } finally { + try { + release() + } catch {} + } +} + +async function openVerifiedBackgroundRecoveryJournal( + flags: 'a' | 'r' | 'r+', +): Promise>> { + const path = getBackgroundSessionRecoveryJournalPath() + const handle = await open(path, flags, 0o600) + try { + const [pathIdentity, handleIdentity] = await Promise.all([ + lstat(path), + handle.stat(), + ]) + if ( + !pathIdentity.isFile() || + !handleIdentity.isFile() || + pathIdentity.dev !== handleIdentity.dev || + pathIdentity.ino !== handleIdentity.ino + ) { + throw new Error('Invalid background recovery journal') + } + return handle + } catch (error) { + await handle.close().catch(() => {}) + throw error + } +} + +function openVerifiedBackgroundRecoveryJournalSync( + flags: 'a' | 'r' | 'r+', +): number { + const path = getBackgroundSessionRecoveryJournalPath() + const fd = openSync(path, flags, 0o600) + try { + const pathIdentity = lstatSync(path) + const handleIdentity = fstatSync(fd) + if ( + !pathIdentity.isFile() || + !handleIdentity.isFile() || + pathIdentity.dev !== handleIdentity.dev || + pathIdentity.ino !== handleIdentity.ino + ) { + throw new Error('Invalid background recovery journal') + } + return fd + } catch (error) { + closeSync(fd) + throw error + } +} + +function backgroundRecoveryJournalRecord( + id: string, + generation?: string, +): string | undefined { + if (!SAFE_ID_RE.test(id)) return undefined + if ( + generation !== undefined && + !isValidBackgroundProcessMarker(generation) + ) { + return undefined + } + const record = `${id}${generation === undefined ? '' : `~${generation}`}\n` + return Buffer.byteLength(record) <= + BACKGROUND_RECOVERY_JOURNAL_RECORD_MAX_BYTES + ? record + : undefined +} + +async function enqueueBackgroundSessionRecovery( + id: string, + generation?: string, +): Promise { + await enqueueBackgroundSessionRecoveryTargets([{ id, generation }]) +} + +async function enqueueBackgroundSessionRecoveryTargets( + targets: readonly BackgroundSessionRecoveryTarget[], +): Promise { + const records = [ + ...new Set( + targets + .map(target => + backgroundRecoveryJournalRecord(target.id, target.generation), + ) + .filter((record): record is string => record !== undefined), + ), + ].join('') + if (!records) return + await withBackgroundRecoveryJournalLock(async () => { + const handle = await openVerifiedBackgroundRecoveryJournal('a') + try { + await handle.writeFile(records) + await handle.sync() + } finally { + await handle.close().catch(() => {}) + } + }) +} + +function enqueueBackgroundSessionRecoverySync( + id: string, + generation?: string, +): void { + const record = backgroundRecoveryJournalRecord(id, generation) + if (!record) return + withBackgroundRecoveryJournalLockSync(() => { + const fd = openVerifiedBackgroundRecoveryJournalSync('a') + try { + writeFileSync(fd, record) + fsyncSync(fd) + } finally { + closeSync(fd) + } + }) +} + +function sameBackgroundRecoveryJournalIdentity( + left: Partial, + right: BackgroundSessionRecoveryJournalIdentity, +): boolean { + return ( + typeof left.dev === 'number' && + typeof left.ino === 'number' && + left.dev === right.dev && + left.ino === right.ino + ) +} + +async function readBackgroundSessionRecoveryCursor( + identity: BackgroundSessionRecoveryJournalIdentity, +): Promise { + try { + if (!(await lstat(getBackgroundSessionRecoveryCursorPath())).isFile()) { + return 0 + } + const parsed = jsonParse( + await readFile(getBackgroundSessionRecoveryCursorPath(), 'utf8'), + ) as Partial + return parsed.version === BACKGROUND_RECOVERY_JOURNAL_VERSION && + sameBackgroundRecoveryJournalIdentity(parsed, identity) && + Number.isSafeInteger(parsed.offset) && + parsed.offset! >= 0 + ? parsed.offset! + : 0 + } catch { + return 0 + } +} + +async function writeBackgroundSessionRecoveryCursor( + offset: number, + identity: BackgroundSessionRecoveryJournalIdentity, +): Promise { + const path = getBackgroundSessionRecoveryCursorPath() + const tmp = join( + getBackgroundSessionsRoot(), + `.recovery-cursor.${process.pid}.${randomUUID()}.tmp`, + ) + const handle = await open(tmp, 'wx', 0o600) + try { + await handle.writeFile( + jsonStringify({ + version: BACKGROUND_RECOVERY_JOURNAL_VERSION, + offset, + ...identity, + } satisfies BackgroundSessionRecoveryCursor), + ) + await handle.sync() + await handle.close() + await rename(tmp, path) + } finally { + await handle.close().catch(() => {}) + await unlink(tmp).catch(() => {}) + } +} + +async function rotateBackgroundSessionRecoveryJournal( + startOffset: number, + expectedIdentity: BackgroundSessionRecoveryJournalIdentity, +): Promise { + const journalPath = getBackgroundSessionRecoveryJournalPath() + const source = await openVerifiedBackgroundRecoveryJournal('r') + const sourceIdentity = await source.stat() + if ( + !sameBackgroundRecoveryJournalIdentity( + sourceIdentity, + expectedIdentity, + ) + ) { + await source.close() + return false + } + const tmp = join( + getBackgroundSessionsRoot(), + `.recovery-journal.${process.pid}.${randomUUID()}.tmp`, + ) + let target: Awaited> | undefined + try { + target = await open(tmp, 'wx', 0o600) + const buffer = Buffer.alloc(64 * 1024) + let position = startOffset + while (true) { + const { bytesRead } = await source.read( + buffer, + 0, + buffer.byteLength, + position, + ) + if (bytesRead === 0) break + await target.writeFile(buffer.subarray(0, bytesRead)) + position += bytesRead + } + await target.sync() + await source.close() + await target.close() + target = undefined + await rename(tmp, journalPath) + await unlink(getBackgroundSessionRecoveryCursorPath()).catch(() => {}) + return true + } finally { + await source.close().catch(() => {}) + await target?.close().catch(() => {}) + await unlink(tmp).catch(() => {}) + } +} + +export async function snapshotBackgroundSessionRecoveryJournal(): Promise< + BackgroundSessionRecoverySnapshot +> { + let snapshotSize = 0 + let snapshotIdentity: BackgroundSessionRecoveryJournalIdentity = { + dev: 0, + ino: 0, + } + await withBackgroundRecoveryJournalLock(async () => { + const handle = await openVerifiedBackgroundRecoveryJournal('r') + try { + const identity = await handle.stat() + snapshotSize = identity.size + snapshotIdentity = { dev: identity.dev, ino: identity.ino } + } finally { + await handle.close().catch(() => {}) + } + }) + return { + retry: enqueueBackgroundSessionRecoveryTargets, + commit: async () => + await withBackgroundRecoveryJournalLock( + async () => + await rotateBackgroundSessionRecoveryJournal( + snapshotSize, + snapshotIdentity, + ), + ), + } +} + +export async function takeBackgroundSessionRecoveryBatch( + maxEntries: number, +): Promise { + if (!Number.isSafeInteger(maxEntries) || maxEntries < 1) { + return { sessionIds: [], terminalFacts: [], commit: async () => {} } + } + const boundedMaxEntries = Math.min( + maxEntries, + BACKGROUND_RECOVERY_JOURNAL_ENTRY_LIMIT, + ) + + let startOffset = 0 + let nextOffset = 0 + let snapshotSize = 0 + let snapshotIdentity: BackgroundSessionRecoveryJournalIdentity = { + dev: 0, + ino: 0, + } + let sessionIds: string[] = [] + const terminalFacts: BackgroundSessionRecoveryTarget[] = [] + await withBackgroundRecoveryJournalLock(async () => { + const handle = await openVerifiedBackgroundRecoveryJournal('r') + try { + const identity = await handle.stat() + snapshotSize = identity.size + snapshotIdentity = { dev: identity.dev, ino: identity.ino } + startOffset = await readBackgroundSessionRecoveryCursor( + snapshotIdentity, + ) + if (startOffset > snapshotSize) startOffset = 0 + nextOffset = startOffset + const maxBytes = + boundedMaxEntries * BACKGROUND_RECOVERY_JOURNAL_RECORD_MAX_BYTES + const buffer = Buffer.alloc(maxBytes) + const { bytesRead } = await handle.read( + buffer, + 0, + maxBytes, + startOffset, + ) + const contents = buffer.subarray(0, bytesRead) + let lineStart = 0 + let linesRead = 0 + const ids = new Set() + while (linesRead < boundedMaxEntries) { + const newline = contents.indexOf(0x0a, lineStart) + if (newline < 0) break + const record = contents + .subarray(lineStart, newline) + .toString('utf8') + const [id, generation, extra] = record.split('~') + if ( + id && + extra === undefined && + backgroundRecoveryJournalRecord(id, generation) !== undefined + ) { + ids.add(id) + terminalFacts.push({ id, generation }) + } + nextOffset = startOffset + newline + 1 + lineStart = newline + 1 + linesRead++ + } + if (linesRead === 0 && bytesRead === maxBytes) { + nextOffset = startOffset + bytesRead + } + sessionIds = [...ids] + } finally { + await handle.close().catch(() => {}) + } + }) + + return { + sessionIds, + terminalFacts, + commit: async (retrySessionIds = [], retryTerminalFacts = []) => { + if (nextOffset === startOffset) return + await withBackgroundRecoveryJournalLock(async () => { + const handle = await openVerifiedBackgroundRecoveryJournal('r') + let identity: Stats + try { + identity = await handle.stat() + } finally { + await handle.close().catch(() => {}) + } + const currentIdentity = { dev: identity.dev, ino: identity.ino } + if ( + !sameBackgroundRecoveryJournalIdentity( + currentIdentity, + snapshotIdentity, + ) + ) { + return + } + const currentOffset = await readBackgroundSessionRecoveryCursor( + currentIdentity, + ) + if (currentOffset !== startOffset) return + const retryIds = new Set(retrySessionIds) + const retryRecords = [ + ...new Set( + [ + ...terminalFacts.filter(target => retryIds.has(target.id)), + ...retryTerminalFacts, + ] + .filter(target => sessionIds.includes(target.id)) + .map(target => + backgroundRecoveryJournalRecord( + target.id, + target.generation, + ), + ) + .filter((record): record is string => record !== undefined), + ), + ].join('') + if (retryRecords) { + const retryHandle = + await openVerifiedBackgroundRecoveryJournal('a') + try { + if ( + !sameBackgroundRecoveryJournalIdentity( + await retryHandle.stat(), + currentIdentity, + ) + ) { + throw new Error( + 'Background recovery journal changed before retry', + ) + } + await retryHandle.writeFile(retryRecords) + await retryHandle.sync() + } finally { + await retryHandle.close().catch(() => {}) + } + } + if (identity.size === snapshotSize && nextOffset >= snapshotSize) { + await rotateBackgroundSessionRecoveryJournal( + nextOffset, + currentIdentity, + ) + return + } + await writeBackgroundSessionRecoveryCursor( + nextOffset, + currentIdentity, + ) + }) + }, + } +} + function metadataPathForId(id: string): string { assertSafeId(id) return join(getBackgroundSessionMetadataDir(), `${id}.json`) } +async function withBackgroundSessionIdLock( + id: string, + operation: () => Promise, +): Promise { + const release = await lockfile.lock( + metadataPathForId(id), + NAME_RESERVATION_LOCK_OPTIONS, + ) + try { + return await operation() + } finally { + await release().catch(() => {}) + } +} + +function withBackgroundSessionIdLockSync( + id: string, + operation: () => T, +): T { + const release = lockfile.lockSync( + metadataPathForId(id), + NAME_RESERVATION_SYNC_LOCK_OPTIONS, + ) + try { + return operation() + } finally { + try { + release() + } catch {} + } +} + function nameReservationPathForName(name: string): string { const digest = createHash('sha256').update(name).digest('hex') return join(getBackgroundSessionNamesDir(), `${digest}.json`) @@ -161,9 +735,20 @@ function nameReservationPathForName(name: string): string { function terminalFactPathForId( id: string, kind: 'natural' | 'killed', + generation?: string, ): string { assertSafeId(id) - return join(getBackgroundSessionTerminalDir(), `${id}.${kind}.json`) + if ( + generation !== undefined && + !isValidBackgroundProcessMarker(generation) + ) { + throw new Error('Invalid background terminal-fact generation') + } + const generationSuffix = generation ? `~${generation}` : '' + return join( + getBackgroundSessionTerminalDir(), + `${id}${generationSuffix}.${kind}.json`, + ) } function assertSafeId(id: string): void { @@ -181,10 +766,130 @@ function isErrno(error: unknown, code: string): boolean { ) } +type CleanupJsonRead = + | { state: 'missing' } + | { state: 'valid'; value: T; identity?: CleanupFileIdentity } + | { state: 'invalid' } + | { state: 'error' } + +type CleanupFileIdentity = { dev: number; ino: number } + +type CleanupDirectorySnapshot = + | { state: 'missing'; path: string } + | { state: 'directory'; path: string; dev: number; ino: number } + +type BackgroundCleanupFileSystem = { + lstatFile: (path: string) => Promise + readTextFile: (path: string) => Promise + unlinkFile: (path: string) => Promise +} + +async function snapshotCleanupDirectory( + path: string, + lstatFile: BackgroundCleanupFileSystem['lstatFile'], +): Promise> { + try { + const stats = await lstatFile(path) + if (!stats.isDirectory() || stats.isSymbolicLink()) { + return { state: 'invalid' } + } + return { + state: 'valid', + value: { state: 'directory', path, dev: stats.dev, ino: stats.ino }, + } + } catch (error) { + return isErrno(error, 'ENOENT') + ? { state: 'valid', value: { state: 'missing', path } } + : { state: 'error' } + } +} + +async function inspectCleanupDirectory( + snapshot: CleanupDirectorySnapshot, + lstatFile: BackgroundCleanupFileSystem['lstatFile'], +): Promise<'same' | 'missing' | 'changed' | 'error'> { + try { + const stats = await lstatFile(snapshot.path) + if (snapshot.state === 'missing') return 'changed' + return stats.isDirectory() && + !stats.isSymbolicLink() && + stats.dev === snapshot.dev && + stats.ino === snapshot.ino + ? 'same' + : 'changed' + } catch (error) { + if (isErrno(error, 'ENOENT')) return 'missing' + return 'error' + } +} + +async function readCleanupJson( + path: string, + validate: (value: unknown) => value is T, + directory: CleanupDirectorySnapshot, + fileSystem: BackgroundCleanupFileSystem, +): Promise> { + const directoryState = await inspectCleanupDirectory( + directory, + fileSystem.lstatFile, + ) + if (directoryState === 'missing') return { state: 'missing' } + if (directoryState !== 'same') return { state: 'error' } + + let before: Stats + try { + before = await fileSystem.lstatFile(path) + if (!before.isFile() || before.isSymbolicLink()) return { state: 'invalid' } + } catch (error) { + return isErrno(error, 'ENOENT') ? { state: 'missing' } : { state: 'error' } + } + + let content: string + try { + content = await fileSystem.readTextFile(path) + } catch (error) { + return isErrno(error, 'ENOENT') ? { state: 'missing' } : { state: 'error' } + } + const latestDirectoryState = await inspectCleanupDirectory( + directory, + fileSystem.lstatFile, + ) + if (latestDirectoryState !== 'same') return { state: 'error' } + try { + const after = await fileSystem.lstatFile(path) + if (!after.isFile() || after.isSymbolicLink()) return { state: 'invalid' } + if (after.dev !== before.dev || after.ino !== before.ino) { + return { state: 'error' } + } + } catch (error) { + return isErrno(error, 'ENOENT') ? { state: 'missing' } : { state: 'error' } + } + let parsed: unknown + try { + parsed = jsonParse(content) + } catch { + return { state: 'invalid' } + } + return validate(parsed) + ? { + state: 'valid', + value: parsed, + identity: { dev: before.dev, ino: before.ino }, + } + : { state: 'invalid' } +} + function iso(now: Date | undefined): string { return (now ?? new Date()).toISOString() } +function parseCanonicalCompletionTimestamp(value: unknown): number | null { + if (typeof value !== 'string') return null + const timestamp = Date.parse(value) + if (!Number.isFinite(timestamp)) return null + return new Date(timestamp).toISOString() === value ? timestamp : null +} + export function getBackgroundSessionLogPaths(id: string): { stdoutLogPath: string stderrLogPath: string @@ -210,9 +915,24 @@ export async function ensureBackgroundSessionDirs(): Promise { }) } -async function writeSession(session: BackgroundSession): Promise { - await ensureBackgroundSessionDirs() - const target = metadataPathForId(session.id) +function isSameBackgroundSessionGeneration( + first: BackgroundSession, + second: BackgroundSession, +): boolean { + return ( + first.id === second.id && + first.pid === second.pid && + first.sessionId === second.sessionId && + first.startedAt === second.startedAt && + first.processMarker === second.processMarker && + first.terminalFactGeneration === second.terminalFactGeneration + ) +} + +async function replaceSessionFile( + session: BackgroundSession, + target: string, +): Promise { const tmp = join( getBackgroundSessionMetadataDir(), `${session.id}.${process.pid}.${randomUUID()}.tmp`, @@ -220,27 +940,61 @@ async function writeSession(session: BackgroundSession): Promise { try { await writeFile(tmp, jsonStringify(session), { flag: 'wx' }) await rename(tmp, target) - if (session.name && isTerminalBackgroundSession(session)) { - await releaseNameReservation(session.name, session.id) - } } catch (error) { await unlink(tmp).catch(() => {}) throw error } } +async function writeSession( + session: BackgroundSession, + expected: BackgroundSession, + whileLocked?: () => void | Promise, +): Promise { + await ensureBackgroundSessionDirs() + const target = metadataPathForId(session.id) + return await withBackgroundSessionIdLock(session.id, async () => { + const current = await readSessionFile(target) + if (!current || !isSameBackgroundSessionGeneration(current, expected)) { + return current + } + await whileLocked?.() + await replaceSessionFile(session, target) + if (session.name && isTerminalBackgroundSession(session)) { + await releaseNameReservation(session.name, session.id) + } + return session + }) +} + async function writeNewSession(session: BackgroundSession): Promise { await ensureBackgroundSessionDirs() - try { - await writeFile(metadataPathForId(session.id), jsonStringify(session), { - flag: 'wx', - }) - } catch (error) { - if (isErrno(error, 'EEXIST')) { - throw new Error(`Background session id "${session.id}" already exists`) + await withBackgroundSessionIdLock(session.id, async () => { + for (const kind of ['natural', 'killed'] as const) { + try { + await lstat( + terminalFactPathForId( + session.id, + kind, + session.terminalFactGeneration, + ), + ) + throw new Error(`Background session id "${session.id}" already exists`) + } catch (error) { + if (!isErrno(error, 'ENOENT')) throw error + } } - throw error - } + try { + await writeFile(metadataPathForId(session.id), jsonStringify(session), { + flag: 'wx', + }) + } catch (error) { + if (isErrno(error, 'EEXIST')) { + throw new Error(`Background session id "${session.id}" already exists`) + } + throw error + } + }) } async function readSessionFile(path: string): Promise { @@ -266,50 +1020,138 @@ async function readNameReservation( ): Promise { try { const parsed = jsonParse(await readFile(path, 'utf8')) - const candidate = parsed as Partial - if ( - parsed && - typeof parsed === 'object' && - typeof candidate.name === 'string' && - typeof candidate.id === 'string' && - SAFE_ID_RE.test(candidate.id) && - (candidate.creatorPid === undefined || - (typeof candidate.creatorPid === 'number' && - Number.isInteger(candidate.creatorPid) && - candidate.creatorPid > 1)) && - (candidate.createdAt === undefined || - typeof candidate.createdAt === 'string') - ) { - return parsed as BackgroundSessionNameReservation - } + if (isBackgroundSessionNameReservation(parsed)) return parsed } catch { // Malformed reservations are treated as recoverable orphans below. } return null } -async function releaseNameReservation( +async function withNameReservationLock( + name: string, + operation: () => Promise, +): Promise { + const release = await lockfile.lock( + nameReservationPathForName(name), + NAME_RESERVATION_LOCK_OPTIONS, + ) + let value: T + try { + value = await operation() + } catch (error) { + await release().catch(() => {}) + throw error + } + await release() + return value +} + +function withNameReservationLockSync(name: string, operation: () => T): T { + const release = lockfile.lockSync( + nameReservationPathForName(name), + NAME_RESERVATION_SYNC_LOCK_OPTIONS, + ) + let value: T + try { + value = operation() + } catch (error) { + try { + release() + } catch {} + throw error + } + release() + return value +} + +function isBackgroundSessionNameReservation( + value: unknown, +): value is BackgroundSessionNameReservation { + if (!value || typeof value !== 'object') return false + const candidate = value as Partial + return ( + typeof candidate.name === 'string' && + typeof candidate.id === 'string' && + SAFE_ID_RE.test(candidate.id) && + (candidate.creatorPid === undefined || + (typeof candidate.creatorPid === 'number' && + Number.isInteger(candidate.creatorPid) && + candidate.creatorPid > 1)) && + (candidate.createdAt === undefined || + typeof candidate.createdAt === 'string') + ) +} + +async function releaseNameReservationUnlocked( name: string, id: string, -): Promise { +): Promise { const path = nameReservationPathForName(name) - const existing = await readNameReservation(path) - if (existing?.id !== id) return - await unlink(path).catch(() => {}) + let existing: BackgroundSessionNameReservation + try { + const parsed = jsonParse(await readFile(path, 'utf8')) + if (!isBackgroundSessionNameReservation(parsed)) return true + existing = parsed + } catch (error) { + return isErrno(error, 'ENOENT') + } + if (existing.name !== name || existing.id !== id) return true + try { + await unlink(path) + return true + } catch (error) { + return isErrno(error, 'ENOENT') + } } -function releaseNameReservationSync(name: string, id: string): void { +async function tryReleaseNameReservation( + name: string, + id: string, +): Promise { + try { + return await withNameReservationLock( + name, + async () => await releaseNameReservationUnlocked(name, id), + ) + } catch { + return false + } +} + +async function releaseNameReservation(name: string, id: string): Promise { + await tryReleaseNameReservation(name, id) +} + +function releaseNameReservationSyncUnlocked( + name: string, + id: string, +): void { const path = nameReservationPathForName(name) try { - const parsed = jsonParse( - readFileSync(path, 'utf8'), - ) as Partial - if (parsed?.id === id) unlinkSync(path) + const parsed = jsonParse(readFileSync(path, 'utf8')) + if ( + !isBackgroundSessionNameReservation(parsed) || + parsed.name !== name || + parsed.id !== id + ) { + return + } + unlinkSync(path) } catch { // Effective terminal-state reads recover stale reservations later. } } +function releaseNameReservationSync(name: string, id: string): void { + try { + withNameReservationLockSync(name, () => { + releaseNameReservationSyncUnlocked(name, id) + }) + } catch { + // A stale reservation is recoverable on the next name claim. + } +} + async function unlinkStaleNameReservation(path: string): Promise { try { await unlink(path) @@ -318,7 +1160,7 @@ async function unlinkStaleNameReservation(path: string): Promise { } } -async function releaseStaleNameReservation( +async function releaseStaleNameReservationUnlocked( name: string, id: string, ): Promise { @@ -354,38 +1196,40 @@ async function reserveBackgroundSessionName( name: string, id: string, ): Promise<() => Promise> { - const path = nameReservationPathForName(name) - const reservation = jsonStringify({ - name, - id, - creatorPid: process.pid, - createdAt: iso(undefined), - }) + return await withNameReservationLock(name, async () => { + const path = nameReservationPathForName(name) + const reservation = jsonStringify({ + name, + id, + creatorPid: process.pid, + createdAt: iso(undefined), + }) - while (true) { - try { - await writeFile(path, reservation, { flag: 'wx' }) - return () => releaseNameReservation(name, id) - } catch (error) { - if (!isErrno(error, 'EEXIST')) throw error - - const existing = await readNameReservation(path) - if (!(await isLiveNameReservation(name, existing))) { - if (existing) { - await releaseStaleNameReservation(name, existing.id) - } else { - await unlinkStaleNameReservation(path) + while (true) { + try { + await writeFile(path, reservation, { flag: 'wx' }) + return () => releaseNameReservation(name, id) + } catch (error) { + if (!isErrno(error, 'EEXIST')) throw error + + const existing = await readNameReservation(path) + if (!(await isLiveNameReservation(name, existing))) { + if (existing) { + await releaseStaleNameReservationUnlocked(name, existing.id) + } else { + await unlinkStaleNameReservation(path) + } + continue } - continue - } - const suffix = - existing && existing.name === name ? ` (${existing.id})` : '' - throw new Error( - `Background session name "${name}" already exists${suffix}`, - ) + const suffix = + existing && existing.name === name ? ` (${existing.id})` : '' + throw new Error( + `Background session name "${name}" already exists${suffix}`, + ) + } } - } + }) } function isStringArray(value: unknown): value is string[] { @@ -434,6 +1278,11 @@ function isBackgroundSession( typeof candidate.sessionId === 'string' && (candidate.processMarker === undefined || isValidBackgroundProcessMarker(candidate.processMarker)) && + (candidate.terminalFactGeneration === undefined || + (candidate.terminalFactGeneration === candidate.processMarker && + isValidBackgroundProcessMarker( + candidate.terminalFactGeneration, + ))) && typeof candidate.startedAt === 'string' && typeof candidate.updatedAt === 'string' && isStringArray(candidate.command) && @@ -462,6 +1311,8 @@ function isBackgroundSessionTerminalFact( typeof candidate.pid !== 'number' || !Number.isInteger(candidate.pid) || candidate.pid <= 0 || + (candidate.generation !== undefined && + !isValidBackgroundProcessMarker(candidate.generation)) || typeof candidate.finishedAt !== 'string' || !isTerminalReason(candidate.terminalReason) || (candidate.exitCode !== undefined && @@ -505,12 +1356,16 @@ function isBackgroundSessionTerminalFact( async function readTerminalFact( id: string, kind: 'natural' | 'killed', + generation?: string, ): Promise { try { const parsed = jsonParse( - await readFile(terminalFactPathForId(id, kind), 'utf8'), + await readFile(terminalFactPathForId(id, kind, generation), 'utf8'), ) - return isBackgroundSessionTerminalFact(parsed, id, kind) ? parsed : null + return isBackgroundSessionTerminalFact(parsed, id, kind) && + parsed.generation === generation + ? parsed + : null } catch { return null } @@ -519,22 +1374,42 @@ async function readTerminalFact( function readTerminalFactSync( id: string, kind: 'natural' | 'killed', + generation?: string, ): BackgroundSessionTerminalFact | null { try { const parsed = jsonParse( - readFileSync(terminalFactPathForId(id, kind), 'utf8'), + readFileSync(terminalFactPathForId(id, kind, generation), 'utf8'), ) - return isBackgroundSessionTerminalFact(parsed, id, kind) ? parsed : null + return isBackgroundSessionTerminalFact(parsed, id, kind) && + parsed.generation === generation + ? parsed + : null } catch { return null } } -async function applyAuthoritativeTerminalFacts( +async function applyAuthoritativeTerminalFacts( + session: BackgroundSession, +): Promise { + const natural = await readTerminalFact( + session.id, + 'natural', + session.terminalFactGeneration, + ) + const killed = await readTerminalFact( + session.id, + 'killed', + session.terminalFactGeneration, + ) + return applyTerminalFacts(session, natural, killed) +} + +function applyTerminalFacts( session: BackgroundSession, -): Promise { - const natural = await readTerminalFact(session.id, 'natural') - const killed = await readTerminalFact(session.id, 'killed') + natural: BackgroundSessionTerminalFact | null, + killed: BackgroundSessionTerminalFact | null, +): BackgroundSession { let effective = session if ( @@ -581,9 +1456,10 @@ async function installTerminalFact( kind: 'natural' | 'killed', ): Promise { await ensureBackgroundSessionDirs() - const target = terminalFactPathForId(fact.id, kind) + const target = terminalFactPathForId(fact.id, kind, fact.generation) const tmp = terminalFactTempPath(fact.id) let handle: Awaited> | undefined + let installed: BackgroundSessionTerminalFact try { handle = await open(tmp, 'wx', 0o600) await handle.writeFile(jsonStringify(fact)) @@ -591,18 +1467,24 @@ async function installTerminalFact( await handle.close() handle = undefined await link(tmp, target) - return fact + installed = fact } catch (error) { if (isErrno(error, 'EEXIST')) { - const existing = await readTerminalFact(fact.id, kind) - if (existing) return existing - throw new Error(`Invalid background session ${kind} terminal fact`) + const existing = await readTerminalFact(fact.id, kind, fact.generation) + if (existing) { + installed = existing + } else { + throw new Error(`Invalid background session ${kind} terminal fact`) + } + } else { + throw error } - throw error } finally { await handle?.close().catch(() => {}) await unlink(tmp).catch(() => {}) } + await enqueueBackgroundSessionRecovery(installed.id, installed.generation).catch(() => {}) + return installed } function installTerminalFactSync( @@ -613,9 +1495,10 @@ function installTerminalFactSync( recursive: true, mode: 0o700, }) - const target = terminalFactPathForId(fact.id, kind) + const target = terminalFactPathForId(fact.id, kind, fact.generation) const tmp = terminalFactTempPath(fact.id) let fd: number | undefined + let installed: BackgroundSessionTerminalFact try { fd = openSync(tmp, 'wx', 0o600) writeFileSync(fd, jsonStringify(fact)) @@ -623,14 +1506,18 @@ function installTerminalFactSync( closeSync(fd) fd = undefined linkSync(tmp, target) - return fact + installed = fact } catch (error) { if (isErrno(error, 'EEXIST')) { - const existing = readTerminalFactSync(fact.id, kind) - if (existing) return existing - throw new Error(`Invalid background session ${kind} terminal fact`) + const existing = readTerminalFactSync(fact.id, kind, fact.generation) + if (existing) { + installed = existing + } else { + throw new Error(`Invalid background session ${kind} terminal fact`) + } + } else { + throw error } - throw error } finally { if (fd !== undefined) { try { @@ -641,9 +1528,22 @@ function installTerminalFactSync( unlinkSync(tmp) } catch {} } + try { + enqueueBackgroundSessionRecoverySync(installed.id, installed.generation) + } catch { + // The terminal fact remains the durable fallback for the daily sweep. + } + return installed } -export async function listBackgroundSessions(): Promise { +type BackgroundSessionRecord = { + stored: BackgroundSession + effective: BackgroundSession +} + +async function listBackgroundSessionRecords(): Promise< + BackgroundSessionRecord[] +> { let entries: string[] try { entries = await readdir(getBackgroundSessionMetadataDir()) @@ -651,16 +1551,895 @@ export async function listBackgroundSessions(): Promise { return [] } - const sessions: BackgroundSession[] = [] + const records: BackgroundSessionRecord[] = [] for (const entry of entries) { if (!entry.endsWith('.json')) continue const session = await readSessionFile( join(getBackgroundSessionMetadataDir(), entry), ) - if (session) sessions.push(await applyAuthoritativeTerminalFacts(session)) + if (session) { + records.push({ + stored: session, + effective: await applyAuthoritativeTerminalFacts(session), + }) + } + } + + return records.sort((a, b) => + a.stored.startedAt.localeCompare(b.stored.startedAt), + ) +} + +export async function listBackgroundSessions(): Promise { + return (await listBackgroundSessionRecords()).map(record => record.effective) +} + +async function readReconciliationTerminalFact( + session: BackgroundSession, + kind: 'natural' | 'killed', + terminalDirectory: CleanupDirectorySnapshot, + fileSystem: BackgroundCleanupFileSystem, +): Promise> { + const generation = session.terminalFactGeneration + return await readCleanupJson( + terminalFactPathForId(session.id, kind, generation), + (value): value is BackgroundSessionTerminalFact => + isBackgroundSessionTerminalFact(value, session.id, kind) && + value.generation === generation && + parseCanonicalCompletionTimestamp(value.finishedAt) !== null, + terminalDirectory, + fileSystem, + ) +} + +function hasSamePersistedTerminalState( + stored: BackgroundSession, + effective: BackgroundSession, +): boolean { + return ( + stored.status === effective.status && + stored.updatedAt === effective.updatedAt && + stored.finishedAt === effective.finishedAt && + stored.exitCode === effective.exitCode && + stored.signal === effective.signal && + stored.terminalReason === effective.terminalReason + ) +} + +export async function reconcileBackgroundSessionTerminalFacts( + options: { + sessionIds?: readonly string[] + onRetry?: (target?: BackgroundSessionRecoveryTarget) => void + terminalScanLimit?: number + } = {}, +): Promise<{ sessionsUpdated: number; errors: number }> { + const result = { + sessionsUpdated: 0, + errors: 0, + } + + const fileSystem: BackgroundCleanupFileSystem = { + lstatFile: lstat, + readTextFile: async path => await readFile(path, 'utf8'), + unlinkFile: unlink, + } + const [metadataRead, terminalRead] = await Promise.all([ + snapshotCleanupDirectory( + getBackgroundSessionMetadataDir(), + fileSystem.lstatFile, + ), + snapshotCleanupDirectory( + getBackgroundSessionTerminalDir(), + fileSystem.lstatFile, + ), + ]) + if (metadataRead.state !== 'valid' || terminalRead.state !== 'valid') { + result.errors += Number(metadataRead.state !== 'valid') + result.errors += Number(terminalRead.state !== 'valid') + if (result.errors > 0) options.onRetry?.() + return result + } + const metadataDirectory = metadataRead.value + const terminalDirectory = terminalRead.value + if ( + metadataDirectory.state === 'missing' || + terminalDirectory.state === 'missing' + ) { + if (result.errors > 0) options.onRetry?.() + return result + } + + let candidateIds: string[] + if (options.sessionIds !== undefined) { + candidateIds = [ + ...new Set(options.sessionIds.filter(id => SAFE_ID_RE.test(id))), + ] + } else { + let entries: Dirent[] + try { + const scanTerminalFacts = options.terminalScanLimit !== undefined + entries = await readDirectoryEntries( + scanTerminalFacts + ? getBackgroundSessionTerminalDir() + : getBackgroundSessionMetadataDir(), + scanTerminalFacts ? options.terminalScanLimit : undefined, + ) + } catch (error) { + if (!isErrno(error, 'ENOENT')) result.errors++ + if (result.errors > 0) options.onRetry?.() + return result + } + const scannedDirectory = + options.terminalScanLimit === undefined + ? metadataDirectory + : terminalDirectory + if ( + (await inspectCleanupDirectory( + scannedDirectory, + fileSystem.lstatFile, + )) !== 'same' + ) { + result.errors++ + if (result.errors > 0) options.onRetry?.() + return result + } + candidateIds = [ + ...new Set( + options.terminalScanLimit === undefined + ? entries + .filter( + entry => entry.isFile() && entry.name.endsWith('.json'), + ) + .map(entry => entry.name.slice(0, -'.json'.length)) + .filter(id => SAFE_ID_RE.test(id)) + : entries + .map(terminalFactCandidateFromEntry) + .filter( + (candidate): candidate is NonNullable => + candidate !== null, + ) + .map(candidate => candidate.id), + ), + ] + } + + for (const id of candidateIds) { + const errorsBefore = result.errors + let retryGeneration: string | undefined + try { + const metadataPath = metadataPathForId(id) + const candidate = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => + isBackgroundSession(value, id), + metadataDirectory, + fileSystem, + ) + if (candidate.state === 'error') { + result.errors++ + continue + } + if (candidate.state !== 'valid' || !candidate.identity) continue + retryGeneration = candidate.value.terminalFactGeneration + + try { + await withBackgroundSessionIdLock(id, async () => { + const natural = await readReconciliationTerminalFact( + candidate.value, + 'natural', + terminalDirectory, + fileSystem, + ) + const killed = await readReconciliationTerminalFact( + candidate.value, + 'killed', + terminalDirectory, + fileSystem, + ) + if (natural.state === 'error' || killed.state === 'error') { + result.errors++ + return + } + + const latest = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => + isBackgroundSession(value, id), + metadataDirectory, + fileSystem, + ) + if (latest.state === 'error') { + result.errors++ + return + } + if ( + latest.state !== 'valid' || + !latest.identity || + latest.identity.dev !== candidate.identity?.dev || + latest.identity.ino !== candidate.identity?.ino || + !isSameBackgroundSessionGeneration( + latest.value, + candidate.value, + ) + ) { + return + } + + const effective = applyTerminalFacts( + latest.value, + natural.state === 'valid' ? natural.value : null, + killed.state === 'valid' ? killed.value : null, + ) + if (!COMPLETED_STATUSES.has(effective.status)) return + + if (!hasSamePersistedTerminalState(latest.value, effective)) { + if ( + (await inspectCleanupDirectory( + metadataDirectory, + fileSystem.lstatFile, + )) !== 'same' + ) { + result.errors++ + return + } + try { + const beforeWrite = await fileSystem.lstatFile(metadataPath) + if ( + !beforeWrite.isFile() || + beforeWrite.isSymbolicLink() || + beforeWrite.dev !== latest.identity.dev || + beforeWrite.ino !== latest.identity.ino + ) { + return + } + } catch (error) { + if (!isErrno(error, 'ENOENT')) result.errors++ + return + } + await replaceSessionFile(effective, metadataPath) + result.sessionsUpdated++ + } + if ( + effective.name && + !(await tryReleaseNameReservation(effective.name, effective.id)) + ) { + result.errors++ + } + }) + } catch { + result.errors++ + } + } finally { + if (result.errors > errorsBefore) { + options.onRetry?.({ id: id, generation: retryGeneration }) + } + } + } + + return result +} + +type CleanupArtifactRemoval = 'removed' | 'missing' | 'error' + +async function removeCleanupArtifact( + path: string, + result: BackgroundCleanupResult, + directory: CleanupDirectorySnapshot, + fileSystem: BackgroundCleanupFileSystem, + expectedIdentity?: CleanupFileIdentity, +): Promise { + const directoryState = await inspectCleanupDirectory( + directory, + fileSystem.lstatFile, + ) + if (directoryState === 'missing') return 'missing' + if (directoryState !== 'same') { + result.errors++ + return 'error' + } + if (expectedIdentity) { + try { + const stats = await fileSystem.lstatFile(path) + if ( + !stats.isFile() || + stats.isSymbolicLink() || + stats.dev !== expectedIdentity.dev || + stats.ino !== expectedIdentity.ino + ) { + result.errors++ + return 'error' + } + } catch (error) { + if (isErrno(error, 'ENOENT')) return 'missing' + result.errors++ + return 'error' + } + } + try { + await fileSystem.unlinkFile(path) + result.artifactsRemoved++ + return 'removed' + } catch (error) { + if (isErrno(error, 'ENOENT')) return 'missing' + result.errors++ + return 'error' + } +} + +function cleanupReadBlocksRemoval( + read: CleanupJsonRead, + result: BackgroundCleanupResult, +): boolean { + if (read.state === 'error') { + result.errors++ + return true + } + return read.state === 'invalid' +} + +function terminalFactCandidateFromEntry( + entry: Dirent, +): { + id: string + kind: 'natural' | 'killed' + generation?: string +} | null { + if (!entry.isFile()) return null + const markedMatch = + /^(.*)~([a-f0-9]{64})\.(natural|killed)\.json$/.exec(entry.name) + const legacyMatch = /^(.*)\.(natural|killed)\.json$/.exec(entry.name) + const match = markedMatch ?? legacyMatch + if (!match) return null + const id = match[1] + const generation = markedMatch?.[2] + const kind = markedMatch?.[3] ?? legacyMatch?.[2] + if (!id || !kind || !SAFE_ID_RE.test(id)) return null + if ( + generation !== undefined && + !isValidBackgroundProcessMarker(generation) + ) { + return null + } + return { + id, + kind: kind as 'natural' | 'killed', + ...(generation ? { generation } : {}), + } +} + +async function cleanupOrphanedTerminalFacts( + cutoffMs: number, + metadataDirectory: CleanupDirectorySnapshot, + terminalDirectory: CleanupDirectorySnapshot, + metadataIds: ReadonlySet, + result: BackgroundCleanupResult, + fileSystem: BackgroundCleanupFileSystem, + maxDirectoryEntries?: number, + targets?: readonly BackgroundSessionRecoveryTarget[], + onRetry?: (target?: BackgroundSessionRecoveryTarget) => void, +): Promise { + const terminalDirectoryState = await inspectCleanupDirectory( + terminalDirectory, + fileSystem.lstatFile, + ) + if (terminalDirectoryState === 'missing') return + if (terminalDirectoryState !== 'same') { + result.errors++ + onRetry?.() + return + } + + let candidates: NonNullable< + ReturnType + >[] + if (targets !== undefined) { + candidates = targets + .filter( + target => + backgroundRecoveryJournalRecord(target.id, target.generation) !== + undefined, + ) + .flatMap(target => [ + { ...target, kind: 'natural' as const }, + { ...target, kind: 'killed' as const }, + ]) + } else { + let entries: Dirent[] + try { + entries = await readDirectoryEntries( + getBackgroundSessionTerminalDir(), + maxDirectoryEntries, + ) + } catch (error) { + if (!isErrno(error, 'ENOENT')) { + result.errors++ + onRetry?.() + } + return + } + const terminalDirectoryAfterRead = await inspectCleanupDirectory( + terminalDirectory, + fileSystem.lstatFile, + ) + if (terminalDirectoryAfterRead !== 'same') { + if (terminalDirectoryAfterRead !== 'missing') { + result.errors++ + onRetry?.() + } + return + } + + candidates = entries + .map(terminalFactCandidateFromEntry) + .filter( + (candidate): candidate is NonNullable => + candidate !== null && !metadataIds.has(candidate.id), + ) + .sort((a, b) => { + const idOrder = a.id.localeCompare(b.id) + if (idOrder !== 0) return idOrder + const markerOrder = (a.generation ?? '').localeCompare( + b.generation ?? '', + ) + return markerOrder !== 0 + ? markerOrder + : a.kind.localeCompare(b.kind) + }) + } + if (candidates.length === 0) return + + for (const candidate of candidates) { + const errorsBefore = result.errors + try { + const factPath = terminalFactPathForId( + candidate.id, + candidate.kind, + candidate.generation, + ) + const fact = await readCleanupJson( + factPath, + (value): value is BackgroundSessionTerminalFact => + isBackgroundSessionTerminalFact( + value, + candidate.id, + candidate.kind, + ) && + value.generation === candidate.generation && + parseCanonicalCompletionTimestamp(value.finishedAt) !== null, + terminalDirectory, + fileSystem, + ) + if (fact.state === 'error') { + result.errors++ + continue + } + if ( + fact.state !== 'valid' || + Date.parse(fact.value.finishedAt) >= cutoffMs + ) { + continue + } + + const metadata = await readCleanupJson( + metadataPathForId(candidate.id), + (value): value is BackgroundSession => + isBackgroundSession(value, candidate.id), + metadataDirectory, + fileSystem, + ) + if (metadata.state === 'error') { + result.errors++ + continue + } + if (metadata.state !== 'missing') continue + + await removeCleanupArtifact( + factPath, + result, + terminalDirectory, + fileSystem, + fact.identity, + ) + } finally { + if (result.errors > errorsBefore) onRetry?.(candidate) + } + } +} + +export async function cleanupBackgroundSessionsBefore( + cutoff: Date, + options: { + lstatFile?: BackgroundCleanupFileSystem['lstatFile'] + readTextFile?: BackgroundCleanupFileSystem['readTextFile'] + unlinkFile?: BackgroundCleanupFileSystem['unlinkFile'] + _beforeMetadataDirectoryReadForTesting?: () => Promise + _beforeArtifactRemovalForTesting?: (id: string) => Promise + _beforeReservationRemovalForTesting?: (path: string) => Promise + sessionIds?: readonly string[] + onRetry?: (target?: BackgroundSessionRecoveryTarget) => void + orphanedTerminalFacts?: readonly BackgroundSessionRecoveryTarget[] + maxDirectoryEntries?: number + } = {}, +): Promise { + const result: BackgroundCleanupResult = { + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + } + const cutoffMs = cutoff.getTime() + if (!Number.isFinite(cutoffMs)) { + result.errors++ + options.onRetry?.() + return result + } + + const fileSystem: BackgroundCleanupFileSystem = { + lstatFile: options.lstatFile ?? lstat, + readTextFile: + options.readTextFile ?? (async path => await readFile(path, 'utf8')), + unlinkFile: options.unlinkFile ?? (async path => await unlink(path)), + } + const directoryReads = await Promise.all( + [ + getBackgroundSessionMetadataDir(), + getBackgroundSessionLogsDir(), + getBackgroundSessionNamesDir(), + getBackgroundSessionTerminalDir(), + ].map( + async path => + await snapshotCleanupDirectory(path, fileSystem.lstatFile), + ), + ) + const [metadataRead, logsRead, namesRead, terminalRead] = directoryReads + const requiredDirectoryReads = [metadataRead, logsRead, terminalRead] + const requiredDirectoryErrors = requiredDirectoryReads.filter( + read => read.state === 'error' || read.state === 'invalid', + ).length + if (requiredDirectoryErrors > 0) { + result.errors += requiredDirectoryErrors + options.onRetry?.() + return result + } + const metadataDirectory = + metadataRead.state === 'valid' ? metadataRead.value : undefined + const logsDirectory = + logsRead.state === 'valid' ? logsRead.value : undefined + const namesDirectory = + namesRead.state === 'valid' ? namesRead.value : undefined + const terminalDirectory = + terminalRead.state === 'valid' ? terminalRead.value : undefined + if (!metadataDirectory || !logsDirectory || !terminalDirectory) { + if (result.errors > 0) options.onRetry?.() + return result + } + if (metadataDirectory.state === 'missing') { + if (options.sessionIds !== undefined && !options.orphanedTerminalFacts) + return result + await cleanupOrphanedTerminalFacts( + cutoffMs, + metadataDirectory, + terminalDirectory, + new Set(), + result, + fileSystem, + options.maxDirectoryEntries, + options.orphanedTerminalFacts, + options.onRetry, + ) + return result + } + const metadataDirectoryState = await inspectCleanupDirectory( + metadataDirectory, + fileSystem.lstatFile, + ) + if (metadataDirectoryState !== 'same') { + if (metadataDirectoryState !== 'missing') result.errors++ + if (result.errors > 0) options.onRetry?.() + return result + } + + let candidateIds: string[] + if (options.sessionIds !== undefined) { + candidateIds = [ + ...new Set(options.sessionIds.filter(id => SAFE_ID_RE.test(id))), + ] + } else { + let entries: Dirent[] + try { + await options._beforeMetadataDirectoryReadForTesting?.() + entries = await readDirectoryEntries( + getBackgroundSessionMetadataDir(), + options.maxDirectoryEntries, + ) + } catch (error) { + if (!isErrno(error, 'ENOENT')) result.errors++ + if (result.errors > 0) options.onRetry?.() + return result + } + const metadataDirectoryAfterRead = await inspectCleanupDirectory( + metadataDirectory, + fileSystem.lstatFile, + ) + if (metadataDirectoryAfterRead !== 'same') { + if (metadataDirectoryAfterRead !== 'missing') result.errors++ + if (result.errors > 0) options.onRetry?.() + return result + } + candidateIds = entries + .filter(entry => entry.isFile() && entry.name.endsWith('.json')) + .map(entry => basename(entry.name, '.json')) + .filter(id => SAFE_ID_RE.test(id)) + } + + const metadataIds = new Set(candidateIds) + + for (const expectedId of candidateIds) { + const errorsBefore = result.errors + let retryGeneration: string | undefined + try { + const metadataPath = metadataPathForId(expectedId) + const metadata = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => + isBackgroundSession(value, expectedId) && + (value.finishedAt === undefined || + parseCanonicalCompletionTimestamp(value.finishedAt) !== null), + metadataDirectory, + fileSystem, + ) + if (metadata.state === 'error') { + result.errors++ + continue + } + if (metadata.state !== 'valid') continue + + const session = metadata.value + retryGeneration = session.terminalFactGeneration + const naturalPath = terminalFactPathForId( + session.id, + 'natural', + session.terminalFactGeneration, + ) + const killedPath = terminalFactPathForId( + session.id, + 'killed', + session.terminalFactGeneration, + ) + const natural = await readCleanupJson( + naturalPath, + (value): value is BackgroundSessionTerminalFact => + isBackgroundSessionTerminalFact(value, session.id, 'natural') && + value.generation === session.terminalFactGeneration && + parseCanonicalCompletionTimestamp(value.finishedAt) !== null, + terminalDirectory, + fileSystem, + ) + const killed = await readCleanupJson( + killedPath, + (value): value is BackgroundSessionTerminalFact => + isBackgroundSessionTerminalFact(value, session.id, 'killed') && + value.generation === session.terminalFactGeneration && + parseCanonicalCompletionTimestamp(value.finishedAt) !== null, + terminalDirectory, + fileSystem, + ) + const naturalBlocksRemoval = cleanupReadBlocksRemoval(natural, result) + const killedBlocksRemoval = cleanupReadBlocksRemoval(killed, result) + if ( + naturalBlocksRemoval || + killedBlocksRemoval || + (natural.state === 'valid' && natural.value.pid !== session.pid) || + (killed.state === 'valid' && killed.value.pid !== session.pid) + ) { + continue + } + + const effective = applyTerminalFacts( + session, + natural.state === 'valid' ? natural.value : null, + killed.state === 'valid' ? killed.value : null, + ) + if ( + effective.status !== 'exited' && + effective.status !== 'failed' && + effective.status !== 'killed' + ) { + continue + } + if (effective.finishedAt === undefined) continue + const finishedAtMs = Date.parse(effective.finishedAt) + if (!Number.isFinite(finishedAtMs) || finishedAtMs >= cutoffMs) + continue + + let reservation: + | CleanupJsonRead + | undefined + if (session.name) { + if (!namesDirectory) { + result.errors++ + continue + } + reservation = await readCleanupJson( + nameReservationPathForName(session.name), + isBackgroundSessionNameReservation, + namesDirectory, + fileSystem, + ) + if (reservation.state === 'error') { + result.errors++ + continue + } + } + + try { + await options._beforeArtifactRemovalForTesting?.(session.id) + await withBackgroundSessionIdLock(session.id, async () => { + const latestMetadata = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => + isBackgroundSession(value, session.id) && + (value.finishedAt === undefined || + parseCanonicalCompletionTimestamp(value.finishedAt) !== + null), + metadataDirectory, + fileSystem, + ) + if (latestMetadata.state === 'error') { + result.errors++ + return + } + if ( + latestMetadata.state !== 'valid' || + !latestMetadata.identity || + !metadata.identity || + latestMetadata.identity.dev !== metadata.identity.dev || + latestMetadata.identity.ino !== metadata.identity.ino || + !isSameBackgroundSessionGeneration( + latestMetadata.value, + session, + ) + ) { + return + } + + const logPaths = getBackgroundSessionLogPaths(session.id) + const stdoutRemoval = await removeCleanupArtifact( + logPaths.stdoutLogPath, + result, + logsDirectory, + fileSystem, + ) + const stderrRemoval = await removeCleanupArtifact( + logPaths.stderrLogPath, + result, + logsDirectory, + fileSystem, + ) + let reservationRemoval: CleanupArtifactRemoval = 'missing' + const sessionName = session.name + if ( + sessionName && + reservation?.state === 'valid' && + namesDirectory + ) { + try { + reservationRemoval = await withNameReservationLock( + sessionName, + async () => { + const latestReservation = await readCleanupJson( + nameReservationPathForName(sessionName), + isBackgroundSessionNameReservation, + namesDirectory, + fileSystem, + ) + if (latestReservation.state === 'error') { + result.errors++ + return 'error' + } + if ( + latestReservation.state !== 'valid' || + latestReservation.value.name !== sessionName || + latestReservation.value.id !== session.id + ) { + return 'missing' + } + const reservationPath = + nameReservationPathForName(sessionName) + await options._beforeReservationRemovalForTesting?.( + reservationPath, + ) + return await removeCleanupArtifact( + reservationPath, + result, + namesDirectory, + fileSystem, + latestReservation.identity, + ) + }, + ) + } catch { + result.errors++ + reservationRemoval = 'error' + } + } + if ( + stdoutRemoval === 'error' || + stderrRemoval === 'error' || + reservationRemoval === 'error' + ) { + return + } + + const metadataRemoval = await removeCleanupArtifact( + metadataPath, + result, + metadataDirectory, + fileSystem, + latestMetadata.identity, + ) + if (metadataRemoval === 'error') return + if (metadataRemoval === 'removed') result.sessionsRemoved++ + + const errorsBeforeFacts = result.errors + if (natural.state === 'valid') { + await removeCleanupArtifact( + naturalPath, + result, + terminalDirectory, + fileSystem, + natural.identity, + ) + } + if (killed.state === 'valid') { + await removeCleanupArtifact( + killedPath, + result, + terminalDirectory, + fileSystem, + killed.identity, + ) + } + if (result.errors > errorsBeforeFacts) { + const target = { + id: session.id, + generation: session.terminalFactGeneration, + } + if (!options.onRetry) { + await enqueueBackgroundSessionRecovery( + target.id, + target.generation, + ) + } + } + }) + } catch { + result.errors++ + } + } finally { + if (result.errors > errorsBefore) { + options.onRetry?.({ id: expectedId, generation: retryGeneration }) + } + } + } + + if ( + options.sessionIds === undefined || + options.orphanedTerminalFacts !== undefined + ) { + await cleanupOrphanedTerminalFacts( + cutoffMs, + metadataDirectory, + terminalDirectory, + options.sessionIds === undefined ? metadataIds : new Set(), + result, + fileSystem, + options.maxDirectoryEntries, + options.orphanedTerminalFacts, + options.onRetry, + ) } - return sessions.sort((a, b) => a.startedAt.localeCompare(b.startedAt)) + return result } export async function readBackgroundSessionForOwner( @@ -711,6 +2490,9 @@ export async function createBackgroundSession( ...(input.processMarker ? { processMarker: input.processMarker } : {}), + ...(input.processMarker + ? { terminalFactGeneration: input.processMarker } + : {}), startedAt: timestamp, updatedAt: timestamp, command: input.command, @@ -792,6 +2574,10 @@ export async function refreshBackgroundSessionStatuses(options?: { session: BackgroundSession, nextStatus: BackgroundSessionStatus, ) => Promise + _whileStatusWriteLockedForTesting?: ( + session: BackgroundSession, + nextStatus: BackgroundSessionStatus, + ) => void | Promise }): Promise { const timestamp = iso(options?.now) const sessions = await listBackgroundSessions() @@ -821,8 +2607,12 @@ export async function refreshBackgroundSessionStatuses(options?: { updatedAt: timestamp, } await options?._beforeStatusWriteForTesting?.(session, nextStatus) - await writeSession(updated) - refreshed.push(await applyAuthoritativeTerminalFacts(updated)) + const persisted = await writeSession(updated, session, () => + options?._whileStatusWriteLockedForTesting?.(session, nextStatus), + ) + if (persisted) { + refreshed.push(await applyAuthoritativeTerminalFacts(persisted)) + } continue } @@ -1020,6 +2810,7 @@ export function isBackgroundSessionProcessAlive( function naturalTerminalFact( id: string, pid: number, + generation: string | undefined, termination: BackgroundSessionNaturalTermination, now: Date | undefined, ): BackgroundSessionTerminalFact { @@ -1035,6 +2826,7 @@ function naturalTerminalFact( version: 1, id, pid, + ...(generation ? { generation } : {}), status: 'failed', finishedAt, terminalReason: 'signal', @@ -1048,6 +2840,7 @@ function naturalTerminalFact( version: 1, id, pid, + ...(generation ? { generation } : {}), status: termination.exitCode === 0 ? 'exited' : 'failed', finishedAt, terminalReason: 'exit_code', @@ -1059,106 +2852,214 @@ function assertNaturalFinalizationOwner( session: BackgroundSession | null, id: string, ownerPid: number, + expectedSession?: BackgroundSession, ): asserts session is BackgroundSession { - if (!session || session.id !== id || session.pid !== ownerPid) { + if ( + !session || + session.id !== id || + session.pid !== ownerPid || + (expectedSession !== undefined && + !isSameBackgroundSessionGeneration(session, expectedSession)) + ) { throw new Error('Background session finalizer does not own this session') } } +type BackgroundSessionNaturalFinalizationOptions = { + ownerPid?: number + now?: Date + expectedSession?: BackgroundSession +} + export async function recordBackgroundSessionNaturalTermination( id: string, termination: BackgroundSessionNaturalTermination, - options: { ownerPid?: number; now?: Date } = {}, + options: BackgroundSessionNaturalFinalizationOptions = {}, ): Promise { assertSafeId(id) const ownerPid = options.ownerPid ?? process.pid - const session = await readSessionFile(metadataPathForId(id)) - assertNaturalFinalizationOwner(session, id, ownerPid) + return await withBackgroundSessionIdLock(id, async () => { + const session = await readSessionFile(metadataPathForId(id)) + assertNaturalFinalizationOwner( + session, + id, + ownerPid, + options.expectedSession, + ) - const effective = await applyAuthoritativeTerminalFacts(session) - if ( - effective.status === 'killed' || - session.status === 'exited' || - session.status === 'failed' - ) { - return effective - } - if ( - session.status !== 'running' && - session.status !== 'unknown' && - session.status !== 'stale' - ) { - // Retain an exhaustive guard so future status additions require a deliberate - // natural-finalization policy. - throw new Error('Background session is not eligible for natural finalization') - } + const effective = await applyAuthoritativeTerminalFacts(session) + if ( + effective.status === 'killed' || + session.status === 'exited' || + session.status === 'failed' + ) { + return effective + } + if ( + session.status !== 'running' && + session.status !== 'unknown' && + session.status !== 'stale' + ) { + // Retain an exhaustive guard so future status additions require a deliberate + // natural-finalization policy. + throw new Error( + 'Background session is not eligible for natural finalization', + ) + } - await installTerminalFact( - naturalTerminalFact(id, ownerPid, termination, options.now), - 'natural', - ) - if (session.name) await releaseNameReservation(session.name, session.id) - return await applyAuthoritativeTerminalFacts(session) + await installTerminalFact( + naturalTerminalFact( + id, + ownerPid, + session.terminalFactGeneration, + termination, + options.now, + ), + 'natural', + ) + if (session.name) await releaseNameReservation(session.name, session.id) + return await applyAuthoritativeTerminalFacts(session) + }) } export function recordBackgroundSessionNaturalTerminationSync( id: string, termination: BackgroundSessionNaturalTermination, - options: { ownerPid?: number; now?: Date } = {}, + options: BackgroundSessionNaturalFinalizationOptions = {}, ): void { assertSafeId(id) const ownerPid = options.ownerPid ?? process.pid - const session = readSessionFileSync(metadataPathForId(id)) - assertNaturalFinalizationOwner(session, id, ownerPid) - if ( - session.status === 'killed' || - session.status === 'exited' || - session.status === 'failed' || - readTerminalFactSync(id, 'killed')?.pid === ownerPid - ) { - return - } - if ( - session.status !== 'running' && - session.status !== 'unknown' && - session.status !== 'stale' - ) { - // Retain an exhaustive guard so future status additions require a deliberate - // natural-finalization policy. - throw new Error('Background session is not eligible for natural finalization') - } + try { + withBackgroundSessionIdLockSync(id, () => { + const session = readSessionFileSync(metadataPathForId(id)) + assertNaturalFinalizationOwner( + session, + id, + ownerPid, + options.expectedSession, + ) + if ( + session.status === 'killed' || + session.status === 'exited' || + session.status === 'failed' || + readTerminalFactSync( + id, + 'killed', + session.terminalFactGeneration, + )?.pid === ownerPid + ) { + return + } + if ( + session.status !== 'running' && + session.status !== 'unknown' && + session.status !== 'stale' + ) { + // Retain an exhaustive guard so future status additions require a deliberate + // natural-finalization policy. + throw new Error( + 'Background session is not eligible for natural finalization', + ) + } - installTerminalFactSync( - naturalTerminalFact(id, ownerPid, termination, options.now), - 'natural', - ) - if (session.name) releaseNameReservationSync(session.name, session.id) + installTerminalFactSync( + naturalTerminalFact( + id, + ownerPid, + session.terminalFactGeneration, + termination, + options.now, + ), + 'natural', + ) + if (session.name) releaseNameReservationSync(session.name, session.id) + }) + } catch (error) { + const expectedSession = options.expectedSession + const generation = expectedSession?.terminalFactGeneration + const hasSafeCompatibilityHandoff = + generation !== undefined || expectedSession?.processMarker !== undefined + if ( + !isErrno(error, 'ELOCKED') || + !expectedSession || + !hasSafeCompatibilityHandoff + ) { + throw error + } + // The exit event cannot wait for another process's metadata lock. A marked + // session can still leave an immutable fact without touching metadata or a + // name reservation. Marker-only metadata predates generation-scoped fact + // paths, but new marked replacements never read its legacy path and a + // markerless replacement cannot reuse the ID while that fact remains. + assertNaturalFinalizationOwner( + expectedSession, + id, + ownerPid, + expectedSession, + ) + if ( + expectedSession.status === 'killed' || + expectedSession.status === 'exited' || + expectedSession.status === 'failed' || + readTerminalFactSync( + id, + 'killed', + generation, + )?.pid === ownerPid + ) { + return + } + installTerminalFactSync( + naturalTerminalFact( + id, + ownerPid, + generation, + termination, + options.now, + ), + 'natural', + ) + } } export async function markBackgroundSessionKilled( target: string, - options?: { now?: Date }, + options?: { + now?: Date + _beforeMarkWriteForTesting?: (session: BackgroundSession) => Promise + }, ): Promise { const session = await resolveBackgroundSession(target) - const rawSession = await readSessionFile(metadataPathForId(session.id)) - if (!rawSession || rawSession.pid !== session.pid) { - throw new Error('Background session changed before it could be marked killed') - } - await installTerminalFact( - { - version: 1, - id: session.id, - pid: session.pid, - status: 'killed', - finishedAt: iso(options?.now), - terminalReason: 'explicit_kill', - }, - 'killed', - ) - if (rawSession.name) { - await releaseNameReservation(rawSession.name, rawSession.id) - } - return await applyAuthoritativeTerminalFacts(rawSession) + await options?._beforeMarkWriteForTesting?.(session) + return await withBackgroundSessionIdLock(session.id, async () => { + const rawSession = await readSessionFile(metadataPathForId(session.id)) + if ( + !rawSession || + !isSameBackgroundSessionGeneration(rawSession, session) + ) { + throw new Error( + 'Background session changed before it could be marked killed', + ) + } + await installTerminalFact( + { + version: 1, + id: rawSession.id, + pid: rawSession.pid, + ...(rawSession.terminalFactGeneration + ? { generation: rawSession.terminalFactGeneration } + : {}), + status: 'killed', + finishedAt: iso(options?.now), + terminalReason: 'explicit_kill', + }, + 'killed', + ) + if (rawSession.name) { + await releaseNameReservation(rawSession.name, rawSession.id) + } + return await applyAuthoritativeTerminalFacts(rawSession) + }) } export async function backgroundSessionLogExists(path: string): Promise { diff --git a/src/cli/bgRouting.ts b/src/cli/bgRouting.ts index 53b5e1e5bc..4000f02cab 100644 --- a/src/cli/bgRouting.ts +++ b/src/cli/bgRouting.ts @@ -4,6 +4,10 @@ export const BACKGROUND_SESSION_ID_ENV = 'OPENCLAUDE_INTERNAL_BACKGROUND_SESSION_ID' export const BACKGROUND_SESSION_LAUNCHER_PID_ENV = 'OPENCLAUDE_INTERNAL_BACKGROUND_LAUNCHER_PID' +export const BACKGROUND_SESSION_CLEANUP_WORKER_ENV = + 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_WORKER' +export const BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV = + 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_OWNER_PID' export const BACKGROUND_PROCESS_MARKER_FLAG = '--openclaude-bg-session-marker' diff --git a/src/entrypoints/cli.test.ts b/src/entrypoints/cli.test.ts index 34d00b61b8..af37983bb0 100644 --- a/src/entrypoints/cli.test.ts +++ b/src/entrypoints/cli.test.ts @@ -18,6 +18,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { Command } from '@commander-js/extra-typings' import { + BACKGROUND_SESSION_CLEANUP_WORKER_ENV, BACKGROUND_SESSION_ID_ENV, BACKGROUND_SESSION_LAUNCHER_PID_ENV, } from '../cli/bgRouting.js' @@ -40,9 +41,15 @@ const mockProfileCheckpoint = mock((_checkpoint: string) => {}) const mockPsHandler = mock(async (_args: string[]) => {}) const mockLogsHandler = mock(async (_args: string[]) => {}) const mockAttachHandler = mock(async (_args: string[]) => {}) -const mockKillHandler = mock(async (_args: string[]) => {}) +const mockKillHandler = mock( + async ( + _args: string[], + _options?: { retentionSettingsReady?: boolean }, + ) => {}, +) const mockHandleBgFlag = mock(async (_args: string[]) => {}) const mockPrepareBackgroundSessionFinalizer = mock(async () => 'installed') +const mockRunBackgroundSessionCleanupWorker = mock(async () => {}) const mockLoadEnvFile = mock((_filePath: string) => ({})) const mockParseProviderEnvFileArgs = mock((_args: string[]) => ({ paths: [] })) const mockReapplyRememberedEnvFileValues = mock(() => {}) @@ -61,6 +68,7 @@ const mockGetProviderValidationError = mock( async (_env: NodeJS.ProcessEnv) => undefined, ) const mockEagerLoadSettingsFromArgs = mock((_args: string[]) => ({ ok: true })) +const mockResetSettingsCache = mock(() => {}) const mockResolveOutOfProcessTeammateProviderFromCliArgs = mock( (_args: string[], _settings: unknown) => undefined, ) @@ -81,6 +89,7 @@ const runtimeMocks = [ mockKillHandler, mockHandleBgFlag, mockPrepareBackgroundSessionFinalizer, + mockRunBackgroundSessionCleanupWorker, mockLoadEnvFile, mockParseProviderEnvFileArgs, mockReapplyRememberedEnvFileValues, @@ -90,6 +99,7 @@ const runtimeMocks = [ mockApplyStartupEnvFromProfile, mockGetProviderValidationError, mockEagerLoadSettingsFromArgs, + mockResetSettingsCache, mockResolveOutOfProcessTeammateProviderFromCliArgs, mockApplyAgentProviderOverrideToEnv, mockGetInitialSettings, @@ -295,18 +305,15 @@ describe('cli.tsx — --provider startup ordering', () => { expect(process.env.GEMINI_MODEL).toBe('gemini-2.0-flash') }) - it('dispatches background session management before config and provider validation', async () => { + it('dispatches background session management before provider validation', async () => { const src = await Bun.file(`${import.meta.dir}/cli.tsx`).text() const bgManagementIndex = src.indexOf("args[0] === 'ps'") - const configEnableIndex = src.indexOf('enableConfigs()') const providerValidationIndex = src.indexOf( 'await validateProviderEnvForStartupOrExit()', ) expect(bgManagementIndex).toBeGreaterThanOrEqual(0) - expect(configEnableIndex).toBeGreaterThanOrEqual(0) expect(providerValidationIndex).toBeGreaterThanOrEqual(0) - expect(bgManagementIndex).toBeLessThan(configEnableIndex) expect(bgManagementIndex).toBeLessThan(providerValidationIndex) }) @@ -340,6 +347,7 @@ const mockImporters = { }), bgFinalizer: async () => ({ prepareBackgroundSessionFinalizer: mockPrepareBackgroundSessionFinalizer, + runBackgroundSessionCleanupWorker: mockRunBackgroundSessionCleanupWorker, }), envFile: async () => ({ loadEnvFile: mockLoadEnvFile, @@ -365,6 +373,9 @@ const mockImporters = { flagSettings: async () => ({ eagerLoadSettingsFromArgs: mockEagerLoadSettingsFromArgs, }), + settingsCache: async () => ({ + resetSettingsCache: mockResetSettingsCache, + }), agentRouting: async () => ({ applyAgentProviderOverrideToEnv: mockApplyAgentProviderOverrideToEnv, resolveOutOfProcessTeammateProviderFromCliArgs: @@ -435,15 +446,44 @@ describe('cli.tsx — background routing behavior', () => { await runCliEntrypoint([command, ...tail], bgOptions) - expect(handler.mock.calls).toEqual([[tail]]) + if (command === 'kill') { + expect(mockKillHandler.mock.calls).toEqual([ + [tail, { retentionSettingsReady: true }], + ]) + } else { + expect(handler.mock.calls).toEqual([[tail]]) + } expect(mockParseProviderEnvFileArgs).not.toHaveBeenCalled() expect(mockHandleBgFlag).not.toHaveBeenCalled() - expect(mockEnableConfigs).not.toHaveBeenCalled() + expect(mockEnableConfigs).toHaveBeenCalledTimes( + command === 'kill' ? 1 : 0, + ) + expect(mockEagerLoadSettingsFromArgs).toHaveBeenCalledTimes( + command === 'kill' ? 1 : 0, + ) expect(mockValidateProviderEnvForStartupOrExit).not.toHaveBeenCalled() expect(mockCliMain).not.toHaveBeenCalled() } }) + it('keeps kill reachable when retention settings cannot be loaded', async () => { + mockEagerLoadSettingsFromArgs.mockImplementationOnce(() => ({ + ok: false, + message: 'missing retention settings', + })) + + await runCliEntrypoint( + ['kill', 'session-1', '--settings', 'missing.json'], + bgOptions, + ) + + expect(mockKillHandler).toHaveBeenCalledWith( + ['session-1', '--settings', 'missing.json'], + { retentionSettingsReady: false }, + ) + expect(mockValidateProviderEnvForStartupOrExit).not.toHaveBeenCalled() + }) + it('establishes background finalizer ownership before any command path', async () => { process.env[BACKGROUND_SESSION_ID_ENV] = 'bg-entrypoint' mockPrepareBackgroundSessionFinalizer.mockImplementationOnce(async () => { @@ -462,6 +502,36 @@ describe('cli.tsx — background routing behavior', () => { expect(mockEnableConfigs).not.toHaveBeenCalled() }) + it('runs an internal cleanup worker before any command path', async () => { + process.env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] = '1' + try { + await runCliEntrypoint(['ps'], bgOptions) + } finally { + delete process.env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] + } + + expect(mockRunBackgroundSessionCleanupWorker).toHaveBeenCalledTimes(1) + expect(mockPrepareBackgroundSessionFinalizer).not.toHaveBeenCalled() + expect(mockPsHandler).not.toHaveBeenCalled() + expect(mockEnableConfigs).toHaveBeenCalledTimes(1) + expect(mockResetSettingsCache).toHaveBeenCalledTimes(1) + expect(mockEagerLoadSettingsFromArgs.mock.calls).toEqual([[['ps']]]) + expect(mockRunBackgroundSessionCleanupWorker).toHaveBeenCalledWith({ + reloadSettings: expect.any(Function), + }) + const workerOptions = ( + mockRunBackgroundSessionCleanupWorker.mock.calls as unknown as Array< + [{ reloadSettings?: () => boolean }] + > + )[0]?.[0] + expect(workerOptions?.reloadSettings?.()).toBe(true) + expect(mockResetSettingsCache).toHaveBeenCalledTimes(2) + expect(mockEagerLoadSettingsFromArgs.mock.calls).toEqual([ + [['ps']], + [['ps']], + ]) + }) + it('routes partial background metadata through the finalizer before dispatch', async () => { process.env[BACKGROUND_SESSION_LAUNCHER_PID_ENV] = '123' try { @@ -487,10 +557,22 @@ describe('cli.tsx — background routing behavior', () => { await runCliEntrypoint([command, '--bg', 'session-1'], bgOptions) - expect(handler.mock.calls).toEqual([[['--bg', 'session-1']]]) + const tail = ['--bg', 'session-1'] + if (command === 'kill') { + expect(mockKillHandler.mock.calls).toEqual([ + [tail, { retentionSettingsReady: true }], + ]) + } else { + expect(handler.mock.calls).toEqual([[tail]]) + } expect(mockParseProviderEnvFileArgs).not.toHaveBeenCalled() expect(mockHandleBgFlag).not.toHaveBeenCalled() - expect(mockEnableConfigs).not.toHaveBeenCalled() + expect(mockEnableConfigs).toHaveBeenCalledTimes( + command === 'kill' ? 1 : 0, + ) + expect(mockEagerLoadSettingsFromArgs).toHaveBeenCalledTimes( + command === 'kill' ? 1 : 0, + ) expect(mockValidateProviderEnvForStartupOrExit).not.toHaveBeenCalled() expect(mockCliMain).not.toHaveBeenCalled() } diff --git a/src/entrypoints/cli.tsx b/src/entrypoints/cli.tsx index 76b27d7610..fd705419b0 100644 --- a/src/entrypoints/cli.tsx +++ b/src/entrypoints/cli.tsx @@ -1,5 +1,6 @@ import { feature } from 'bun:bundle'; import { + BACKGROUND_SESSION_CLEANUP_WORKER_ENV, BACKGROUND_SESSION_ID_ENV, BACKGROUND_SESSION_LAUNCHER_PID_ENV, } from '../cli/bgRouting.js' @@ -265,6 +266,9 @@ type CliEntrypointImporters = { flagSettings: () => Promise< typeof import('../utils/settings/flagSettings.js') > + settingsCache: () => Promise< + typeof import('../utils/settings/settingsCache.js') + > agentRouting: () => Promise< typeof import('../services/api/agentRouting.js') > @@ -289,6 +293,7 @@ const defaultCliEntrypointImporters: CliEntrypointImporters = { providerProfile: () => import('../utils/providerProfile.js'), providerValidation: () => import('../utils/providerValidation.js'), flagSettings: () => import('../utils/settings/flagSettings.js'), + settingsCache: () => import('../utils/settings/settingsCache.js'), agentRouting: () => import('../services/api/agentRouting.js'), settings: () => import('../utils/settings/settings.js'), cliArgs: () => import('../utils/cliArgs.js'), @@ -319,6 +324,20 @@ export async function main( options: CliEntrypointOptions = {}, ): Promise { const importers = getCliEntrypointImporters(options.importers) + if (process.env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] === '1') { + const { enableConfigs } = await importers.config() + enableConfigs() + const { eagerLoadSettingsFromArgs } = await importers.flagSettings() + const { resetSettingsCache } = await importers.settingsCache() + const reloadSettings = () => { + resetSettingsCache() + return eagerLoadSettingsFromArgs(args).ok + } + if (!reloadSettings()) return + const { runBackgroundSessionCleanupWorker } = await importers.bgFinalizer() + await runBackgroundSessionCleanupWorker({ reloadSettings }) + return + } // The detached CLI is the registered background-session PID. Establish // exact registry ownership and install its terminal finalizer before any // fast path or startup validation can call process.exit(). The private env @@ -368,7 +387,13 @@ export async function main( await bg.attachHandler(args.slice(1)); break; case 'kill': - await bg.killHandler(args.slice(1)); + { + const { enableConfigs } = await importers.config() + enableConfigs() + const { eagerLoadSettingsFromArgs } = await importers.flagSettings() + const retentionSettingsReady = eagerLoadSettingsFromArgs(args).ok + await bg.killHandler(args.slice(1), { retentionSettingsReady }); + } break; } return; diff --git a/src/skills/bundled/updateConfig.ts b/src/skills/bundled/updateConfig.ts index 990cfe5636..922644719f 100644 --- a/src/skills/bundled/updateConfig.ts +++ b/src/skills/bundled/updateConfig.ts @@ -114,7 +114,7 @@ Plugin syntax: \`plugin-name@source\` where source is \`claude-code-marketplace\ ### Other Settings - \`language\`: Preferred response language (e.g., "japanese") -- \`cleanupPeriodDays\`: Days to keep transcripts (default: 30; 0 disables persistence entirely) +- \`cleanupPeriodDays\`: Days to keep local session artifacts, including transcripts and completed background sessions (default: 30; 0 disables transcript persistence and removes background artifacts after completion) - \`respectGitignore\`: Whether to respect .gitignore (default: true) - \`spinnerTipsEnabled\`: Show tips in spinner - \`spinnerVerbs\`: Customize spinner verbs (\`{ "mode": "append" | "replace", "verbs": [...] }\`) diff --git a/src/utils/backgroundHousekeeping.test.ts b/src/utils/backgroundHousekeeping.test.ts new file mode 100644 index 0000000000..53c9753102 --- /dev/null +++ b/src/utils/backgroundHousekeeping.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from 'bun:test' +import { + startBackgroundHousekeeping, + startBackgroundSessionReconciliation, +} from './backgroundHousekeeping.js' + +describe('background session housekeeping', () => { + it('starts recurring background-session reconciliation', async () => { + let callback: (() => void) | undefined + let runs = 0 + let retentionSchedules = 0 + startBackgroundHousekeeping({ + backgroundSessionReconciliation: { + cleanup: async () => { + runs++ + }, + setInterval: scheduledCallback => { + callback = scheduledCallback + return { unref: () => {} } + }, + }, + backgroundSessionRetention: { + setInterval: () => { + retentionSchedules++ + return { unref: () => {} } + }, + }, + _reconciliationOnlyForTesting: true, + }) + + callback?.() + await new Promise(resolve => setTimeout(resolve, 0)) + expect(runs).toBe(1) + expect(retentionSchedules).toBe(0) + }) + + it('schedules recurring positive-retention cleanup every 24 hours', async () => { + let callback: (() => void) | undefined + let intervalMs: number | undefined + let runs = 0 + startBackgroundHousekeeping({ + backgroundSessionReconciliation: { + setInterval: () => ({ unref: () => {} }), + }, + backgroundSessionRetention: { + cleanup: async () => { + runs++ + }, + setInterval: (scheduledCallback, scheduledIntervalMs) => { + callback = scheduledCallback + intervalMs = scheduledIntervalMs + return { unref: () => {} } + }, + }, + _backgroundSessionTimersOnlyForTesting: true, + }) + + expect(intervalMs).toBe(24 * 60 * 60 * 1000) + callback?.() + await new Promise(resolve => setTimeout(resolve, 0)) + expect(runs).toBe(1) + }) + + it('runs recurring reconciliation without overlapping passes', async () => { + let callback: (() => void) | undefined + let intervalMs: number | undefined + let unrefCalls = 0 + let runs = 0 + let releaseFirstRun!: () => void + const firstRun = new Promise(resolve => { + releaseFirstRun = resolve + }) + + startBackgroundSessionReconciliation({ + cleanup: async () => { + runs++ + if (runs === 1) await firstRun + }, + setInterval: (scheduledCallback, scheduledIntervalMs) => { + callback = scheduledCallback + intervalMs = scheduledIntervalMs + return { + unref: () => { + unrefCalls++ + }, + } + }, + }) + + expect(intervalMs).toBe(60_000) + expect(unrefCalls).toBe(1) + callback?.() + callback?.() + await Promise.resolve() + expect(runs).toBe(1) + + releaseFirstRun() + await firstRun + await new Promise(resolve => setTimeout(resolve, 0)) + callback?.() + await Promise.resolve() + expect(runs).toBe(2) + }) +}) diff --git a/src/utils/backgroundHousekeeping.ts b/src/utils/backgroundHousekeeping.ts index fa56721324..da2b7bf513 100644 --- a/src/utils/backgroundHousekeeping.ts +++ b/src/utils/backgroundHousekeeping.ts @@ -15,20 +15,95 @@ const registerProtocolModule = feature('LODESTONE') import { getIsInteractive, getLastInteractionTime } from '../bootstrap/state.js' import { + cleanupBackgroundSessionRetentionInBackground, + cleanupBackgroundSessionsInBackground, cleanupNpmCacheForAnthropicPackages, cleanupOldMessageFilesInBackground, cleanupOldVersionsThrottled, } from './cleanup.js' +import { logError } from './log.js' import { cleanupOldVersions } from './nativeInstaller/index.js' import { autoUpdateMarketplacesAndPluginsInBackground } from './plugins/pluginAutoupdate.js' // 24 hours in milliseconds const RECURRING_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000 +// Terminal facts are a durable handoff from background children. The cleanup +// coordinator globally throttles this minute trigger and streams only a bounded +// fact/metadata sample, without rerunning the age-based retention sweep. +const BACKGROUND_SESSION_RECONCILIATION_INTERVAL_MS = 60 * 1000 // 10 minutes after start. const DELAY_VERY_SLOW_OPERATIONS_THAT_HAPPEN_EVERY_SESSION = 10 * 60 * 1000 -export function startBackgroundHousekeeping(): void { +type BackgroundSessionScheduleOptions = { + cleanup?: () => Promise + setInterval?: ( + callback: () => void, + intervalMs: number, + ) => { unref(): unknown } + _onPassFinishedForTesting?: () => void +} + +type BackgroundHousekeepingOptions = { + backgroundSessionReconciliation?: BackgroundSessionScheduleOptions + backgroundSessionRetention?: BackgroundSessionScheduleOptions + _reconciliationOnlyForTesting?: boolean + _backgroundSessionTimersOnlyForTesting?: boolean +} + +function scheduleNonOverlappingBackgroundCleanup( + cleanup: () => Promise, + intervalMs: number, + options: BackgroundSessionScheduleOptions, +): void { + const scheduleInterval = + options.setInterval ?? + ((callback, intervalMs) => setInterval(callback, intervalMs)) + let running = false + const interval = scheduleInterval(() => { + if (running) return + running = true + void Promise.resolve() + .then(() => cleanup()) + .catch(error => logError(error as Error)) + .finally(() => { + running = false + options._onPassFinishedForTesting?.() + }) + }, intervalMs) + interval.unref() +} + +function startBackgroundSessionRetention( + options: BackgroundSessionScheduleOptions = {}, +): void { + scheduleNonOverlappingBackgroundCleanup( + options.cleanup ?? cleanupBackgroundSessionRetentionInBackground, + RECURRING_CLEANUP_INTERVAL_MS, + options, + ) +} + +export function startBackgroundSessionReconciliation( + options: BackgroundSessionScheduleOptions = {}, +): void { + scheduleNonOverlappingBackgroundCleanup( + options.cleanup ?? cleanupBackgroundSessionsInBackground, + BACKGROUND_SESSION_RECONCILIATION_INTERVAL_MS, + options, + ) +} + +export function startBackgroundHousekeeping( + options: BackgroundHousekeepingOptions = {}, +): void { + startBackgroundSessionReconciliation( + options.backgroundSessionReconciliation, + ) + if (options._reconciliationOnlyForTesting) return + startBackgroundSessionRetention(options.backgroundSessionRetention) + if (options._backgroundSessionTimersOnlyForTesting) return + void initMagicDocs() void initSkillImprovement() if (feature('EXTRACT_MEMORIES')) { @@ -39,7 +114,6 @@ export function startBackgroundHousekeeping(): void { if (feature('LODESTONE') && getIsInteractive()) { void registerProtocolModule!.ensureDeepLinkProtocolRegistered() } - let needsCleanup = true async function runVerySlowOps(): Promise { // If the user did something in the last minute, don't make them wait for these slow operations to run. diff --git a/src/utils/cleanup.test.ts b/src/utils/cleanup.test.ts index 1926a0f06d..3306f63453 100644 --- a/src/utils/cleanup.test.ts +++ b/src/utils/cleanup.test.ts @@ -1,19 +1,211 @@ import { afterEach, describe, expect, test } from 'bun:test' -import { mkdir, rm, stat, utimes, writeFile } from 'fs/promises' +import { + mkdir, + chmod, + readdir, + readFile, + rename, + rm, + stat, + utimes, + writeFile, +} from 'fs/promises' import { tmpdir } from 'os' import { join } from 'path' +import { + _setBackgroundSessionsRootForTesting, + createBackgroundSession, + recordBackgroundSessionNaturalTermination, +} from '../cli/bgRegistry.js' import { cleanupOldSessionFilesInProjectsDir } from './cleanup.js' import { NodeFsOperations } from './fsOperations.js' const tempDirs: string[] = [] +const FIXTURE_OUTPUT_LIMIT_BYTES = 64 * 1024 +// The production minute-recovery contract caps each pass at 256 records. +const BACKGROUND_RECOVERY_ENTRY_LIMIT = 256 afterEach(async () => { + _setBackgroundSessionsRootForTesting(undefined) await Promise.all( - tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })), + tempDirs.splice(0).map(async dir => { + await chmod(join(dir, 'bg-sessions', 'terminal'), 0o755).catch(() => {}) + await rm(dir, { recursive: true, force: true }) + }), ) }) +async function readBoundedFixtureOutput( + stream: ReadableStream, + label: string, +): Promise { + const reader = stream.getReader() + const decoder = new TextDecoder() + let bytes = 0 + let text = '' + try { + while (true) { + const { done, value } = await reader.read() + if (done) break + bytes += value.byteLength + if (bytes > FIXTURE_OUTPUT_LIMIT_BYTES) { + throw new Error( + `${label} exceeded ${FIXTURE_OUTPUT_LIMIT_BYTES} bytes`, + ) + } + text += decoder.decode(value, { stream: true }) + } + return text + decoder.decode() + } finally { + reader.releaseLock() + } +} + +async function runCleanupFixture( + configDir: string, + mode: + | 'once' + | 'pass-outcome' + | 'marker-failure' + | 'periodic-policy-recheck' + | 'periodic-recovery' + | 'periodic-recovery-bounded' + | 'periodic-retention' + | 'periodic-retention-policy-recheck' + | 'post-finalization-gate' + | 'post-finalization-policy-recheck' + | 'post-finalization-exact-cutoff' + | 'post-finalization-timeout' + | 'explicit-kill' = 'once', + fixtureEnv: NodeJS.ProcessEnv = {}, + expectedError?: string, +): Promise> { + const fixture = join(import.meta.dir, 'cleanupBackgroundSessions.fixture.ts') + const { USER_TYPE: _userType, ...inheritedEnv } = process.env + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 15_000) + const child = Bun.spawn([process.execPath, fixture], { + cwd: configDir, + env: { + ...inheritedEnv, + HOME: configDir, + XDG_CACHE_HOME: join(configDir, 'cache'), + OPENCLAUDE_CONFIG_DIR: configDir, + OPENCLAUDE_CLEANUP_FIXTURE_MODE: mode, + NODE_ENV: 'test', + ...fixtureEnv, + }, + signal: controller.signal, + stdout: 'pipe', + stderr: 'pipe', + }) + + try { + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + readBoundedFixtureOutput( + child.stdout as ReadableStream, + 'cleanup fixture stdout', + ), + readBoundedFixtureOutput( + child.stderr as ReadableStream, + 'cleanup fixture stderr', + ), + ]) + if (expectedError !== undefined) { + expect(exitCode, stderr).toBe(1) + expect(stdout).toBe('') + expect(stderr).toContain( + `background cleanup fixture failed: Error: ${expectedError}`, + ) + expect(stderr).toMatch( + /\bat .*cleanupBackgroundSessions\.fixture\.ts:\d+:\d+/, + ) + return {} + } + expect(exitCode, stderr).toBe(0) + const result = JSON.parse(stdout) as Record + if (mode === 'once') expect(result).toEqual({ completed: true }) + return result + } finally { + clearTimeout(timeout) + controller.abort() + } +} + +async function createCompletedBackgroundSession( + configDir: string, + id: string, + finishedAt: Date, + name?: string, + processMarker?: string, +): Promise { + const root = join(configDir, 'bg-sessions') + _setBackgroundSessionsRootForTesting(root) + await createBackgroundSession({ + id, + name, + processMarker, + pid: process.pid, + cwd: configDir, + command: ['openclaude', '--print', 'fixture'], + sessionId: `${id}-conversation`, + now: new Date(finishedAt.getTime() - 60_000), + }) + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + { ownerPid: process.pid, now: finishedAt }, + ) + _setBackgroundSessionsRootForTesting(undefined) + return join(root, 'sessions', `${id}.json`) +} + +async function runConcurrentPeriodicRecovery( + configDir: string, +): Promise { + const fixture = join(import.meta.dir, 'cleanupBackgroundSessions.fixture.ts') + const { NODE_ENV: _nodeEnv, USER_TYPE: _userType, ...inheritedEnv } = + process.env + const runRecovery = async (): Promise => { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 15_000) + const child = Bun.spawn([process.execPath, fixture], { + cwd: configDir, + env: { + ...inheritedEnv, + HOME: configDir, + XDG_CACHE_HOME: join(configDir, 'cache'), + OPENCLAUDE_CONFIG_DIR: configDir, + OPENCLAUDE_CLEANUP_FIXTURE_MODE: 'periodic-recovery', + }, + signal: controller.signal, + stdout: 'pipe', + stderr: 'pipe', + }) + try { + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + readBoundedFixtureOutput( + child.stdout as ReadableStream, + 'periodic recovery stdout', + ), + readBoundedFixtureOutput( + child.stderr as ReadableStream, + 'periodic recovery stderr', + ), + ]) + expect(exitCode, stderr).toBe(0) + return (JSON.parse(stdout) as { result: string }).result + } finally { + clearTimeout(timeout) + controller.abort() + } + } + return await Promise.all([runRecovery(), runRecovery()]) +} + describe('cleanupOldSessionFiles', () => { test('removes old replay sidecars while preserving non-session files', async () => { const projectsDir = join( @@ -46,3 +238,988 @@ describe('cleanupOldSessionFiles', () => { expect((await stat(keepPath)).isFile()).toBe(true) }) }) + +describe('cleanupOldMessageFilesInBackground', () => { + for (const limit of [ + undefined, + '', + '0', + '-1', + '1.5', + 'invalid', + '9007199254740992', + ]) { + test(`rejects invalid bounded recovery fixture limit ${JSON.stringify(limit)}`, async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-invalid-limit-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const id = 'bg-invalid-limit' + const metadata = await createCompletedBackgroundSession( + configDir, + id, + new Date(0), + ) + const root = join(configDir, 'bg-sessions') + const journal = join(root, '.recovery-journal') + const before = await readFile(journal, 'utf8') + + await runCleanupFixture( + configDir, + 'periodic-recovery-bounded', + { OPENCLAUDE_CLEANUP_RECOVERY_LIMIT: limit }, + 'OPENCLAUDE_CLEANUP_RECOVERY_LIMIT must be a positive safe integer', + ) + + expect(await Bun.file(metadata).exists()).toBe(true) + expect( + await Bun.file(join(root, 'logs', `${id}.out.log`)).exists(), + ).toBe(true) + expect(await readFile(journal, 'utf8')).toBe(before) + expect(await Bun.file(join(root, '.recovery-pass')).exists()).toBe(false) + expect( + await Bun.file(join(root, '.recovery-cursor.json')).exists(), + ).toBe(false) + }, 30_000) + } + + test( + 'globally throttles and bounds concurrent prompt-recovery passes', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-bounded-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + const root = join(configDir, 'bg-sessions') + const metadataDir = join(root, 'sessions') + const logsDir = join(root, 'logs') + await Promise.all([ + mkdir(metadataDir, { recursive: true }), + mkdir(logsDir, { recursive: true }), + mkdir(join(root, 'names'), { recursive: true }), + mkdir(join(root, 'terminal'), { recursive: true }), + ]) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const finishedAt = new Date(Date.now() - 60_000).toISOString() + const ids = Array.from( + { length: 300 }, + (_, index) => `bg-bounded-${String(index).padStart(3, '0')}`, + ) + await Promise.all( + ids.map(async id => { + const stdoutLogPath = join(logsDir, `${id}.out.log`) + const stderrLogPath = join(logsDir, `${id}.err.log`) + await Promise.all([ + writeFile( + join(metadataDir, `${id}.json`), + JSON.stringify({ + id, + pid: 999_999, + cwd: configDir, + status: 'exited', + sessionId: `${id}-conversation`, + startedAt: new Date(Date.now() - 120_000).toISOString(), + updatedAt: finishedAt, + command: ['openclaude', '--print', 'fixture'], + stdoutLogPath, + stderrLogPath, + finishedAt, + exitCode: 0, + terminalReason: 'exit_code', + }), + ), + writeFile(stdoutLogPath, ''), + writeFile(stderrLogPath, ''), + ]) + }), + ) + await writeFile(join(root, '.recovery-journal'), `${ids.join('\n')}\n`) + + expect((await runConcurrentPeriodicRecovery(configDir)).sort()).toEqual([ + 'ran', + 'skipped', + ]) + expect( + (await readdir(metadataDir)).filter(name => name.endsWith('.json')), + ).toHaveLength(ids.length - BACKGROUND_RECOVERY_ENTRY_LIMIT) + }, + 30_000, + ) + + test( + 'advances bounded recovery past malformed entries across process restarts', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-progress-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + const root = join(configDir, 'bg-sessions') + const metadataDir = join(root, 'sessions') + const logsDir = join(root, 'logs') + await Promise.all([ + mkdir(metadataDir, { recursive: true }), + mkdir(logsDir, { recursive: true }), + mkdir(join(root, 'names'), { recursive: true }), + mkdir(join(root, 'terminal'), { recursive: true }), + ]) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + for (const id of ['bg-retained-a', 'bg-retained-b']) { + await writeFile(join(metadataDir, `${id}.json`), '{') + } + const targetId = 'bg-progress-target' + const targetPath = await createCompletedBackgroundSession( + configDir, + targetId, + new Date(Date.now() - 60_000), + ) + await writeFile( + join(root, '.recovery-journal'), + `bg-retained-a\nbg-retained-b\n${targetId}\n`, + ) + expect( + await runCleanupFixture( + configDir, + 'periodic-recovery-bounded', + { OPENCLAUDE_CLEANUP_RECOVERY_LIMIT: '2' }, + ), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(targetPath).exists()).toBe(true) + await rm(join(root, '.recovery-pass'), { force: true }) + + expect( + await runCleanupFixture( + configDir, + 'periodic-recovery-bounded', + { OPENCLAUDE_CLEANUP_RECOVERY_LIMIT: '2' }, + ), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(targetPath).exists()).toBe(false) + }, + 30_000, + ) + + for (const trigger of [ + 'periodic-recovery', + 'periodic-retention', + ] as const) { + for (const timing of ['before', 'after'] as const) { + test(`does not acknowledge policy rejected ${timing} the ${trigger} lock`, async () => { + const configDir = join( + tmpdir(), + `openclaude-policy-outcome-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + const settingsPath = join(configDir, 'settings.json') + const validPolicy = JSON.stringify({ cleanupPeriodDays: 0 }) + await writeFile( + settingsPath, + timing === 'before' + ? JSON.stringify({ cleanupPeriodDays: 'invalid' }) + : validPolicy, + ) + const id = 'bg-rejected-policy' + const metadataPath = await createCompletedBackgroundSession( + configDir, + id, + new Date(0), + ) + const root = join(configDir, 'bg-sessions') + const marker = join( + root, + trigger === 'periodic-recovery' + ? '.recovery-pass' + : '.retention-pass', + ) + const journal = join(root, '.recovery-journal') + const before = await readFile(journal, 'utf8') + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + OPENCLAUDE_CLEANUP_REJECT_AFTER_LOCK: + timing === 'after' ? '1' : '0', + }), + ).toEqual({ result: 'invalid-policy' }) + expect(await Bun.file(metadataPath).exists()).toBe(true) + expect(await Bun.file(marker).exists()).toBe(false) + expect( + await Bun.file(join(root, '.recovery-cursor.json')).exists(), + ).toBe(false) + expect(await readFile(journal, 'utf8')).toBe(before) + + await writeFile(settingsPath, validPolicy) + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(metadataPath).exists()).toBe(false) + expect(await readFile(journal, 'utf8')).toBe('') + expect(await Bun.file(marker).exists()).toBe(true) + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'skipped' }) + await rm(marker) + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(marker).exists()).toBe(true) + }, 30_000) + } + + for (const failure of [ + 'names-directory', + 'log-unlink', + 'terminal-unlink', + 'marked-terminal-unlink', + ] as const) { + const terminalFailure = failure.endsWith('terminal-unlink') + test.skipIf( + terminalFailure && + (process.platform === 'win32' || process.geteuid?.() === 0), + )(`retries ${failure} after a partial ${trigger} pass in a new process`, async () => { + const configDir = join( + tmpdir(), + `openclaude-retry-outcome-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const root = join(configDir, 'bg-sessions') + const id = 'bg-retryable' + const metadata = await createCompletedBackgroundSession( + configDir, + id, + new Date(0), + 'retryable', + failure === 'marked-terminal-unlink' ? 'a'.repeat(64) : undefined, + ) + const eligible = await createCompletedBackgroundSession( + configDir, + 'bg-independent', + new Date(0), + ) + const malformed = join(root, 'sessions', 'bg-malformed.json') + await writeFile(malformed, '{invalid') + const journal = join(root, '.recovery-journal') + // A full sweep must also recover work whose original journal append failed. + await writeFile( + journal, + trigger === 'periodic-retention' + ? '' + : `bg-malformed\n${id}\nbg-independent\n`, + ) + const brokenPath = terminalFailure + ? join(root, 'terminal') + : failure === 'names-directory' + ? join(root, 'names') + : join(root, 'logs', `${id}.out.log`) + if (terminalFailure) { + await chmod(brokenPath, 0o555) + } else { + await rm(brokenPath, { recursive: true, force: true }) + if (failure === 'names-directory') + await writeFile(brokenPath, 'not a directory') + else await mkdir(brokenPath) + } + const marker = join( + root, + trigger === 'periodic-recovery' + ? '.recovery-pass' + : '.retention-pass', + ) + + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'partial' }) + expect(await Bun.file(metadata).exists()).toBe(!terminalFailure) + expect(await Bun.file(eligible).exists()).toBe(false) + expect(await readFile(malformed, 'utf8')).toBe('{invalid') + expect(await readFile(journal, 'utf8')).toContain(id) + if ( + failure === 'names-directory' && + trigger === 'periodic-recovery' + ) { + expect(await readFile(journal, 'utf8')).toBe(`${id}\n`) + } + expect(await Bun.file(marker).exists()).toBe(true) + + if (terminalFailure) await chmod(brokenPath, 0o755) + else { + await rm(brokenPath, { recursive: true, force: true }) + if (failure === 'names-directory') await mkdir(brokenPath) + } + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'skipped' }) + await rm(marker) + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: terminalFailure + ? 'periodic-recovery' + : trigger, + }), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(metadata).exists()).toBe(false) + expect(await readdir(join(root, 'terminal'))).toEqual([]) + expect(await readFile(journal, 'utf8')).toBe('') + expect( + await Bun.file(join(root, '.recovery-cursor.json')).exists(), + ).toBe(false) + expect( + await Bun.file( + terminalFailure ? join(root, '.recovery-pass') : marker, + ).exists(), + ).toBe(true) + }, 30_000) + } + } + + for (const trigger of [ + 'periodic-recovery', + 'periodic-retention', + ] as const) { + test(`acknowledges unnamed work despite a broken names directory during ${trigger}`, async () => { + const configDir = join( + tmpdir(), + `openclaude-unnamed-retry-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const metadata = await createCompletedBackgroundSession( + configDir, + 'bg-unnamed', + new Date(0), + ) + const root = join(configDir, 'bg-sessions') + await rm(join(root, 'names'), { recursive: true, force: true }) + await writeFile(join(root, 'names'), 'not a directory') + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(metadata).exists()).toBe(false) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe('') + }, 30_000) + } + + test('queues recent reconciliation failures that were absent from the journal', async () => { + const configDir = join( + tmpdir(), + `openclaude-reconcile-retry-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const id = 'bg-recent-reconcile' + const generation = 'b'.repeat(64) + const metadata = await createCompletedBackgroundSession( + configDir, + id, + new Date(), + 'recent-reconcile', + generation, + ) + const root = join(configDir, 'bg-sessions') + await writeFile(join(root, '.recovery-journal'), '') + await rm(join(root, 'names'), { recursive: true, force: true }) + await writeFile(join(root, 'names'), 'not a directory') + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ + result: 'partial', + }) + expect(await Bun.file(metadata).exists()).toBe(true) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe( + `${id}~${generation}\n`, + ) + await rm(join(root, 'names')) + await mkdir(join(root, 'names')) + await rm(join(root, '.retention-pass')) + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ + result: 'ran', + }) + expect(await Bun.file(metadata).exists()).toBe(true) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe('') + }, 30_000) + + test('reports an interrupted full sweep without acknowledging its snapshot', async () => { + const configDir = join( + tmpdir(), + `openclaude-failed-outcome-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const metadata = await createCompletedBackgroundSession( + configDir, + 'bg-before-interruption', + new Date(0), + ) + const root = join(configDir, 'bg-sessions') + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: 'periodic-retention', + OPENCLAUDE_CLEANUP_APPEND_AFTER_SNAPSHOT: '1', + OPENCLAUDE_CLEANUP_THROW_AFTER_SNAPSHOT: '1', + }), + ).toEqual({ result: 'failed' }) + expect(await Bun.file(metadata).exists()).toBe(true) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe( + 'bg-before-interruption\nbg-snapshot-appended\n', + ) + expect(await Bun.file(join(root, '.retention-pass')).exists()).toBe(false) + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ + result: 'ran', + }) + expect(await Bun.file(metadata).exists()).toBe(false) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe('') + expect(await readdir(join(root, 'terminal'))).toEqual([]) + }, 30_000) + + test('does not throttle a full sweep when its retry inventory is unreadable', async () => { + const configDir = join( + tmpdir(), + `openclaude-failed-inventory-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const metadata = await createCompletedBackgroundSession( + configDir, + 'bg-unreadable-inventory', + new Date(0), + ) + const root = join(configDir, 'bg-sessions') + const directory = join(root, 'sessions') + const saved = join(root, 'saved-sessions') + await rename(directory, saved) + await writeFile(directory, 'not a directory') + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ + result: 'failed', + }) + expect(await Bun.file(join(root, '.retention-pass')).exists()).toBe(false) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe( + 'bg-unreadable-inventory\n', + ) + await rm(directory) + await rename(saved, directory) + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ + result: 'ran', + }) + expect(await Bun.file(metadata).exists()).toBe(false) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe('') + }, 30_000) + + test('acknowledges only the full-sweep snapshot while preserving a concurrent append', async () => { + const configDir = join( + tmpdir(), + `openclaude-snapshot-outcome-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const metadata = await createCompletedBackgroundSession( + configDir, + 'bg-snapshot-old', + new Date(0), + ) + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: 'periodic-retention', + OPENCLAUDE_CLEANUP_APPEND_AFTER_SNAPSHOT: '1', + }), + ).toEqual({ result: 'ran' }) + const root = join(configDir, 'bg-sessions') + expect(await Bun.file(metadata).exists()).toBe(false) + expect( + await Bun.file( + join(root, 'sessions', 'bg-snapshot-appended.json'), + ).exists(), + ).toBe(true) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe( + 'bg-snapshot-appended\n', + ) + expect(await Bun.file(join(root, '.retention-pass')).exists()).toBe(true) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + expect(await runCleanupFixture(configDir, 'periodic-recovery')).toEqual({ + result: 'ran', + }) + expect( + await Bun.file( + join(root, 'sessions', 'bg-snapshot-appended.json'), + ).exists(), + ).toBe(false) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe('') + }, 30_000) + + for (const scenario of [ + { label: 'positive', setting: 30, result: 'ran' }, + { label: 'invalid', setting: 'invalid', result: 'invalid-policy' }, + ] as const) { + test( + `keeps periodic recovery non-destructive with ${scenario.label} retention`, + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-periodic-${scenario.label}-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: scenario.setting }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + `bg-periodic-${scenario.label}`, + new Date(Date.now() - 60_000), + ) + + expect( + await runCleanupFixture(configDir, 'periodic-recovery'), + ).toEqual({ result: scenario.result }) + expect(await Bun.file(metadataPath).exists()).toBe(true) + expect( + await Bun.file( + join(configDir, 'bg-sessions', '.recovery-pass'), + ).exists(), + ).toBe(scenario.result === 'ran') + }, + 30_000, + ) + } + + test( + 'compacts the recovery journal after a positive-retention full sweep', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-journal-compact-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-journal-compact', + new Date(Date.now() - 60_000), + ) + const journalPath = join( + configDir, + 'bg-sessions', + '.recovery-journal', + ) + expect((await readFile(journalPath, 'utf8')).length).toBeGreaterThan(0) + + expect( + await runCleanupFixture(configDir, 'periodic-retention'), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(metadataPath).exists()).toBe(true) + expect(await readFile(journalPath, 'utf8')).toBe('') + }, + 30_000, + ) + + test( + 'fails the recurring retention owner closed on project MCP errors', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-daily-invalid-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + await writeFile(join(configDir, '.mcp.json'), '{invalid') + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-daily-invalid-policy', + new Date(Date.now() - 31 * 24 * 60 * 60 * 1000), + ) + + expect( + await runCleanupFixture(configDir, 'periodic-retention'), + ).toEqual({ result: 'invalid-policy' }) + expect(await Bun.file(metadataPath).exists()).toBe(true) + expect( + await Bun.file( + join(configDir, 'bg-sessions', '.retention-pass'), + ).exists(), + ).toBe(false) + }, + 30_000, + ) + + test( + 'rechecks project MCP errors after acquiring the daily retention lock', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-daily-recheck-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-daily-policy-recheck', + new Date(Date.now() - 31 * 24 * 60 * 60 * 1000), + ) + + expect( + await runCleanupFixture( + configDir, + 'periodic-retention-policy-recheck', + ), + ).toEqual({ result: 'invalid-policy' }) + expect(await Bun.file(metadataPath).exists()).toBe(true) + expect(await Bun.file(join(configDir, 'bg-sessions', '.retention-pass')).exists()).toBe(false) + await rm(join(configDir, '.mcp.json')) + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ result: 'ran' }) + expect(await Bun.file(metadataPath).exists()).toBe(false) + }, + 30_000, + ) + + test( + 'rechecks periodic retention policy after acquiring the global lock', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-periodic-recheck-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect( + await runCleanupFixture(configDir, 'periodic-policy-recheck'), + ).toEqual({ result: 'ran', metadataPresent: true }) + }, + 30_000, + ) + + test( + 'continues later cleanup stages when the retention marker cannot be created', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-marker-failure-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect( + await runCleanupFixture(configDir, 'marker-failure'), + ).toEqual({ completed: true, planPresent: false }) + }, + 30_000, + ) + + test( + 'removes old completed background-session artifacts through the scheduler', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-scheduler-old', + new Date(Date.now() - 31 * 24 * 60 * 60 * 1000), + ) + + await runCleanupFixture(configDir) + + expect(await Bun.file(metadataPath).exists()).toBe(false) + expect( + await Bun.file( + join(configDir, 'bg-sessions', '.retention-pass'), + ).exists(), + ).toBe(true) + }, + 30_000, + ) + + test( + 'uses cleanupPeriodDays zero for completed background sessions', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-zero-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-scheduler-zero', + new Date(Date.now() - 1_000), + ) + + await runCleanupFixture(configDir) + + expect(await Bun.file(metadataPath).exists()).toBe(false) + }, + 30_000, + ) + + test( + 'includes the finalization millisecond in zero-day cleanup', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-exact-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect( + await runCleanupFixture( + configDir, + 'post-finalization-exact-cutoff', + ), + ).toEqual({ + waits: 1, + metadataPresent: false, + reservationPresent: false, + stdoutPresent: false, + stderrPresent: false, + }) + }, + 30_000, + ) + + test( + 'retains zero-day artifacts when worker ownership exit is unobserved', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-timeout-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect( + await runCleanupFixture(configDir, 'post-finalization-timeout'), + ).toEqual({ + waits: 1, + metadataPresent: true, + reservationPresent: false, + stdoutPresent: true, + stderrPresent: true, + }) + }, + 30_000, + ) + + test( + 'rechecks the originating settings policy after process handoff', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-policy-recheck-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect( + await runCleanupFixture( + configDir, + 'post-finalization-policy-recheck', + ), + ).toEqual({ + result: 'skipped', + waits: 1, + reloads: 1, + metadataPresent: true, + }) + }, + 30_000, + ) + + test( + 'reclaims zero-day artifacts after an explicit kill transition', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-kill-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect(await runCleanupFixture(configDir, 'explicit-kill')).toEqual({ + metadataPresent: false, + stdoutPresent: false, + stderrPresent: false, + }) + }, + 30_000, + ) + + for (const setting of [30, 'invalid'] as const) { + test( + `does not wait for post-finalization cleanup with ${String(setting)} retention`, + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-gate-${String(setting)}-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: setting }), + ) + + expect( + await runCleanupFixture(configDir, 'post-finalization-gate'), + ).toEqual({ waits: 0 }) + }, + 30_000, + ) + } + + test( + 'preserves background artifacts when explicit cleanupPeriodDays is invalid', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-invalid-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 'invalid' }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-scheduler-invalid', + new Date('2026-06-01T00:00:00.000Z'), + ) + + await runCleanupFixture(configDir) + + expect(await Bun.file(metadataPath).exists()).toBe(true) + }, + 30_000, + ) + + test( + 'retains recent background sessions under a nonzero cleanup period', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-recent-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const oldMetadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-scheduler-old-pair', + new Date(Date.now() - 31 * 24 * 60 * 60 * 1000), + ) + const recentMetadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-scheduler-recent', + new Date(Date.now() - 60_000), + ) + + await runCleanupFixture(configDir) + + expect(await Bun.file(oldMetadataPath).exists()).toBe(false) + expect(await Bun.file(recentMetadataPath).exists()).toBe(true) + }, + 30_000, + ) +}) diff --git a/src/utils/cleanup.ts b/src/utils/cleanup.ts index e9375b049a..b4c3edb47d 100644 --- a/src/utils/cleanup.ts +++ b/src/utils/cleanup.ts @@ -1,6 +1,13 @@ import * as fs from 'fs/promises' import { homedir } from 'os' -import { join } from 'path' +import { dirname, join } from 'path' +import { + cleanupBackgroundSessionsBefore, + reconcileBackgroundSessionTerminalFacts, + snapshotBackgroundSessionRecoveryJournal, + takeBackgroundSessionRecoveryBatch, + type BackgroundSessionRecoveryTarget, +} from '../cli/bgRegistry.js' import { logEvent } from '../services/analytics/index.js' import { CACHE_PATHS } from './cachePaths.js' import { logForDebugging } from './debug.js' @@ -15,12 +22,49 @@ import { getDefaultPlansDirectory } from './plans.js' import { getSettingsWithAllErrors } from './settings/allErrors.js' import { getSettings_DEPRECATED, + getSettingsWithErrors, rawSettingsContainsKey, } from './settings/settings.js' +import { resetSettingsCache } from './settings/settingsCache.js' import { TOOL_RESULTS_SUBDIR } from './toolResultStorage.js' import { cleanupStaleAgentWorktrees } from './worktree.js' const DEFAULT_CLEANUP_PERIOD_DAYS = 30 +const BACKGROUND_RECOVERY_INTERVAL_MS = 60 * 1000 +const BACKGROUND_RETENTION_SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000 +const BACKGROUND_RECOVERY_ENTRY_LIMIT = 256 + +export type BackgroundSessionRetentionTrigger = + | 'natural-finalization' + | 'explicit-kill' + | 'periodic-recovery' + | 'periodic-retention' + +export type BackgroundSessionRetentionResult = + | 'ran' + | 'partial' + | 'failed' + | 'skipped' + | 'invalid-policy' + +type BackgroundSessionRetentionRequest = + | { + trigger: 'natural-finalization' + sessionId: string + waitForProcessesToExit: () => Promise + reloadSettings?: () => boolean + } + | { trigger: 'explicit-kill'; sessionId: string } + | { + trigger: 'periodic-recovery' + _afterThrottleLockForTesting?: () => Promise + _recoveryBatchLimitForTesting?: number + } + | { + trigger: 'periodic-retention' + _afterThrottleLockForTesting?: () => Promise + _afterJournalSnapshotForTesting?: () => Promise + } function getCutoffDate(): Date { const settings = getSettings_DEPRECATED() || {} @@ -588,8 +632,7 @@ export async function cleanupOldMessageFilesInBackground(): Promise { // If settings have validation errors but the user explicitly set cleanupPeriodDays, // skip cleanup entirely rather than falling back to the default (30 days). // This prevents accidentally deleting files when the user intended a different retention period. - const { errors } = getSettingsWithAllErrors() - if (errors.length > 0 && rawSettingsContainsKey('cleanupPeriodDays')) { + if (cleanupPeriodSettingIsInvalid()) { logForDebugging( 'Skipping cleanup: settings have validation errors but cleanupPeriodDays was explicitly set. Fix settings errors to enable cleanup.', ) @@ -598,6 +641,7 @@ export async function cleanupOldMessageFilesInBackground(): Promise { await cleanupOldMessageFiles() await cleanupOldSessionFiles() + await runBackgroundSessionRetention({ trigger: 'periodic-retention' }) await cleanupOldPlanFiles() await cleanupOldFileHistoryBackups() await cleanupOldSessionEnvDirs() @@ -612,3 +656,265 @@ export async function cleanupOldMessageFilesInBackground(): Promise { await cleanupNpmCacheForAnthropicPackages() } } + +function readBackgroundRetentionPeriod( + options: { fresh?: boolean } = {}, +): + | { state: 'valid'; days: number } + | { state: 'invalid' } { + try { + if (options.fresh) resetSettingsCache() + const { errors } = getSettingsWithErrors() + if (errors.length > 0 && rawSettingsContainsKey('cleanupPeriodDays')) { + return { state: 'invalid' } + } + const days = + getSettings_DEPRECATED()?.cleanupPeriodDays ?? + DEFAULT_CLEANUP_PERIOD_DAYS + return Number.isFinite(days) && days >= 0 + ? { state: 'valid', days } + : { state: 'invalid' } + } catch { + return { state: 'invalid' } + } +} + +function logBackgroundReconciliation( + reconciliation: Awaited< + ReturnType + >, +): void { + if (reconciliation.errors > 0) { + logForDebugging( + `Background session reconciliation: ${reconciliation.sessionsUpdated} sessions, ${reconciliation.errors} errors`, + ) + } +} + +function logBackgroundCleanup( + backgroundResult: Awaited< + ReturnType + >, +): void { + if ( + backgroundResult.sessionsRemoved > 0 || + backgroundResult.artifactsRemoved > 0 || + backgroundResult.errors > 0 + ) { + logForDebugging( + `Background session cleanup: ${backgroundResult.sessionsRemoved} sessions, ${backgroundResult.artifactsRemoved} artifacts, ${backgroundResult.errors} errors`, + ) + } +} + +async function runWithBackgroundRetentionThrottle( + markerName: string, + intervalMs: number, + operation: () => Promise< + Exclude + >, + afterLock?: () => Promise, +): Promise { + const markerPath = join( + getClaudeConfigHomeDir(), + 'bg-sessions', + markerName, + ) + const markerIsFresh = async (): Promise => { + try { + return Date.now() - (await fs.stat(markerPath)).mtimeMs < intervalMs + } catch { + return false + } + } + let locked = false + try { + if (await markerIsFresh()) return 'skipped' + await fs.mkdir(dirname(markerPath), { recursive: true }) + await lockfile.lock(markerPath, { retries: 0, realpath: false }) + locked = true + } catch { + return 'skipped' + } + try { + if (await markerIsFresh()) return 'skipped' + await afterLock?.() + const outcome = await operation() + if (outcome !== 'ran' && outcome !== 'partial') return outcome + try { + await fs.writeFile(markerPath, new Date().toISOString()) + } catch (error) { + logError(error instanceof Error ? error : new Error(String(error))) + } + return outcome + } catch (error) { + logError(error instanceof Error ? error : new Error(String(error))) + return 'failed' + } finally { + if (locked) { + await lockfile.unlock(markerPath, { realpath: false }).catch(() => {}) + } + } +} + +export async function runBackgroundSessionRetention( + request: BackgroundSessionRetentionRequest, +): Promise { + let policy = readBackgroundRetentionPeriod() + if (policy.state === 'invalid') return 'invalid-policy' + + if (request.trigger === 'natural-finalization') { + if (policy.days !== 0) return 'skipped' + if (!(await request.waitForProcessesToExit())) return 'skipped' + if (request.reloadSettings && !request.reloadSettings()) { + return 'invalid-policy' + } + policy = readBackgroundRetentionPeriod() + if (policy.state === 'invalid') return 'invalid-policy' + if (policy.days !== 0) return 'skipped' + } + + if ( + request.trigger === 'natural-finalization' || + request.trigger === 'explicit-kill' + ) { + const reconciliation = await reconcileBackgroundSessionTerminalFacts({ + sessionIds: [request.sessionId], + }) + logBackgroundReconciliation(reconciliation) + if (policy.days === 0) { + const cleanup = await cleanupBackgroundSessionsBefore( + new Date(Date.now() + 1), + { sessionIds: [request.sessionId] }, + ) + logBackgroundCleanup(cleanup) + } + return 'ran' + } + + if (request.trigger === 'periodic-recovery') { + return await runWithBackgroundRetentionThrottle( + '.recovery-pass', + BACKGROUND_RECOVERY_INTERVAL_MS, + async () => { + const currentPolicy = readBackgroundRetentionPeriod({ fresh: true }) + if (currentPolicy.state === 'invalid') return 'invalid-policy' + if (currentPolicy.days !== 0) return 'ran' + const batch = await takeBackgroundSessionRecoveryBatch( + request._recoveryBatchLimitForTesting ?? + BACKGROUND_RECOVERY_ENTRY_LIMIT, + ) + const retrySessionIds: string[] = [] + const retryTerminalFacts: typeof batch.terminalFacts = [] + for (const id of batch.sessionIds) { + // Attribute retryable errors to their ID, so retained entries cannot + // hold unrelated work at the front of the bounded journal. + const reconciliation = + await reconcileBackgroundSessionTerminalFacts({ + sessionIds: [id], + }) + logBackgroundReconciliation(reconciliation) + const cleanup = await cleanupBackgroundSessionsBefore( + new Date(Date.now() + 1), + { + sessionIds: [id], + orphanedTerminalFacts: batch.terminalFacts.filter( + target => target.id === id, + ), + onRetry: target => { + if (target) retryTerminalFacts.push(target) + }, + }, + ) + logBackgroundCleanup(cleanup) + if (reconciliation.errors > 0 || cleanup.errors > 0) { + retrySessionIds.push(id) + } + } + // Save retries before publishing the cursor that acknowledges this batch. + await batch.commit(retrySessionIds, retryTerminalFacts) + return retrySessionIds.length > 0 ? 'partial' : 'ran' + }, + request._afterThrottleLockForTesting, + ) + } + + return await runWithBackgroundRetentionThrottle( + '.retention-pass', + BACKGROUND_RETENTION_SWEEP_INTERVAL_MS, + async () => { + resetSettingsCache() + if (cleanupPeriodSettingIsInvalid()) return 'invalid-policy' + const currentPolicy = readBackgroundRetentionPeriod({ fresh: true }) + if (currentPolicy.state === 'invalid') return 'invalid-policy' + const cutoff = new Date( + Date.now() - currentPolicy.days * 24 * 60 * 60 * 1000, + ) + const journal = await snapshotBackgroundSessionRecoveryJournal() + await request._afterJournalSnapshotForTesting?.() + const retryTargets: BackgroundSessionRecoveryTarget[] = [] + let retryInventoryComplete = true + const onRetry = (target?: BackgroundSessionRecoveryTarget) => { + if (target) retryTargets.push(target) + else retryInventoryComplete = false + } + const reconciliation = await reconcileBackgroundSessionTerminalFacts({ + onRetry, + }) + logBackgroundReconciliation(reconciliation) + const cleanup = await cleanupBackgroundSessionsBefore(cutoff, { + onRetry, + }) + logBackgroundCleanup(cleanup) + // A full sweep owns the snapshot only when all its work succeeded. + // On partial failure leave it discoverable by a later process. + if (reconciliation.errors > 0 || cleanup.errors > 0) { + await journal.retry(retryTargets) + return retryInventoryComplete ? 'partial' : 'failed' + } + await journal.commit() + return 'ran' + }, + request._afterThrottleLockForTesting, + ) +} + +function cleanupPeriodSettingIsInvalid(): boolean { + try { + const { errors } = getSettingsWithAllErrors() + return errors.length > 0 && rawSettingsContainsKey('cleanupPeriodDays') + } catch { + return true + } +} + +export async function cleanupBackgroundSessionsInBackground(): Promise< + BackgroundSessionRetentionResult +> { + return await runBackgroundSessionRetention({ + trigger: 'periodic-recovery', + }) +} + +export async function cleanupBackgroundSessionRetentionInBackground(): Promise< + BackgroundSessionRetentionResult +> { + resetSettingsCache() + if (cleanupPeriodSettingIsInvalid()) return 'invalid-policy' + return await runBackgroundSessionRetention({ + trigger: 'periodic-retention', + }) +} + +export async function cleanupBackgroundSessionsAfterFinalization( + sessionId: string, + waitForProcessesToExit: () => Promise, + reloadSettings?: () => boolean, +): Promise { + return await runBackgroundSessionRetention({ + trigger: 'natural-finalization', + sessionId, + waitForProcessesToExit, + ...(reloadSettings ? { reloadSettings } : {}), + }) +} diff --git a/src/utils/cleanupBackgroundSessions.fixture.ts b/src/utils/cleanupBackgroundSessions.fixture.ts new file mode 100644 index 0000000000..a9b0a838cc --- /dev/null +++ b/src/utils/cleanupBackgroundSessions.fixture.ts @@ -0,0 +1,380 @@ +import { createHash } from 'node:crypto' +import { mkdir, utimes } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { + createBackgroundSession, + recordBackgroundSessionNaturalTerminationSync, +} from '../cli/bgRegistry.js' +import { killHandler } from '../cli/bg.js' +import { enableConfigs } from './config.js' +import { eagerLoadSettingsFromArgs } from './settings/flagSettings.js' +import { + cleanupBackgroundSessionsAfterFinalization, + cleanupBackgroundSessionsInBackground, + cleanupOldMessageFilesInBackground, + runBackgroundSessionRetention, +} from './cleanup.js' + +try { + const mode = process.env.OPENCLAUDE_CLEANUP_FIXTURE_MODE + if (mode === 'pass-outcome') { + enableConfigs() + const settingsPath = join( + process.env.OPENCLAUDE_CONFIG_DIR!, + 'settings.json', + ) + const settingsResult = eagerLoadSettingsFromArgs([ + '--settings', + settingsPath, + ]) + if (!settingsResult.ok) throw new Error(settingsResult.message) + const trigger = + process.env.OPENCLAUDE_CLEANUP_TRIGGER === 'periodic-retention' + ? 'periodic-retention' + : 'periodic-recovery' + const afterLock = async () => { + if (process.env.OPENCLAUDE_CLEANUP_REJECT_AFTER_LOCK === '1') { + await Bun.write( + settingsPath, + JSON.stringify({ cleanupPeriodDays: 'invalid' }), + ) + } + } + const result = await runBackgroundSessionRetention( + trigger === 'periodic-recovery' + ? { trigger, _afterThrottleLockForTesting: afterLock } + : { + trigger, + _afterThrottleLockForTesting: afterLock, + _afterJournalSnapshotForTesting: async () => { + if ( + process.env.OPENCLAUDE_CLEANUP_APPEND_AFTER_SNAPSHOT !== '1' + ) + return + const session = await createBackgroundSession({ + id: 'bg-snapshot-appended', + pid: process.pid, + cwd: process.cwd(), + command: ['openclaude', '--print', 'fixture'], + sessionId: 'snapshot-appended-conversation', + }) + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { + ownerPid: process.pid, + expectedSession: session, + }, + ) + if ( + process.env.OPENCLAUDE_CLEANUP_THROW_AFTER_SNAPSHOT === '1' + ) { + throw new Error('intentional full-sweep interruption') + } + }, + }, + ) + process.stdout.write(`${JSON.stringify({ result })}\n`) + } else if ( + mode === 'periodic-retention' || + mode === 'periodic-retention-policy-recheck' + ) { + enableConfigs() + const settingsResult = eagerLoadSettingsFromArgs([ + '--settings', + join(process.env.OPENCLAUDE_CONFIG_DIR!, 'settings.json'), + ]) + if (!settingsResult.ok) throw new Error(settingsResult.message) + const { cleanupBackgroundSessionRetentionInBackground } = + await import('./cleanup.js') + process.stdout.write( + `${JSON.stringify({ + result: + mode === 'periodic-retention' + ? await cleanupBackgroundSessionRetentionInBackground() + : await runBackgroundSessionRetention({ + trigger: 'periodic-retention', + _afterThrottleLockForTesting: async () => { + await Bun.write( + join(process.cwd(), '.mcp.json'), + '{invalid', + ) + }, + }), + })}\n`, + ) + } else if ( + mode === 'periodic-recovery' || + mode === 'periodic-recovery-bounded' + ) { + enableConfigs() + const settingsResult = eagerLoadSettingsFromArgs([ + '--settings', + join(process.env.OPENCLAUDE_CONFIG_DIR!, 'settings.json'), + ]) + if (!settingsResult.ok) throw new Error(settingsResult.message) + process.stdout.write( + `${JSON.stringify({ + result: + mode === 'periodic-recovery' + ? await cleanupBackgroundSessionsInBackground() + : await runBackgroundSessionRetention({ + trigger: 'periodic-recovery', + _recoveryBatchLimitForTesting: (() => { + const limit = Number( + process.env.OPENCLAUDE_CLEANUP_RECOVERY_LIMIT, + ) + if (!Number.isSafeInteger(limit) || limit < 1) { + throw new Error( + 'OPENCLAUDE_CLEANUP_RECOVERY_LIMIT must be a positive safe integer', + ) + } + return limit + })(), + }), + })}\n`, + ) + } else if (mode === 'periodic-policy-recheck') { + enableConfigs() + const settingsPath = join( + process.env.OPENCLAUDE_CONFIG_DIR!, + 'settings.json', + ) + const settingsResult = eagerLoadSettingsFromArgs([ + '--settings', + settingsPath, + ]) + if (!settingsResult.ok) throw new Error(settingsResult.message) + const session = await createBackgroundSession({ + id: 'bg-periodic-policy-recheck', + pid: process.pid, + cwd: process.cwd(), + command: ['openclaude', '--print', 'fixture'], + sessionId: 'bg-periodic-policy-recheck-conversation', + }) + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { + ownerPid: process.pid, + expectedSession: session, + now: new Date(Date.now() - 1_000), + }, + ) + const result = await runBackgroundSessionRetention({ + trigger: 'periodic-recovery', + _afterThrottleLockForTesting: async () => { + await Bun.write( + settingsPath, + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + }, + }) + process.stdout.write( + `${JSON.stringify({ + result, + metadataPresent: await Bun.file( + join( + dirname(dirname(session.stdoutLogPath)), + 'sessions', + `${session.id}.json`, + ), + ).exists(), + })}\n`, + ) + } else if (mode === 'marker-failure') { + enableConfigs() + const settingsPath = join( + process.env.OPENCLAUDE_CONFIG_DIR!, + 'settings.json', + ) + const settingsResult = eagerLoadSettingsFromArgs([ + '--settings', + settingsPath, + ]) + if (!settingsResult.ok) throw new Error(settingsResult.message) + const plansDir = join(process.env.OPENCLAUDE_CONFIG_DIR!, 'plans') + const planPath = join(plansDir, 'old-plan.md') + await mkdir(plansDir, { recursive: true }) + await Bun.write(planPath, 'old plan') + const old = new Date('2000-01-01T00:00:00.000Z') + await utimes(planPath, old, old) + await Bun.write( + join(process.env.OPENCLAUDE_CONFIG_DIR!, 'bg-sessions'), + 'not a directory', + ) + await cleanupOldMessageFilesInBackground() + process.stdout.write( + `${JSON.stringify({ + completed: true, + planPresent: await Bun.file(planPath).exists(), + })}\n`, + ) + } else if (mode === 'post-finalization-policy-recheck') { + const session = await createBackgroundSession({ + id: 'bg-post-finalization-policy-recheck', + pid: process.pid, + cwd: process.cwd(), + command: ['openclaude', '--print', 'fixture'], + sessionId: 'bg-post-finalization-policy-recheck-conversation', + }) + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { + ownerPid: process.pid, + expectedSession: session, + now: new Date(Date.now() - 1_000), + }, + ) + const settingsPath = join( + process.env.OPENCLAUDE_CONFIG_DIR!, + 'settings.json', + ) + let waits = 0 + let reloads = 0 + const result = await cleanupBackgroundSessionsAfterFinalization( + session.id, + async () => { + waits++ + await Bun.write( + settingsPath, + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + return true + }, + () => { + reloads++ + return eagerLoadSettingsFromArgs(['--settings', settingsPath]).ok + }, + ) + process.stdout.write( + `${JSON.stringify({ + result, + waits, + reloads, + metadataPresent: await Bun.file( + join( + dirname(dirname(session.stdoutLogPath)), + 'sessions', + `${session.id}.json`, + ), + ).exists(), + })}\n`, + ) + } else if (mode === 'post-finalization-gate') { + let waits = 0 + await cleanupBackgroundSessionsAfterFinalization( + 'bg-post-finalization-gate', + async () => { + waits++ + return true + }, + ) + process.stdout.write(`${JSON.stringify({ waits })}\n`) + } else if ( + mode === 'post-finalization-exact-cutoff' || + mode === 'post-finalization-timeout' + ) { + const exactNow = Date.now() + const processMarker = 'd'.repeat(64) + const session = await createBackgroundSession({ + id: 'bg-post-finalization-exact', + name: 'post-finalization-exact', + pid: process.pid, + cwd: process.cwd(), + command: [ + 'openclaude', + `--openclaude-bg-session-marker=${processMarker}`, + '--print', + 'fixture', + ], + sessionId: 'bg-post-finalization-exact-conversation', + processMarker, + now: new Date(exactNow - 1_000), + }) + const root = dirname(dirname(session.stdoutLogPath)) + const metadataPath = join(root, 'sessions', `${session.id}.json`) + const reservationPath = join( + root, + 'names', + `${createHash('sha256').update(session.name!).digest('hex')}.json`, + ) + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { + ownerPid: process.pid, + expectedSession: session, + now: new Date(exactNow), + }, + ) + const originalDateNow = Date.now + let waits = 0 + try { + Date.now = () => exactNow + await cleanupBackgroundSessionsAfterFinalization( + session.id, + async () => { + waits++ + return mode === 'post-finalization-exact-cutoff' + }, + ) + } finally { + Date.now = originalDateNow + } + process.stdout.write( + `${JSON.stringify({ + waits, + metadataPresent: await Bun.file(metadataPath).exists(), + reservationPresent: await Bun.file(reservationPath).exists(), + stdoutPresent: await Bun.file(session.stdoutLogPath).exists(), + stderrPresent: await Bun.file(session.stderrLogPath).exists(), + })}\n`, + ) + } else if (mode === 'explicit-kill') { + const session = await createBackgroundSession({ + id: 'bg-explicit-kill-cleanup', + pid: process.pid, + cwd: process.cwd(), + command: ['openclaude', '--print', 'fixture'], + sessionId: 'bg-explicit-kill-cleanup-conversation', + }) + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { + ownerPid: process.pid, + expectedSession: session, + now: new Date(Date.now() - 1_000), + }, + ) + const originalConsoleLog = console.log + try { + console.log = () => {} + await killHandler([session.id]) + } finally { + console.log = originalConsoleLog + } + const root = dirname(dirname(session.stdoutLogPath)) + process.stdout.write( + `${JSON.stringify({ + metadataPresent: await Bun.file( + join(root, 'sessions', `${session.id}.json`), + ).exists(), + stdoutPresent: await Bun.file(session.stdoutLogPath).exists(), + stderrPresent: await Bun.file(session.stderrLogPath).exists(), + })}\n`, + ) + } else { + await cleanupOldMessageFilesInBackground() + process.stdout.write(`${JSON.stringify({ completed: true })}\n`) + } +} catch (error) { + process.stderr.write( + `background cleanup fixture failed: ${ + error instanceof Error ? (error.stack ?? error.message) : String(error) + }\n`, + ) + process.exitCode = 1 +} diff --git a/src/utils/settings/types.ts b/src/utils/settings/types.ts index 121a17c3c0..aa6eeab6ae 100644 --- a/src/utils/settings/types.ts +++ b/src/utils/settings/types.ts @@ -451,7 +451,7 @@ export const SettingsSchema = lazySchema(() => .int() .optional() .describe( - 'Number of days to retain chat transcripts (default: 30). Setting to 0 disables session persistence entirely: no transcripts are written and existing transcripts are deleted at startup.', + 'Number of days to retain local session artifacts, including chat transcripts and completed background-session metadata and logs (default: 30). Setting to 0 disables transcript persistence entirely and removes completed background-session artifacts after they finish.', ), env: EnvironmentVariablesSchema() .optional() diff --git a/src/utils/settings/validationTips.ts b/src/utils/settings/validationTips.ts index f5a8428b80..d3f78c889e 100644 --- a/src/utils/settings/validationTips.ts +++ b/src/utils/settings/validationTips.ts @@ -47,7 +47,7 @@ const TIP_MATCHERS: TipMatcher[] = [ ctx.expected === '0', tip: { suggestion: - 'Must be 0 or greater. Set a positive number for days to retain transcripts (default is 30). Setting 0 disables session persistence entirely: no transcripts are written and existing transcripts are deleted at startup.', + 'Must be 0 or greater. Set a positive number for days to retain local session artifacts, including transcripts and completed background sessions (default is 30). Setting 0 disables transcript persistence and removes completed background-session artifacts after they finish.', }, }, {