From 4c3d963088fc7a2effed7cb4a202baddc6277e7f Mon Sep 17 00:00:00 2001 From: chioarub Date: Mon, 24 Aug 2026 09:33:43 +0300 Subject: [PATCH 01/13] fix(bg): clean up old completed session artifacts --- src/cli/bgRegistry.cleanup.test.ts | 929 ++++++++++++++++++ src/cli/bgRegistry.ts | 625 +++++++++++- src/skills/bundled/updateConfig.ts | 2 +- src/utils/cleanup.test.ts | 174 ++++ src/utils/cleanup.ts | 13 + .../cleanupBackgroundSessions.fixture.ts | 9 + src/utils/settings/types.ts | 2 +- src/utils/settings/validationTips.ts | 2 +- 8 files changed, 1706 insertions(+), 50 deletions(-) create mode 100644 src/cli/bgRegistry.cleanup.test.ts create mode 100644 src/utils/cleanupBackgroundSessions.fixture.ts diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts new file mode 100644 index 0000000000..44bbcff7cf --- /dev/null +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -0,0 +1,929 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { createHash } from 'node:crypto' +import { + lstat, + mkdir, + mkdtemp, + readFile, + rm, + symlink, + unlink, + writeFile, +} from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + _setBackgroundSessionsRootForTesting, + cleanupBackgroundSessionsBefore, + createBackgroundSession, + markBackgroundSessionKilled, + recordBackgroundSessionNaturalTermination, + refreshBackgroundSessionStatuses, + type BackgroundSession, + type BackgroundSessionNaturalTermination, +} from './bgRegistry.js' + +const CUTOFF = new Date('2026-07-01T00:00:00.000Z') +const OLD_FINISH = new Date('2026-06-01T00:00:00.000Z') +const RECENT_FINISH = new Date('2026-07-01T00:00:01.000Z') + +describe('background session retention cleanup', () => { + let configDir: string + let root: string + let nextPid: number + + function paths(id: string): { + metadata: string + stdout: string + stderr: string + natural: string + killed: string + } { + return { + metadata: join(root, 'sessions', `${id}.json`), + stdout: join(root, 'logs', `${id}.out.log`), + stderr: join(root, 'logs', `${id}.err.log`), + natural: join(root, 'terminal', `${id}.natural.json`), + killed: join(root, 'terminal', `${id}.killed.json`), + } + } + + function reservationPath(name: string): string { + const digest = createHash('sha256').update(name).digest('hex') + return join(root, 'names', `${digest}.json`) + } + + async function exists(path: string): Promise { + try { + await lstat(path) + return true + } catch { + return false + } + } + + async function createRunning( + id: string, + options: { name?: string; startedAt?: Date } = {}, + ): Promise { + return await createBackgroundSession({ + id, + ...(options.name ? { name: options.name } : {}), + pid: nextPid++, + cwd: configDir, + command: ['openclaude', '--print', id], + sessionId: `${id}-conversation`, + now: options.startedAt ?? new Date(OLD_FINISH.getTime() - 60_000), + }) + } + + async function finishNaturally( + session: BackgroundSession, + termination: BackgroundSessionNaturalTermination, + finishedAt: Date = OLD_FINISH, + ): Promise { + await recordBackgroundSessionNaturalTermination(session.id, termination, { + ownerPid: session.pid, + now: finishedAt, + }) + } + + async function writeRawSession(options: { + id: string + status: BackgroundSession['status'] + finishedAt?: string + name?: string + stdoutLogPath?: string + stderrLogPath?: string + createLogs?: boolean + processMarker?: string + }): Promise { + const idPaths = paths(options.id) + await Promise.all([ + mkdir(join(root, 'sessions'), { recursive: true }), + mkdir(join(root, 'logs'), { recursive: true }), + mkdir(join(root, 'terminal'), { recursive: true }), + mkdir(join(root, 'names'), { recursive: true }), + ]) + if (options.createLogs !== false) { + await writeFile(idPaths.stdout, 'stdout') + await writeFile(idPaths.stderr, 'stderr') + } + const session: BackgroundSession = { + id: options.id, + ...(options.name ? { name: options.name } : {}), + pid: nextPid++, + cwd: configDir, + status: options.status, + sessionId: `${options.id}-conversation`, + ...(options.processMarker !== undefined + ? { processMarker: options.processMarker } + : {}), + startedAt: new Date(OLD_FINISH.getTime() - 60_000).toISOString(), + updatedAt: options.finishedAt ?? OLD_FINISH.toISOString(), + command: ['openclaude', '--print', options.id], + stdoutLogPath: options.stdoutLogPath ?? idPaths.stdout, + stderrLogPath: options.stderrLogPath ?? idPaths.stderr, + ...(options.finishedAt ? { finishedAt: options.finishedAt } : {}), + } + await writeFile(idPaths.metadata, JSON.stringify(session)) + return session + } + + async function writeReservation(name: string, id: string): Promise { + await mkdir(join(root, 'names'), { recursive: true }) + await writeFile( + reservationPath(name), + JSON.stringify({ name, id, creatorPid: process.pid }), + ) + } + + function deniedError(): NodeJS.ErrnoException { + return Object.assign(new Error('denied'), { code: 'EACCES' }) + } + + beforeEach(async () => { + configDir = await mkdtemp(join(tmpdir(), 'openclaude-bg-cleanup-')) + root = join(configDir, 'bg-sessions') + nextPid = 20_000 + _setBackgroundSessionsRootForTesting(root) + }) + + afterEach(async () => { + _setBackgroundSessionsRootForTesting(undefined) + await rm(configDir, { recursive: true, force: true }) + }) + + for (const scenario of [ + { + name: 'old natural exit zero', + id: 'bg-exit-zero', + termination: { exitCode: 0 } as const, + }, + { + name: 'old natural nonzero failure', + id: 'bg-exit-nonzero', + termination: { exitCode: 23 } as const, + }, + { + name: 'old signal failure', + id: 'bg-signal', + termination: { signal: 'SIGTERM' } as const, + }, + ]) { + it(`removes ${scenario.name} artifacts`, async () => { + const session = await createRunning(scenario.id) + await finishNaturally(session, scenario.termination) + + const result = await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(result).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + for (const path of Object.values(paths(scenario.id))) { + expect(await exists(path)).toBe(false) + } + }) + } + + it('removes old explicit-kill artifacts', async () => { + const session = await createRunning('bg-killed') + await markBackgroundSessionKilled(session.id, { now: OLD_FINISH }) + + const result = await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(result).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await exists(paths(session.id).killed)).toBe(false) + }) + + it('retains recent completed sessions', async () => { + const session = await createRunning('bg-recent') + await finishNaturally(session, { exitCode: 0 }, RECENT_FINISH) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).natural)).toBe(true) + }) + + it('retains running, unknown, and stale sessions', async () => { + const running = await createRunning('bg-running') + const unknown = await createRunning('bg-unknown') + const stale = await createRunning('bg-stale') + await refreshBackgroundSessionStatuses({ + isProcessAlive: pid => pid !== stale.pid, + getProcessCommand: pid => + pid === running.pid ? running.command.join(' ') : null, + now: OLD_FINISH, + }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + for (const session of [running, unknown, stale]) { + expect(await exists(paths(session.id).metadata)).toBe(true) + } + }) + + it('retains completed metadata without a valid finishedAt', async () => { + const missing = await writeRawSession({ + id: 'bg-missing-finished', + status: 'exited', + }) + const invalid = await writeRawSession({ + id: 'bg-invalid-finished', + status: 'failed', + finishedAt: 'not-a-date', + }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(missing.id).metadata)).toBe(true) + expect(await exists(paths(invalid.id).metadata)).toBe(true) + }) + + it('removes an old completed session with a valid process marker', async () => { + const session = await writeRawSession({ + id: 'bg-valid-process-marker', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + processMarker: 'a'.repeat(64), + }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 3, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(false) + }) + + it('retains metadata with an invalid process marker', async () => { + const session = await writeRawSession({ + id: 'bg-invalid-process-marker', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + processMarker: 'not-a-valid-marker', + }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).stdout)).toBe(true) + }) + + it('retains a completion exactly at the cutoff', async () => { + const session = await writeRawSession({ + id: 'bg-at-cutoff', + status: 'exited', + finishedAt: CUTOFF.toISOString(), + }) + + await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(await exists(paths(session.id).metadata)).toBe(true) + }) + + it('uses authoritative facts to clean stale metadata', async () => { + const session = await createRunning('bg-stale-with-fact') + await refreshBackgroundSessionStatuses({ + isProcessAlive: () => false, + now: new Date(OLD_FINISH.getTime() - 1_000), + }) + await finishNaturally(session, { exitCode: 0 }) + + const result = await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(result.sessionsRemoved).toBe(1) + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await exists(paths(session.id).natural)).toBe(false) + }) + + it('ignores malformed metadata without deleting same-looking logs', async () => { + const id = 'bg-malformed' + const idPaths = paths(id) + await mkdir(join(root, 'sessions'), { recursive: true }) + await mkdir(join(root, 'logs'), { recursive: true }) + await writeFile(idPaths.metadata, '{bad json') + await writeFile(idPaths.stdout, 'keep stdout') + await writeFile(idPaths.stderr, 'keep stderr') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(idPaths.stdout)).toBe(true) + expect(await exists(idPaths.stderr)).toBe(true) + }) + + it('does not follow symlinked metadata outside the registry root', async () => { + const id = 'bg-symlink-metadata' + const idPaths = paths(id) + const externalMetadata = join(configDir, 'external-metadata.json') + await mkdir(join(root, 'sessions'), { recursive: true }) + await mkdir(join(root, 'logs'), { recursive: true }) + await writeFile( + externalMetadata, + JSON.stringify({ + id, + pid: 123, + cwd: configDir, + status: 'exited', + sessionId: 'external-conversation', + startedAt: '2026-05-31T23:59:00.000Z', + updatedAt: OLD_FINISH.toISOString(), + finishedAt: OLD_FINISH.toISOString(), + command: ['openclaude'], + stdoutLogPath: idPaths.stdout, + stderrLogPath: idPaths.stderr, + }), + ) + await symlink(externalMetadata, idPaths.metadata) + await writeFile(idPaths.stdout, 'keep stdout') + await writeFile(idPaths.stderr, 'keep stderr') + + await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(await exists(externalMetadata)).toBe(true) + expect(await exists(idPaths.metadata)).toBe(true) + expect(await exists(idPaths.stdout)).toBe(true) + }) + + it('does not follow a symlinked metadata directory outside the registry root', async () => { + const id = 'bg-symlink-sessions-dir' + const idPaths = paths(id) + const externalSessions = join(configDir, 'external-sessions') + await mkdir(externalSessions, { recursive: true }) + await mkdir(join(root, 'logs'), { recursive: true }) + await mkdir(join(root, 'terminal'), { recursive: true }) + await mkdir(join(root, 'names'), { recursive: true }) + await writeFile( + join(externalSessions, `${id}.json`), + JSON.stringify({ + id, + pid: 123, + cwd: configDir, + status: 'exited', + sessionId: 'external-conversation', + startedAt: '2026-05-31T23:59:00.000Z', + updatedAt: OLD_FINISH.toISOString(), + finishedAt: OLD_FINISH.toISOString(), + command: ['openclaude'], + stdoutLogPath: idPaths.stdout, + stderrLogPath: idPaths.stderr, + }), + ) + await symlink(externalSessions, join(root, 'sessions'), 'dir') + await writeFile(idPaths.stdout, 'keep stdout') + await writeFile(idPaths.stderr, 'keep stderr') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 1, + }) + expect(await exists(join(externalSessions, `${id}.json`))).toBe(true) + expect(await exists(idPaths.stdout)).toBe(true) + expect(await exists(idPaths.stderr)).toBe(true) + }) + + it('does not follow symlinked artifact directories outside the registry root', async () => { + const externalLogs = join(configDir, 'external-logs') + const session = await writeRawSession({ + id: 'bg-symlink-logs-dir', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + createLogs: false, + }) + await rm(join(root, 'logs'), { recursive: true, force: true }) + await mkdir(externalLogs, { recursive: true }) + await writeFile(join(externalLogs, `${session.id}.out.log`), 'outside') + await writeFile(join(externalLogs, `${session.id}.err.log`), 'outside') + await symlink(externalLogs, join(root, 'logs'), 'dir') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 1, + }) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect( + await exists(join(externalLogs, `${session.id}.out.log`)), + ).toBe(true) + expect( + await exists(join(externalLogs, `${session.id}.err.log`)), + ).toBe(true) + }) + + it('stops when the metadata directory identity changes during cleanup', async () => { + const session = await writeRawSession({ + id: 'bg-sessions-dir-swapped', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + const sessionsDir = join(root, 'sessions') + const externalSessions = join(configDir, 'swapped-sessions') + await mkdir(externalSessions, { recursive: true }) + await writeFile( + join(externalSessions, `${session.id}.json`), + await readFile(paths(session.id).metadata, 'utf8'), + ) + let swapped = false + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + _beforeMetadataDirectoryReadForTesting: async () => { + swapped = true + await rm(sessionsDir, { recursive: true, force: true }) + await symlink(externalSessions, sessionsDir, 'dir') + }, + }) + + expect(result).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 1, + }) + expect(swapped).toBe(true) + expect( + await exists(join(externalSessions, `${session.id}.json`)), + ).toBe(true) + expect(await exists(paths(session.id).stdout)).toBe(true) + }) + + it('never trusts persisted external log paths', async () => { + const externalStdout = join(configDir, 'external.out.log') + const externalStderr = join(configDir, 'external.err.log') + await writeFile(externalStdout, 'external stdout') + await writeFile(externalStderr, 'external stderr') + const session = await writeRawSession({ + id: 'bg-external-logs', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + stdoutLogPath: externalStdout, + stderrLogPath: externalStderr, + }) + + await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await Bun.file(externalStdout).text()).toBe('external stdout') + expect(await Bun.file(externalStderr).text()).toBe('external stderr') + }) + + it('unlinks symlinked expected logs without touching their targets', async () => { + const id = 'bg-symlink-log' + const external = join(configDir, 'external-target.log') + await writeFile(external, 'outside') + const session = await writeRawSession({ + id, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + createLogs: false, + }) + await symlink(external, paths(id).stdout) + await writeFile(paths(id).stderr, 'stderr') + + await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(await exists(paths(session.id).stdout)).toBe(false) + expect(await Bun.file(external).text()).toBe('outside') + }) + + it('preserves malformed, wrong-owner, and symlinked terminal facts', async () => { + const malformed = await writeRawSession({ + id: 'bg-malformed-fact', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + await writeFile(paths(malformed.id).killed, '{bad json') + + const wrongOwner = await writeRawSession({ + id: 'bg-wrong-owner-fact', + status: 'failed', + finishedAt: OLD_FINISH.toISOString(), + }) + await writeFile( + paths(wrongOwner.id).natural, + JSON.stringify({ + version: 1, + id: wrongOwner.id, + pid: wrongOwner.pid + 1, + status: 'failed', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 1, + }), + ) + + const symlinked = await writeRawSession({ + id: 'bg-symlink-fact', + status: 'killed', + finishedAt: OLD_FINISH.toISOString(), + }) + const externalFact = join(configDir, 'external-fact.json') + await writeFile( + externalFact, + JSON.stringify({ + version: 1, + id: symlinked.id, + pid: symlinked.pid, + status: 'killed', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'explicit_kill', + }), + ) + await symlink(externalFact, paths(symlinked.id).killed) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + for (const session of [malformed, wrongOwner, symlinked]) { + expect(await exists(paths(session.id).metadata)).toBe(true) + } + expect(await exists(externalFact)).toBe(true) + }) + + it('preserves terminal facts with invalid timestamps', async () => { + const session = await writeRawSession({ + id: 'bg-invalid-fact-time', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + await writeFile( + paths(session.id).natural, + JSON.stringify({ + version: 1, + id: session.id, + pid: session.pid, + status: 'exited', + finishedAt: 'not-a-date', + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).natural)).toBe(true) + }) + + it('treats missing artifacts as idempotent success', async () => { + const session = await writeRawSession({ + id: 'bg-missing-artifacts', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + createLogs: false, + }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 1, + errors: 0, + }) + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(session.id).metadata)).toBe(false) + }) + + it('counts one unlink failure and continues with unrelated sessions', async () => { + const blocked = await createRunning('bg-blocked-unlink') + await finishNaturally(blocked, { exitCode: 0 }) + const removable = await createRunning('bg-other-removable') + await finishNaturally(removable, { exitCode: 0 }) + const blockedStdout = paths(blocked.id).stdout + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === blockedStdout) throw deniedError() + await unlink(path) + }, + }) + + expect(result.errors).toBe(1) + expect(result.sessionsRemoved).toBe(1) + expect(await exists(paths(blocked.id).metadata)).toBe(true) + expect(await exists(paths(blocked.id).natural)).toBe(true) + expect(await exists(paths(removable.id).metadata)).toBe(false) + + expect((await cleanupBackgroundSessionsBefore(CUTOFF)).errors).toBe(0) + expect(await exists(paths(blocked.id).metadata)).toBe(false) + }) + + it('counts a metadata read error and continues with unrelated sessions', async () => { + const blocked = await writeRawSession({ + id: 'bg-metadata-read-error', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + const removable = await writeRawSession({ + id: 'bg-after-metadata-read-error', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + const blockedMetadata = paths(blocked.id).metadata + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + readTextFile: async path => { + if (path === blockedMetadata) throw deniedError() + return await readFile(path, 'utf8') + }, + }) + + expect(result.errors).toBe(1) + expect(result.sessionsRemoved).toBe(1) + expect(await exists(blockedMetadata)).toBe(true) + expect(await exists(paths(blocked.id).stdout)).toBe(true) + expect(await exists(paths(removable.id).metadata)).toBe(false) + }) + + it('counts a terminal-fact read error and retains the session', async () => { + const session = await createRunning('bg-fact-read-error') + await finishNaturally(session, { exitCode: 0 }) + const naturalFact = paths(session.id).natural + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + readTextFile: async path => { + if (path === naturalFact) throw deniedError() + return await readFile(path, 'utf8') + }, + }) + + expect(result.errors).toBe(1) + expect(result.sessionsRemoved).toBe(0) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(naturalFact)).toBe(true) + }) + + it('keeps authoritative facts retryable after metadata unlink fails', async () => { + const session = await createRunning('bg-metadata-retry') + await finishNaturally(session, { exitCode: 0 }) + const metadataPath = paths(session.id).metadata + + const first = await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === metadataPath) throw deniedError() + await unlink(path) + }, + }) + + expect(first.errors).toBe(1) + expect(first.sessionsRemoved).toBe(0) + expect(await exists(metadataPath)).toBe(true) + expect(await exists(paths(session.id).natural)).toBe(true) + + const retry = await cleanupBackgroundSessionsBefore(CUTOFF) + expect(retry.sessionsRemoved).toBe(1) + expect(await exists(metadataPath)).toBe(false) + expect(await exists(paths(session.id).natural)).toBe(false) + }) + + it('removes only a reservation that still belongs to the session', async () => { + const matching = await createRunning('bg-matching-name', { + name: 'matching-name', + }) + await finishNaturally(matching, { exitCode: 0 }) + await writeReservation(matching.name!, matching.id) + + const reassigned = await createRunning('bg-reassigned-name', { + name: 'reassigned-name', + }) + await finishNaturally(reassigned, { exitCode: 0 }) + await writeReservation(reassigned.name!, 'bg-new-owner') + + await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(await exists(reservationPath(matching.name!))).toBe(false) + expect(await exists(reservationPath(reassigned.name!))).toBe(true) + expect(await exists(paths(reassigned.id).metadata)).toBe(false) + }) + + it('does not remove a reservation replaced during artifact cleanup', async () => { + const session = await createRunning('bg-reservation-replaced', { + name: 'reservation-replaced', + }) + await finishNaturally(session, { exitCode: 0 }) + await writeReservation(session.name!, session.id) + let replaced = false + + await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + await unlink(path) + if (!replaced && path === paths(session.id).stdout) { + replaced = true + await writeReservation(session.name!, 'bg-new-owner') + } + }, + }) + + expect( + JSON.parse(await readFile(reservationPath(session.name!), 'utf8')), + ).toMatchObject({ id: 'bg-new-owner' }) + expect(await exists(paths(session.id).metadata)).toBe(false) + }) + + it('does not remove a reservation generation replaced before unlink', async () => { + const session = await createRunning('bg-reservation-generation', { + name: 'reservation-generation', + }) + await finishNaturally(session, { exitCode: 0 }) + await writeReservation(session.name!, session.id) + const target = reservationPath(session.name!) + let replaced = false + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + _beforeReservationRemovalForTesting: async path => { + if (path === target) { + replaced = true + await unlink(target) + await writeReservation(session.name!, 'bg-new-generation') + } + }, + }) + + expect(result.errors).toBe(1) + expect(replaced).toBe(true) + expect(result.sessionsRemoved).toBe(0) + expect( + JSON.parse(await readFile(target, 'utf8')), + ).toMatchObject({ id: 'bg-new-generation' }) + expect(await exists(paths(session.id).metadata)).toBe(true) + }) + + it('keeps a replacement created after final reservation validation', async () => { + const session = await createRunning('bg-reservation-final-window', { + name: 'reservation-final-window', + }) + await finishNaturally(session, { exitCode: 0 }) + await writeReservation(session.name!, session.id) + const target = reservationPath(session.name!) + let replacementPromise: Promise | undefined + + await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === target) { + replacementPromise = createBackgroundSession({ + id: 'bg-reservation-final-replacement', + name: session.name, + pid: nextPid++, + cwd: configDir, + command: ['openclaude', '--print', 'replacement'], + sessionId: 'replacement-conversation', + }) + } + await unlink(path) + }, + }) + + expect(replacementPromise).toBeDefined() + const replacement = await replacementPromise! + expect( + JSON.parse(await readFile(target, 'utf8')), + ).toMatchObject({ id: replacement.id }) + }) + + it('counts reservation unlink failure and keeps metadata for retry', async () => { + const session = await createRunning('bg-reservation-error', { + name: 'reservation-error', + }) + await finishNaturally(session, { exitCode: 0 }) + await writeReservation(session.name!, session.id) + const target = reservationPath(session.name!) + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === target) throw deniedError() + await unlink(path) + }, + }) + + expect(result.errors).toBe(1) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).natural)).toBe(true) + expect(await exists(target)).toBe(true) + }) + + it('counts a reservation read error and retains the session', async () => { + const session = await createRunning('bg-reservation-read-error', { + name: 'reservation-read-error', + }) + await finishNaturally(session, { exitCode: 0 }) + await writeReservation(session.name!, session.id) + const target = reservationPath(session.name!) + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + readTextFile: async path => { + if (path === target) throw deniedError() + return await readFile(path, 'utf8') + }, + }) + + expect(result.errors).toBe(1) + expect(result.sessionsRemoved).toBe(0) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).natural)).toBe(true) + expect(await exists(target)).toBe(true) + }) + + it('preserves a malformed reservation while cleaning owned artifacts', async () => { + const session = await createRunning('bg-malformed-reservation', { + name: 'malformed-reservation', + }) + await finishNaturally(session, { exitCode: 0 }) + const target = reservationPath(session.name!) + await writeFile(target, '{bad json') + + const result = await cleanupBackgroundSessionsBefore(CUTOFF) + + expect(result).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + expect(await exists(target)).toBe(true) + expect(await exists(paths(session.id).metadata)).toBe(false) + }) + + it('reports sessions-directory errors without throwing', async () => { + await mkdir(root, { recursive: true }) + await writeFile(join(root, 'sessions'), 'not a directory') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 1, + }) + }) + + it('treats a missing sessions directory as an empty registry', async () => { + await mkdir(root, { recursive: true }) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + }) + + it('retains eligible sessions when the terminal directory is invalid', async () => { + const session = await writeRawSession({ + id: 'bg-invalid-terminal-directory', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + await rm(join(root, 'terminal'), { recursive: true, force: true }) + await writeFile(join(root, 'terminal'), 'not a directory') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 1, + }) + expect(await exists(paths(session.id).metadata)).toBe(true) + expect(await exists(paths(session.id).stdout)).toBe(true) + }) + + it('cleans an unnamed session when only the names directory is invalid', async () => { + const session = await writeRawSession({ + id: 'bg-invalid-names-directory', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + await rm(join(root, 'names'), { recursive: true, force: true }) + await writeFile(join(root, 'names'), 'not a directory') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 3, + errors: 1, + }) + expect(await exists(paths(session.id).metadata)).toBe(false) + }) +}) diff --git a/src/cli/bgRegistry.ts b/src/cli/bgRegistry.ts index ae9782d0b5..c3d5478d5e 100644 --- a/src/cli/bgRegistry.ts +++ b/src/cli/bgRegistry.ts @@ -1,5 +1,6 @@ import { link, + lstat, mkdir, open, readFile, @@ -18,6 +19,8 @@ import { readFileSync, unlinkSync, writeFileSync, + type Dirent, + type Stats, } from 'node:fs' import { createHash, randomUUID } from 'node:crypto' import { basename, join } from 'node:path' @@ -26,6 +29,7 @@ import { getProcessCommand, isProcessRunning, } from '../utils/genericProcessUtils.js' +import * as lockfile from '../utils/lockfile.js' import { jsonParse, jsonStringify } from '../utils/slowOperations.js' import { backgroundProcessMarkerToken, @@ -104,6 +108,12 @@ type BackgroundSessionNameReservation = { createdAt?: string } +export type BackgroundCleanupResult = { + sessionsRemoved: number + artifactsRemoved: number + errors: number +} + const TERMINAL_STATUSES = new Set([ 'exited', 'failed', @@ -117,6 +127,20 @@ const ALL_STATUSES = new Set([ ]) const SAFE_ID_RE = /^[A-Za-z0-9._-]+$/ const SAFE_SIGNAL_RE = /^SIG[A-Z0-9]{1,24}$/ +const NAME_RESERVATION_LOCK_OPTIONS = { + realpath: false, + retries: { + retries: 20, + factor: 1, + minTimeout: 5, + maxTimeout: 25, + randomize: true, + }, +} satisfies NonNullable[1]> +const NAME_RESERVATION_SYNC_LOCK_OPTIONS = { + realpath: false, + retries: 0, +} satisfies NonNullable[1]> let backgroundSessionsRootForTesting: string | undefined export function _setBackgroundSessionsRootForTesting( @@ -181,6 +205,119 @@ function isErrno(error: unknown, code: string): boolean { ) } +type CleanupJsonRead = + | { state: 'missing' } + | { state: 'valid'; value: T; identity?: CleanupFileIdentity } + | { state: 'invalid' } + | { state: 'error' } + +type CleanupFileIdentity = { dev: number; ino: number } + +type CleanupDirectorySnapshot = + | { state: 'missing'; path: string } + | { state: 'directory'; path: string; dev: number; ino: number } + +type BackgroundCleanupFileSystem = { + lstatFile: (path: string) => Promise + readTextFile: (path: string) => Promise + unlinkFile: (path: string) => Promise +} + +async function snapshotCleanupDirectory( + path: string, + lstatFile: BackgroundCleanupFileSystem['lstatFile'], +): Promise> { + try { + const stats = await lstatFile(path) + if (!stats.isDirectory() || stats.isSymbolicLink()) { + return { state: 'invalid' } + } + return { + state: 'valid', + value: { state: 'directory', path, dev: stats.dev, ino: stats.ino }, + } + } catch (error) { + return isErrno(error, 'ENOENT') + ? { state: 'valid', value: { state: 'missing', path } } + : { state: 'error' } + } +} + +async function inspectCleanupDirectory( + snapshot: CleanupDirectorySnapshot, + lstatFile: BackgroundCleanupFileSystem['lstatFile'], +): Promise<'same' | 'missing' | 'changed' | 'error'> { + try { + const stats = await lstatFile(snapshot.path) + if (snapshot.state === 'missing') return 'changed' + return stats.isDirectory() && + !stats.isSymbolicLink() && + stats.dev === snapshot.dev && + stats.ino === snapshot.ino + ? 'same' + : 'changed' + } catch (error) { + if (isErrno(error, 'ENOENT')) return 'missing' + return 'error' + } +} + +async function readCleanupJson( + path: string, + validate: (value: unknown) => value is T, + directory: CleanupDirectorySnapshot, + fileSystem: BackgroundCleanupFileSystem, +): Promise> { + const directoryState = await inspectCleanupDirectory( + directory, + fileSystem.lstatFile, + ) + if (directoryState === 'missing') return { state: 'missing' } + if (directoryState !== 'same') return { state: 'error' } + + let before: Stats + try { + before = await fileSystem.lstatFile(path) + if (!before.isFile() || before.isSymbolicLink()) return { state: 'invalid' } + } catch (error) { + return isErrno(error, 'ENOENT') ? { state: 'missing' } : { state: 'error' } + } + + let content: string + try { + content = await fileSystem.readTextFile(path) + } catch (error) { + return isErrno(error, 'ENOENT') ? { state: 'missing' } : { state: 'error' } + } + const latestDirectoryState = await inspectCleanupDirectory( + directory, + fileSystem.lstatFile, + ) + if (latestDirectoryState !== 'same') return { state: 'error' } + try { + const after = await fileSystem.lstatFile(path) + if (!after.isFile() || after.isSymbolicLink()) return { state: 'invalid' } + if (after.dev !== before.dev || after.ino !== before.ino) { + return { state: 'error' } + } + } catch (error) { + return isErrno(error, 'ENOENT') ? { state: 'missing' } : { state: 'error' } + } + let parsed: unknown + try { + parsed = jsonParse(content) + } catch { + return { state: 'invalid' } + } + return validate(parsed) + ? { + state: 'valid', + value: parsed, + identity: { dev: before.dev, ino: before.ino }, + } + : { state: 'invalid' } +} + function iso(now: Date | undefined): string { return (now ?? new Date()).toISOString() } @@ -266,29 +403,69 @@ async function readNameReservation( ): Promise { try { const parsed = jsonParse(await readFile(path, 'utf8')) - const candidate = parsed as Partial - if ( - parsed && - typeof parsed === 'object' && - typeof candidate.name === 'string' && - typeof candidate.id === 'string' && - SAFE_ID_RE.test(candidate.id) && - (candidate.creatorPid === undefined || - (typeof candidate.creatorPid === 'number' && - Number.isInteger(candidate.creatorPid) && - candidate.creatorPid > 1)) && - (candidate.createdAt === undefined || - typeof candidate.createdAt === 'string') - ) { - return parsed as BackgroundSessionNameReservation - } + if (isBackgroundSessionNameReservation(parsed)) return parsed } catch { // Malformed reservations are treated as recoverable orphans below. } return null } -async function releaseNameReservation( +async function withNameReservationLock( + name: string, + operation: () => Promise, +): Promise { + const release = await lockfile.lock( + nameReservationPathForName(name), + NAME_RESERVATION_LOCK_OPTIONS, + ) + let value: T + try { + value = await operation() + } catch (error) { + await release().catch(() => {}) + throw error + } + await release() + return value +} + +function withNameReservationLockSync(name: string, operation: () => T): T { + const release = lockfile.lockSync( + nameReservationPathForName(name), + NAME_RESERVATION_SYNC_LOCK_OPTIONS, + ) + let value: T + try { + value = operation() + } catch (error) { + try { + release() + } catch {} + throw error + } + release() + return value +} + +function isBackgroundSessionNameReservation( + value: unknown, +): value is BackgroundSessionNameReservation { + if (!value || typeof value !== 'object') return false + const candidate = value as Partial + return ( + typeof candidate.name === 'string' && + typeof candidate.id === 'string' && + SAFE_ID_RE.test(candidate.id) && + (candidate.creatorPid === undefined || + (typeof candidate.creatorPid === 'number' && + Number.isInteger(candidate.creatorPid) && + candidate.creatorPid > 1)) && + (candidate.createdAt === undefined || + typeof candidate.createdAt === 'string') + ) +} + +async function releaseNameReservationUnlocked( name: string, id: string, ): Promise { @@ -298,7 +475,16 @@ async function releaseNameReservation( await unlink(path).catch(() => {}) } -function releaseNameReservationSync(name: string, id: string): void { +async function releaseNameReservation( + name: string, + id: string, +): Promise { + await withNameReservationLock(name, async () => { + await releaseNameReservationUnlocked(name, id) + }).catch(() => {}) +} + +function releaseNameReservationSyncUnlocked(name: string, id: string): void { const path = nameReservationPathForName(name) try { const parsed = jsonParse( @@ -310,6 +496,16 @@ function releaseNameReservationSync(name: string, id: string): void { } } +function releaseNameReservationSync(name: string, id: string): void { + try { + withNameReservationLockSync(name, () => { + releaseNameReservationSyncUnlocked(name, id) + }) + } catch { + // A stale reservation is recoverable on the next name claim. + } +} + async function unlinkStaleNameReservation(path: string): Promise { try { await unlink(path) @@ -318,7 +514,7 @@ async function unlinkStaleNameReservation(path: string): Promise { } } -async function releaseStaleNameReservation( +async function releaseStaleNameReservationUnlocked( name: string, id: string, ): Promise { @@ -354,38 +550,40 @@ async function reserveBackgroundSessionName( name: string, id: string, ): Promise<() => Promise> { - const path = nameReservationPathForName(name) - const reservation = jsonStringify({ - name, - id, - creatorPid: process.pid, - createdAt: iso(undefined), - }) + return await withNameReservationLock(name, async () => { + const path = nameReservationPathForName(name) + const reservation = jsonStringify({ + name, + id, + creatorPid: process.pid, + createdAt: iso(undefined), + }) - while (true) { - try { - await writeFile(path, reservation, { flag: 'wx' }) - return () => releaseNameReservation(name, id) - } catch (error) { - if (!isErrno(error, 'EEXIST')) throw error - - const existing = await readNameReservation(path) - if (!(await isLiveNameReservation(name, existing))) { - if (existing) { - await releaseStaleNameReservation(name, existing.id) - } else { - await unlinkStaleNameReservation(path) + while (true) { + try { + await writeFile(path, reservation, { flag: 'wx' }) + return () => releaseNameReservation(name, id) + } catch (error) { + if (!isErrno(error, 'EEXIST')) throw error + + const existing = await readNameReservation(path) + if (!(await isLiveNameReservation(name, existing))) { + if (existing) { + await releaseStaleNameReservationUnlocked(name, existing.id) + } else { + await unlinkStaleNameReservation(path) + } + continue } - continue - } - const suffix = - existing && existing.name === name ? ` (${existing.id})` : '' - throw new Error( - `Background session name "${name}" already exists${suffix}`, - ) + const suffix = + existing && existing.name === name ? ` (${existing.id})` : '' + throw new Error( + `Background session name "${name}" already exists${suffix}`, + ) + } } - } + }) } function isStringArray(value: unknown): value is string[] { @@ -535,6 +733,14 @@ async function applyAuthoritativeTerminalFacts( ): Promise { const natural = await readTerminalFact(session.id, 'natural') const killed = await readTerminalFact(session.id, 'killed') + return applyTerminalFacts(session, natural, killed) +} + +function applyTerminalFacts( + session: BackgroundSession, + natural: BackgroundSessionTerminalFact | null, + killed: BackgroundSessionTerminalFact | null, +): BackgroundSession { let effective = session if ( @@ -663,6 +869,331 @@ export async function listBackgroundSessions(): Promise { return sessions.sort((a, b) => a.startedAt.localeCompare(b.startedAt)) } +type CleanupArtifactRemoval = 'removed' | 'missing' | 'error' + +async function removeCleanupArtifact( + path: string, + result: BackgroundCleanupResult, + directory: CleanupDirectorySnapshot, + fileSystem: BackgroundCleanupFileSystem, + expectedIdentity?: CleanupFileIdentity, +): Promise { + const directoryState = await inspectCleanupDirectory( + directory, + fileSystem.lstatFile, + ) + if (directoryState === 'missing') return 'missing' + if (directoryState !== 'same') { + result.errors++ + return 'error' + } + if (expectedIdentity) { + try { + const stats = await fileSystem.lstatFile(path) + if ( + !stats.isFile() || + stats.isSymbolicLink() || + stats.dev !== expectedIdentity.dev || + stats.ino !== expectedIdentity.ino + ) { + result.errors++ + return 'error' + } + } catch (error) { + if (isErrno(error, 'ENOENT')) return 'missing' + result.errors++ + return 'error' + } + } + try { + await fileSystem.unlinkFile(path) + result.artifactsRemoved++ + return 'removed' + } catch (error) { + if (isErrno(error, 'ENOENT')) return 'missing' + result.errors++ + return 'error' + } +} + +function cleanupReadBlocksRemoval( + read: CleanupJsonRead, + result: BackgroundCleanupResult, +): boolean { + if (read.state === 'error') { + result.errors++ + return true + } + return read.state === 'invalid' +} + +export async function cleanupBackgroundSessionsBefore( + cutoff: Date, + options: { + lstatFile?: BackgroundCleanupFileSystem['lstatFile'] + readTextFile?: BackgroundCleanupFileSystem['readTextFile'] + unlinkFile?: BackgroundCleanupFileSystem['unlinkFile'] + _beforeMetadataDirectoryReadForTesting?: () => Promise + _beforeReservationRemovalForTesting?: (path: string) => Promise + } = {}, +): Promise { + const result: BackgroundCleanupResult = { + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + } + const cutoffMs = cutoff.getTime() + if (!Number.isFinite(cutoffMs)) { + result.errors++ + return result + } + + const fileSystem: BackgroundCleanupFileSystem = { + lstatFile: options.lstatFile ?? lstat, + readTextFile: + options.readTextFile ?? (async path => await readFile(path, 'utf8')), + unlinkFile: options.unlinkFile ?? (async path => await unlink(path)), + } + const directoryReads = await Promise.all( + [ + getBackgroundSessionMetadataDir(), + getBackgroundSessionLogsDir(), + getBackgroundSessionNamesDir(), + getBackgroundSessionTerminalDir(), + ].map(async path => + await snapshotCleanupDirectory(path, fileSystem.lstatFile), + ), + ) + const [metadataRead, logsRead, namesRead, terminalRead] = directoryReads + const requiredDirectoryReads = [metadataRead, logsRead, terminalRead] + const requiredDirectoryErrors = requiredDirectoryReads.filter( + read => read.state === 'error' || read.state === 'invalid', + ).length + if (requiredDirectoryErrors > 0) { + result.errors += requiredDirectoryErrors + return result + } + if (namesRead.state === 'error' || namesRead.state === 'invalid') { + result.errors++ + } + const metadataDirectory = + metadataRead.state === 'valid' ? metadataRead.value : undefined + const logsDirectory = + logsRead.state === 'valid' ? logsRead.value : undefined + const namesDirectory = + namesRead.state === 'valid' ? namesRead.value : undefined + const terminalDirectory = + terminalRead.state === 'valid' ? terminalRead.value : undefined + if ( + !metadataDirectory || + !logsDirectory || + !terminalDirectory || + metadataDirectory.state === 'missing' + ) { + return result + } + const metadataDirectoryState = await inspectCleanupDirectory( + metadataDirectory, + fileSystem.lstatFile, + ) + if (metadataDirectoryState !== 'same') { + if (metadataDirectoryState !== 'missing') result.errors++ + return result + } + + let entries: Dirent[] + try { + await options._beforeMetadataDirectoryReadForTesting?.() + entries = await readdir(getBackgroundSessionMetadataDir(), { + withFileTypes: true, + }) + } catch (error) { + if (!isErrno(error, 'ENOENT')) result.errors++ + return result + } + const metadataDirectoryAfterRead = await inspectCleanupDirectory( + metadataDirectory, + fileSystem.lstatFile, + ) + if (metadataDirectoryAfterRead !== 'same') { + if (metadataDirectoryAfterRead !== 'missing') result.errors++ + return result + } + + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith('.json')) continue + const metadataPath = join(getBackgroundSessionMetadataDir(), entry.name) + const expectedId = basename(entry.name, '.json') + const metadata = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => + isBackgroundSession(value, expectedId), + metadataDirectory, + fileSystem, + ) + if (metadata.state === 'error') { + result.errors++ + continue + } + if (metadata.state !== 'valid') continue + + const session = metadata.value + const natural = await readCleanupJson( + terminalFactPathForId(session.id, 'natural'), + (value): value is BackgroundSessionTerminalFact => + isBackgroundSessionTerminalFact(value, session.id, 'natural') && + Number.isFinite(Date.parse(value.finishedAt)), + terminalDirectory, + fileSystem, + ) + const killed = await readCleanupJson( + terminalFactPathForId(session.id, 'killed'), + (value): value is BackgroundSessionTerminalFact => + isBackgroundSessionTerminalFact(value, session.id, 'killed') && + Number.isFinite(Date.parse(value.finishedAt)), + terminalDirectory, + fileSystem, + ) + const naturalBlocksRemoval = cleanupReadBlocksRemoval(natural, result) + const killedBlocksRemoval = cleanupReadBlocksRemoval(killed, result) + if ( + naturalBlocksRemoval || + killedBlocksRemoval || + (natural.state === 'valid' && natural.value.pid !== session.pid) || + (killed.state === 'valid' && killed.value.pid !== session.pid) + ) { + continue + } + + const effective = applyTerminalFacts( + session, + natural.state === 'valid' ? natural.value : null, + killed.state === 'valid' ? killed.value : null, + ) + if ( + effective.status !== 'exited' && + effective.status !== 'failed' && + effective.status !== 'killed' + ) { + continue + } + if (effective.finishedAt === undefined) continue + const finishedAtMs = Date.parse(effective.finishedAt) + if (!Number.isFinite(finishedAtMs) || finishedAtMs >= cutoffMs) continue + + let reservation: + | CleanupJsonRead + | undefined + if (session.name) { + if (!namesDirectory) continue + reservation = await readCleanupJson( + nameReservationPathForName(session.name), + isBackgroundSessionNameReservation, + namesDirectory, + fileSystem, + ) + if (reservation.state === 'error') { + result.errors++ + continue + } + } + + const logPaths = getBackgroundSessionLogPaths(session.id) + const stdoutRemoval = await removeCleanupArtifact( + logPaths.stdoutLogPath, + result, + logsDirectory, + fileSystem, + ) + const stderrRemoval = await removeCleanupArtifact( + logPaths.stderrLogPath, + result, + logsDirectory, + fileSystem, + ) + let reservationRemoval: CleanupArtifactRemoval = 'missing' + const sessionName = session.name + if (sessionName && reservation?.state === 'valid' && namesDirectory) { + try { + reservationRemoval = await withNameReservationLock( + sessionName, + async () => { + const latestReservation = await readCleanupJson( + nameReservationPathForName(sessionName), + isBackgroundSessionNameReservation, + namesDirectory, + fileSystem, + ) + if (latestReservation.state === 'error') { + result.errors++ + return 'error' + } + if ( + latestReservation.state !== 'valid' || + latestReservation.value.name !== sessionName || + latestReservation.value.id !== session.id + ) { + return 'missing' + } + const reservationPath = nameReservationPathForName(sessionName) + await options._beforeReservationRemovalForTesting?.( + reservationPath, + ) + return await removeCleanupArtifact( + reservationPath, + result, + namesDirectory, + fileSystem, + latestReservation.identity, + ) + }, + ) + } catch { + result.errors++ + reservationRemoval = 'error' + } + } + if ( + stdoutRemoval === 'error' || + stderrRemoval === 'error' || + reservationRemoval === 'error' + ) { + continue + } + + const metadataRemoval = await removeCleanupArtifact( + metadataPath, + result, + metadataDirectory, + fileSystem, + metadata.identity, + ) + if (metadataRemoval === 'error') continue + if (metadataRemoval === 'removed') result.sessionsRemoved++ + + if (natural.state === 'valid') { + await removeCleanupArtifact( + terminalFactPathForId(session.id, 'natural'), + result, + terminalDirectory, + fileSystem, + natural.identity, + ) + } + if (killed.state === 'valid') { + await removeCleanupArtifact( + terminalFactPathForId(session.id, 'killed'), + result, + terminalDirectory, + fileSystem, + killed.identity, + ) + } + } + + return result +} + export async function readBackgroundSessionForOwner( id: string, ): Promise { diff --git a/src/skills/bundled/updateConfig.ts b/src/skills/bundled/updateConfig.ts index 990cfe5636..922644719f 100644 --- a/src/skills/bundled/updateConfig.ts +++ b/src/skills/bundled/updateConfig.ts @@ -114,7 +114,7 @@ Plugin syntax: \`plugin-name@source\` where source is \`claude-code-marketplace\ ### Other Settings - \`language\`: Preferred response language (e.g., "japanese") -- \`cleanupPeriodDays\`: Days to keep transcripts (default: 30; 0 disables persistence entirely) +- \`cleanupPeriodDays\`: Days to keep local session artifacts, including transcripts and completed background sessions (default: 30; 0 disables transcript persistence and removes background artifacts after completion) - \`respectGitignore\`: Whether to respect .gitignore (default: true) - \`spinnerTipsEnabled\`: Show tips in spinner - \`spinnerVerbs\`: Customize spinner verbs (\`{ "mode": "append" | "replace", "verbs": [...] }\`) diff --git a/src/utils/cleanup.test.ts b/src/utils/cleanup.test.ts index 1926a0f06d..2a5693a8e3 100644 --- a/src/utils/cleanup.test.ts +++ b/src/utils/cleanup.test.ts @@ -3,17 +3,79 @@ import { mkdir, rm, stat, utimes, writeFile } from 'fs/promises' import { tmpdir } from 'os' import { join } from 'path' +import { + _setBackgroundSessionsRootForTesting, + createBackgroundSession, + recordBackgroundSessionNaturalTermination, +} from '../cli/bgRegistry.js' import { cleanupOldSessionFilesInProjectsDir } from './cleanup.js' import { NodeFsOperations } from './fsOperations.js' const tempDirs: string[] = [] afterEach(async () => { + _setBackgroundSessionsRootForTesting(undefined) await Promise.all( tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })), ) }) +async function runCleanupFixture(configDir: string): Promise { + const fixture = join(import.meta.dir, 'cleanupBackgroundSessions.fixture.ts') + const { USER_TYPE: _userType, ...inheritedEnv } = process.env + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 15_000) + const child = Bun.spawn([process.execPath, fixture], { + cwd: configDir, + env: { + ...inheritedEnv, + HOME: configDir, + XDG_CACHE_HOME: join(configDir, 'cache'), + OPENCLAUDE_CONFIG_DIR: configDir, + NODE_ENV: 'test', + }, + signal: controller.signal, + stdout: 'pipe', + stderr: 'pipe', + }) + + try { + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]) + expect(exitCode, stderr).toBe(0) + expect(JSON.parse(stdout)).toEqual({ completed: true }) + } finally { + clearTimeout(timeout) + } +} + +async function createCompletedBackgroundSession( + configDir: string, + id: string, + finishedAt: Date, +): Promise { + const root = join(configDir, 'bg-sessions') + _setBackgroundSessionsRootForTesting(root) + await createBackgroundSession({ + id, + pid: process.pid, + cwd: configDir, + command: ['openclaude', '--print', 'fixture'], + sessionId: `${id}-conversation`, + now: new Date(finishedAt.getTime() - 60_000), + }) + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + { ownerPid: process.pid, now: finishedAt }, + ) + _setBackgroundSessionsRootForTesting(undefined) + return join(root, 'sessions', `${id}.json`) +} + describe('cleanupOldSessionFiles', () => { test('removes old replay sidecars while preserving non-session files', async () => { const projectsDir = join( @@ -46,3 +108,115 @@ describe('cleanupOldSessionFiles', () => { expect((await stat(keepPath)).isFile()).toBe(true) }) }) + +describe('cleanupOldMessageFilesInBackground', () => { + test( + 'removes old completed background-session artifacts through the scheduler', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-scheduler-old', + new Date(Date.now() - 31 * 24 * 60 * 60 * 1000), + ) + + await runCleanupFixture(configDir) + + expect(await Bun.file(metadataPath).exists()).toBe(false) + }, + 30_000, + ) + + test( + 'uses cleanupPeriodDays zero for completed background sessions', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-zero-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-scheduler-zero', + new Date(Date.now() - 1_000), + ) + + await runCleanupFixture(configDir) + + expect(await Bun.file(metadataPath).exists()).toBe(false) + }, + 30_000, + ) + + test( + 'preserves background artifacts when explicit cleanupPeriodDays is invalid', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-invalid-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 'invalid' }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-scheduler-invalid', + new Date('2026-06-01T00:00:00.000Z'), + ) + + await runCleanupFixture(configDir) + + expect(await Bun.file(metadataPath).exists()).toBe(true) + }, + 30_000, + ) + + test( + 'retains recent background sessions under a nonzero cleanup period', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-recent-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const oldMetadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-scheduler-old-pair', + new Date(Date.now() - 31 * 24 * 60 * 60 * 1000), + ) + const recentMetadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-scheduler-recent', + new Date(Date.now() - 60_000), + ) + + await runCleanupFixture(configDir) + + expect(await Bun.file(oldMetadataPath).exists()).toBe(false) + expect(await Bun.file(recentMetadataPath).exists()).toBe(true) + }, + 30_000, + ) +}) diff --git a/src/utils/cleanup.ts b/src/utils/cleanup.ts index e9375b049a..66258cc045 100644 --- a/src/utils/cleanup.ts +++ b/src/utils/cleanup.ts @@ -1,6 +1,7 @@ import * as fs from 'fs/promises' import { homedir } from 'os' import { join } from 'path' +import { cleanupBackgroundSessionsBefore } from '../cli/bgRegistry.js' import { logEvent } from '../services/analytics/index.js' import { CACHE_PATHS } from './cachePaths.js' import { logForDebugging } from './debug.js' @@ -598,6 +599,18 @@ export async function cleanupOldMessageFilesInBackground(): Promise { await cleanupOldMessageFiles() await cleanupOldSessionFiles() + const backgroundResult = await cleanupBackgroundSessionsBefore( + getCutoffDate(), + ) + if ( + backgroundResult.sessionsRemoved > 0 || + backgroundResult.artifactsRemoved > 0 || + backgroundResult.errors > 0 + ) { + logForDebugging( + `Background session cleanup: ${backgroundResult.sessionsRemoved} sessions, ${backgroundResult.artifactsRemoved} artifacts, ${backgroundResult.errors} errors`, + ) + } await cleanupOldPlanFiles() await cleanupOldFileHistoryBackups() await cleanupOldSessionEnvDirs() diff --git a/src/utils/cleanupBackgroundSessions.fixture.ts b/src/utils/cleanupBackgroundSessions.fixture.ts new file mode 100644 index 0000000000..4db38942be --- /dev/null +++ b/src/utils/cleanupBackgroundSessions.fixture.ts @@ -0,0 +1,9 @@ +import { cleanupOldMessageFilesInBackground } from './cleanup.js' + +try { + await cleanupOldMessageFilesInBackground() + process.stdout.write(`${JSON.stringify({ completed: true })}\n`) +} catch { + process.stderr.write('background cleanup fixture failed\n') + process.exitCode = 1 +} diff --git a/src/utils/settings/types.ts b/src/utils/settings/types.ts index 121a17c3c0..aa6eeab6ae 100644 --- a/src/utils/settings/types.ts +++ b/src/utils/settings/types.ts @@ -451,7 +451,7 @@ export const SettingsSchema = lazySchema(() => .int() .optional() .describe( - 'Number of days to retain chat transcripts (default: 30). Setting to 0 disables session persistence entirely: no transcripts are written and existing transcripts are deleted at startup.', + 'Number of days to retain local session artifacts, including chat transcripts and completed background-session metadata and logs (default: 30). Setting to 0 disables transcript persistence entirely and removes completed background-session artifacts after they finish.', ), env: EnvironmentVariablesSchema() .optional() diff --git a/src/utils/settings/validationTips.ts b/src/utils/settings/validationTips.ts index f5a8428b80..d3f78c889e 100644 --- a/src/utils/settings/validationTips.ts +++ b/src/utils/settings/validationTips.ts @@ -47,7 +47,7 @@ const TIP_MATCHERS: TipMatcher[] = [ ctx.expected === '0', tip: { suggestion: - 'Must be 0 or greater. Set a positive number for days to retain transcripts (default is 30). Setting 0 disables session persistence entirely: no transcripts are written and existing transcripts are deleted at startup.', + 'Must be 0 or greater. Set a positive number for days to retain local session artifacts, including transcripts and completed background sessions (default is 30). Setting 0 disables transcript persistence and removes completed background-session artifacts after they finish.', }, }, { From 342bb478211e970aa512fc613b6bee2b3c27c7bc Mon Sep 17 00:00:00 2001 From: chioarub Date: Mon, 24 Aug 2026 10:09:34 +0300 Subject: [PATCH 02/13] fix(bg): reclaim orphaned terminal facts --- src/cli/bgRegistry.cleanup.test.ts | 151 +++++++++++++++++++++++---- src/cli/bgRegistry.ts | 160 ++++++++++++++++++++++++++++- 2 files changed, 291 insertions(+), 20 deletions(-) diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts index 44bbcff7cf..3b373d5c50 100644 --- a/src/cli/bgRegistry.cleanup.test.ts +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -705,6 +705,130 @@ describe('background session retention cleanup', () => { expect(await exists(paths(session.id).natural)).toBe(false) }) + it('reclaims an old terminal fact after metadata-first partial cleanup', async () => { + const session = await createRunning('bg-terminal-fact-retry') + await finishNaturally(session, { exitCode: 0 }) + const naturalFact = paths(session.id).natural + + const first = await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === naturalFact) throw deniedError() + await unlink(path) + }, + }) + + expect(first.errors).toBe(1) + expect(first.sessionsRemoved).toBe(1) + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await exists(naturalFact)).toBe(true) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 1, + errors: 0, + }) + expect(await exists(naturalFact)).toBe(false) + }) + + it('reclaims an old killed fact after metadata-first partial cleanup', async () => { + const session = await createRunning('bg-killed-fact-retry') + await markBackgroundSessionKilled(session.id, { now: OLD_FINISH }) + const killedFact = paths(session.id).killed + + const first = await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === killedFact) throw deniedError() + await unlink(path) + }, + }) + + expect(first.errors).toBe(1) + expect(first.sessionsRemoved).toBe(1) + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await exists(killedFact)).toBe(true) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 1, + errors: 0, + }) + expect(await exists(killedFact)).toBe(false) + }) + + it('retains recent and malformed orphaned terminal facts', async () => { + const recent = await createRunning('bg-recent-orphan') + await finishNaturally(recent, { exitCode: 0 }, RECENT_FINISH) + await unlink(paths(recent.id).metadata) + + const malformed = await createRunning('bg-malformed-orphan') + await finishNaturally(malformed, { exitCode: 0 }) + await unlink(paths(malformed.id).metadata) + await writeFile(paths(malformed.id).natural, '{bad json') + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(recent.id).natural)).toBe(true) + expect(await exists(paths(malformed.id).natural)).toBe(true) + }) + + it('advances the bounded scan past preserved orphaned facts', async () => { + const terminalDir = join(root, 'terminal') + await mkdir(terminalDir, { recursive: true }) + const preservedFacts = Array.from({ length: 256 }, (_, index) => { + const id = `bg-preserved-orphan-${index.toString().padStart(3, '0')}` + return writeFile( + paths(id).natural, + JSON.stringify({ + version: 1, + id, + pid: nextPid++, + status: 'exited', + finishedAt: RECENT_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + }) + await Promise.all(preservedFacts) + const oldId = 'zz-bg-old-orphan' + await writeFile( + paths(oldId).natural, + JSON.stringify({ + version: 1, + id: oldId, + pid: nextPid++, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + const scanOptions = { + _orphanedTerminalFactScanStartForTesting: 0, + } + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, scanOptions), + ).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(oldId).natural)).toBe(true) + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, scanOptions), + ).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 1, + errors: 0, + }) + expect(await exists(paths(oldId).natural)).toBe(false) + }) + it('removes only a reservation that still belongs to the session', async () => { const matching = await createRunning('bg-matching-name', { name: 'matching-name', @@ -777,33 +901,24 @@ describe('background session retention cleanup', () => { expect(await exists(paths(session.id).metadata)).toBe(true) }) - it('keeps a replacement created after final reservation validation', async () => { + it('allows a replacement to claim the name after cleanup', async () => { const session = await createRunning('bg-reservation-final-window', { name: 'reservation-final-window', }) await finishNaturally(session, { exitCode: 0 }) await writeReservation(session.name!, session.id) const target = reservationPath(session.name!) - let replacementPromise: Promise | undefined - await cleanupBackgroundSessionsBefore(CUTOFF, { - unlinkFile: async path => { - if (path === target) { - replacementPromise = createBackgroundSession({ - id: 'bg-reservation-final-replacement', - name: session.name, - pid: nextPid++, - cwd: configDir, - command: ['openclaude', '--print', 'replacement'], - sessionId: 'replacement-conversation', - }) - } - await unlink(path) - }, + await cleanupBackgroundSessionsBefore(CUTOFF) + const replacement = await createBackgroundSession({ + id: 'bg-reservation-final-replacement', + name: session.name, + pid: nextPid++, + cwd: configDir, + command: ['openclaude', '--print', 'replacement'], + sessionId: 'replacement-conversation', }) - expect(replacementPromise).toBeDefined() - const replacement = await replacementPromise! expect( JSON.parse(await readFile(target, 'utf8')), ).toMatchObject({ id: replacement.id }) diff --git a/src/cli/bgRegistry.ts b/src/cli/bgRegistry.ts index c3d5478d5e..d45acce840 100644 --- a/src/cli/bgRegistry.ts +++ b/src/cli/bgRegistry.ts @@ -127,6 +127,7 @@ const ALL_STATUSES = new Set([ ]) const SAFE_ID_RE = /^[A-Za-z0-9._-]+$/ const SAFE_SIGNAL_RE = /^SIG[A-Z0-9]{1,24}$/ +const ORPHANED_TERMINAL_FACT_SCAN_LIMIT = 256 const NAME_RESERVATION_LOCK_OPTIONS = { realpath: false, retries: { @@ -142,11 +143,13 @@ const NAME_RESERVATION_SYNC_LOCK_OPTIONS = { retries: 0, } satisfies NonNullable[1]> let backgroundSessionsRootForTesting: string | undefined +let orphanedTerminalFactScanOffset = 0 export function _setBackgroundSessionsRootForTesting( root: string | undefined, ): void { backgroundSessionsRootForTesting = root?.normalize('NFC') + orphanedTerminalFactScanOffset = 0 } function getBackgroundSessionsRoot(): string { @@ -927,6 +930,130 @@ function cleanupReadBlocksRemoval( return read.state === 'invalid' } +function terminalFactCandidateFromEntry( + entry: Dirent, +): { id: string; kind: 'natural' | 'killed' } | null { + if (!entry.isFile()) return null + const match = /^(.*)\.(natural|killed)\.json$/.exec(entry.name) + if (!match) return null + const [, id, kind] = match + if (!id || !kind || !SAFE_ID_RE.test(id)) return null + return { id, kind: kind as 'natural' | 'killed' } +} + +async function cleanupOrphanedTerminalFacts( + cutoffMs: number, + metadataDirectory: CleanupDirectorySnapshot, + terminalDirectory: CleanupDirectorySnapshot, + metadataIds: ReadonlySet, + result: BackgroundCleanupResult, + fileSystem: BackgroundCleanupFileSystem, + scanStartSeed: number = cutoffMs, +): Promise { + const terminalDirectoryState = await inspectCleanupDirectory( + terminalDirectory, + fileSystem.lstatFile, + ) + if (terminalDirectoryState === 'missing') return + if (terminalDirectoryState !== 'same') { + result.errors++ + return + } + + let entries: Dirent[] + try { + entries = await readdir(getBackgroundSessionTerminalDir(), { + withFileTypes: true, + }) + } catch (error) { + if (!isErrno(error, 'ENOENT')) result.errors++ + return + } + const terminalDirectoryAfterRead = await inspectCleanupDirectory( + terminalDirectory, + fileSystem.lstatFile, + ) + if (terminalDirectoryAfterRead !== 'same') { + if (terminalDirectoryAfterRead !== 'missing') result.errors++ + return + } + + const candidates = entries + .map(terminalFactCandidateFromEntry) + .filter( + (candidate): candidate is NonNullable => + candidate !== null && !metadataIds.has(candidate.id), + ) + .sort((a, b) => { + const idOrder = a.id.localeCompare(b.id) + return idOrder !== 0 ? idOrder : a.kind.localeCompare(b.kind) + }) + if (candidates.length === 0) return + + const scanCount = Math.min( + candidates.length, + ORPHANED_TERMINAL_FACT_SCAN_LIMIT, + ) + const effectiveScanStartSeed = Number.isFinite(scanStartSeed) + ? scanStartSeed + : cutoffMs + const normalizedSeed = + ((Math.trunc(effectiveScanStartSeed) % candidates.length) + + candidates.length) % + candidates.length + const startIndex = + (normalizedSeed + orphanedTerminalFactScanOffset) % candidates.length + orphanedTerminalFactScanOffset = + (orphanedTerminalFactScanOffset + scanCount) % candidates.length + for (let index = 0; index < scanCount; index++) { + const candidate = candidates[(startIndex + index) % candidates.length] + if (!candidate) continue + const factPath = terminalFactPathForId(candidate.id, candidate.kind) + const fact = await readCleanupJson( + factPath, + (value): value is BackgroundSessionTerminalFact => + isBackgroundSessionTerminalFact( + value, + candidate.id, + candidate.kind, + ) && Number.isFinite(Date.parse(value.finishedAt)), + terminalDirectory, + fileSystem, + ) + if (fact.state === 'error') { + result.errors++ + continue + } + if ( + fact.state !== 'valid' || + Date.parse(fact.value.finishedAt) >= cutoffMs + ) { + continue + } + + const metadata = await readCleanupJson( + metadataPathForId(candidate.id), + (value): value is BackgroundSession => + isBackgroundSession(value, candidate.id), + metadataDirectory, + fileSystem, + ) + if (metadata.state === 'error') { + result.errors++ + continue + } + if (metadata.state !== 'missing') continue + + await removeCleanupArtifact( + factPath, + result, + terminalDirectory, + fileSystem, + fact.identity, + ) + } +} + export async function cleanupBackgroundSessionsBefore( cutoff: Date, options: { @@ -935,6 +1062,7 @@ export async function cleanupBackgroundSessionsBefore( unlinkFile?: BackgroundCleanupFileSystem['unlinkFile'] _beforeMetadataDirectoryReadForTesting?: () => Promise _beforeReservationRemovalForTesting?: (path: string) => Promise + _orphanedTerminalFactScanStartForTesting?: number } = {}, ): Promise { const result: BackgroundCleanupResult = { @@ -987,11 +1115,22 @@ export async function cleanupBackgroundSessionsBefore( if ( !metadataDirectory || !logsDirectory || - !terminalDirectory || - metadataDirectory.state === 'missing' + !terminalDirectory ) { return result } + if (metadataDirectory.state === 'missing') { + await cleanupOrphanedTerminalFacts( + cutoffMs, + metadataDirectory, + terminalDirectory, + new Set(), + result, + fileSystem, + options._orphanedTerminalFactScanStartForTesting, + ) + return result + } const metadataDirectoryState = await inspectCleanupDirectory( metadataDirectory, fileSystem.lstatFile, @@ -1020,6 +1159,13 @@ export async function cleanupBackgroundSessionsBefore( return result } + const metadataIds = new Set( + entries + .filter(entry => entry.isFile() && entry.name.endsWith('.json')) + .map(entry => basename(entry.name, '.json')) + .filter(id => SAFE_ID_RE.test(id)), + ) + for (const entry of entries) { if (!entry.isFile() || !entry.name.endsWith('.json')) continue const metadataPath = join(getBackgroundSessionMetadataDir(), entry.name) @@ -1191,6 +1337,16 @@ export async function cleanupBackgroundSessionsBefore( } } + await cleanupOrphanedTerminalFacts( + cutoffMs, + metadataDirectory, + terminalDirectory, + metadataIds, + result, + fileSystem, + options._orphanedTerminalFactScanStartForTesting, + ) + return result } From 7c19b0cf4ff2447166d7b1636a60dbbfddf3e3b0 Mon Sep 17 00:00:00 2001 From: chioarub Date: Mon, 31 Aug 2026 08:21:55 +0300 Subject: [PATCH 03/13] test(bg): stabilize reservation cleanup handoff --- src/cli/bgRegistry.cleanup.test.ts | 144 +++++++++++++++++++++++------ 1 file changed, 118 insertions(+), 26 deletions(-) diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts index 3b373d5c50..33a9fa9fae 100644 --- a/src/cli/bgRegistry.cleanup.test.ts +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { afterEach, beforeEach, describe, expect, it, spyOn } from 'bun:test' import { createHash } from 'node:crypto' import { lstat, @@ -12,6 +12,7 @@ import { } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import * as nameReservationLock from '../utils/lockfile.js' import { _setBackgroundSessionsRootForTesting, cleanupBackgroundSessionsBefore, @@ -873,33 +874,124 @@ describe('background session retention cleanup', () => { expect(await exists(paths(session.id).metadata)).toBe(false) }) - it('does not remove a reservation generation replaced before unlink', async () => { - const session = await createRunning('bg-reservation-generation', { - name: 'reservation-generation', - }) - await finishNaturally(session, { exitCode: 0 }) - await writeReservation(session.name!, session.id) - const target = reservationPath(session.name!) - let replaced = false + it( + 'serializes cleanup with a replacement using the supported name writer', + async () => { + const session = await createRunning('bg-reservation-generation', { + name: 'reservation-generation', + }) + await finishNaturally(session, { exitCode: 0 }) + await writeReservation(session.name!, session.id) + const target = reservationPath(session.name!) + let replacementPromise: Promise | undefined + let replacementStarted = false + let cleanupLockAcquired = false + let cleanupLockReleased = false + let replacementLockAttempted = false + let targetLockCalls = 0 + let signalCleanupLockRelease!: () => void + const cleanupLockRelease = new Promise(resolve => { + signalCleanupLockRelease = resolve + }) + let signalReplacementLockAttempt!: () => void + const replacementLockAttempt = new Promise(resolve => { + signalReplacementLockAttempt = resolve + }) + const originalLock = nameReservationLock.lock + const lockSpy = spyOn(nameReservationLock, 'lock').mockImplementation( + async (path, options) => { + if (path !== target) return await originalLock(path, options) + targetLockCalls++ + if (!replacementStarted) { + cleanupLockAcquired = true + const release = await originalLock(path, options) + return async () => { + try { + await release() + } finally { + cleanupLockReleased = true + signalCleanupLockRelease() + } + } + } + + replacementLockAttempted = true + signalReplacementLockAttempt() + await cleanupLockRelease + return await originalLock(path, options) + }, + ) - const result = await cleanupBackgroundSessionsBefore(CUTOFF, { - _beforeReservationRemovalForTesting: async path => { - if (path === target) { - replaced = true - await unlink(target) - await writeReservation(session.name!, 'bg-new-generation') + const result = await (async () => { + try { + return await cleanupBackgroundSessionsBefore(CUTOFF, { + _beforeReservationRemovalForTesting: async path => { + if (path !== target) return + replacementStarted = true + replacementPromise = createBackgroundSession({ + id: 'bg-new-generation', + name: session.name, + pid: nextPid++, + cwd: configDir, + command: ['openclaude', '--print', 'replacement'], + sessionId: 'replacement-conversation', + }) + let timeout: ReturnType | undefined + try { + await Promise.race([ + replacementLockAttempt, + new Promise((_, reject) => { + timeout = setTimeout( + () => + reject( + new Error('replacement lock barrier timed out'), + ), + 5_000, + ) + timeout.unref?.() + }), + ]) + } finally { + if (timeout !== undefined) clearTimeout(timeout) + } + }, + }) + } finally { + lockSpy.mockRestore() } - }, - }) - - expect(result.errors).toBe(1) - expect(replaced).toBe(true) - expect(result.sessionsRemoved).toBe(0) - expect( - JSON.parse(await readFile(target, 'utf8')), - ).toMatchObject({ id: 'bg-new-generation' }) - expect(await exists(paths(session.id).metadata)).toBe(true) - }) + })() + + expect({ + cleanupLockAcquired, + cleanupLockReleased, + replacementLockAttempted, + replacementStarted, + targetLockCalls, + }).toEqual({ + cleanupLockAcquired: true, + cleanupLockReleased: true, + replacementLockAttempted: true, + replacementStarted: true, + targetLockCalls: 2, + }) + expect(result).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 5, + errors: 0, + }) + expect(replacementPromise).toBeDefined() + if (!replacementPromise) { + throw new Error('replacement writer did not run') + } + const replacement = await replacementPromise + expect( + JSON.parse(await readFile(target, 'utf8')), + ).toMatchObject({ id: replacement.id }) + expect(await exists(paths(session.id).metadata)).toBe(false) + expect(await exists(paths(replacement.id).metadata)).toBe(true) + }, + 10_000, + ) it('allows a replacement to claim the name after cleanup', async () => { const session = await createRunning('bg-reservation-final-window', { From 9c938ba1b60d8778a91b0f4ab8d143c0a55b02b8 Mon Sep 17 00:00:00 2001 From: chioarub Date: Mon, 31 Aug 2026 09:02:26 +0300 Subject: [PATCH 04/13] test(bg): clarify cleanup race invariants --- src/cli/bgRegistry.cleanup.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts index 33a9fa9fae..4ee4dd086f 100644 --- a/src/cli/bgRegistry.cleanup.test.ts +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -808,6 +808,7 @@ describe('background session retention cleanup', () => { }), ) + // The fixed seed selects the initial index; later calls resume from the module-level offset. const scanOptions = { _orphanedTerminalFactScanStartForTesting: 0, } @@ -927,6 +928,11 @@ describe('background session retention cleanup', () => { return await cleanupBackgroundSessionsBefore(CUTOFF, { _beforeReservationRemovalForTesting: async path => { if (path !== target) return + if (!cleanupLockAcquired) { + throw new Error( + 'cleanup did not acquire the name lock before reservation removal', + ) + } replacementStarted = true replacementPromise = createBackgroundSession({ id: 'bg-new-generation', From 7a7170eb564d37be9a79ada94fa4a0f308ef2bd8 Mon Sep 17 00:00:00 2001 From: chioarub Date: Mon, 31 Aug 2026 19:38:44 +0300 Subject: [PATCH 05/13] fix(bg): validate cleanup completion timestamps --- src/cli/bgRegistry.cleanup.test.ts | 139 +++++++++++++++++++++++++++++ src/cli/bgRegistry.ts | 17 +++- 2 files changed, 152 insertions(+), 4 deletions(-) diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts index 4ee4dd086f..3dd419315f 100644 --- a/src/cli/bgRegistry.cleanup.test.ts +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -27,6 +27,10 @@ import { const CUTOFF = new Date('2026-07-01T00:00:00.000Z') const OLD_FINISH = new Date('2026-06-01T00:00:00.000Z') const RECENT_FINISH = new Date('2026-07-01T00:00:01.000Z') +const NONCANONICAL_FINISHED_AT_CASES = [ + ['normalized-date', '2026-02-30'], + ['numeric-date', '0'], +] as const describe('background session retention cleanup', () => { let configDir: string @@ -258,6 +262,51 @@ describe('background session retention cleanup', () => { expect(await exists(paths(invalid.id).metadata)).toBe(true) }) + it('retains completed metadata with noncanonical finishedAt values', async () => { + const sessions: BackgroundSession[] = [] + for (const [suffix, finishedAt] of NONCANONICAL_FINISHED_AT_CASES) { + const id = `bg-metadata-${suffix}` + const name = `metadata-${suffix}` + const session = await writeRawSession({ + id, + name, + status: 'running', + finishedAt, + }) + await writeReservation(name, id) + await writeFile( + paths(id).natural, + JSON.stringify({ + version: 1, + id, + pid: session.pid, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + sessions.push(session) + } + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + for (const session of sessions) { + for (const path of [ + paths(session.id).metadata, + paths(session.id).stdout, + paths(session.id).stderr, + paths(session.id).natural, + reservationPath(session.name!), + ]) { + expect(await exists(path)).toBe(true) + } + } + }) + it('removes an old completed session with a valid process marker', async () => { const session = await writeRawSession({ id: 'bg-valid-process-marker', @@ -593,6 +642,96 @@ describe('background session retention cleanup', () => { expect(await exists(paths(session.id).natural)).toBe(true) }) + it('preserves terminal facts with noncanonical timestamps', async () => { + const sessions: BackgroundSession[] = [] + for (const [suffix, finishedAt] of NONCANONICAL_FINISHED_AT_CASES) { + for (const malformedKind of ['natural', 'killed'] as const) { + const id = `bg-${malformedKind}-${suffix}` + const name = `${malformedKind}-${suffix}` + const session = await writeRawSession({ id, name, status: 'running' }) + await writeReservation(name, id) + await writeFile( + paths(id).natural, + JSON.stringify({ + version: 1, + id, + pid: session.pid, + status: 'exited', + finishedAt: + malformedKind === 'natural' + ? finishedAt + : OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + await writeFile( + paths(id).killed, + JSON.stringify({ + version: 1, + id, + pid: session.pid, + status: 'killed', + finishedAt: + malformedKind === 'killed' + ? finishedAt + : OLD_FINISH.toISOString(), + terminalReason: 'explicit_kill', + }), + ) + sessions.push(session) + } + } + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + for (const session of sessions) { + for (const path of [ + ...Object.values(paths(session.id)), + reservationPath(session.name!), + ]) { + expect(await exists(path)).toBe(true) + } + } + }) + + it('preserves orphaned terminal facts with noncanonical timestamps', async () => { + await mkdir(join(root, 'terminal'), { recursive: true }) + const factPaths: string[] = [] + for (const [suffix, finishedAt] of NONCANONICAL_FINISHED_AT_CASES) { + for (const kind of ['natural', 'killed'] as const) { + const id = `bg-orphan-${kind}-${suffix}` + const factPath = paths(id)[kind] + await writeFile( + factPath, + JSON.stringify({ + version: 1, + id, + pid: nextPid++, + status: kind === 'natural' ? 'exited' : 'killed', + finishedAt, + terminalReason: + kind === 'natural' ? 'exit_code' : 'explicit_kill', + ...(kind === 'natural' ? { exitCode: 0 } : {}), + }), + ) + factPaths.push(factPath) + } + } + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + for (const factPath of factPaths) { + expect(await exists(factPath)).toBe(true) + } + }) + it('treats missing artifacts as idempotent success', async () => { const session = await writeRawSession({ id: 'bg-missing-artifacts', diff --git a/src/cli/bgRegistry.ts b/src/cli/bgRegistry.ts index d45acce840..70b4aac56d 100644 --- a/src/cli/bgRegistry.ts +++ b/src/cli/bgRegistry.ts @@ -325,6 +325,13 @@ function iso(now: Date | undefined): string { return (now ?? new Date()).toISOString() } +function parseCanonicalCompletionTimestamp(value: unknown): number | null { + if (typeof value !== 'string') return null + const timestamp = Date.parse(value) + if (!Number.isFinite(timestamp)) return null + return new Date(timestamp).toISOString() === value ? timestamp : null +} + export function getBackgroundSessionLogPaths(id: string): { stdoutLogPath: string stderrLogPath: string @@ -1016,7 +1023,7 @@ async function cleanupOrphanedTerminalFacts( value, candidate.id, candidate.kind, - ) && Number.isFinite(Date.parse(value.finishedAt)), + ) && parseCanonicalCompletionTimestamp(value.finishedAt) !== null, terminalDirectory, fileSystem, ) @@ -1173,7 +1180,9 @@ export async function cleanupBackgroundSessionsBefore( const metadata = await readCleanupJson( metadataPath, (value): value is BackgroundSession => - isBackgroundSession(value, expectedId), + isBackgroundSession(value, expectedId) && + (value.finishedAt === undefined || + parseCanonicalCompletionTimestamp(value.finishedAt) !== null), metadataDirectory, fileSystem, ) @@ -1188,7 +1197,7 @@ export async function cleanupBackgroundSessionsBefore( terminalFactPathForId(session.id, 'natural'), (value): value is BackgroundSessionTerminalFact => isBackgroundSessionTerminalFact(value, session.id, 'natural') && - Number.isFinite(Date.parse(value.finishedAt)), + parseCanonicalCompletionTimestamp(value.finishedAt) !== null, terminalDirectory, fileSystem, ) @@ -1196,7 +1205,7 @@ export async function cleanupBackgroundSessionsBefore( terminalFactPathForId(session.id, 'killed'), (value): value is BackgroundSessionTerminalFact => isBackgroundSessionTerminalFact(value, session.id, 'killed') && - Number.isFinite(Date.parse(value.finishedAt)), + parseCanonicalCompletionTimestamp(value.finishedAt) !== null, terminalDirectory, fileSystem, ) From af1942a660f4e02f6d0e865ac825757e7d24f84b Mon Sep 17 00:00:00 2001 From: chioarub Date: Mon, 31 Aug 2026 22:17:44 +0300 Subject: [PATCH 06/13] fix(bg): isolate reused session generations --- src/cli/bgRegistry.cleanup.test.ts | 175 +++++++++- src/cli/bgRegistry.test.ts | 298 ++++++++++++++++- src/cli/bgRegistry.ts | 507 +++++++++++++++++++++-------- 3 files changed, 832 insertions(+), 148 deletions(-) diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts index 3dd419315f..96133d231d 100644 --- a/src/cli/bgRegistry.cleanup.test.ts +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -13,6 +13,7 @@ import { import { tmpdir } from 'node:os' import { join } from 'node:path' import * as nameReservationLock from '../utils/lockfile.js' +import { backgroundProcessMarkerToken } from './bgRouting.js' import { _setBackgroundSessionsRootForTesting, cleanupBackgroundSessionsBefore, @@ -20,6 +21,7 @@ import { markBackgroundSessionKilled, recordBackgroundSessionNaturalTermination, refreshBackgroundSessionStatuses, + resolveBackgroundSession, type BackgroundSession, type BackgroundSessionNaturalTermination, } from './bgRegistry.js' @@ -27,6 +29,8 @@ import { const CUTOFF = new Date('2026-07-01T00:00:00.000Z') const OLD_FINISH = new Date('2026-06-01T00:00:00.000Z') const RECENT_FINISH = new Date('2026-07-01T00:00:01.000Z') +const OLD_PROCESS_MARKER = 'a'.repeat(64) +const REPLACEMENT_PROCESS_MARKER = 'b'.repeat(64) const NONCANONICAL_FINISHED_AT_CASES = [ ['normalized-date', '2026-02-30'], ['numeric-date', '0'], @@ -58,6 +62,14 @@ describe('background session retention cleanup', () => { return join(root, 'names', `${digest}.json`) } + function markedTerminalFactPath( + id: string, + kind: 'natural' | 'killed', + processMarker: string, + ): string { + return join(root, 'terminal', `${id}~${processMarker}.${kind}.json`) + } + async function exists(path: string): Promise { try { await lstat(path) @@ -194,8 +206,25 @@ describe('background session retention cleanup', () => { } it('removes old explicit-kill artifacts', async () => { - const session = await createRunning('bg-killed') + const session = await createBackgroundSession({ + id: 'bg-killed', + pid: nextPid++, + cwd: configDir, + command: [ + 'openclaude', + backgroundProcessMarkerToken(OLD_PROCESS_MARKER), + '--print', + 'killed generation', + ], + sessionId: 'killed-generation-conversation', + processMarker: OLD_PROCESS_MARKER, + }) await markBackgroundSessionKilled(session.id, { now: OLD_FINISH }) + const killedFact = markedTerminalFactPath( + session.id, + 'killed', + OLD_PROCESS_MARKER, + ) const result = await cleanupBackgroundSessionsBefore(CUTOFF) @@ -205,7 +234,7 @@ describe('background session retention cleanup', () => { errors: 0, }) expect(await exists(paths(session.id).metadata)).toBe(false) - expect(await exists(paths(session.id).killed)).toBe(false) + expect(await exists(killedFact)).toBe(false) }) it('retains recent completed sessions', async () => { @@ -732,6 +761,36 @@ describe('background session retention cleanup', () => { } }) + it('preserves an orphaned fact whose generation does not match its path', async () => { + await mkdir(join(root, 'terminal'), { recursive: true }) + const id = 'bg-orphan-generation-mismatch' + const factPath = markedTerminalFactPath( + id, + 'natural', + OLD_PROCESS_MARKER, + ) + await writeFile( + factPath, + JSON.stringify({ + version: 1, + id, + pid: nextPid++, + generation: REPLACEMENT_PROCESS_MARKER, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(factPath)).toBe(true) + }) + it('treats missing artifacts as idempotent success', async () => { const session = await writeRawSession({ id: 'bg-missing-artifacts', @@ -870,6 +929,118 @@ describe('background session retention cleanup', () => { expect(await exists(naturalFact)).toBe(false) }) + for (const legacyFact of [false, true]) { + const suffix = legacyFact ? 'legacy' : 'marked' + it(`isolates a retained ${suffix} terminal fact from a same-ID replacement generation`, async () => { + const id = `bg-terminal-fact-generation-${suffix}` + const oldSession = await createBackgroundSession({ + id, + pid: nextPid++, + cwd: configDir, + command: [ + 'openclaude', + backgroundProcessMarkerToken(OLD_PROCESS_MARKER), + '--print', + 'old generation', + ], + sessionId: `old-generation-${suffix}`, + processMarker: OLD_PROCESS_MARKER, + }) + const retainedFact = legacyFact + ? paths(id).natural + : markedTerminalFactPath(id, 'natural', OLD_PROCESS_MARKER) + if (legacyFact) { + const metadata = JSON.parse( + await readFile(paths(id).metadata, 'utf8'), + ) as Record + delete metadata.terminalFactGeneration + await writeFile(paths(id).metadata, JSON.stringify(metadata)) + await writeFile( + retainedFact, + JSON.stringify({ + version: 1, + id, + pid: oldSession.pid, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + } else { + await finishNaturally(oldSession, { exitCode: 0 }) + } + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + unlinkFile: async path => { + if (path === retainedFact) throw deniedError() + await unlink(path) + }, + }), + ).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 3, + errors: 1, + }) + expect(await exists(paths(id).metadata)).toBe(false) + expect(await exists(retainedFact)).toBe(true) + + const replacement = await createBackgroundSession({ + id, + pid: nextPid++, + cwd: configDir, + command: [ + 'openclaude', + backgroundProcessMarkerToken(REPLACEMENT_PROCESS_MARKER), + '--print', + 'replacement generation', + ], + sessionId: `replacement-generation-${suffix}`, + processMarker: REPLACEMENT_PROCESS_MARKER, + }) + await recordBackgroundSessionNaturalTermination( + replacement.id, + { exitCode: 23 }, + { + ownerPid: replacement.pid, + now: OLD_FINISH, + }, + ) + + expect(await resolveBackgroundSession(id)).toMatchObject({ + id, + pid: replacement.pid, + status: 'failed', + finishedAt: OLD_FINISH.toISOString(), + exitCode: 23, + }) + const replacementFact = markedTerminalFactPath( + id, + 'natural', + REPLACEMENT_PROCESS_MARKER, + ) + expect(await exists(retainedFact)).toBe(true) + expect(await exists(replacementFact)).toBe(true) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + expect(await exists(paths(id).metadata)).toBe(false) + expect(await exists(replacementFact)).toBe(false) + expect(await exists(retainedFact)).toBe(true) + + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 1, + errors: 0, + }) + expect(await exists(retainedFact)).toBe(false) + }) + } + it('reclaims an old killed fact after metadata-first partial cleanup', async () => { const session = await createRunning('bg-killed-fact-retry') await markBackgroundSessionKilled(session.id, { now: OLD_FINISH }) diff --git a/src/cli/bgRegistry.test.ts b/src/cli/bgRegistry.test.ts index 90983089a7..2a1130ba78 100644 --- a/src/cli/bgRegistry.test.ts +++ b/src/cli/bgRegistry.test.ts @@ -6,6 +6,7 @@ import { join } from 'node:path' import * as backgroundSessionRegistry from './bgRegistry.js' import { _setBackgroundSessionsRootForTesting, + cleanupBackgroundSessionsBefore, createBackgroundSession, isBackgroundSessionProcessAlive, isTerminalBackgroundSession, @@ -42,8 +43,15 @@ describe('background session registry', () => { function terminalFactPath( id: string, kind: 'natural' | 'killed', + generation?: string, ): string { - return join(configDir, 'bg-sessions', 'terminal', `${id}.${kind}.json`) + const generationSuffix = generation ? `~${generation}` : '' + return join( + configDir, + 'bg-sessions', + 'terminal', + `${id}${generationSuffix}.${kind}.json`, + ) } async function writeTerminalFact( @@ -114,6 +122,7 @@ describe('background session registry', () => { model: 'gpt-5', sessionId: 'conversation-1', processMarker: TEST_PROCESS_MARKER, + terminalFactGeneration: TEST_PROCESS_MARKER, startedAt: '2026-06-15T08:00:00.000Z', updatedAt: '2026-06-15T08:00:00.000Z', command: [ @@ -158,6 +167,7 @@ describe('background session registry', () => { const [session] = await listBackgroundSessions() expect(session?.id).toBe('bg-legacy') expect(session?.processMarker).toBeUndefined() + expect(session?.terminalFactGeneration).toBeUndefined() }) it('rejects malformed process markers on creation', async () => { @@ -211,6 +221,28 @@ describe('background session registry', () => { ) }), ) + await writeFile( + join( + configDir, + 'bg-sessions', + 'sessions', + 'bg-mismatched-terminal-generation.json', + ), + JSON.stringify({ + id: 'bg-mismatched-terminal-generation', + pid: 999, + cwd: '/repo', + status: 'running', + sessionId: 'conversation-mismatched-terminal-generation', + processMarker: TEST_PROCESS_MARKER, + terminalFactGeneration: OTHER_PROCESS_MARKER, + startedAt: '2026-06-15T08:00:00.000Z', + updatedAt: '2026-06-15T08:00:00.000Z', + command: ['openclaude', '--print', 'work'], + stdoutLogPath: '/tmp/stdout.log', + stderrLogPath: '/tmp/stderr.log', + }), + ) expect(await listBackgroundSessions()).toEqual([]) }) @@ -572,6 +604,27 @@ describe('background session registry', () => { expect((await resolveBackgroundSession('bg-collision')).name).toBe('first') }) + it('keeps a markerless id unavailable while a terminal fact remains', async () => { + await writeTerminalFact('bg-retained-fact', 'natural', { + pid: 111, + status: 'exited', + finishedAt: '2026-06-15T08:04:00.000Z', + terminalReason: 'exit_code', + exitCode: 0, + }) + + await expect( + createBackgroundSession({ + id: 'bg-retained-fact', + pid: 222, + cwd: '/repo', + command: ['openclaude', '--print', 'replacement'], + sessionId: 'conversation-replacement', + }), + ).rejects.toThrow('already exists') + expect(await listBackgroundSessions()).toEqual([]) + }) + it('rejects non-positive pids at creation', async () => { await expect( createBackgroundSession({ @@ -1012,8 +1065,14 @@ describe('background session registry', () => { id: 'bg-concurrent-natural', pid: 346, cwd: '/repo', - command: ['openclaude', '--print', 'work'], + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], sessionId: 'conversation-concurrent-natural', + processMarker: TEST_PROCESS_MARKER, }) const results = await Promise.all([ @@ -1123,13 +1182,174 @@ describe('background session registry', () => { }) }) + it('does not let a stale refresh overwrite a same-ID replacement generation', async () => { + const id = 'bg-refresh-generation-race' + const name = 'refresh-generation-race' + const oldSession = await createBackgroundSession({ + id, + name, + pid: 360, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'old generation', + ], + sessionId: 'conversation-refresh-generation-old', + processMarker: TEST_PROCESS_MARKER, + }) + let replacement: BackgroundSession | undefined + + const refreshed = await refreshBackgroundSessionStatuses({ + isProcessAlive: () => false, + _beforeStatusWriteForTesting: async () => { + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + { + ownerPid: oldSession.pid, + now: new Date('2026-06-01T00:00:00.000Z'), + }, + ) + await cleanupBackgroundSessionsBefore( + new Date('2026-07-01T00:00:00.000Z'), + ) + replacement = await createBackgroundSession({ + id, + name, + pid: 361, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(OTHER_PROCESS_MARKER), + '--print', + 'replacement generation', + ], + sessionId: 'conversation-refresh-generation-replacement', + processMarker: OTHER_PROCESS_MARKER, + }) + }, + }) + + if (!replacement) throw new Error('replacement session was not created') + expect(refreshed).toHaveLength(1) + expect(refreshed[0]).toMatchObject({ + id, + pid: replacement.pid, + status: 'running', + processMarker: OTHER_PROCESS_MARKER, + terminalFactGeneration: OTHER_PROCESS_MARKER, + }) + expect(await Bun.file(nameReservationPath(name)).json()).toMatchObject({ + id, + }) + + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 23 }, + { + ownerPid: replacement.pid, + now: new Date('2026-07-01T00:00:01.000Z'), + }, + ) + expect(await resolveBackgroundSession(id)).toMatchObject({ + pid: replacement.pid, + status: 'failed', + exitCode: 23, + }) + }) + + it('does not let a stale kill mark a same-ID replacement generation', async () => { + const id = 'bg-kill-generation-race' + const name = 'kill-generation-race' + const reusedPid = 362 + const oldSession = await createBackgroundSession({ + id, + name, + pid: reusedPid, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'old generation', + ], + sessionId: 'conversation-kill-generation-old', + processMarker: TEST_PROCESS_MARKER, + }) + let replacement: BackgroundSession | undefined + + await expect( + markBackgroundSessionKilled(id, { + _beforeMarkWriteForTesting: async () => { + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + { + ownerPid: oldSession.pid, + now: new Date('2026-06-01T00:00:00.000Z'), + }, + ) + await cleanupBackgroundSessionsBefore( + new Date('2026-07-01T00:00:00.000Z'), + ) + replacement = await createBackgroundSession({ + id, + name, + pid: reusedPid, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(OTHER_PROCESS_MARKER), + '--print', + 'replacement generation', + ], + sessionId: 'conversation-kill-generation-replacement', + processMarker: OTHER_PROCESS_MARKER, + }) + }, + }), + ).rejects.toThrow('changed before it could be marked killed') + + if (!replacement) throw new Error('replacement session was not created') + expect(await Bun.file(nameReservationPath(name)).json()).toMatchObject({ + id, + }) + expect( + await Bun.file( + terminalFactPath(id, 'killed', OTHER_PROCESS_MARKER), + ).exists(), + ).toBe(false) + + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 9 }, + { + ownerPid: replacement.pid, + now: new Date('2026-07-01T00:00:01.000Z'), + }, + ) + expect(await resolveBackgroundSession(id)).toMatchObject({ + pid: replacement.pid, + status: 'failed', + exitCode: 9, + }) + }) + it('records a bounded observed signal without inventing an exit code', async () => { await createBackgroundSession({ id: 'bg-observed-signal', pid: 344, cwd: '/repo', - command: ['openclaude', '--print', 'work'], + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], sessionId: 'conversation-observed-signal', + processMarker: TEST_PROCESS_MARKER, }) const failed = await recordBackgroundSessionNaturalTermination( 'bg-observed-signal', @@ -1145,6 +1365,55 @@ describe('background session registry', () => { expect('exitCode' in failed).toBe(false) }) + it('keeps the first marked sync terminal fact immutable', async () => { + await createBackgroundSession({ + id: 'bg-marked-sync-natural', + pid: 348, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], + sessionId: 'conversation-marked-sync-natural', + processMarker: TEST_PROCESS_MARKER, + }) + recordBackgroundSessionNaturalTerminationSync( + 'bg-marked-sync-natural', + { exitCode: 19 }, + { + ownerPid: 348, + now: new Date('2026-06-15T08:09:10.000Z'), + }, + ) + recordBackgroundSessionNaturalTerminationSync( + 'bg-marked-sync-natural', + { exitCode: 0 }, + { + ownerPid: 348, + now: new Date('2026-06-15T08:09:20.000Z'), + }, + ) + + expect(await resolveBackgroundSession('bg-marked-sync-natural')).toMatchObject( + { + status: 'failed', + finishedAt: '2026-06-15T08:09:10.000Z', + exitCode: 19, + }, + ) + expect( + await Bun.file( + terminalFactPath( + 'bg-marked-sync-natural', + 'natural', + TEST_PROCESS_MARKER, + ), + ).json(), + ).toMatchObject({ generation: TEST_PROCESS_MARKER, exitCode: 19 }) + }) + it('does not let a late natural finalizer overwrite an explicit kill fact', async () => { await createBackgroundSession({ id: 'bg-killed-absorbing', @@ -1179,8 +1448,14 @@ describe('background session registry', () => { id: 'bg-sync-killed-absorbing', pid: 347, cwd: '/repo', - command: ['openclaude', '--print', 'work'], + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], sessionId: 'conversation-sync-killed-absorbing', + processMarker: TEST_PROCESS_MARKER, }) await markBackgroundSessionKilled('bg-sync-killed-absorbing', { now: new Date('2026-06-15T08:09:30.000Z'), @@ -1201,9 +1476,22 @@ describe('background session registry', () => { }) expect( await Bun.file( - terminalFactPath('bg-sync-killed-absorbing', 'natural'), + terminalFactPath( + 'bg-sync-killed-absorbing', + 'natural', + TEST_PROCESS_MARKER, + ), ).exists(), ).toBe(false) + expect( + await Bun.file( + terminalFactPath( + 'bg-sync-killed-absorbing', + 'killed', + TEST_PROCESS_MARKER, + ), + ).json(), + ).toMatchObject({ generation: TEST_PROCESS_MARKER }) }) it('keeps killed strongest when kill and natural completion race', async () => { diff --git a/src/cli/bgRegistry.ts b/src/cli/bgRegistry.ts index 70b4aac56d..d9f674789b 100644 --- a/src/cli/bgRegistry.ts +++ b/src/cli/bgRegistry.ts @@ -54,6 +54,7 @@ export type BackgroundSession = { model?: string sessionId: string processMarker?: string + terminalFactGeneration?: string startedAt: string updatedAt: string command: string[] @@ -74,6 +75,7 @@ type BackgroundSessionTerminalFact = { version: 1 id: string pid: number + generation?: string status: 'exited' | 'failed' | 'killed' finishedAt: string terminalReason: BackgroundSessionTerminalReason @@ -180,6 +182,38 @@ function metadataPathForId(id: string): string { return join(getBackgroundSessionMetadataDir(), `${id}.json`) } +async function withBackgroundSessionIdLock( + id: string, + operation: () => Promise, +): Promise { + const release = await lockfile.lock( + metadataPathForId(id), + NAME_RESERVATION_LOCK_OPTIONS, + ) + try { + return await operation() + } finally { + await release().catch(() => {}) + } +} + +function withBackgroundSessionIdLockSync( + id: string, + operation: () => T, +): T { + const release = lockfile.lockSync( + metadataPathForId(id), + NAME_RESERVATION_SYNC_LOCK_OPTIONS, + ) + try { + return operation() + } finally { + try { + release() + } catch {} + } +} + function nameReservationPathForName(name: string): string { const digest = createHash('sha256').update(name).digest('hex') return join(getBackgroundSessionNamesDir(), `${digest}.json`) @@ -188,9 +222,20 @@ function nameReservationPathForName(name: string): string { function terminalFactPathForId( id: string, kind: 'natural' | 'killed', + generation?: string, ): string { assertSafeId(id) - return join(getBackgroundSessionTerminalDir(), `${id}.${kind}.json`) + if ( + generation !== undefined && + !isValidBackgroundProcessMarker(generation) + ) { + throw new Error('Invalid background terminal-fact generation') + } + const generationSuffix = generation ? `~${generation}` : '' + return join( + getBackgroundSessionTerminalDir(), + `${id}${generationSuffix}.${kind}.json`, + ) } function assertSafeId(id: string): void { @@ -357,37 +402,77 @@ export async function ensureBackgroundSessionDirs(): Promise { }) } -async function writeSession(session: BackgroundSession): Promise { +function isSameBackgroundSessionGeneration( + first: BackgroundSession, + second: BackgroundSession, +): boolean { + return ( + first.id === second.id && + first.pid === second.pid && + first.sessionId === second.sessionId && + first.startedAt === second.startedAt && + first.processMarker === second.processMarker && + first.terminalFactGeneration === second.terminalFactGeneration + ) +} + +async function writeSession( + session: BackgroundSession, + expected: BackgroundSession, +): Promise { await ensureBackgroundSessionDirs() const target = metadataPathForId(session.id) - const tmp = join( - getBackgroundSessionMetadataDir(), - `${session.id}.${process.pid}.${randomUUID()}.tmp`, - ) - try { - await writeFile(tmp, jsonStringify(session), { flag: 'wx' }) - await rename(tmp, target) - if (session.name && isTerminalBackgroundSession(session)) { - await releaseNameReservation(session.name, session.id) + return await withBackgroundSessionIdLock(session.id, async () => { + const current = await readSessionFile(target) + if (!current || !isSameBackgroundSessionGeneration(current, expected)) { + return current } - } catch (error) { - await unlink(tmp).catch(() => {}) - throw error - } + const tmp = join( + getBackgroundSessionMetadataDir(), + `${session.id}.${process.pid}.${randomUUID()}.tmp`, + ) + try { + await writeFile(tmp, jsonStringify(session), { flag: 'wx' }) + await rename(tmp, target) + if (session.name && isTerminalBackgroundSession(session)) { + await releaseNameReservation(session.name, session.id) + } + return session + } catch (error) { + await unlink(tmp).catch(() => {}) + throw error + } + }) } async function writeNewSession(session: BackgroundSession): Promise { await ensureBackgroundSessionDirs() - try { - await writeFile(metadataPathForId(session.id), jsonStringify(session), { - flag: 'wx', - }) - } catch (error) { - if (isErrno(error, 'EEXIST')) { - throw new Error(`Background session id "${session.id}" already exists`) + await withBackgroundSessionIdLock(session.id, async () => { + for (const kind of ['natural', 'killed'] as const) { + try { + await lstat( + terminalFactPathForId( + session.id, + kind, + session.terminalFactGeneration, + ), + ) + throw new Error(`Background session id "${session.id}" already exists`) + } catch (error) { + if (!isErrno(error, 'ENOENT')) throw error + } } - throw error - } + try { + await writeFile(metadataPathForId(session.id), jsonStringify(session), { + flag: 'wx', + }) + } catch (error) { + if (isErrno(error, 'EEXIST')) { + throw new Error(`Background session id "${session.id}" already exists`) + } + throw error + } + }) } async function readSessionFile(path: string): Promise { @@ -642,6 +727,11 @@ function isBackgroundSession( typeof candidate.sessionId === 'string' && (candidate.processMarker === undefined || isValidBackgroundProcessMarker(candidate.processMarker)) && + (candidate.terminalFactGeneration === undefined || + (candidate.terminalFactGeneration === candidate.processMarker && + isValidBackgroundProcessMarker( + candidate.terminalFactGeneration, + ))) && typeof candidate.startedAt === 'string' && typeof candidate.updatedAt === 'string' && isStringArray(candidate.command) && @@ -670,6 +760,8 @@ function isBackgroundSessionTerminalFact( typeof candidate.pid !== 'number' || !Number.isInteger(candidate.pid) || candidate.pid <= 0 || + (candidate.generation !== undefined && + !isValidBackgroundProcessMarker(candidate.generation)) || typeof candidate.finishedAt !== 'string' || !isTerminalReason(candidate.terminalReason) || (candidate.exitCode !== undefined && @@ -713,12 +805,16 @@ function isBackgroundSessionTerminalFact( async function readTerminalFact( id: string, kind: 'natural' | 'killed', + generation?: string, ): Promise { try { const parsed = jsonParse( - await readFile(terminalFactPathForId(id, kind), 'utf8'), + await readFile(terminalFactPathForId(id, kind, generation), 'utf8'), ) - return isBackgroundSessionTerminalFact(parsed, id, kind) ? parsed : null + return isBackgroundSessionTerminalFact(parsed, id, kind) && + parsed.generation === generation + ? parsed + : null } catch { return null } @@ -727,12 +823,16 @@ async function readTerminalFact( function readTerminalFactSync( id: string, kind: 'natural' | 'killed', + generation?: string, ): BackgroundSessionTerminalFact | null { try { const parsed = jsonParse( - readFileSync(terminalFactPathForId(id, kind), 'utf8'), + readFileSync(terminalFactPathForId(id, kind, generation), 'utf8'), ) - return isBackgroundSessionTerminalFact(parsed, id, kind) ? parsed : null + return isBackgroundSessionTerminalFact(parsed, id, kind) && + parsed.generation === generation + ? parsed + : null } catch { return null } @@ -741,8 +841,16 @@ function readTerminalFactSync( async function applyAuthoritativeTerminalFacts( session: BackgroundSession, ): Promise { - const natural = await readTerminalFact(session.id, 'natural') - const killed = await readTerminalFact(session.id, 'killed') + const natural = await readTerminalFact( + session.id, + 'natural', + session.terminalFactGeneration, + ) + const killed = await readTerminalFact( + session.id, + 'killed', + session.terminalFactGeneration, + ) return applyTerminalFacts(session, natural, killed) } @@ -797,7 +905,7 @@ async function installTerminalFact( kind: 'natural' | 'killed', ): Promise { await ensureBackgroundSessionDirs() - const target = terminalFactPathForId(fact.id, kind) + const target = terminalFactPathForId(fact.id, kind, fact.generation) const tmp = terminalFactTempPath(fact.id) let handle: Awaited> | undefined try { @@ -810,7 +918,7 @@ async function installTerminalFact( return fact } catch (error) { if (isErrno(error, 'EEXIST')) { - const existing = await readTerminalFact(fact.id, kind) + const existing = await readTerminalFact(fact.id, kind, fact.generation) if (existing) return existing throw new Error(`Invalid background session ${kind} terminal fact`) } @@ -829,7 +937,7 @@ function installTerminalFactSync( recursive: true, mode: 0o700, }) - const target = terminalFactPathForId(fact.id, kind) + const target = terminalFactPathForId(fact.id, kind, fact.generation) const tmp = terminalFactTempPath(fact.id) let fd: number | undefined try { @@ -842,7 +950,7 @@ function installTerminalFactSync( return fact } catch (error) { if (isErrno(error, 'EEXIST')) { - const existing = readTerminalFactSync(fact.id, kind) + const existing = readTerminalFactSync(fact.id, kind, fact.generation) if (existing) return existing throw new Error(`Invalid background session ${kind} terminal fact`) } @@ -939,13 +1047,32 @@ function cleanupReadBlocksRemoval( function terminalFactCandidateFromEntry( entry: Dirent, -): { id: string; kind: 'natural' | 'killed' } | null { +): { + id: string + kind: 'natural' | 'killed' + generation?: string +} | null { if (!entry.isFile()) return null - const match = /^(.*)\.(natural|killed)\.json$/.exec(entry.name) + const markedMatch = + /^(.*)~([a-f0-9]{64})\.(natural|killed)\.json$/.exec(entry.name) + const legacyMatch = /^(.*)\.(natural|killed)\.json$/.exec(entry.name) + const match = markedMatch ?? legacyMatch if (!match) return null - const [, id, kind] = match + const id = match[1] + const generation = markedMatch?.[2] + const kind = markedMatch?.[3] ?? legacyMatch?.[2] if (!id || !kind || !SAFE_ID_RE.test(id)) return null - return { id, kind: kind as 'natural' | 'killed' } + if ( + generation !== undefined && + !isValidBackgroundProcessMarker(generation) + ) { + return null + } + return { + id, + kind: kind as 'natural' | 'killed', + ...(generation ? { generation } : {}), + } } async function cleanupOrphanedTerminalFacts( @@ -993,7 +1120,11 @@ async function cleanupOrphanedTerminalFacts( ) .sort((a, b) => { const idOrder = a.id.localeCompare(b.id) - return idOrder !== 0 ? idOrder : a.kind.localeCompare(b.kind) + if (idOrder !== 0) return idOrder + const markerOrder = (a.generation ?? '').localeCompare( + b.generation ?? '', + ) + return markerOrder !== 0 ? markerOrder : a.kind.localeCompare(b.kind) }) if (candidates.length === 0) return @@ -1015,7 +1146,11 @@ async function cleanupOrphanedTerminalFacts( for (let index = 0; index < scanCount; index++) { const candidate = candidates[(startIndex + index) % candidates.length] if (!candidate) continue - const factPath = terminalFactPathForId(candidate.id, candidate.kind) + const factPath = terminalFactPathForId( + candidate.id, + candidate.kind, + candidate.generation, + ) const fact = await readCleanupJson( factPath, (value): value is BackgroundSessionTerminalFact => @@ -1023,7 +1158,9 @@ async function cleanupOrphanedTerminalFacts( value, candidate.id, candidate.kind, - ) && parseCanonicalCompletionTimestamp(value.finishedAt) !== null, + ) && + value.generation === candidate.generation && + parseCanonicalCompletionTimestamp(value.finishedAt) !== null, terminalDirectory, fileSystem, ) @@ -1193,18 +1330,30 @@ export async function cleanupBackgroundSessionsBefore( if (metadata.state !== 'valid') continue const session = metadata.value + const naturalPath = terminalFactPathForId( + session.id, + 'natural', + session.terminalFactGeneration, + ) + const killedPath = terminalFactPathForId( + session.id, + 'killed', + session.terminalFactGeneration, + ) const natural = await readCleanupJson( - terminalFactPathForId(session.id, 'natural'), + naturalPath, (value): value is BackgroundSessionTerminalFact => isBackgroundSessionTerminalFact(value, session.id, 'natural') && + value.generation === session.terminalFactGeneration && parseCanonicalCompletionTimestamp(value.finishedAt) !== null, terminalDirectory, fileSystem, ) const killed = await readCleanupJson( - terminalFactPathForId(session.id, 'killed'), + killedPath, (value): value is BackgroundSessionTerminalFact => isBackgroundSessionTerminalFact(value, session.id, 'killed') && + value.generation === session.terminalFactGeneration && parseCanonicalCompletionTimestamp(value.finishedAt) !== null, terminalDirectory, fileSystem, @@ -1316,33 +1465,63 @@ export async function cleanupBackgroundSessionsBefore( continue } - const metadataRemoval = await removeCleanupArtifact( - metadataPath, - result, - metadataDirectory, - fileSystem, - metadata.identity, - ) - if (metadataRemoval === 'error') continue - if (metadataRemoval === 'removed') result.sessionsRemoved++ - - if (natural.state === 'valid') { - await removeCleanupArtifact( - terminalFactPathForId(session.id, 'natural'), - result, - terminalDirectory, - fileSystem, - natural.identity, - ) - } - if (killed.state === 'valid') { - await removeCleanupArtifact( - terminalFactPathForId(session.id, 'killed'), - result, - terminalDirectory, - fileSystem, - killed.identity, - ) + try { + await withBackgroundSessionIdLock(session.id, async () => { + const latestMetadata = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => + isBackgroundSession(value, session.id) && + (value.finishedAt === undefined || + parseCanonicalCompletionTimestamp(value.finishedAt) !== null), + metadataDirectory, + fileSystem, + ) + if (latestMetadata.state === 'error') { + result.errors++ + return + } + if ( + latestMetadata.state !== 'valid' || + !latestMetadata.identity || + !metadata.identity || + latestMetadata.identity.dev !== metadata.identity.dev || + latestMetadata.identity.ino !== metadata.identity.ino || + !isSameBackgroundSessionGeneration(latestMetadata.value, session) + ) { + return + } + + const metadataRemoval = await removeCleanupArtifact( + metadataPath, + result, + metadataDirectory, + fileSystem, + latestMetadata.identity, + ) + if (metadataRemoval === 'error') return + if (metadataRemoval === 'removed') result.sessionsRemoved++ + + if (natural.state === 'valid') { + await removeCleanupArtifact( + naturalPath, + result, + terminalDirectory, + fileSystem, + natural.identity, + ) + } + if (killed.state === 'valid') { + await removeCleanupArtifact( + killedPath, + result, + terminalDirectory, + fileSystem, + killed.identity, + ) + } + }) + } catch { + result.errors++ } } @@ -1407,6 +1586,9 @@ export async function createBackgroundSession( ...(input.processMarker ? { processMarker: input.processMarker } : {}), + ...(input.processMarker + ? { terminalFactGeneration: input.processMarker } + : {}), startedAt: timestamp, updatedAt: timestamp, command: input.command, @@ -1517,8 +1699,10 @@ export async function refreshBackgroundSessionStatuses(options?: { updatedAt: timestamp, } await options?._beforeStatusWriteForTesting?.(session, nextStatus) - await writeSession(updated) - refreshed.push(await applyAuthoritativeTerminalFacts(updated)) + const persisted = await writeSession(updated, session) + if (persisted) { + refreshed.push(await applyAuthoritativeTerminalFacts(persisted)) + } continue } @@ -1716,6 +1900,7 @@ export function isBackgroundSessionProcessAlive( function naturalTerminalFact( id: string, pid: number, + generation: string | undefined, termination: BackgroundSessionNaturalTermination, now: Date | undefined, ): BackgroundSessionTerminalFact { @@ -1731,6 +1916,7 @@ function naturalTerminalFact( version: 1, id, pid, + ...(generation ? { generation } : {}), status: 'failed', finishedAt, terminalReason: 'signal', @@ -1744,6 +1930,7 @@ function naturalTerminalFact( version: 1, id, pid, + ...(generation ? { generation } : {}), status: termination.exitCode === 0 ? 'exited' : 'failed', finishedAt, terminalReason: 'exit_code', @@ -1768,33 +1955,43 @@ export async function recordBackgroundSessionNaturalTermination( ): Promise { assertSafeId(id) const ownerPid = options.ownerPid ?? process.pid - const session = await readSessionFile(metadataPathForId(id)) - assertNaturalFinalizationOwner(session, id, ownerPid) + return await withBackgroundSessionIdLock(id, async () => { + const session = await readSessionFile(metadataPathForId(id)) + assertNaturalFinalizationOwner(session, id, ownerPid) - const effective = await applyAuthoritativeTerminalFacts(session) - if ( - effective.status === 'killed' || - session.status === 'exited' || - session.status === 'failed' - ) { - return effective - } - if ( - session.status !== 'running' && - session.status !== 'unknown' && - session.status !== 'stale' - ) { - // Retain an exhaustive guard so future status additions require a deliberate - // natural-finalization policy. - throw new Error('Background session is not eligible for natural finalization') - } + const effective = await applyAuthoritativeTerminalFacts(session) + if ( + effective.status === 'killed' || + session.status === 'exited' || + session.status === 'failed' + ) { + return effective + } + if ( + session.status !== 'running' && + session.status !== 'unknown' && + session.status !== 'stale' + ) { + // Retain an exhaustive guard so future status additions require a deliberate + // natural-finalization policy. + throw new Error( + 'Background session is not eligible for natural finalization', + ) + } - await installTerminalFact( - naturalTerminalFact(id, ownerPid, termination, options.now), - 'natural', - ) - if (session.name) await releaseNameReservation(session.name, session.id) - return await applyAuthoritativeTerminalFacts(session) + await installTerminalFact( + naturalTerminalFact( + id, + ownerPid, + session.terminalFactGeneration, + termination, + options.now, + ), + 'natural', + ) + if (session.name) await releaseNameReservation(session.name, session.id) + return await applyAuthoritativeTerminalFacts(session) + }) } export function recordBackgroundSessionNaturalTerminationSync( @@ -1804,57 +2001,85 @@ export function recordBackgroundSessionNaturalTerminationSync( ): void { assertSafeId(id) const ownerPid = options.ownerPid ?? process.pid - const session = readSessionFileSync(metadataPathForId(id)) - assertNaturalFinalizationOwner(session, id, ownerPid) - if ( - session.status === 'killed' || - session.status === 'exited' || - session.status === 'failed' || - readTerminalFactSync(id, 'killed')?.pid === ownerPid - ) { - return - } - if ( - session.status !== 'running' && - session.status !== 'unknown' && - session.status !== 'stale' - ) { - // Retain an exhaustive guard so future status additions require a deliberate - // natural-finalization policy. - throw new Error('Background session is not eligible for natural finalization') - } + withBackgroundSessionIdLockSync(id, () => { + const session = readSessionFileSync(metadataPathForId(id)) + assertNaturalFinalizationOwner(session, id, ownerPid) + if ( + session.status === 'killed' || + session.status === 'exited' || + session.status === 'failed' || + readTerminalFactSync( + id, + 'killed', + session.terminalFactGeneration, + )?.pid === ownerPid + ) { + return + } + if ( + session.status !== 'running' && + session.status !== 'unknown' && + session.status !== 'stale' + ) { + // Retain an exhaustive guard so future status additions require a deliberate + // natural-finalization policy. + throw new Error( + 'Background session is not eligible for natural finalization', + ) + } - installTerminalFactSync( - naturalTerminalFact(id, ownerPid, termination, options.now), - 'natural', - ) - if (session.name) releaseNameReservationSync(session.name, session.id) + installTerminalFactSync( + naturalTerminalFact( + id, + ownerPid, + session.terminalFactGeneration, + termination, + options.now, + ), + 'natural', + ) + if (session.name) releaseNameReservationSync(session.name, session.id) + }) } export async function markBackgroundSessionKilled( target: string, - options?: { now?: Date }, + options?: { + now?: Date + _beforeMarkWriteForTesting?: (session: BackgroundSession) => Promise + }, ): Promise { const session = await resolveBackgroundSession(target) - const rawSession = await readSessionFile(metadataPathForId(session.id)) - if (!rawSession || rawSession.pid !== session.pid) { - throw new Error('Background session changed before it could be marked killed') - } - await installTerminalFact( - { - version: 1, - id: session.id, - pid: session.pid, - status: 'killed', - finishedAt: iso(options?.now), - terminalReason: 'explicit_kill', - }, - 'killed', - ) - if (rawSession.name) { - await releaseNameReservation(rawSession.name, rawSession.id) - } - return await applyAuthoritativeTerminalFacts(rawSession) + await options?._beforeMarkWriteForTesting?.(session) + return await withBackgroundSessionIdLock(session.id, async () => { + const rawSession = await readSessionFile(metadataPathForId(session.id)) + if ( + !rawSession || + !isSameBackgroundSessionGeneration(rawSession, session) + ) { + throw new Error( + 'Background session changed before it could be marked killed', + ) + } + await installTerminalFact( + { + version: 1, + id: rawSession.id, + pid: rawSession.pid, + ...(rawSession.terminalFactGeneration + ? { generation: rawSession.terminalFactGeneration } + : {}), + status: 'killed', + finishedAt: iso(options?.now), + terminalReason: 'explicit_kill', + }, + 'killed', + ) + if (rawSession.name) { + await releaseNameReservation(rawSession.name, rawSession.id) + } + return await applyAuthoritativeTerminalFacts(rawSession) + }) } export async function backgroundSessionLogExists(path: string): Promise { From f597a1be84bb3d29ca0c691c8f3f25acf57f3da7 Mon Sep 17 00:00:00 2001 From: chioarub Date: Tue, 1 Sep 2026 02:52:22 +0300 Subject: [PATCH 07/13] fix(bg): recover terminal session handoffs --- src/cli/bgFinalizer.test.ts | 47 +- src/cli/bgFinalizer.ts | 21 +- src/cli/bgRegistry.cleanup.test.ts | 92 ++++ src/cli/bgRegistry.test.ts | 402 ++++++++++++++++ src/cli/bgRegistry.ts | 448 +++++++++++++++--- src/utils/backgroundHousekeeping.test.ts | 69 +++ src/utils/backgroundHousekeeping.ts | 52 +- src/utils/cleanup.test.ts | 66 ++- src/utils/cleanup.ts | 37 +- .../cleanupBackgroundSessions.fixture.ts | 101 +++- 10 files changed, 1256 insertions(+), 79 deletions(-) create mode 100644 src/utils/backgroundHousekeeping.test.ts diff --git a/src/cli/bgFinalizer.test.ts b/src/cli/bgFinalizer.test.ts index 7af3edae76..1283f3904f 100644 --- a/src/cli/bgFinalizer.test.ts +++ b/src/cli/bgFinalizer.test.ts @@ -182,12 +182,14 @@ describe('background session finalizer', () => { let exitListener: ((code: number) => void) | undefined const finalized: number[] = [] const finalizedSync: number[] = [] + const registeredSession = ownedSession('bg-owned', 500) + let finalizationOwner: BackgroundSession | undefined const preparation = await prepareBackgroundSessionFinalizer({ env, pid: 500, readSession: async () => - ++reads < 3 ? null : ownedSession('bg-owned', 500), + ++reads < 3 ? null : registeredSession, isLauncherAlive: () => true, sleep: async () => {}, registrationWaitMs: 10, @@ -202,8 +204,9 @@ describe('background session finalizer', () => { onExit: listener => { exitListener = listener }, - finalize: async (_id, termination) => { + finalize: async (_id, termination, options) => { finalized.push(termination.exitCode ?? -1) + finalizationOwner = options?.expectedSession return ownedSession('bg-owned', 500) }, finalizeSync: (_id, termination) => { @@ -225,6 +228,46 @@ describe('background session finalizer', () => { } expect(finalized).toEqual([7]) expect(finalizedSync).toEqual([]) + expect(finalizationOwner).toEqual(registeredSession) + }) + + it('passes the registered generation to the synchronous exit fallback', async () => { + let cleanup: (() => void | Promise) | undefined + let exitListener: ((code: number) => void) | undefined + const registeredSession: BackgroundSession = { + ...ownedSession('bg-sync-generation', 500), + processMarker: 'a'.repeat(64), + terminalFactGeneration: 'a'.repeat(64), + } + let syncOwner: BackgroundSession | undefined + + await prepareBackgroundSessionFinalizer({ + env: { + [BACKGROUND_SESSION_ID_ENV]: registeredSession.id, + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: '123', + }, + pid: registeredSession.pid, + readSession: async () => registeredSession, + isLauncherAlive: () => true, + registerCleanup: fn => { + cleanup = fn + return () => {} + }, + onBeforeExit: () => {}, + onExit: listener => { + exitListener = listener + }, + finalize: async () => { + throw Object.assign(new Error('contended'), { code: 'ELOCKED' }) + }, + finalizeSync: (_id, _termination, options) => { + syncOwner = options?.expectedSession + }, + }) + + await cleanup?.() + exitListener?.(23) + expect(syncOwner).toEqual(registeredSession) }) it('keeps the original exit code when both persistence paths fail', async () => { diff --git a/src/cli/bgFinalizer.ts b/src/cli/bgFinalizer.ts index a6afc63360..b7c887fb14 100644 --- a/src/cli/bgFinalizer.ts +++ b/src/cli/bgFinalizer.ts @@ -120,25 +120,25 @@ async function waitForOwnedSession( | 'registrationPollMs' > >, -): Promise<'owned' | 'mismatch' | 'timeout'> { +): Promise { const attempts = Math.max( 1, Math.ceil(options.registrationWaitMs / options.registrationPollMs), ) for (let attempt = 0; attempt < attempts; attempt += 1) { const session = await options.readSession(id) - if (session) return session.pid === ownerPid ? 'owned' : 'mismatch' + if (session) return session.pid === ownerPid ? session : 'mismatch' if (!options.isLauncherAlive(launcherPid)) { const finalSession = await options.readSession(id) if (finalSession) { - return finalSession.pid === ownerPid ? 'owned' : 'mismatch' + return finalSession.pid === ownerPid ? finalSession : 'mismatch' } return 'timeout' } await options.sleep(options.registrationPollMs) } const session = await options.readSession(id) - if (session) return session.pid === ownerPid ? 'owned' : 'mismatch' + if (session) return session.pid === ownerPid ? session : 'mismatch' return 'timeout' } @@ -159,7 +159,7 @@ export async function prepareBackgroundSessionFinalizer( } const ownerPid = options.pid ?? process.pid - const ownership = await waitForOwnedSession(id, ownerPid, launcherPid, { + const ownedSession = await waitForOwnedSession(id, ownerPid, launcherPid, { readSession: options.readSession ?? readBackgroundSessionForOwner, isLauncherAlive: options.isLauncherAlive ?? isBackgroundLauncherAlive, sleep: @@ -170,11 +170,11 @@ export async function prepareBackgroundSessionFinalizer( registrationPollMs: options.registrationPollMs ?? DEFAULT_REGISTRATION_POLL_MS, }) - if (ownership === 'mismatch') { + if (ownedSession === 'mismatch') { scrubRoutingEnvironment(env) return 'invalid-routing' } - if (ownership === 'timeout') { + if (ownedSession === 'timeout') { scrubRoutingEnvironment(env) throw new Error('Background session registration was not established') } @@ -199,7 +199,10 @@ export async function prepareBackgroundSessionFinalizer( const finalizeAwaited = async () => { if (finalized) return try { - await finalize(id, currentTermination(), { ownerPid }) + await finalize(id, currentTermination(), { + ownerPid, + expectedSession: ownedSession, + }) finalized = true } catch (error) { reportFinalizationFailure(debug, error) @@ -221,7 +224,7 @@ export async function prepareBackgroundSessionFinalizer( finalizeSync( id, signal === undefined ? { exitCode: code } : { signal }, - { ownerPid }, + { ownerPid, expectedSession: ownedSession }, ) finalized = true } catch (error) { diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts index 96133d231d..ac197cdd27 100644 --- a/src/cli/bgRegistry.cleanup.test.ts +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -20,6 +20,7 @@ import { createBackgroundSession, markBackgroundSessionKilled, recordBackgroundSessionNaturalTermination, + reconcileBackgroundSessionTerminalFacts, refreshBackgroundSessionStatuses, resolveBackgroundSession, type BackgroundSession, @@ -485,6 +486,97 @@ describe('background session retention cleanup', () => { expect(await exists(idPaths.stderr)).toBe(true) }) + it('does not reconcile through symlinked metadata outside the registry root', async () => { + const id = 'bg-symlink-reconciliation' + const idPaths = paths(id) + const externalMetadata = join(configDir, 'external-reconciliation.json') + await mkdir(join(root, 'sessions'), { recursive: true }) + await mkdir(join(root, 'terminal'), { recursive: true }) + const stored = { + id, + pid: 124, + cwd: configDir, + status: 'stale', + sessionId: 'external-reconciliation-conversation', + processMarker: OLD_PROCESS_MARKER, + terminalFactGeneration: OLD_PROCESS_MARKER, + startedAt: '2026-05-31T23:59:00.000Z', + updatedAt: OLD_FINISH.toISOString(), + command: ['openclaude'], + stdoutLogPath: idPaths.stdout, + stderrLogPath: idPaths.stderr, + } + await writeFile(externalMetadata, JSON.stringify(stored)) + await symlink(externalMetadata, idPaths.metadata) + await writeFile( + markedTerminalFactPath(id, 'natural', OLD_PROCESS_MARKER), + JSON.stringify({ + version: 1, + id, + pid: stored.pid, + generation: OLD_PROCESS_MARKER, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 0, + errors: 0, + }) + expect(JSON.parse(await readFile(externalMetadata, 'utf8'))).toEqual( + stored, + ) + expect((await lstat(idPaths.metadata)).isSymbolicLink()).toBe(true) + }) + + it('does not reconcile through a symlinked metadata directory', async () => { + const id = 'bg-symlink-reconciliation-dir' + const externalSessions = join(configDir, 'external-reconciliation-sessions') + const externalMetadata = join(externalSessions, `${id}.json`) + await mkdir(externalSessions, { recursive: true }) + await mkdir(join(root, 'terminal'), { recursive: true }) + const stored = { + id, + pid: 125, + cwd: configDir, + status: 'stale', + sessionId: 'external-reconciliation-dir-conversation', + processMarker: OLD_PROCESS_MARKER, + terminalFactGeneration: OLD_PROCESS_MARKER, + startedAt: '2026-05-31T23:59:00.000Z', + updatedAt: OLD_FINISH.toISOString(), + command: ['openclaude'], + stdoutLogPath: paths(id).stdout, + stderrLogPath: paths(id).stderr, + } + await writeFile(externalMetadata, JSON.stringify(stored)) + await symlink(externalSessions, join(root, 'sessions'), 'dir') + await writeFile( + markedTerminalFactPath(id, 'natural', OLD_PROCESS_MARKER), + JSON.stringify({ + version: 1, + id, + pid: stored.pid, + generation: OLD_PROCESS_MARKER, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 0, + errors: 1, + }) + expect(JSON.parse(await readFile(externalMetadata, 'utf8'))).toEqual( + stored, + ) + }) + it('does not follow symlinked artifact directories outside the registry root', async () => { const externalLogs = join(configDir, 'external-logs') const session = await writeRawSession({ diff --git a/src/cli/bgRegistry.test.ts b/src/cli/bgRegistry.test.ts index 2a1130ba78..fa6254e7fd 100644 --- a/src/cli/bgRegistry.test.ts +++ b/src/cli/bgRegistry.test.ts @@ -3,6 +3,7 @@ import { createHash } from 'node:crypto' import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import * as lockfile from '../utils/lockfile.js' import * as backgroundSessionRegistry from './bgRegistry.js' import { _setBackgroundSessionsRootForTesting, @@ -12,6 +13,7 @@ import { isTerminalBackgroundSession, listBackgroundSessions, markBackgroundSessionKilled, + readBackgroundSessionForOwner, refreshBackgroundSessionStatuses, resolveBackgroundSession, verifyBackgroundSessionProcessIdentity, @@ -28,6 +30,7 @@ const backgroundProcessMarkerToken = (marker: string) => `--openclaude-bg-session-marker=${marker}` const { + reconcileBackgroundSessionTerminalFacts, recordBackgroundSessionNaturalTermination, recordBackgroundSessionNaturalTerminationSync, } = backgroundSessionRegistry @@ -1337,6 +1340,80 @@ describe('background session registry', () => { }) }) + it('does not let a late finalizer write a same-ID replacement generation', async () => { + const id = 'bg-finalizer-generation-race' + const reusedPid = 363 + const oldSession = await createBackgroundSession({ + id, + pid: reusedPid, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'old generation', + ], + sessionId: 'conversation-finalizer-generation-old', + processMarker: TEST_PROCESS_MARKER, + }) + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + { + ownerPid: reusedPid, + expectedSession: oldSession, + now: new Date('2026-06-01T00:00:00.000Z'), + }, + ) + await cleanupBackgroundSessionsBefore( + new Date('2026-07-01T00:00:00.000Z'), + ) + const replacement = await createBackgroundSession({ + id, + pid: reusedPid, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(OTHER_PROCESS_MARKER), + '--print', + 'replacement generation', + ], + sessionId: 'conversation-finalizer-generation-replacement', + processMarker: OTHER_PROCESS_MARKER, + }) + + await expect( + recordBackgroundSessionNaturalTermination( + id, + { exitCode: 17 }, + { ownerPid: reusedPid, expectedSession: oldSession }, + ), + ).rejects.toThrow('does not own this session') + expect(() => + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 17 }, + { ownerPid: reusedPid, expectedSession: oldSession }, + ), + ).toThrow('does not own this session') + expect( + await Bun.file( + terminalFactPath(id, 'natural', OTHER_PROCESS_MARKER), + ).exists(), + ).toBe(false) + + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 7 }, + { ownerPid: reusedPid, expectedSession: replacement }, + ) + expect(await resolveBackgroundSession(id)).toMatchObject({ + processMarker: OTHER_PROCESS_MARKER, + status: 'failed', + exitCode: 7, + }) + }) + it('records a bounded observed signal without inventing an exit code', async () => { await createBackgroundSession({ id: 'bg-observed-signal', @@ -1414,6 +1491,331 @@ describe('background session registry', () => { ).toMatchObject({ generation: TEST_PROCESS_MARKER, exitCode: 19 }) }) + it('writes a durable marked fact when the metadata lock is contended', async () => { + const id = 'bg-contended-sync-fact' + const name = 'contended-sync-fact' + const session = await createBackgroundSession({ + id, + name, + pid: 350, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], + sessionId: 'conversation-contended-sync-fact', + processMarker: TEST_PROCESS_MARKER, + }) + const metadataPath = join( + configDir, + 'bg-sessions', + 'sessions', + `${id}.json`, + ) + const release = await lockfile.lock(metadataPath, { realpath: false }) + try { + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 19 }, + { + ownerPid: session.pid, + expectedSession: session, + now: new Date('2026-06-15T08:09:24.000Z'), + }, + ) + } finally { + await release() + } + + expect( + await Bun.file( + terminalFactPath(id, 'natural', TEST_PROCESS_MARKER), + ).json(), + ).toMatchObject({ + generation: TEST_PROCESS_MARKER, + status: 'failed', + exitCode: 19, + }) + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(true) + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 1, + errors: 0, + }) + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(false) + }) + + it('does not bypass a contended metadata lock for a markerless session', async () => { + const id = 'bg-contended-sync-legacy' + const session = await createBackgroundSession({ + id, + pid: 351, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: 'conversation-contended-sync-legacy', + }) + const metadataPath = join( + configDir, + 'bg-sessions', + 'sessions', + `${id}.json`, + ) + const release = await lockfile.lock(metadataPath, { realpath: false }) + try { + expect(() => + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 0 }, + { ownerPid: session.pid, expectedSession: session }, + ), + ).toThrow('Lock file is already being held') + } finally { + await release() + } + expect(await Bun.file(terminalFactPath(id, 'natural')).exists()).toBe( + false, + ) + }) + + it('uses the isolated legacy fact path for a marker-only sync handoff', async () => { + const id = 'bg-contended-sync-marker-only' + const session = await createBackgroundSession({ + id, + pid: 352, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], + sessionId: 'conversation-contended-sync-marker-only', + processMarker: TEST_PROCESS_MARKER, + }) + const metadataPath = join( + configDir, + 'bg-sessions', + 'sessions', + `${id}.json`, + ) + const metadata = (await Bun.file(metadataPath).json()) as Record< + string, + unknown + > + delete metadata.terminalFactGeneration + await writeFile(metadataPath, JSON.stringify(metadata)) + const markerOnlySession = await readBackgroundSessionForOwner(id) + if (!markerOnlySession) throw new Error('marker-only session was not read') + expect(markerOnlySession.terminalFactGeneration).toBeUndefined() + + const release = await lockfile.lock(metadataPath, { realpath: false }) + try { + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 31 }, + { + ownerPid: session.pid, + expectedSession: markerOnlySession, + now: new Date('2026-06-15T08:09:26.000Z'), + }, + ) + } finally { + await release() + } + + expect( + await Bun.file( + terminalFactPath(id, 'natural'), + ).json(), + ).toMatchObject({ + status: 'failed', + exitCode: 31, + }) + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 1, + errors: 0, + }) + expect(await Bun.file(metadataPath).json()).toMatchObject({ + status: 'failed', + exitCode: 31, + }) + }) + + it('reconciles a sync finalization after refresh holds the metadata lock', async () => { + const id = 'bg-contended-sync-natural' + const name = 'contended-sync-natural' + const session = await createBackgroundSession({ + id, + name, + pid: 349, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], + sessionId: 'conversation-contended-sync-natural', + processMarker: TEST_PROCESS_MARKER, + }) + const metadataPath = join( + configDir, + 'bg-sessions', + 'sessions', + `${id}.json`, + ) + expect( + await refreshBackgroundSessionStatuses({ + isProcessAlive: () => false, + _whileStatusWriteLockedForTesting: () => { + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 23 }, + { + ownerPid: session.pid, + expectedSession: session, + now: new Date('2026-06-15T08:09:25.000Z'), + }, + ) + }, + }), + ).toMatchObject([{ status: 'failed', exitCode: 23 }]) + + expect(await Bun.file(metadataPath).json()).toMatchObject({ + status: 'stale', + }) + expect( + await Bun.file( + terminalFactPath(id, 'natural', TEST_PROCESS_MARKER), + ).json(), + ).toMatchObject({ + generation: TEST_PROCESS_MARKER, + status: 'failed', + exitCode: 23, + }) + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 1, + errors: 0, + }) + expect(await Bun.file(metadataPath).json()).toMatchObject({ + status: 'failed', + exitCode: 23, + finishedAt: '2026-06-15T08:09:25.000Z', + }) + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(false) + + expect( + await cleanupBackgroundSessionsBefore( + new Date('2026-07-01T00:00:00.000Z'), + ), + ).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + expect(await Bun.file(metadataPath).exists()).toBe(false) + }) + + it('reconciles stronger terminal facts and retries reservation release', async () => { + const id = 'bg-terminal-reconciliation-retry' + const name = 'terminal-reconciliation-retry' + const session = await createBackgroundSession({ + id, + name, + pid: 353, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + 'work', + ], + sessionId: 'conversation-terminal-reconciliation-retry', + processMarker: TEST_PROCESS_MARKER, + }) + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 5 }, + { ownerPid: session.pid }, + ) + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 1, + errors: 0, + }) + await markBackgroundSessionKilled(id, { + now: new Date('2026-06-15T08:09:27.000Z'), + }) + await writeNameReservation(name, { id }) + + const release = await lockfile.lock(nameReservationPath(name), { + realpath: false, + }) + try { + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 1, + errors: 1, + }) + } finally { + await release() + } + expect( + await Bun.file( + join(configDir, 'bg-sessions', 'sessions', `${id}.json`), + ).json(), + ).toMatchObject({ + status: 'killed', + terminalReason: 'explicit_kill', + }) + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(true) + + expect(await reconcileBackgroundSessionTerminalFacts()).toEqual({ + sessionsUpdated: 0, + errors: 0, + }) + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(false) + }) + + for (const sync of [false, true]) { + it(`preserves a mismatched reservation name during ${sync ? 'sync' : 'async'} finalization`, async () => { + const suffix = sync ? 'sync' : 'async' + const id = `bg-mismatched-reservation-${suffix}` + const name = `mismatched-reservation-${suffix}` + const session = await createBackgroundSession({ + id, + name, + pid: sync ? 354 : 355, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: `conversation-mismatched-reservation-${suffix}`, + }) + await writeFile( + nameReservationPath(name), + JSON.stringify({ name: `${name}-other`, id }), + ) + + if (sync) { + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 0 }, + { ownerPid: session.pid }, + ) + } else { + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + { ownerPid: session.pid }, + ) + } + + expect(await Bun.file(nameReservationPath(name)).exists()).toBe(true) + expect(await Bun.file(nameReservationPath(name)).json()).toEqual({ + name: `${name}-other`, + id, + }) + }) + } + it('does not let a late natural finalizer overwrite an explicit kill fact', async () => { await createBackgroundSession({ id: 'bg-killed-absorbing', diff --git a/src/cli/bgRegistry.ts b/src/cli/bgRegistry.ts index d9f674789b..46b7162424 100644 --- a/src/cli/bgRegistry.ts +++ b/src/cli/bgRegistry.ts @@ -127,6 +127,11 @@ const ALL_STATUSES = new Set([ 'unknown', ...TERMINAL_STATUSES, ]) +const COMPLETED_STATUSES = new Set([ + 'exited', + 'failed', + 'killed', +]) const SAFE_ID_RE = /^[A-Za-z0-9._-]+$/ const SAFE_SIGNAL_RE = /^SIG[A-Z0-9]{1,24}$/ const ORPHANED_TERMINAL_FACT_SCAN_LIMIT = 256 @@ -416,9 +421,27 @@ function isSameBackgroundSessionGeneration( ) } +async function replaceSessionFile( + session: BackgroundSession, + target: string, +): Promise { + const tmp = join( + getBackgroundSessionMetadataDir(), + `${session.id}.${process.pid}.${randomUUID()}.tmp`, + ) + try { + await writeFile(tmp, jsonStringify(session), { flag: 'wx' }) + await rename(tmp, target) + } catch (error) { + await unlink(tmp).catch(() => {}) + throw error + } +} + async function writeSession( session: BackgroundSession, expected: BackgroundSession, + whileLocked?: () => void | Promise, ): Promise { await ensureBackgroundSessionDirs() const target = metadataPathForId(session.id) @@ -427,21 +450,12 @@ async function writeSession( if (!current || !isSameBackgroundSessionGeneration(current, expected)) { return current } - const tmp = join( - getBackgroundSessionMetadataDir(), - `${session.id}.${process.pid}.${randomUUID()}.tmp`, - ) - try { - await writeFile(tmp, jsonStringify(session), { flag: 'wx' }) - await rename(tmp, target) - if (session.name && isTerminalBackgroundSession(session)) { - await releaseNameReservation(session.name, session.id) - } - return session - } catch (error) { - await unlink(tmp).catch(() => {}) - throw error + await whileLocked?.() + await replaceSessionFile(session, target) + if (session.name && isTerminalBackgroundSession(session)) { + await releaseNameReservation(session.name, session.id) } + return session }) } @@ -563,29 +577,58 @@ function isBackgroundSessionNameReservation( async function releaseNameReservationUnlocked( name: string, id: string, -): Promise { +): Promise { const path = nameReservationPathForName(name) - const existing = await readNameReservation(path) - if (existing?.id !== id) return - await unlink(path).catch(() => {}) + let existing: BackgroundSessionNameReservation + try { + const parsed = jsonParse(await readFile(path, 'utf8')) + if (!isBackgroundSessionNameReservation(parsed)) return true + existing = parsed + } catch (error) { + return isErrno(error, 'ENOENT') + } + if (existing.name !== name || existing.id !== id) return true + try { + await unlink(path) + return true + } catch (error) { + return isErrno(error, 'ENOENT') + } } -async function releaseNameReservation( +async function tryReleaseNameReservation( name: string, id: string, -): Promise { - await withNameReservationLock(name, async () => { - await releaseNameReservationUnlocked(name, id) - }).catch(() => {}) +): Promise { + try { + return await withNameReservationLock( + name, + async () => await releaseNameReservationUnlocked(name, id), + ) + } catch { + return false + } } -function releaseNameReservationSyncUnlocked(name: string, id: string): void { +async function releaseNameReservation(name: string, id: string): Promise { + await tryReleaseNameReservation(name, id) +} + +function releaseNameReservationSyncUnlocked( + name: string, + id: string, +): void { const path = nameReservationPathForName(name) try { - const parsed = jsonParse( - readFileSync(path, 'utf8'), - ) as Partial - if (parsed?.id === id) unlinkSync(path) + const parsed = jsonParse(readFileSync(path, 'utf8')) + if ( + !isBackgroundSessionNameReservation(parsed) || + parsed.name !== name || + parsed.id !== id + ) { + return + } + unlinkSync(path) } catch { // Effective terminal-state reads recover stale reservations later. } @@ -967,7 +1010,14 @@ function installTerminalFactSync( } } -export async function listBackgroundSessions(): Promise { +type BackgroundSessionRecord = { + stored: BackgroundSession + effective: BackgroundSession +} + +async function listBackgroundSessionRecords(): Promise< + BackgroundSessionRecord[] +> { let entries: string[] try { entries = await readdir(getBackgroundSessionMetadataDir()) @@ -975,16 +1025,220 @@ export async function listBackgroundSessions(): Promise { return [] } - const sessions: BackgroundSession[] = [] + const records: BackgroundSessionRecord[] = [] for (const entry of entries) { if (!entry.endsWith('.json')) continue const session = await readSessionFile( join(getBackgroundSessionMetadataDir(), entry), ) - if (session) sessions.push(await applyAuthoritativeTerminalFacts(session)) + if (session) { + records.push({ + stored: session, + effective: await applyAuthoritativeTerminalFacts(session), + }) + } } - return sessions.sort((a, b) => a.startedAt.localeCompare(b.startedAt)) + return records.sort((a, b) => + a.stored.startedAt.localeCompare(b.stored.startedAt), + ) +} + +export async function listBackgroundSessions(): Promise { + return (await listBackgroundSessionRecords()).map(record => record.effective) +} + +async function readReconciliationTerminalFact( + session: BackgroundSession, + kind: 'natural' | 'killed', + terminalDirectory: CleanupDirectorySnapshot, + fileSystem: BackgroundCleanupFileSystem, +): Promise> { + const generation = session.terminalFactGeneration + return await readCleanupJson( + terminalFactPathForId(session.id, kind, generation), + (value): value is BackgroundSessionTerminalFact => + isBackgroundSessionTerminalFact(value, session.id, kind) && + value.generation === generation && + parseCanonicalCompletionTimestamp(value.finishedAt) !== null, + terminalDirectory, + fileSystem, + ) +} + +function hasSamePersistedTerminalState( + stored: BackgroundSession, + effective: BackgroundSession, +): boolean { + return ( + stored.status === effective.status && + stored.updatedAt === effective.updatedAt && + stored.finishedAt === effective.finishedAt && + stored.exitCode === effective.exitCode && + stored.signal === effective.signal && + stored.terminalReason === effective.terminalReason + ) +} + +export async function reconcileBackgroundSessionTerminalFacts(): Promise< + { sessionsUpdated: number; errors: number } +> { + const result = { + sessionsUpdated: 0, + errors: 0, + } + + const fileSystem: BackgroundCleanupFileSystem = { + lstatFile: lstat, + readTextFile: async path => await readFile(path, 'utf8'), + unlinkFile: unlink, + } + const [metadataRead, terminalRead] = await Promise.all([ + snapshotCleanupDirectory( + getBackgroundSessionMetadataDir(), + fileSystem.lstatFile, + ), + snapshotCleanupDirectory( + getBackgroundSessionTerminalDir(), + fileSystem.lstatFile, + ), + ]) + if (metadataRead.state !== 'valid' || terminalRead.state !== 'valid') { + result.errors += Number(metadataRead.state !== 'valid') + result.errors += Number(terminalRead.state !== 'valid') + return result + } + const metadataDirectory = metadataRead.value + const terminalDirectory = terminalRead.value + if ( + metadataDirectory.state === 'missing' || + terminalDirectory.state === 'missing' + ) { + return result + } + + let entries: Dirent[] + try { + entries = await readdir(getBackgroundSessionMetadataDir(), { + withFileTypes: true, + }) + } catch (error) { + if (!isErrno(error, 'ENOENT')) result.errors++ + return result + } + if ( + (await inspectCleanupDirectory( + metadataDirectory, + fileSystem.lstatFile, + )) !== 'same' + ) { + result.errors++ + return result + } + + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith('.json')) continue + const id = entry.name.slice(0, -'.json'.length) + if (!SAFE_ID_RE.test(id)) continue + const metadataPath = metadataPathForId(id) + const candidate = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => isBackgroundSession(value, id), + metadataDirectory, + fileSystem, + ) + if (candidate.state === 'error') { + result.errors++ + continue + } + if (candidate.state !== 'valid' || !candidate.identity) continue + + try { + await withBackgroundSessionIdLock(id, async () => { + const natural = await readReconciliationTerminalFact( + candidate.value, + 'natural', + terminalDirectory, + fileSystem, + ) + const killed = await readReconciliationTerminalFact( + candidate.value, + 'killed', + terminalDirectory, + fileSystem, + ) + if (natural.state === 'error' || killed.state === 'error') { + result.errors++ + return + } + + const latest = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => + isBackgroundSession(value, id), + metadataDirectory, + fileSystem, + ) + if (latest.state === 'error') { + result.errors++ + return + } + if ( + latest.state !== 'valid' || + !latest.identity || + latest.identity.dev !== candidate.identity?.dev || + latest.identity.ino !== candidate.identity?.ino || + !isSameBackgroundSessionGeneration(latest.value, candidate.value) + ) { + return + } + + const effective = applyTerminalFacts( + latest.value, + natural.state === 'valid' ? natural.value : null, + killed.state === 'valid' ? killed.value : null, + ) + if (!COMPLETED_STATUSES.has(effective.status)) return + + if (!hasSamePersistedTerminalState(latest.value, effective)) { + if ( + (await inspectCleanupDirectory( + metadataDirectory, + fileSystem.lstatFile, + )) !== 'same' + ) { + result.errors++ + return + } + try { + const beforeWrite = await fileSystem.lstatFile(metadataPath) + if ( + !beforeWrite.isFile() || + beforeWrite.isSymbolicLink() || + beforeWrite.dev !== latest.identity.dev || + beforeWrite.ino !== latest.identity.ino + ) { + return + } + } catch (error) { + if (!isErrno(error, 'ENOENT')) result.errors++ + return + } + await replaceSessionFile(effective, metadataPath) + result.sessionsUpdated++ + } + if ( + effective.name && + !(await tryReleaseNameReservation(effective.name, effective.id)) + ) { + result.errors++ + } + }) + } catch { + result.errors++ + } + } + return result } type CleanupArtifactRemoval = 'removed' | 'missing' | 'error' @@ -1670,6 +1924,10 @@ export async function refreshBackgroundSessionStatuses(options?: { session: BackgroundSession, nextStatus: BackgroundSessionStatus, ) => Promise + _whileStatusWriteLockedForTesting?: ( + session: BackgroundSession, + nextStatus: BackgroundSessionStatus, + ) => void | Promise }): Promise { const timestamp = iso(options?.now) const sessions = await listBackgroundSessions() @@ -1699,7 +1957,9 @@ export async function refreshBackgroundSessionStatuses(options?: { updatedAt: timestamp, } await options?._beforeStatusWriteForTesting?.(session, nextStatus) - const persisted = await writeSession(updated, session) + const persisted = await writeSession(updated, session, () => + options?._whileStatusWriteLockedForTesting?.(session, nextStatus), + ) if (persisted) { refreshed.push(await applyAuthoritativeTerminalFacts(persisted)) } @@ -1942,22 +2202,40 @@ function assertNaturalFinalizationOwner( session: BackgroundSession | null, id: string, ownerPid: number, + expectedSession?: BackgroundSession, ): asserts session is BackgroundSession { - if (!session || session.id !== id || session.pid !== ownerPid) { + if ( + !session || + session.id !== id || + session.pid !== ownerPid || + (expectedSession !== undefined && + !isSameBackgroundSessionGeneration(session, expectedSession)) + ) { throw new Error('Background session finalizer does not own this session') } } +type BackgroundSessionNaturalFinalizationOptions = { + ownerPid?: number + now?: Date + expectedSession?: BackgroundSession +} + export async function recordBackgroundSessionNaturalTermination( id: string, termination: BackgroundSessionNaturalTermination, - options: { ownerPid?: number; now?: Date } = {}, + options: BackgroundSessionNaturalFinalizationOptions = {}, ): Promise { assertSafeId(id) const ownerPid = options.ownerPid ?? process.pid return await withBackgroundSessionIdLock(id, async () => { const session = await readSessionFile(metadataPathForId(id)) - assertNaturalFinalizationOwner(session, id, ownerPid) + assertNaturalFinalizationOwner( + session, + id, + ownerPid, + options.expectedSession, + ) const effective = await applyAuthoritativeTerminalFacts(session) if ( @@ -1997,49 +2275,101 @@ export async function recordBackgroundSessionNaturalTermination( export function recordBackgroundSessionNaturalTerminationSync( id: string, termination: BackgroundSessionNaturalTermination, - options: { ownerPid?: number; now?: Date } = {}, + options: BackgroundSessionNaturalFinalizationOptions = {}, ): void { assertSafeId(id) const ownerPid = options.ownerPid ?? process.pid - withBackgroundSessionIdLockSync(id, () => { - const session = readSessionFileSync(metadataPathForId(id)) - assertNaturalFinalizationOwner(session, id, ownerPid) + try { + withBackgroundSessionIdLockSync(id, () => { + const session = readSessionFileSync(metadataPathForId(id)) + assertNaturalFinalizationOwner( + session, + id, + ownerPid, + options.expectedSession, + ) + if ( + session.status === 'killed' || + session.status === 'exited' || + session.status === 'failed' || + readTerminalFactSync( + id, + 'killed', + session.terminalFactGeneration, + )?.pid === ownerPid + ) { + return + } + if ( + session.status !== 'running' && + session.status !== 'unknown' && + session.status !== 'stale' + ) { + // Retain an exhaustive guard so future status additions require a deliberate + // natural-finalization policy. + throw new Error( + 'Background session is not eligible for natural finalization', + ) + } + + installTerminalFactSync( + naturalTerminalFact( + id, + ownerPid, + session.terminalFactGeneration, + termination, + options.now, + ), + 'natural', + ) + if (session.name) releaseNameReservationSync(session.name, session.id) + }) + } catch (error) { + const expectedSession = options.expectedSession + const generation = expectedSession?.terminalFactGeneration + const hasSafeCompatibilityHandoff = + generation !== undefined || expectedSession?.processMarker !== undefined if ( - session.status === 'killed' || - session.status === 'exited' || - session.status === 'failed' || + !isErrno(error, 'ELOCKED') || + !expectedSession || + !hasSafeCompatibilityHandoff + ) { + throw error + } + // The exit event cannot wait for another process's metadata lock. A marked + // session can still leave an immutable fact without touching metadata or a + // name reservation. Marker-only metadata predates generation-scoped fact + // paths, but new marked replacements never read its legacy path and a + // markerless replacement cannot reuse the ID while that fact remains. + assertNaturalFinalizationOwner( + expectedSession, + id, + ownerPid, + expectedSession, + ) + if ( + expectedSession.status === 'killed' || + expectedSession.status === 'exited' || + expectedSession.status === 'failed' || readTerminalFactSync( id, 'killed', - session.terminalFactGeneration, + generation, )?.pid === ownerPid ) { return } - if ( - session.status !== 'running' && - session.status !== 'unknown' && - session.status !== 'stale' - ) { - // Retain an exhaustive guard so future status additions require a deliberate - // natural-finalization policy. - throw new Error( - 'Background session is not eligible for natural finalization', - ) - } - installTerminalFactSync( naturalTerminalFact( id, ownerPid, - session.terminalFactGeneration, + generation, termination, options.now, ), 'natural', ) - if (session.name) releaseNameReservationSync(session.name, session.id) - }) + } } export async function markBackgroundSessionKilled( diff --git a/src/utils/backgroundHousekeeping.test.ts b/src/utils/backgroundHousekeeping.test.ts new file mode 100644 index 0000000000..7614ae738e --- /dev/null +++ b/src/utils/backgroundHousekeeping.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'bun:test' +import { + startBackgroundHousekeeping, + startBackgroundSessionReconciliation, +} from './backgroundHousekeeping.js' + +describe('background session housekeeping', () => { + it('starts recurring background-session reconciliation', async () => { + let callback: (() => void) | undefined + let runs = 0 + startBackgroundHousekeeping({ + backgroundSessionReconciliation: { + cleanup: async () => { + runs++ + }, + setInterval: scheduledCallback => { + callback = scheduledCallback + return { unref: () => {} } + }, + }, + _reconciliationOnlyForTesting: true, + }) + + callback?.() + await new Promise(resolve => setTimeout(resolve, 0)) + expect(runs).toBe(1) + }) + + it('runs recurring reconciliation without overlapping passes', async () => { + let callback: (() => void) | undefined + let intervalMs: number | undefined + let unrefCalls = 0 + let runs = 0 + let releaseFirstRun!: () => void + const firstRun = new Promise(resolve => { + releaseFirstRun = resolve + }) + + startBackgroundSessionReconciliation({ + cleanup: async () => { + runs++ + if (runs === 1) await firstRun + }, + setInterval: (scheduledCallback, scheduledIntervalMs) => { + callback = scheduledCallback + intervalMs = scheduledIntervalMs + return { + unref: () => { + unrefCalls++ + }, + } + }, + }) + + expect(intervalMs).toBe(60_000) + expect(unrefCalls).toBe(1) + callback?.() + callback?.() + await Promise.resolve() + expect(runs).toBe(1) + + releaseFirstRun() + await firstRun + await new Promise(resolve => setTimeout(resolve, 0)) + callback?.() + await Promise.resolve() + expect(runs).toBe(2) + }) +}) diff --git a/src/utils/backgroundHousekeeping.ts b/src/utils/backgroundHousekeeping.ts index fa56721324..857ed76019 100644 --- a/src/utils/backgroundHousekeeping.ts +++ b/src/utils/backgroundHousekeeping.ts @@ -15,20 +15,69 @@ const registerProtocolModule = feature('LODESTONE') import { getIsInteractive, getLastInteractionTime } from '../bootstrap/state.js' import { + cleanupBackgroundSessionsInBackground, cleanupNpmCacheForAnthropicPackages, cleanupOldMessageFilesInBackground, cleanupOldVersionsThrottled, } from './cleanup.js' +import { logError } from './log.js' import { cleanupOldVersions } from './nativeInstaller/index.js' import { autoUpdateMarketplacesAndPluginsInBackground } from './plugins/pluginAutoupdate.js' // 24 hours in milliseconds const RECURRING_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000 +// Terminal facts are a durable handoff from background children. Polling this +// small registry once a minute gives zero-day retention a prompt later pass +// without rerunning the full message and cache cleanup pipeline. +const BACKGROUND_SESSION_RECONCILIATION_INTERVAL_MS = 60 * 1000 // 10 minutes after start. const DELAY_VERY_SLOW_OPERATIONS_THAT_HAPPEN_EVERY_SESSION = 10 * 60 * 1000 -export function startBackgroundHousekeeping(): void { +type BackgroundSessionReconciliationOptions = { + cleanup?: () => Promise + setInterval?: ( + callback: () => void, + intervalMs: number, + ) => { unref(): unknown } + _onPassFinishedForTesting?: () => void +} + +type BackgroundHousekeepingOptions = { + backgroundSessionReconciliation?: BackgroundSessionReconciliationOptions + _reconciliationOnlyForTesting?: boolean +} + +export function startBackgroundSessionReconciliation( + options: BackgroundSessionReconciliationOptions = {}, +): void { + const cleanup = options.cleanup ?? cleanupBackgroundSessionsInBackground + const scheduleInterval = + options.setInterval ?? + ((callback, intervalMs) => setInterval(callback, intervalMs)) + let running = false + const interval = scheduleInterval(() => { + if (running) return + running = true + void Promise.resolve() + .then(() => cleanup()) + .catch(error => logError(error as Error)) + .finally(() => { + running = false + options._onPassFinishedForTesting?.() + }) + }, BACKGROUND_SESSION_RECONCILIATION_INTERVAL_MS) + interval.unref() +} + +export function startBackgroundHousekeeping( + options: BackgroundHousekeepingOptions = {}, +): void { + startBackgroundSessionReconciliation( + options.backgroundSessionReconciliation, + ) + if (options._reconciliationOnlyForTesting) return + void initMagicDocs() void initSkillImprovement() if (feature('EXTRACT_MEMORIES')) { @@ -39,7 +88,6 @@ export function startBackgroundHousekeeping(): void { if (feature('LODESTONE') && getIsInteractive()) { void registerProtocolModule!.ensureDeepLinkProtocolRegistered() } - let needsCleanup = true async function runVerySlowOps(): Promise { // If the user did something in the last minute, don't make them wait for these slow operations to run. diff --git a/src/utils/cleanup.test.ts b/src/utils/cleanup.test.ts index 2a5693a8e3..47732ec259 100644 --- a/src/utils/cleanup.test.ts +++ b/src/utils/cleanup.test.ts @@ -20,7 +20,10 @@ afterEach(async () => { ) }) -async function runCleanupFixture(configDir: string): Promise { +async function runCleanupFixture( + configDir: string, + mode: 'once' | 'post-completion' = 'once', +): Promise> { const fixture = join(import.meta.dir, 'cleanupBackgroundSessions.fixture.ts') const { USER_TYPE: _userType, ...inheritedEnv } = process.env const controller = new AbortController() @@ -32,6 +35,7 @@ async function runCleanupFixture(configDir: string): Promise { HOME: configDir, XDG_CACHE_HOME: join(configDir, 'cache'), OPENCLAUDE_CONFIG_DIR: configDir, + OPENCLAUDE_CLEANUP_FIXTURE_MODE: mode, NODE_ENV: 'test', }, signal: controller.signal, @@ -46,7 +50,9 @@ async function runCleanupFixture(configDir: string): Promise { new Response(child.stderr).text(), ]) expect(exitCode, stderr).toBe(0) - expect(JSON.parse(stdout)).toEqual({ completed: true }) + const result = JSON.parse(stdout) as Record + if (mode === 'once') expect(result).toEqual({ completed: true }) + return result } finally { clearTimeout(timeout) } @@ -162,6 +168,34 @@ describe('cleanupOldMessageFilesInBackground', () => { 30_000, ) + test( + 'reclaims a zero-day session that completes after the initial sweep', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-post-completion-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect( + await runCleanupFixture(configDir, 'post-completion'), + ).toEqual({ + presentAfterInitialSweep: true, + metadataPresentAfterCompletion: false, + metadataStatusAfterCompletion: null, + reservationPresentAfterCompletion: false, + stdoutPresentAfterCompletion: false, + stderrPresentAfterCompletion: false, + }) + }, + 30_000, + ) + test( 'preserves background artifacts when explicit cleanupPeriodDays is invalid', async () => { @@ -188,6 +222,34 @@ describe('cleanupOldMessageFilesInBackground', () => { 30_000, ) + test( + 'keeps post-completion artifacts when recurring retention is disabled by invalid settings', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-invalid-post-completion-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 'invalid' }), + ) + + expect( + await runCleanupFixture(configDir, 'post-completion'), + ).toEqual({ + presentAfterInitialSweep: true, + metadataPresentAfterCompletion: true, + metadataStatusAfterCompletion: 'running', + reservationPresentAfterCompletion: true, + stdoutPresentAfterCompletion: true, + stderrPresentAfterCompletion: true, + }) + }, + 30_000, + ) + test( 'retains recent background sessions under a nonzero cleanup period', async () => { diff --git a/src/utils/cleanup.ts b/src/utils/cleanup.ts index 66258cc045..d0e0e62041 100644 --- a/src/utils/cleanup.ts +++ b/src/utils/cleanup.ts @@ -1,7 +1,10 @@ import * as fs from 'fs/promises' import { homedir } from 'os' import { join } from 'path' -import { cleanupBackgroundSessionsBefore } from '../cli/bgRegistry.js' +import { + cleanupBackgroundSessionsBefore, + reconcileBackgroundSessionTerminalFacts, +} from '../cli/bgRegistry.js' import { logEvent } from '../services/analytics/index.js' import { CACHE_PATHS } from './cachePaths.js' import { logForDebugging } from './debug.js' @@ -589,8 +592,7 @@ export async function cleanupOldMessageFilesInBackground(): Promise { // If settings have validation errors but the user explicitly set cleanupPeriodDays, // skip cleanup entirely rather than falling back to the default (30 days). // This prevents accidentally deleting files when the user intended a different retention period. - const { errors } = getSettingsWithAllErrors() - if (errors.length > 0 && rawSettingsContainsKey('cleanupPeriodDays')) { + if (cleanupPeriodSettingIsInvalid()) { logForDebugging( 'Skipping cleanup: settings have validation errors but cleanupPeriodDays was explicitly set. Fix settings errors to enable cleanup.', ) @@ -625,3 +627,32 @@ export async function cleanupOldMessageFilesInBackground(): Promise { await cleanupNpmCacheForAnthropicPackages() } } + +function cleanupPeriodSettingIsInvalid(): boolean { + const { errors } = getSettingsWithAllErrors() + return errors.length > 0 && rawSettingsContainsKey('cleanupPeriodDays') +} + +export async function cleanupBackgroundSessionsInBackground(): Promise { + if (cleanupPeriodSettingIsInvalid()) { + return + } + const reconciliation = await reconcileBackgroundSessionTerminalFacts() + if (reconciliation.errors > 0) { + logForDebugging( + `Background session reconciliation: ${reconciliation.sessionsUpdated} sessions, ${reconciliation.errors} errors`, + ) + } + const backgroundResult = await cleanupBackgroundSessionsBefore( + getCutoffDate(), + ) + if ( + backgroundResult.sessionsRemoved > 0 || + backgroundResult.artifactsRemoved > 0 || + backgroundResult.errors > 0 + ) { + logForDebugging( + `Background session cleanup: ${backgroundResult.sessionsRemoved} sessions, ${backgroundResult.artifactsRemoved} artifacts, ${backgroundResult.errors} errors`, + ) + } +} diff --git a/src/utils/cleanupBackgroundSessions.fixture.ts b/src/utils/cleanupBackgroundSessions.fixture.ts index 4db38942be..172aed97b6 100644 --- a/src/utils/cleanupBackgroundSessions.fixture.ts +++ b/src/utils/cleanupBackgroundSessions.fixture.ts @@ -1,8 +1,105 @@ +import { createHash } from 'node:crypto' +import { dirname, join } from 'node:path' +import * as lockfile from './lockfile.js' +import { + createBackgroundSession, + recordBackgroundSessionNaturalTerminationSync, +} from '../cli/bgRegistry.js' +import { startBackgroundHousekeeping } from './backgroundHousekeeping.js' import { cleanupOldMessageFilesInBackground } from './cleanup.js' try { - await cleanupOldMessageFilesInBackground() - process.stdout.write(`${JSON.stringify({ completed: true })}\n`) + if (process.env.OPENCLAUDE_CLEANUP_FIXTURE_MODE === 'post-completion') { + const processMarker = 'c'.repeat(64) + const session = await createBackgroundSession({ + id: 'bg-post-completion', + name: 'post-completion', + pid: process.pid, + cwd: process.cwd(), + command: [ + 'openclaude', + `--openclaude-bg-session-marker=${processMarker}`, + '--print', + 'fixture', + ], + sessionId: 'bg-post-completion-conversation', + processMarker, + }) + const root = dirname(dirname(session.stdoutLogPath)) + const metadataPath = join(root, 'sessions', `${session.id}.json`) + const reservationPath = join( + root, + 'names', + `${createHash('sha256').update(session.name!).digest('hex')}.json`, + ) + let scheduledCallback: (() => void) | undefined + let finishPass: (() => void) | undefined + startBackgroundHousekeeping({ + backgroundSessionReconciliation: { + setInterval: callback => { + scheduledCallback = callback + return { unref: () => {} } + }, + _onPassFinishedForTesting: () => finishPass?.(), + }, + _reconciliationOnlyForTesting: true, + }) + const runScheduledPass = async (): Promise => { + if (!scheduledCallback) { + throw new Error('background reconciliation was not scheduled') + } + await new Promise(resolve => { + finishPass = resolve + scheduledCallback?.() + }) + finishPass = undefined + } + + await runScheduledPass() + const presentAfterInitialSweep = await Bun.file( + session.stdoutLogPath, + ).exists() + const release = await lockfile.lock(metadataPath, { realpath: false }) + try { + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { + ownerPid: process.pid, + expectedSession: session, + now: new Date(Date.now() - 1_000), + }, + ) + } finally { + await release() + } + await runScheduledPass() + const metadataPresentAfterCompletion = await Bun.file( + metadataPath, + ).exists() + const storedMetadata = metadataPresentAfterCompletion + ? ((await Bun.file(metadataPath).json()) as { status?: unknown }) + : undefined + process.stdout.write( + `${JSON.stringify({ + presentAfterInitialSweep, + metadataPresentAfterCompletion, + metadataStatusAfterCompletion: storedMetadata?.status ?? null, + reservationPresentAfterCompletion: await Bun.file( + reservationPath, + ).exists(), + stdoutPresentAfterCompletion: await Bun.file( + session.stdoutLogPath, + ).exists(), + stderrPresentAfterCompletion: await Bun.file( + session.stderrLogPath, + ).exists(), + })}\n`, + ) + } else { + await cleanupOldMessageFilesInBackground() + process.stdout.write(`${JSON.stringify({ completed: true })}\n`) + } } catch { process.stderr.write('background cleanup fixture failed\n') process.exitCode = 1 From 84c33cdb2d49342919388b62a2cdd2e31b7d8af2 Mon Sep 17 00:00:00 2001 From: chioarub Date: Tue, 1 Sep 2026 08:41:26 +0300 Subject: [PATCH 08/13] fix(bg): retain zero-day cleanup ownership --- src/cli/bg.ts | 8 + src/cli/bgFinalizer.fixture.ts | 38 +- src/cli/bgFinalizer.test.ts | 359 +++++++++++++++++- src/cli/bgFinalizer.ts | 149 +++++++- src/cli/bgRegistry.test.ts | 10 +- src/cli/bgRouting.ts | 4 + src/entrypoints/cli.test.ts | 20 +- src/entrypoints/cli.tsx | 8 + src/utils/cleanup.test.ts | 111 +++++- src/utils/cleanup.ts | 20 +- .../cleanupBackgroundSessions.fixture.ts | 107 +++++- 11 files changed, 816 insertions(+), 18 deletions(-) diff --git a/src/cli/bg.ts b/src/cli/bg.ts index c2a29c213e..d8bf64db85 100644 --- a/src/cli/bg.ts +++ b/src/cli/bg.ts @@ -1054,6 +1054,14 @@ export async function killHandler( `Failed to kill background session ${session.id}: ${errorMessage(error)}`, ) }) + try { + const { cleanupBackgroundSessionsAfterFinalization } = await import( + '../utils/cleanup.js' + ) + await cleanupBackgroundSessionsAfterFinalization(async () => true) + } catch { + // A retention failure must not replace the successful kill outcome. + } console.log(`Killed background session ${killed.id}.`) } diff --git a/src/cli/bgFinalizer.fixture.ts b/src/cli/bgFinalizer.fixture.ts index 72072a950d..999d568832 100644 --- a/src/cli/bgFinalizer.fixture.ts +++ b/src/cli/bgFinalizer.fixture.ts @@ -1,14 +1,30 @@ import { writeFile } from 'node:fs/promises' import { handleBgFlag } from './bg.js' -import { prepareBackgroundSessionFinalizer } from './bgFinalizer.js' +import * as backgroundFinalizer from './bgFinalizer.js' import { noteBackgroundSessionTerminationSignal } from '../utils/backgroundSessionTermination.js' const invocation = process.argv.slice(2) -if (invocation[0] === 'launcher') { - await handleBgFlag(['--bg', invocation[1] ?? 'success']) +const cleanupWorkerEnv = 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_WORKER' +if (process.env[cleanupWorkerEnv] === '1') { + const runCleanupWorker = ( + backgroundFinalizer as typeof backgroundFinalizer & { + runBackgroundSessionCleanupWorker?: () => Promise + } + ).runBackgroundSessionCleanupWorker + if (!runCleanupWorker) { + throw new Error('background cleanup worker is unavailable') + } + await runCleanupWorker() +} else if (invocation[0] === 'launcher') { + const name = invocation[2] + await handleBgFlag([ + '--bg', + ...(name ? ['--name', name] : []), + invocation[1] ?? 'success', + ]) } else { const mode = invocation.at(-1) - await prepareBackgroundSessionFinalizer() + await backgroundFinalizer.prepareBackgroundSessionFinalizer() if (mode === 'throw') { throw new Error('intentional background finalizer fixture failure') @@ -30,6 +46,20 @@ if (invocation[0] === 'launcher') { } const readyPath = process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_READY if (readyPath) await writeFile(readyPath, 'ready') + if (mode === 'controlled' || mode === 'controlled-exit') { + const releasePath = process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_RELEASE + if (!releasePath) { + throw new Error('controlled fixture release path is missing') + } + const deadline = Date.now() + 5_000 + while (!(await Bun.file(releasePath).exists())) { + if (Date.now() >= deadline) { + throw new Error('controlled fixture release timed out') + } + await new Promise(resolve => setTimeout(resolve, 10)) + } + if (mode === 'controlled-exit') process.exit(0) + } if (mode === 'wait' || mode === 'sigint' || mode === 'sigterm') { setInterval(() => {}, 1_000) } diff --git a/src/cli/bgFinalizer.test.ts b/src/cli/bgFinalizer.test.ts index 1283f3904f..96c0335dd4 100644 --- a/src/cli/bgFinalizer.test.ts +++ b/src/cli/bgFinalizer.test.ts @@ -1,9 +1,11 @@ import { once } from 'node:events' import { spawn } from 'node:child_process' -import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises' +import { createHash } from 'node:crypto' +import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { basename, join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { isProcessRunning } from '../utils/genericProcessUtils.js' import { BACKGROUND_SESSION_ID_ENV, BACKGROUND_SESSION_LAUNCHER_PID_ENV, @@ -25,6 +27,10 @@ import { const fixturePath = join(import.meta.dir, 'bgFinalizer.fixture.ts') const installedLauncherPath = join(import.meta.dir, '../../bin/openclaude') +const BACKGROUND_SESSION_CLEANUP_WORKER_ENV = + 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_WORKER' +const BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV = + 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_OWNER_PID' describe('background session finalizer', () => { let configDir: string @@ -212,6 +218,7 @@ describe('background session finalizer', () => { finalizeSync: (_id, termination) => { finalizedSync.push(termination.exitCode ?? -1) }, + startCleanupWorker: () => {}, }) expect(preparation).toBe('installed') @@ -231,6 +238,63 @@ describe('background session finalizer', () => { expect(finalizationOwner).toEqual(registeredSession) }) + it('starts a detached cleanup owner after awaited finalization', async () => { + let beforeExitListener: (() => void | Promise) | undefined + const order: string[] = [] + + await prepareBackgroundSessionFinalizer({ + env: { + [BACKGROUND_SESSION_ID_ENV]: 'bg-launcher-handoff', + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: '123', + }, + pid: 500, + readSession: async () => ownedSession('bg-launcher-handoff', 500), + registerCleanup: () => () => {}, + onBeforeExit: listener => { + beforeExitListener = listener + }, + onExit: () => {}, + finalize: async () => { + order.push('finalize') + return ownedSession('bg-launcher-handoff', 500) + }, + startCleanupWorker: (ownerPid, launcherPid) => { + order.push(`worker:${ownerPid}:${launcherPid}`) + }, + }) + + await beforeExitListener?.() + expect(order).toEqual(['finalize', 'worker:500:123']) + }) + + it('retains cleanup when the detached worker cannot observe process exit', async () => { + const { runBackgroundSessionCleanupWorker } = await import( + './bgFinalizer.js' + ) + const env = { + [BACKGROUND_SESSION_CLEANUP_WORKER_ENV]: '1', + [BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV]: '500', + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: '123', + } + let observedExit: boolean | undefined + + await runBackgroundSessionCleanupWorker({ + env, + isProcessAlive: () => true, + sleep: async () => {}, + waitMs: 2, + pollMs: 1, + cleanup: async waitForProcessesToExit => { + observedExit = await waitForProcessesToExit() + }, + }) + + expect(observedExit).toBe(false) + expect(env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV]).toBeUndefined() + expect(env[BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV]).toBeUndefined() + expect(env[BACKGROUND_SESSION_LAUNCHER_PID_ENV]).toBeUndefined() + }) + it('passes the registered generation to the synchronous exit fallback', async () => { let cleanup: (() => void | Promise) | undefined let exitListener: ((code: number) => void) | undefined @@ -240,6 +304,7 @@ describe('background session finalizer', () => { terminalFactGeneration: 'a'.repeat(64), } let syncOwner: BackgroundSession | undefined + const cleanupWorkers: Array<[number, number]> = [] await prepareBackgroundSessionFinalizer({ env: { @@ -260,14 +325,19 @@ describe('background session finalizer', () => { finalize: async () => { throw Object.assign(new Error('contended'), { code: 'ELOCKED' }) }, + debug: () => {}, finalizeSync: (_id, _termination, options) => { syncOwner = options?.expectedSession }, + startCleanupWorker: (ownerPid, launcherPid) => { + cleanupWorkers.push([ownerPid, launcherPid]) + }, }) await cleanup?.() exitListener?.(23) expect(syncOwner).toEqual(registeredSession) + expect(cleanupWorkers).toEqual([[registeredSession.pid, 123]]) }) it('keeps the original exit code when both persistence paths fail', async () => { @@ -317,6 +387,40 @@ describe('background session finalizer', () => { expect(diagnostics.join('\n')).not.toContain('private sync details') }) + it('reports post-finalization cleanup failure without changing the outcome', async () => { + let cleanup: (() => void | Promise) | undefined + const diagnostics: string[] = [] + + await prepareBackgroundSessionFinalizer({ + env: { + [BACKGROUND_SESSION_ID_ENV]: 'bg-cleanup-failure', + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: '123', + }, + pid: 500, + readSession: async () => ownedSession('bg-cleanup-failure', 500), + isLauncherAlive: () => false, + registerCleanup: fn => { + cleanup = fn + return () => {} + }, + onBeforeExit: () => {}, + onExit: () => {}, + finalize: async () => ownedSession('bg-cleanup-failure', 500), + startCleanupWorker: () => { + throw Object.assign(new Error('private cleanup path'), { code: 'EIO' }) + }, + debug: message => { + diagnostics.push(message) + }, + }) + + await cleanup?.() + expect(diagnostics).toEqual([ + 'Background session post-finalization cleanup failed (EIO)', + ]) + expect(diagnostics[0]).not.toContain('private cleanup path') + }) + it('records an observed shutdown signal instead of a successful exit code', async () => { let cleanup: (() => void | Promise) | undefined let termination: { exitCode?: number; signal?: string } | undefined @@ -342,6 +446,7 @@ describe('background session finalizer', () => { termination = observed return ownedSession('bg-observed-sigint', 500) }, + startCleanupWorker: () => {}, }) await cleanup?.() @@ -381,13 +486,21 @@ describe('background session finalizer', () => { } async function waitForFile(path: string): Promise { - for (let attempt = 0; attempt < 200; attempt += 1) { + for (let attempt = 0; attempt < 500; attempt += 1) { if (await Bun.file(path).exists()) return - await new Promise(resolve => setTimeout(resolve, 5)) + await new Promise(resolve => setTimeout(resolve, 10)) } throw new Error('fixture readiness file was not created') } + async function waitForProcessStop(pid: number): Promise { + for (let attempt = 0; attempt < 500; attempt += 1) { + if (!isProcessRunning(pid)) return true + await new Promise(resolve => setTimeout(resolve, 10)) + } + return !isProcessRunning(pid) + } + async function runBuiltCliSession( id: string, args: string[], @@ -549,6 +662,244 @@ describe('background session finalizer', () => { }) } + async function runZeroRetentionFinalizerScenario( + mode: 'controlled' | 'controlled-exit', + ): Promise { + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const homeDir = join(configDir, 'home') + const cacheDir = join(configDir, 'cache') + await mkdir(homeDir, { recursive: true }) + await mkdir(cacheDir, { recursive: true }) + const name = `zero-retention-${mode}` + const readyPath = join(configDir, `${mode}.ready`) + const releasePath = join(configDir, `${mode}.release`) + const launcher = spawn( + process.execPath, + [fixturePath, 'launcher', mode, name], + { + cwd: configDir, + env: { + ...process.env, + HOME: homeDir, + XDG_CACHE_HOME: cacheDir, + OPENCLAUDE_CONFIG_DIR: configDir, + OPENCLAUDE_BG_FINALIZER_FIXTURE_READY: readyPath, + OPENCLAUDE_BG_FINALIZER_FIXTURE_RELEASE: releasePath, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + let stdout = '' + let stderr = '' + launcher.stdout?.setEncoding('utf8') + launcher.stderr?.setEncoding('utf8') + launcher.stdout?.on('data', chunk => { + stdout += chunk + }) + launcher.stderr?.on('data', chunk => { + stderr += chunk + }) + const launcherClose = once(launcher, 'close') + let childPid: number | undefined + try { + const [launcherCode] = (await launcherClose) as [number] + expect(launcherCode).toBe(0) + expect(stderr).toBe('') + const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] + const id = stdoutLogPath + ? basename(stdoutLogPath, '.out.log') + : undefined + const parsedChildPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + expect(id).toBeDefined() + expect(Number.isSafeInteger(parsedChildPid)).toBe(true) + if (!id || !Number.isSafeInteger(parsedChildPid)) { + throw new Error('detached launch output omitted its session identity') + } + childPid = parsedChildPid + const reservationDigest = createHash('sha256') + .update(name) + .digest('hex') + const artifacts = [ + join(sessionsRoot, 'sessions', `${id}.json`), + join(sessionsRoot, 'logs', `${id}.out.log`), + join(sessionsRoot, 'logs', `${id}.err.log`), + join(sessionsRoot, 'names', `${reservationDigest}.json`), + ] + await waitForFile(readyPath) + expect(isProcessRunning(childPid)).toBe(true) + expect( + await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), + ), + ).toEqual([true, true, true, true]) + await writeFile(releasePath, 'release') + + expect(await waitForProcessStop(childPid)).toBe(true) + for (let attempt = 0; attempt < 500; attempt += 1) { + if ( + ( + await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), + ) + ).every(exists => !exists) + ) { + break + } + await new Promise(resolve => setTimeout(resolve, 10)) + } + expect( + await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), + ), + ).toEqual([false, false, false, false]) + expect( + (await readdir(join(sessionsRoot, 'terminal'))).filter(file => + file.startsWith(`${id}~`), + ), + ).toEqual([]) + } finally { + await writeFile(releasePath, 'release').catch(() => {}) + if (launcher.exitCode === null && launcher.signalCode === null) { + launcher.kill('SIGKILL') + await launcherClose.catch(() => {}) + } + if (childPid !== undefined && isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGTERM') + } catch {} + if (!(await waitForProcessStop(childPid))) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + } + } + } + + for (const mode of ['controlled', 'controlled-exit'] as const) { + it( + `reclaims a short-lived zero-retention ${mode} launch without a recurring pass`, + async () => runZeroRetentionFinalizerScenario(mode), + 30_000, + ) + } + + it( + 'reclaims zero-retention artifacts after a built provider-validation exit', + async () => { + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const homeDir = join(configDir, 'home') + const cacheDir = join(configDir, 'cache') + await mkdir(homeDir, { recursive: true }) + await mkdir(cacheDir, { recursive: true }) + const name = 'node-provider-exit' + const childEnv: NodeJS.ProcessEnv = { + ...process.env, + HOME: homeDir, + XDG_CACHE_HOME: cacheDir, + OPENCLAUDE_CONFIG_DIR: configDir, + } + for (const key of [ + 'ANTHROPIC_API_KEY', + 'OPENAI_API_KEY', + 'OPENAI_API_KEYS', + 'GEMINI_API_KEY', + ]) { + delete childEnv[key] + } + delete childEnv.OPENCLAUDE_DISABLE_CLI_ENTRYPOINT_AUTO_RUN + const launcher = spawn( + 'node', + [ + installedLauncherPath, + '--bg', + '--name', + name, + '--provider', + 'openai', + '--print', + 'noop', + ], + { + cwd: configDir, + env: childEnv, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + let stdout = '' + let stderr = '' + launcher.stdout?.setEncoding('utf8') + launcher.stderr?.setEncoding('utf8') + launcher.stdout?.on('data', chunk => { + stdout += chunk + }) + launcher.stderr?.on('data', chunk => { + stderr += chunk + }) + const [launcherCode] = (await once(launcher, 'close')) as [number] + expect(launcherCode).toBe(0) + expect(stderr).toBe('') + const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] + const id = stdoutLogPath + ? basename(stdoutLogPath, '.out.log') + : undefined + const childPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + expect(id).toBeDefined() + expect(Number.isSafeInteger(childPid)).toBe(true) + if (!id || !Number.isSafeInteger(childPid)) { + throw new Error('built launch output omitted its session identity') + } + try { + expect(await waitForProcessStop(childPid)).toBe(true) + + const reservationDigest = createHash('sha256') + .update(name) + .digest('hex') + const artifacts = [ + join(sessionsRoot, 'sessions', `${id}.json`), + join(sessionsRoot, 'logs', `${id}.out.log`), + join(sessionsRoot, 'logs', `${id}.err.log`), + join(sessionsRoot, 'names', `${reservationDigest}.json`), + ] + for (let attempt = 0; attempt < 500; attempt += 1) { + if ( + ( + await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), + ) + ).every(exists => !exists) + ) { + break + } + await new Promise(resolve => setTimeout(resolve, 10)) + } + expect( + await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), + ), + ).toEqual([false, false, false, false]) + expect( + (await readdir(join(sessionsRoot, 'terminal'))).filter(file => + file.startsWith(`${id}~`), + ), + ).toEqual([]) + } finally { + if (isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + } + }, + 30_000, + ) + it.skipIf(process.platform === 'win32')( 'recognizes a live built marked child through the production command probe', async () => { diff --git a/src/cli/bgFinalizer.ts b/src/cli/bgFinalizer.ts index b7c887fb14..57e3288766 100644 --- a/src/cli/bgFinalizer.ts +++ b/src/cli/bgFinalizer.ts @@ -1,3 +1,4 @@ +import { spawn } from 'node:child_process' import { isProcessRunning } from '../utils/genericProcessUtils.js' import { logForDebugging } from '../utils/debug.js' import { registerCleanup } from '../utils/cleanupRegistry.js' @@ -12,11 +13,15 @@ import { type BackgroundSession, } from './bgRegistry.js' import { + BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV, + BACKGROUND_SESSION_CLEANUP_WORKER_ENV, BACKGROUND_SESSION_ID_ENV, BACKGROUND_SESSION_LAUNCHER_PID_ENV, } from './bgRouting.js' export { + BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV, + BACKGROUND_SESSION_CLEANUP_WORKER_ENV, BACKGROUND_SESSION_ID_ENV, BACKGROUND_SESSION_LAUNCHER_PID_ENV, } from './bgRouting.js' @@ -24,6 +29,8 @@ export { const SAFE_ID_RE = /^[A-Za-z0-9._-]+$/ const DEFAULT_REGISTRATION_WAIT_MS = 5_000 const DEFAULT_REGISTRATION_POLL_MS = 10 +const DEFAULT_CLEANUP_WORKER_WAIT_MS = 10_000 +const DEFAULT_CLEANUP_WORKER_POLL_MS = 25 type PrepareBackgroundSessionFinalizerOptions = { env?: NodeJS.ProcessEnv @@ -38,6 +45,7 @@ type PrepareBackgroundSessionFinalizerOptions = { onExit?: (listener: (code: number) => void) => void finalize?: typeof recordBackgroundSessionNaturalTermination finalizeSync?: typeof recordBackgroundSessionNaturalTerminationSync + startCleanupWorker?: (ownerPid: number, launcherPid: number) => void getObservedSignal?: () => ObservedBackgroundSessionSignal | undefined debug?: (message: string) => void } @@ -63,8 +71,9 @@ function defaultDebug(message: string): void { logForDebugging(message, { level: 'error' }) } -function currentProcessExitCode(): number { - const value = process.exitCode +function normalizeProcessExitCode( + value: string | number | null | undefined, +): number { if (value === undefined) return 0 const parsed = typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : value @@ -75,6 +84,10 @@ function currentProcessExitCode(): number { : 1 } +function currentProcessExitCode(): number { + return normalizeProcessExitCode(process.exitCode) +} + function reportFinalizationFailure( debug: (message: string) => void, error: unknown, @@ -88,6 +101,19 @@ function reportFinalizationFailure( } } +function reportPostFinalizationCleanupFailure( + debug: (message: string) => void, + error: unknown, +): void { + try { + debug( + `Background session post-finalization cleanup failed (${boundedFailureKind(error)})`, + ) + } catch { + // Diagnostics must never replace the child process's original outcome. + } +} + function parsePositivePid(value: string | undefined): number | undefined { if (!value || !/^\d+$/.test(value)) return undefined const parsed = Number(value) @@ -142,6 +168,100 @@ async function waitForOwnedSession( return 'timeout' } +type BackgroundSessionCleanupWorkerStartOptions = { + ownerPid: number + launcherPid: number + env?: NodeJS.ProcessEnv + execPath?: string + execArgv?: string[] + entrypoint?: string + spawnProcess?: typeof spawn +} + +export function startBackgroundSessionCleanupWorker( + options: BackgroundSessionCleanupWorkerStartOptions, +): void { + const entrypoint = options.entrypoint ?? process.argv[1] + if (!entrypoint) { + throw new Error('Background cleanup worker entrypoint is unavailable') + } + const workerEnv = { ...(options.env ?? process.env) } + delete workerEnv[BACKGROUND_SESSION_ID_ENV] + workerEnv[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] = '1' + workerEnv[BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV] = String( + options.ownerPid, + ) + workerEnv[BACKGROUND_SESSION_LAUNCHER_PID_ENV] = String(options.launcherPid) + const child = (options.spawnProcess ?? spawn)( + options.execPath ?? process.execPath, + [...(options.execArgv ?? process.execArgv), entrypoint], + { + detached: true, + env: workerEnv, + stdio: 'ignore', + }, + ) + child.once('error', () => {}) + child.unref() +} + +type RunBackgroundSessionCleanupWorkerOptions = { + env?: NodeJS.ProcessEnv + isProcessAlive?: (pid: number) => boolean + sleep?: (ms: number) => Promise + waitMs?: number + pollMs?: number + cleanup?: ( + waitForProcessesToExit: () => Promise, + ) => Promise +} + +async function runDefaultPostFinalizationCleanup( + waitForProcessesToExit: () => Promise, +): Promise { + const { cleanupBackgroundSessionsAfterFinalization } = await import( + '../utils/cleanup.js' + ) + await cleanupBackgroundSessionsAfterFinalization(waitForProcessesToExit) +} + +export async function runBackgroundSessionCleanupWorker( + options: RunBackgroundSessionCleanupWorkerOptions = {}, +): Promise { + const env = options.env ?? process.env + if (env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] !== '1') return + const ownerPid = parsePositivePid( + env[BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV], + ) + const launcherPid = parsePositivePid( + env[BACKGROUND_SESSION_LAUNCHER_PID_ENV], + ) + delete env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] + delete env[BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV] + delete env[BACKGROUND_SESSION_LAUNCHER_PID_ENV] + delete env[BACKGROUND_SESSION_ID_ENV] + if (ownerPid === undefined || launcherPid === undefined) return + + const isProcessAlive = options.isProcessAlive ?? isBackgroundLauncherAlive + const sleep = + options.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))) + const waitMs = options.waitMs ?? DEFAULT_CLEANUP_WORKER_WAIT_MS + const pollMs = options.pollMs ?? DEFAULT_CLEANUP_WORKER_POLL_MS + const pids = [...new Set([ownerPid, launcherPid])] + const waitForProcessesToExit = async (): Promise => { + const attempts = Math.max(1, Math.ceil(waitMs / pollMs)) + for (let attempt = 0; attempt < attempts; attempt += 1) { + if (pids.every(pid => !isProcessAlive(pid))) return true + await sleep(pollMs) + } + return pids.every(pid => !isProcessAlive(pid)) + } + + await (options.cleanup ?? runDefaultPostFinalizationCleanup)( + waitForProcessesToExit, + ) +} + export async function prepareBackgroundSessionFinalizer( options: PrepareBackgroundSessionFinalizerOptions = {}, ): Promise { @@ -187,8 +307,23 @@ export async function prepareBackgroundSessionFinalizer( const getObservedSignal = options.getObservedSignal ?? beginBackgroundSessionSignalTracking() const debug = options.debug ?? defaultDebug + const startCleanupWorker = + options.startCleanupWorker ?? + ((cleanupOwnerPid, cleanupLauncherPid) => + startBackgroundSessionCleanupWorker({ + ownerPid: cleanupOwnerPid, + launcherPid: cleanupLauncherPid, + })) let finalized = false + const startPostFinalizationCleanup = () => { + try { + startCleanupWorker(ownerPid, launcherPid) + } catch (error) { + reportPostFinalizationCleanupFailure(debug, error) + } + } + const currentTermination = () => { const signal = getObservedSignal() return signal === undefined @@ -206,7 +341,9 @@ export async function prepareBackgroundSessionFinalizer( finalized = true } catch (error) { reportFinalizationFailure(debug, error) + return } + startPostFinalizationCleanup() } const registerFinalizerCleanup = options.registerCleanup ?? registerCleanup registerFinalizerCleanup(finalizeAwaited) @@ -229,11 +366,19 @@ export async function prepareBackgroundSessionFinalizer( finalized = true } catch (error) { reportFinalizationFailure(debug, error) + return } + startPostFinalizationCleanup() } if (options.onExit) { options.onExit(onExit) } else { + const originalExit = process.exit + process.exit = ((code?: string | number | null) => { + onExit(normalizeProcessExitCode(code ?? process.exitCode)) + return originalExit(code) + }) as typeof process.exit + process.once('uncaughtExceptionMonitor', () => onExit(1)) process.once('exit', onExit) } diff --git a/src/cli/bgRegistry.test.ts b/src/cli/bgRegistry.test.ts index fa6254e7fd..19dce0dc31 100644 --- a/src/cli/bgRegistry.test.ts +++ b/src/cli/bgRegistry.test.ts @@ -1563,13 +1563,17 @@ describe('background session registry', () => { ) const release = await lockfile.lock(metadataPath, { realpath: false }) try { - expect(() => + let error: unknown + try { recordBackgroundSessionNaturalTerminationSync( id, { exitCode: 0 }, { ownerPid: session.pid, expectedSession: session }, - ), - ).toThrow('Lock file is already being held') + ) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ code: 'ELOCKED' }) } finally { await release() } diff --git a/src/cli/bgRouting.ts b/src/cli/bgRouting.ts index 53b5e1e5bc..4000f02cab 100644 --- a/src/cli/bgRouting.ts +++ b/src/cli/bgRouting.ts @@ -4,6 +4,10 @@ export const BACKGROUND_SESSION_ID_ENV = 'OPENCLAUDE_INTERNAL_BACKGROUND_SESSION_ID' export const BACKGROUND_SESSION_LAUNCHER_PID_ENV = 'OPENCLAUDE_INTERNAL_BACKGROUND_LAUNCHER_PID' +export const BACKGROUND_SESSION_CLEANUP_WORKER_ENV = + 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_WORKER' +export const BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV = + 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_OWNER_PID' export const BACKGROUND_PROCESS_MARKER_FLAG = '--openclaude-bg-session-marker' diff --git a/src/entrypoints/cli.test.ts b/src/entrypoints/cli.test.ts index 34d00b61b8..4269c54473 100644 --- a/src/entrypoints/cli.test.ts +++ b/src/entrypoints/cli.test.ts @@ -18,6 +18,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { Command } from '@commander-js/extra-typings' import { + BACKGROUND_SESSION_CLEANUP_WORKER_ENV, BACKGROUND_SESSION_ID_ENV, BACKGROUND_SESSION_LAUNCHER_PID_ENV, } from '../cli/bgRouting.js' @@ -43,6 +44,7 @@ const mockAttachHandler = mock(async (_args: string[]) => {}) const mockKillHandler = mock(async (_args: string[]) => {}) const mockHandleBgFlag = mock(async (_args: string[]) => {}) const mockPrepareBackgroundSessionFinalizer = mock(async () => 'installed') +const mockRunBackgroundSessionCleanupWorker = mock(async () => {}) const mockLoadEnvFile = mock((_filePath: string) => ({})) const mockParseProviderEnvFileArgs = mock((_args: string[]) => ({ paths: [] })) const mockReapplyRememberedEnvFileValues = mock(() => {}) @@ -81,6 +83,7 @@ const runtimeMocks = [ mockKillHandler, mockHandleBgFlag, mockPrepareBackgroundSessionFinalizer, + mockRunBackgroundSessionCleanupWorker, mockLoadEnvFile, mockParseProviderEnvFileArgs, mockReapplyRememberedEnvFileValues, @@ -298,7 +301,7 @@ describe('cli.tsx — --provider startup ordering', () => { it('dispatches background session management before config and provider validation', async () => { const src = await Bun.file(`${import.meta.dir}/cli.tsx`).text() const bgManagementIndex = src.indexOf("args[0] === 'ps'") - const configEnableIndex = src.indexOf('enableConfigs()') + const configEnableIndex = src.indexOf('enableConfigs()', bgManagementIndex) const providerValidationIndex = src.indexOf( 'await validateProviderEnvForStartupOrExit()', ) @@ -340,6 +343,7 @@ const mockImporters = { }), bgFinalizer: async () => ({ prepareBackgroundSessionFinalizer: mockPrepareBackgroundSessionFinalizer, + runBackgroundSessionCleanupWorker: mockRunBackgroundSessionCleanupWorker, }), envFile: async () => ({ loadEnvFile: mockLoadEnvFile, @@ -462,6 +466,20 @@ describe('cli.tsx — background routing behavior', () => { expect(mockEnableConfigs).not.toHaveBeenCalled() }) + it('runs an internal cleanup worker before any command path', async () => { + process.env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] = '1' + try { + await runCliEntrypoint(['ps'], bgOptions) + } finally { + delete process.env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] + } + + expect(mockRunBackgroundSessionCleanupWorker).toHaveBeenCalledTimes(1) + expect(mockPrepareBackgroundSessionFinalizer).not.toHaveBeenCalled() + expect(mockPsHandler).not.toHaveBeenCalled() + expect(mockEnableConfigs).toHaveBeenCalledTimes(1) + }) + it('routes partial background metadata through the finalizer before dispatch', async () => { process.env[BACKGROUND_SESSION_LAUNCHER_PID_ENV] = '123' try { diff --git a/src/entrypoints/cli.tsx b/src/entrypoints/cli.tsx index 76b27d7610..76df5f828d 100644 --- a/src/entrypoints/cli.tsx +++ b/src/entrypoints/cli.tsx @@ -1,5 +1,6 @@ import { feature } from 'bun:bundle'; import { + BACKGROUND_SESSION_CLEANUP_WORKER_ENV, BACKGROUND_SESSION_ID_ENV, BACKGROUND_SESSION_LAUNCHER_PID_ENV, } from '../cli/bgRouting.js' @@ -319,6 +320,13 @@ export async function main( options: CliEntrypointOptions = {}, ): Promise { const importers = getCliEntrypointImporters(options.importers) + if (process.env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] === '1') { + const { enableConfigs } = await importers.config() + enableConfigs() + const { runBackgroundSessionCleanupWorker } = await importers.bgFinalizer() + await runBackgroundSessionCleanupWorker() + return + } // The detached CLI is the registered background-session PID. Establish // exact registry ownership and install its terminal finalizer before any // fast path or startup validation can call process.exit(). The private env diff --git a/src/utils/cleanup.test.ts b/src/utils/cleanup.test.ts index 47732ec259..631658d29d 100644 --- a/src/utils/cleanup.test.ts +++ b/src/utils/cleanup.test.ts @@ -22,7 +22,13 @@ afterEach(async () => { async function runCleanupFixture( configDir: string, - mode: 'once' | 'post-completion' = 'once', + mode: + | 'once' + | 'post-completion' + | 'post-finalization-gate' + | 'post-finalization-exact-cutoff' + | 'post-finalization-timeout' + | 'explicit-kill' = 'once', ): Promise> { const fixture = join(import.meta.dir, 'cleanupBackgroundSessions.fixture.ts') const { USER_TYPE: _userType, ...inheritedEnv } = process.env @@ -168,6 +174,109 @@ describe('cleanupOldMessageFilesInBackground', () => { 30_000, ) + test( + 'includes the finalization millisecond in zero-day cleanup', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-exact-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect( + await runCleanupFixture( + configDir, + 'post-finalization-exact-cutoff', + ), + ).toEqual({ + waits: 1, + metadataPresent: false, + reservationPresent: false, + stdoutPresent: false, + stderrPresent: false, + }) + }, + 30_000, + ) + + test( + 'retains zero-day artifacts when worker ownership exit is unobserved', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-timeout-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect( + await runCleanupFixture(configDir, 'post-finalization-timeout'), + ).toEqual({ + waits: 1, + metadataPresent: true, + reservationPresent: false, + stdoutPresent: true, + stderrPresent: true, + }) + }, + 30_000, + ) + + test( + 'reclaims zero-day artifacts after an explicit kill transition', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-kill-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect(await runCleanupFixture(configDir, 'explicit-kill')).toEqual({ + metadataPresent: false, + stdoutPresent: false, + stderrPresent: false, + }) + }, + 30_000, + ) + + for (const setting of [30, 'invalid'] as const) { + test( + `does not wait for post-finalization cleanup with ${String(setting)} retention`, + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-gate-${String(setting)}-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: setting }), + ) + + expect( + await runCleanupFixture(configDir, 'post-finalization-gate'), + ).toEqual({ waits: 0 }) + }, + 30_000, + ) + } + test( 'reclaims a zero-day session that completes after the initial sweep', async () => { diff --git a/src/utils/cleanup.ts b/src/utils/cleanup.ts index d0e0e62041..02da4498dd 100644 --- a/src/utils/cleanup.ts +++ b/src/utils/cleanup.ts @@ -637,6 +637,10 @@ export async function cleanupBackgroundSessionsInBackground(): Promise { if (cleanupPeriodSettingIsInvalid()) { return } + await cleanupBackgroundSessionsAtCutoff(getCutoffDate()) +} + +async function cleanupBackgroundSessionsAtCutoff(cutoff: Date): Promise { const reconciliation = await reconcileBackgroundSessionTerminalFacts() if (reconciliation.errors > 0) { logForDebugging( @@ -644,7 +648,7 @@ export async function cleanupBackgroundSessionsInBackground(): Promise { ) } const backgroundResult = await cleanupBackgroundSessionsBefore( - getCutoffDate(), + cutoff, ) if ( backgroundResult.sessionsRemoved > 0 || @@ -656,3 +660,17 @@ export async function cleanupBackgroundSessionsInBackground(): Promise { ) } } + +export async function cleanupBackgroundSessionsAfterFinalization( + waitForProcessesToExit: () => Promise, +): Promise { + if (cleanupPeriodSettingIsInvalid()) return + const cleanupPeriodDays = + getSettings_DEPRECATED()?.cleanupPeriodDays ?? DEFAULT_CLEANUP_PERIOD_DAYS + if (cleanupPeriodDays !== 0) return + + if (!(await waitForProcessesToExit())) return + if (cleanupPeriodSettingIsInvalid()) return + if (getSettings_DEPRECATED()?.cleanupPeriodDays !== 0) return + await cleanupBackgroundSessionsAtCutoff(new Date(Date.now() + 1)) +} diff --git a/src/utils/cleanupBackgroundSessions.fixture.ts b/src/utils/cleanupBackgroundSessions.fixture.ts index 172aed97b6..d88fa1188f 100644 --- a/src/utils/cleanupBackgroundSessions.fixture.ts +++ b/src/utils/cleanupBackgroundSessions.fixture.ts @@ -5,11 +5,16 @@ import { createBackgroundSession, recordBackgroundSessionNaturalTerminationSync, } from '../cli/bgRegistry.js' +import { killHandler } from '../cli/bg.js' import { startBackgroundHousekeeping } from './backgroundHousekeeping.js' -import { cleanupOldMessageFilesInBackground } from './cleanup.js' +import { + cleanupBackgroundSessionsAfterFinalization, + cleanupOldMessageFilesInBackground, +} from './cleanup.js' try { - if (process.env.OPENCLAUDE_CLEANUP_FIXTURE_MODE === 'post-completion') { + const mode = process.env.OPENCLAUDE_CLEANUP_FIXTURE_MODE + if (mode === 'post-completion') { const processMarker = 'c'.repeat(64) const session = await createBackgroundSession({ id: 'bg-post-completion', @@ -96,6 +101,104 @@ try { ).exists(), })}\n`, ) + } else if (mode === 'post-finalization-gate') { + let waits = 0 + await cleanupBackgroundSessionsAfterFinalization(async () => { + waits++ + return true + }) + process.stdout.write(`${JSON.stringify({ waits })}\n`) + } else if ( + mode === 'post-finalization-exact-cutoff' || + mode === 'post-finalization-timeout' + ) { + const exactNow = Date.now() + const processMarker = 'd'.repeat(64) + const session = await createBackgroundSession({ + id: 'bg-post-finalization-exact', + name: 'post-finalization-exact', + pid: process.pid, + cwd: process.cwd(), + command: [ + 'openclaude', + `--openclaude-bg-session-marker=${processMarker}`, + '--print', + 'fixture', + ], + sessionId: 'bg-post-finalization-exact-conversation', + processMarker, + now: new Date(exactNow - 1_000), + }) + const root = dirname(dirname(session.stdoutLogPath)) + const metadataPath = join(root, 'sessions', `${session.id}.json`) + const reservationPath = join( + root, + 'names', + `${createHash('sha256').update(session.name!).digest('hex')}.json`, + ) + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { + ownerPid: process.pid, + expectedSession: session, + now: new Date(exactNow), + }, + ) + const originalDateNow = Date.now + let waits = 0 + try { + Date.now = () => exactNow + await cleanupBackgroundSessionsAfterFinalization(async () => { + waits++ + return mode === 'post-finalization-exact-cutoff' + }) + } finally { + Date.now = originalDateNow + } + process.stdout.write( + `${JSON.stringify({ + waits, + metadataPresent: await Bun.file(metadataPath).exists(), + reservationPresent: await Bun.file(reservationPath).exists(), + stdoutPresent: await Bun.file(session.stdoutLogPath).exists(), + stderrPresent: await Bun.file(session.stderrLogPath).exists(), + })}\n`, + ) + } else if (mode === 'explicit-kill') { + const session = await createBackgroundSession({ + id: 'bg-explicit-kill-cleanup', + pid: process.pid, + cwd: process.cwd(), + command: ['openclaude', '--print', 'fixture'], + sessionId: 'bg-explicit-kill-cleanup-conversation', + }) + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { + ownerPid: process.pid, + expectedSession: session, + now: new Date(Date.now() - 1_000), + }, + ) + const originalConsoleLog = console.log + try { + console.log = () => {} + await killHandler([session.id]) + } finally { + console.log = originalConsoleLog + } + const root = dirname(dirname(session.stdoutLogPath)) + process.stdout.write( + `${JSON.stringify({ + metadataPresent: await Bun.file( + join(root, 'sessions', `${session.id}.json`), + ).exists(), + stdoutPresent: await Bun.file(session.stdoutLogPath).exists(), + stderrPresent: await Bun.file(session.stderrLogPath).exists(), + })}\n`, + ) } else { await cleanupOldMessageFilesInBackground() process.stdout.write(`${JSON.stringify({ completed: true })}\n`) From 24833730082fd9375e3bcfcd20d2a52c130923d9 Mon Sep 17 00:00:00 2001 From: chioarub Date: Tue, 1 Sep 2026 09:16:42 +0300 Subject: [PATCH 09/13] fix(bg): preserve handled exception outcomes --- src/cli/bgFinalizer.fixture.ts | 23 ++++- src/cli/bgFinalizer.test.ts | 159 ++++++++++++++++++++------------- src/cli/bgFinalizer.ts | 1 - 3 files changed, 118 insertions(+), 65 deletions(-) diff --git a/src/cli/bgFinalizer.fixture.ts b/src/cli/bgFinalizer.fixture.ts index 999d568832..5c56e6b0eb 100644 --- a/src/cli/bgFinalizer.fixture.ts +++ b/src/cli/bgFinalizer.fixture.ts @@ -45,7 +45,28 @@ if (process.env[cleanupWorkerEnv] === '1') { }) } const readyPath = process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_READY - if (readyPath) await writeFile(readyPath, 'ready') + if (mode === 'handled-throw') { + const releasePath = process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_RELEASE + if (!readyPath || !releasePath) { + throw new Error('handled-throw fixture routing is missing') + } + process.once('uncaughtException', async () => { + await writeFile(readyPath, 'ready') + const deadline = Date.now() + 5_000 + while (!(await Bun.file(releasePath).exists())) { + if (Date.now() >= deadline) { + process.exit(24) + } + await new Promise(resolve => setTimeout(resolve, 10)) + } + process.exit(23) + }) + queueMicrotask(() => { + throw new Error('intentional handled fixture failure') + }) + } else if (readyPath) { + await writeFile(readyPath, 'ready') + } if (mode === 'controlled' || mode === 'controlled-exit') { const releasePath = process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_RELEASE if (!releasePath) { diff --git a/src/cli/bgFinalizer.test.ts b/src/cli/bgFinalizer.test.ts index 96c0335dd4..6e8656c490 100644 --- a/src/cli/bgFinalizer.test.ts +++ b/src/cli/bgFinalizer.test.ts @@ -457,7 +457,15 @@ describe('background session finalizer', () => { }) async function runFixture( - mode: 'success' | 'fail' | 'throw' | 'wait' | 'sigint' | 'sigterm', + mode: + | 'success' + | 'fail' + | 'throw' + | 'handled-throw' + | 'wait' + | 'sigint' + | 'sigterm', + releasePath?: string, ): Promise<{ id: string child: ReturnType @@ -472,6 +480,9 @@ describe('background session finalizer', () => { [BACKGROUND_SESSION_ID_ENV]: id, [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: String(process.pid), OPENCLAUDE_BG_FINALIZER_FIXTURE_READY: readyPath, + ...(releasePath + ? { OPENCLAUDE_BG_FINALIZER_FIXTURE_RELEASE: releasePath } + : {}), }, stdio: ['ignore', 'ignore', 'ignore'], }) @@ -501,6 +512,40 @@ describe('background session finalizer', () => { return !isProcessRunning(pid) } + function sessionArtifactPaths(id: string, name: string): string[] { + const reservationDigest = createHash('sha256').update(name).digest('hex') + return [ + join(sessionsRoot, 'sessions', `${id}.json`), + join(sessionsRoot, 'logs', `${id}.out.log`), + join(sessionsRoot, 'logs', `${id}.err.log`), + join(sessionsRoot, 'names', `${reservationDigest}.json`), + ] + } + + async function expectSessionArtifactsReclaimed( + id: string, + name: string, + ): Promise { + const artifacts = sessionArtifactPaths(id, name) + for (let attempt = 0; attempt < 500; attempt += 1) { + const present = await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), + ) + if (present.every(exists => !exists)) break + await new Promise(resolve => setTimeout(resolve, 10)) + } + expect( + await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), + ), + ).toEqual([false, false, false, false]) + expect( + (await readdir(join(sessionsRoot, 'terminal'))).filter(file => + file.startsWith(`${id}~`), + ), + ).toEqual([]) + } + async function runBuiltCliSession( id: string, args: string[], @@ -600,6 +645,53 @@ describe('background session finalizer', () => { }) } + it( + 'keeps a handled uncaught exception live until the actual exit', + async () => { + const releasePath = join(configDir, 'handled-throw.release') + const { id, child, readyPath } = await runFixture( + 'handled-throw', + releasePath, + ) + const childExit = once(child, 'exit') as Promise< + [number | null, NodeJS.Signals | null] + > + const forceTimer = setTimeout(() => child.kill('SIGKILL'), 10_000) + try { + await waitForFile(readyPath) + expect(child.pid).toBeDefined() + expect(isProcessRunning(child.pid!)).toBe(true) + expect( + (await listBackgroundSessions()).find(item => item.id === id), + ).toMatchObject({ + id, + status: 'running', + }) + + await writeFile(releasePath, 'release') + const [code, signal] = await childExit + expect(signal).toBeNull() + expect(code).toBe(23) + expect( + (await listBackgroundSessions()).find(item => item.id === id), + ).toMatchObject({ + id, + status: 'failed', + exitCode: 23, + terminalReason: 'exit_code', + }) + } finally { + clearTimeout(forceTimer) + await writeFile(releasePath, 'release').catch(() => {}) + if (child.exitCode === null && child.signalCode === null) { + child.kill('SIGKILL') + await childExit.catch(() => {}) + } + } + }, + 30_000, + ) + for (const expectation of [ { mode: 'sigint' as const, @@ -719,15 +811,7 @@ describe('background session finalizer', () => { throw new Error('detached launch output omitted its session identity') } childPid = parsedChildPid - const reservationDigest = createHash('sha256') - .update(name) - .digest('hex') - const artifacts = [ - join(sessionsRoot, 'sessions', `${id}.json`), - join(sessionsRoot, 'logs', `${id}.out.log`), - join(sessionsRoot, 'logs', `${id}.err.log`), - join(sessionsRoot, 'names', `${reservationDigest}.json`), - ] + const artifacts = sessionArtifactPaths(id, name) await waitForFile(readyPath) expect(isProcessRunning(childPid)).toBe(true) expect( @@ -738,28 +822,7 @@ describe('background session finalizer', () => { await writeFile(releasePath, 'release') expect(await waitForProcessStop(childPid)).toBe(true) - for (let attempt = 0; attempt < 500; attempt += 1) { - if ( - ( - await Promise.all( - artifacts.map(async path => await Bun.file(path).exists()), - ) - ).every(exists => !exists) - ) { - break - } - await new Promise(resolve => setTimeout(resolve, 10)) - } - expect( - await Promise.all( - artifacts.map(async path => await Bun.file(path).exists()), - ), - ).toEqual([false, false, false, false]) - expect( - (await readdir(join(sessionsRoot, 'terminal'))).filter(file => - file.startsWith(`${id}~`), - ), - ).toEqual([]) + await expectSessionArtifactsReclaimed(id, name) } finally { await writeFile(releasePath, 'release').catch(() => {}) if (launcher.exitCode === null && launcher.signalCode === null) { @@ -858,37 +921,7 @@ describe('background session finalizer', () => { try { expect(await waitForProcessStop(childPid)).toBe(true) - const reservationDigest = createHash('sha256') - .update(name) - .digest('hex') - const artifacts = [ - join(sessionsRoot, 'sessions', `${id}.json`), - join(sessionsRoot, 'logs', `${id}.out.log`), - join(sessionsRoot, 'logs', `${id}.err.log`), - join(sessionsRoot, 'names', `${reservationDigest}.json`), - ] - for (let attempt = 0; attempt < 500; attempt += 1) { - if ( - ( - await Promise.all( - artifacts.map(async path => await Bun.file(path).exists()), - ) - ).every(exists => !exists) - ) { - break - } - await new Promise(resolve => setTimeout(resolve, 10)) - } - expect( - await Promise.all( - artifacts.map(async path => await Bun.file(path).exists()), - ), - ).toEqual([false, false, false, false]) - expect( - (await readdir(join(sessionsRoot, 'terminal'))).filter(file => - file.startsWith(`${id}~`), - ), - ).toEqual([]) + await expectSessionArtifactsReclaimed(id, name) } finally { if (isProcessRunning(childPid)) { try { diff --git a/src/cli/bgFinalizer.ts b/src/cli/bgFinalizer.ts index 57e3288766..96fa3e4cbd 100644 --- a/src/cli/bgFinalizer.ts +++ b/src/cli/bgFinalizer.ts @@ -378,7 +378,6 @@ export async function prepareBackgroundSessionFinalizer( onExit(normalizeProcessExitCode(code ?? process.exitCode)) return originalExit(code) }) as typeof process.exit - process.once('uncaughtExceptionMonitor', () => onExit(1)) process.once('exit', onExit) } From 8b950062f2aa65ad04d46b220389632503a03301 Mon Sep 17 00:00:00 2001 From: chioarub Date: Tue, 1 Sep 2026 23:25:32 +0300 Subject: [PATCH 10/13] fix(bg): unify retention lifecycle cleanup --- src/cli/bg.ts | 20 +- src/cli/bgFinalizer.test.ts | 583 +++++++++++++++--- src/cli/bgFinalizer.ts | 128 +++- src/cli/bgRegistry.cleanup.test.ts | 188 ++++++ src/cli/bgRegistry.test.ts | 61 ++ src/cli/bgRegistry.ts | 333 ++++++---- src/entrypoints/cli.test.ts | 82 ++- src/entrypoints/cli.tsx | 21 +- src/utils/backgroundHousekeeping.ts | 8 +- src/utils/cleanup.test.ts | 321 ++++++++-- src/utils/cleanup.ts | 264 ++++++-- .../cleanupBackgroundSessions.fixture.ts | 230 ++++--- 12 files changed, 1825 insertions(+), 414 deletions(-) diff --git a/src/cli/bg.ts b/src/cli/bg.ts index d8bf64db85..02321b27bf 100644 --- a/src/cli/bg.ts +++ b/src/cli/bg.ts @@ -1043,6 +1043,7 @@ export async function attachHandler( export async function killHandler( args: string[] | string | undefined, + options: { retentionSettingsReady?: boolean } = {}, ): Promise { const target = normalizeArgs(args)[0] if (!target) fail('Usage: openclaude kill ') @@ -1054,13 +1055,18 @@ export async function killHandler( `Failed to kill background session ${session.id}: ${errorMessage(error)}`, ) }) - try { - const { cleanupBackgroundSessionsAfterFinalization } = await import( - '../utils/cleanup.js' - ) - await cleanupBackgroundSessionsAfterFinalization(async () => true) - } catch { - // A retention failure must not replace the successful kill outcome. + if (options.retentionSettingsReady !== false) { + try { + const { runBackgroundSessionRetention } = await import( + '../utils/cleanup.js' + ) + await runBackgroundSessionRetention({ + trigger: 'explicit-kill', + sessionId: killed.id, + }) + } catch { + // A retention failure must not replace the successful kill outcome. + } } console.log(`Killed background session ${killed.id}.`) } diff --git a/src/cli/bgFinalizer.test.ts b/src/cli/bgFinalizer.test.ts index 6e8656c490..a89125aa34 100644 --- a/src/cli/bgFinalizer.test.ts +++ b/src/cli/bgFinalizer.test.ts @@ -10,6 +10,7 @@ import { BACKGROUND_SESSION_ID_ENV, BACKGROUND_SESSION_LAUNCHER_PID_ENV, prepareBackgroundSessionFinalizer, + startBackgroundSessionCleanupWorker, } from './bgFinalizer.js' import { buildBackgroundChildProcessConfig } from './bg.js' import { @@ -31,6 +32,7 @@ const BACKGROUND_SESSION_CLEANUP_WORKER_ENV = 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_WORKER' const BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV = 'OPENCLAUDE_INTERNAL_BACKGROUND_CLEANUP_OWNER_PID' +const FIXTURE_OUTPUT_LIMIT_BYTES = 64 * 1024 describe('background session finalizer', () => { let configDir: string @@ -241,6 +243,7 @@ describe('background session finalizer', () => { it('starts a detached cleanup owner after awaited finalization', async () => { let beforeExitListener: (() => void | Promise) | undefined const order: string[] = [] + let workerSettingsArgs: string[] | undefined await prepareBackgroundSessionFinalizer({ env: { @@ -258,13 +261,72 @@ describe('background session finalizer', () => { order.push('finalize') return ownedSession('bg-launcher-handoff', 500) }, - startCleanupWorker: (ownerPid, launcherPid) => { - order.push(`worker:${ownerPid}:${launcherPid}`) + settingsArgs: ['--settings', 'retention.json'], + settingsCwd: configDir, + startCleanupWorker: ( + sessionId, + ownerPid, + launcherPid, + settingsArgs, + ) => { + order.push(`worker:${sessionId}:${ownerPid}:${launcherPid}`) + workerSettingsArgs = settingsArgs }, }) await beforeExitListener?.() - expect(order).toEqual(['finalize', 'worker:500:123']) + expect(order).toEqual([ + 'finalize', + 'worker:bg-launcher-handoff:500:123', + ]) + expect(workerSettingsArgs).toEqual([ + '--settings', + join(configDir, 'retention.json'), + ]) + }) + + it('passes only settings-source inputs to the detached cleanup worker', () => { + let spawnedArgs: readonly string[] | undefined + let spawnedEnv: NodeJS.ProcessEnv | undefined + let unrefCalls = 0 + startBackgroundSessionCleanupWorker({ + sessionId: 'bg-worker-settings', + ownerPid: 500, + launcherPid: 123, + execPath: 'node', + execArgv: ['--expose-gc'], + entrypoint: '/openclaude/cli.mjs', + settingsArgs: [ + '--provider', + 'openai', + '--settings={"cleanupPeriodDays":0,"apiKeyHelper":"private"}', + '--setting-sources', + 'project,user', + ], + spawnProcess: ((_command, args, options) => { + spawnedArgs = args + spawnedEnv = options.env + return { + once: () => {}, + unref: () => { + unrefCalls++ + }, + } + }) as unknown as typeof spawn, + }) + + expect(spawnedArgs).toEqual([ + '--expose-gc', + '/openclaude/cli.mjs', + '--settings', + '{"cleanupPeriodDays":0}', + '--setting-sources', + 'project,user', + ]) + expect(spawnedEnv?.[BACKGROUND_SESSION_ID_ENV]).toBe( + 'bg-worker-settings', + ) + expect(unrefCalls).toBe(1) }) it('retains cleanup when the detached worker cannot observe process exit', async () => { @@ -274,6 +336,7 @@ describe('background session finalizer', () => { const env = { [BACKGROUND_SESSION_CLEANUP_WORKER_ENV]: '1', [BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV]: '500', + [BACKGROUND_SESSION_ID_ENV]: 'bg-worker-timeout', [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: '123', } let observedExit: boolean | undefined @@ -284,7 +347,7 @@ describe('background session finalizer', () => { sleep: async () => {}, waitMs: 2, pollMs: 1, - cleanup: async waitForProcessesToExit => { + cleanup: async (_sessionId, waitForProcessesToExit) => { observedExit = await waitForProcessesToExit() }, }) @@ -304,7 +367,7 @@ describe('background session finalizer', () => { terminalFactGeneration: 'a'.repeat(64), } let syncOwner: BackgroundSession | undefined - const cleanupWorkers: Array<[number, number]> = [] + const cleanupWorkers: Array<[string, number, number]> = [] await prepareBackgroundSessionFinalizer({ env: { @@ -329,15 +392,17 @@ describe('background session finalizer', () => { finalizeSync: (_id, _termination, options) => { syncOwner = options?.expectedSession }, - startCleanupWorker: (ownerPid, launcherPid) => { - cleanupWorkers.push([ownerPid, launcherPid]) + startCleanupWorker: (sessionId, ownerPid, launcherPid) => { + cleanupWorkers.push([sessionId, ownerPid, launcherPid]) }, }) await cleanup?.() exitListener?.(23) expect(syncOwner).toEqual(registeredSession) - expect(cleanupWorkers).toEqual([[registeredSession.pid, 123]]) + expect(cleanupWorkers).toEqual([ + [registeredSession.id, registeredSession.pid, 123], + ]) }) it('keeps the original exit code when both persistence paths fail', async () => { @@ -546,6 +611,174 @@ describe('background session finalizer', () => { ).toEqual([]) } + async function expectSessionArtifactsRetained( + id: string, + name: string, + stabilityMs: number = 1_000, + ): Promise { + const artifacts = sessionArtifactPaths(id, name) + const deadline = Date.now() + stabilityMs + do { + expect( + await Promise.all( + artifacts.map(async path => await Bun.file(path).exists()), + ), + ).toEqual([true, true, true, false]) + await new Promise(resolve => setTimeout(resolve, 20)) + } while (Date.now() < deadline) + } + + async function waitForChildClose( + child: ReturnType, + label: string, + timeoutMs: number = 15_000, + ): Promise { + const close = once(child, 'close') as Promise<[number | null]> + let timedOut = false + let timer: ReturnType | undefined + try { + return await Promise.race([ + close.then(([code]) => code), + new Promise((_, reject) => { + timer = setTimeout(() => { + timedOut = true + child.kill('SIGKILL') + reject(new Error(`${label} exceeded ${timeoutMs}ms`)) + }, timeoutMs) + timer.unref?.() + }), + ]) + } finally { + if (timer !== undefined) clearTimeout(timer) + if (timedOut) { + await Promise.race([ + close.catch(() => [null] as [null]), + new Promise(resolve => setTimeout(resolve, 2_000)), + ]) + } + } + } + + function captureBoundedOutput( + child: ReturnType, + stream: NodeJS.ReadableStream | null, + label: string, + ): { read: () => string; assertWithinLimit: () => void } { + let text = '' + let bytes = 0 + let exceeded = false + stream?.setEncoding('utf8') + stream?.on('data', chunk => { + if (exceeded) return + const value = String(chunk) + bytes += Buffer.byteLength(value) + if (bytes > FIXTURE_OUTPUT_LIMIT_BYTES) { + exceeded = true + child.kill('SIGKILL') + return + } + text += value + }) + return { + read: () => text, + assertWithinLimit: () => { + if (exceeded) { + throw new Error( + `${label} exceeded ${FIXTURE_OUTPUT_LIMIT_BYTES} bytes`, + ) + } + }, + } + } + + async function runBuiltBackgroundProviderExit( + name: string, + retentionArgs: string[] = [], + ): Promise<{ id: string; childPid: number }> { + const homeDir = join(configDir, 'home') + const cacheDir = join(configDir, 'cache') + await mkdir(homeDir, { recursive: true }) + await mkdir(cacheDir, { recursive: true }) + const childEnv: NodeJS.ProcessEnv = { + ...process.env, + HOME: homeDir, + XDG_CACHE_HOME: cacheDir, + OPENCLAUDE_CONFIG_DIR: configDir, + } + for (const key of [ + 'ANTHROPIC_API_KEY', + 'OPENAI_API_KEY', + 'OPENAI_API_KEYS', + 'GEMINI_API_KEY', + 'NODE_ENV', + ]) { + delete childEnv[key] + } + delete childEnv.OPENCLAUDE_DISABLE_CLI_ENTRYPOINT_AUTO_RUN + const launcher = spawn( + 'node', + [ + installedLauncherPath, + '--bg', + '--name', + name, + ...retentionArgs, + '--provider', + 'openai', + '--print', + 'noop', + ], + { + cwd: configDir, + env: childEnv, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + const stdoutCapture = captureBoundedOutput( + launcher, + launcher.stdout, + 'built background launcher stdout', + ) + const stderrCapture = captureBoundedOutput( + launcher, + launcher.stderr, + 'built background launcher stderr', + ) + let launcherCode: number | null + try { + launcherCode = await waitForChildClose( + launcher, + 'built background launcher', + ) + stdoutCapture.assertWithinLimit() + stderrCapture.assertWithinLimit() + } catch (error) { + const stdout = stdoutCapture.read() + const childPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + if (Number.isSafeInteger(childPid) && isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + throw error + } + const stdout = stdoutCapture.read() + const stderr = stderrCapture.read() + expect(launcherCode).toBe(0) + expect(stderr).toBe('') + const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] + const id = stdoutLogPath + ? basename(stdoutLogPath, '.out.log') + : undefined + const childPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + expect(id).toBeDefined() + expect(Number.isSafeInteger(childPid)).toBe(true) + if (!id || !Number.isSafeInteger(childPid)) { + throw new Error('built launch output omitted its session identity') + } + return { id, childPid } + } + async function runBuiltCliSession( id: string, args: string[], @@ -610,22 +843,128 @@ describe('background session finalizer', () => { }, stdio: ['ignore', 'pipe', 'pipe'], }) - let stdout = '' - let stderr = '' - launcher.stdout?.setEncoding('utf8') - launcher.stderr?.setEncoding('utf8') - launcher.stdout?.on('data', chunk => { - stdout += chunk - }) - launcher.stderr?.on('data', chunk => { - stderr += chunk - }) + const stdoutCapture = captureBoundedOutput( + launcher, + launcher.stdout, + 'detached launcher stdout', + ) + const stderrCapture = captureBoundedOutput( + launcher, + launcher.stderr, + 'detached launcher stderr', + ) const [code] = (await once(launcher, 'exit')) as [number] + stdoutCapture.assertWithinLimit() + stderrCapture.assertWithinLimit() + const stdout = stdoutCapture.read() + const stderr = stderrCapture.read() expect(code).toBe(0) expect(stderr).toBe('') return { stdout, session: await waitForTerminalSession() } } + async function runWaitingDetachedLaunch( + name: string, + ): Promise<{ id: string; childPid: number; readyPath: string }> { + const readyPath = join(configDir, `${name}.ready`) + const launcher = spawn( + process.execPath, + [fixturePath, 'launcher', 'wait', name], + { + cwd: configDir, + env: { + ...process.env, + OPENCLAUDE_CONFIG_DIR: configDir, + OPENCLAUDE_BG_FINALIZER_FIXTURE_READY: readyPath, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + const stdoutCapture = captureBoundedOutput( + launcher, + launcher.stdout, + 'detached wait launcher stdout', + ) + const stderrCapture = captureBoundedOutput( + launcher, + launcher.stderr, + 'detached wait launcher stderr', + ) + let launcherCode: number | null + try { + launcherCode = await waitForChildClose( + launcher, + 'detached wait launcher', + ) + stdoutCapture.assertWithinLimit() + stderrCapture.assertWithinLimit() + } catch (error) { + const stdout = stdoutCapture.read() + const childPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + if (Number.isSafeInteger(childPid) && isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + throw error + } + const stdout = stdoutCapture.read() + const stderr = stderrCapture.read() + expect(launcherCode).toBe(0) + expect(stderr).toBe('') + const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] + const id = stdoutLogPath + ? basename(stdoutLogPath, '.out.log') + : undefined + const childPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + expect(id).toBeDefined() + expect(Number.isSafeInteger(childPid)).toBe(true) + if (!id || !Number.isSafeInteger(childPid)) { + throw new Error('detached wait launch omitted its session identity') + } + await waitForFile(readyPath) + return { id, childPid, readyPath } + } + + async function runBuiltKill( + id: string, + settingsArgs: string[] = [], + ): Promise { + const killEnv: NodeJS.ProcessEnv = { + ...process.env, + OPENCLAUDE_CONFIG_DIR: configDir, + } + delete killEnv.NODE_ENV + delete killEnv.OPENCLAUDE_DISABLE_CLI_ENTRYPOINT_AUTO_RUN + const kill = spawn( + 'node', + [installedLauncherPath, 'kill', id, ...settingsArgs], + { + cwd: configDir, + env: killEnv, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + const stdoutCapture = captureBoundedOutput( + kill, + kill.stdout, + 'built kill stdout', + ) + const stderrCapture = captureBoundedOutput( + kill, + kill.stderr, + 'built kill stderr', + ) + const code = await waitForChildClose(kill, 'built kill command') + stdoutCapture.assertWithinLimit() + stderrCapture.assertWithinLimit() + const stdout = stdoutCapture.read() + const stderr = stderrCapture.read() + expect(code, stderr).toBe(0) + expect(stderr).toBe('') + return stdout + } + for (const expectation of [ { mode: 'success' as const, status: 'exited', exitCode: 0 }, { mode: 'fail' as const, status: 'failed', exitCode: 23 }, @@ -645,6 +984,75 @@ describe('background session finalizer', () => { }) } + for (const scenario of [ + { label: 'zero-day', days: 0, reclaimed: true, settingsArgs: [] }, + { label: 'positive', days: 30, reclaimed: false, settingsArgs: [] }, + { + label: 'failed retention bootstrap', + days: 0, + reclaimed: false, + settingsArgs: ['--settings', 'missing-retention-settings.json'], + }, + ]) { + it( + `keeps built explicit kill successful with ${scenario.label} retention`, + async () => { + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: scenario.days }), + ) + const name = `built-kill-${scenario.label.replaceAll(' ', '-')}` + const { id, childPid } = await runWaitingDetachedLaunch(name) + try { + expect(isProcessRunning(childPid)).toBe(true) + expect(await runBuiltKill(id, scenario.settingsArgs)).toContain( + `Killed background session ${id}.`, + ) + expect(await waitForProcessStop(childPid)).toBe(true) + if (scenario.reclaimed) { + await expectSessionArtifactsReclaimed(id, name) + } else { + await expectSessionArtifactsRetained(id, name) + } + } finally { + if (isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + } + }, + 30_000, + ) + } + + it( + 'keeps built kill retention independent of malformed project MCP config', + async () => { + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + await writeFile(join(configDir, '.mcp.json'), '{invalid') + const name = 'built-kill-malformed-mcp' + const { id, childPid } = await runWaitingDetachedLaunch(name) + try { + expect(await runBuiltKill(id)).toContain( + `Killed background session ${id}.`, + ) + expect(await waitForProcessStop(childPid)).toBe(true) + await expectSessionArtifactsReclaimed(id, name) + } finally { + if (isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + } + }, + 30_000, + ) + it( 'keeps a handled uncaught exception live until the actual exit', async () => { @@ -857,67 +1265,8 @@ describe('background session finalizer', () => { join(configDir, 'settings.json'), JSON.stringify({ cleanupPeriodDays: 0 }), ) - const homeDir = join(configDir, 'home') - const cacheDir = join(configDir, 'cache') - await mkdir(homeDir, { recursive: true }) - await mkdir(cacheDir, { recursive: true }) const name = 'node-provider-exit' - const childEnv: NodeJS.ProcessEnv = { - ...process.env, - HOME: homeDir, - XDG_CACHE_HOME: cacheDir, - OPENCLAUDE_CONFIG_DIR: configDir, - } - for (const key of [ - 'ANTHROPIC_API_KEY', - 'OPENAI_API_KEY', - 'OPENAI_API_KEYS', - 'GEMINI_API_KEY', - ]) { - delete childEnv[key] - } - delete childEnv.OPENCLAUDE_DISABLE_CLI_ENTRYPOINT_AUTO_RUN - const launcher = spawn( - 'node', - [ - installedLauncherPath, - '--bg', - '--name', - name, - '--provider', - 'openai', - '--print', - 'noop', - ], - { - cwd: configDir, - env: childEnv, - stdio: ['ignore', 'pipe', 'pipe'], - }, - ) - let stdout = '' - let stderr = '' - launcher.stdout?.setEncoding('utf8') - launcher.stderr?.setEncoding('utf8') - launcher.stdout?.on('data', chunk => { - stdout += chunk - }) - launcher.stderr?.on('data', chunk => { - stderr += chunk - }) - const [launcherCode] = (await once(launcher, 'close')) as [number] - expect(launcherCode).toBe(0) - expect(stderr).toBe('') - const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] - const id = stdoutLogPath - ? basename(stdoutLogPath, '.out.log') - : undefined - const childPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) - expect(id).toBeDefined() - expect(Number.isSafeInteger(childPid)).toBe(true) - if (!id || !Number.isSafeInteger(childPid)) { - throw new Error('built launch output omitted its session identity') - } + const { id, childPid } = await runBuiltBackgroundProviderExit(name) try { expect(await waitForProcessStop(childPid)).toBe(true) @@ -933,6 +1282,84 @@ describe('background session finalizer', () => { 30_000, ) + for (const scenario of [ + { + label: 'settings file retains against global zero-day policy', + globalDays: 0, + prepare: async () => { + const path = join(configDir, 'retain-settings.json') + await writeFile(path, JSON.stringify({ cleanupPeriodDays: 30 })) + return ['--settings', path] + }, + reclaimed: false, + }, + { + label: 'inline settings enable zero-day against global retention', + globalDays: 30, + prepare: async () => [ + '--settings', + JSON.stringify({ cleanupPeriodDays: 0 }), + ], + reclaimed: true, + }, + { + label: 'project-only sources retain against global zero-day policy', + globalDays: 0, + prepare: async () => { + await mkdir(join(configDir, '.openclaude'), { recursive: true }) + await writeFile( + join(configDir, '.openclaude', 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + return ['--setting-sources', 'project'] + }, + reclaimed: false, + }, + { + label: 'project-only sources enable zero-day against global retention', + globalDays: 30, + prepare: async () => { + await mkdir(join(configDir, '.openclaude'), { recursive: true }) + await writeFile( + join(configDir, '.openclaude', 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + return ['--setting-sources', 'project'] + }, + reclaimed: true, + }, + ]) { + it( + `preserves ${scenario.label} across the cleanup-worker handoff`, + async () => { + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: scenario.globalDays }), + ) + const name = `worker-policy-${scenario.globalDays}-${scenario.reclaimed ? 'clean' : 'retain'}` + const { id, childPid } = await runBuiltBackgroundProviderExit( + name, + await scenario.prepare(), + ) + try { + expect(await waitForProcessStop(childPid)).toBe(true) + if (scenario.reclaimed) { + await expectSessionArtifactsReclaimed(id, name) + } else { + await expectSessionArtifactsRetained(id, name) + } + } finally { + if (isProcessRunning(childPid)) { + try { + process.kill(childPid, 'SIGKILL') + } catch {} + } + } + }, + 30_000, + ) + } + it.skipIf(process.platform === 'win32')( 'recognizes a live built marked child through the production command probe', async () => { diff --git a/src/cli/bgFinalizer.ts b/src/cli/bgFinalizer.ts index 96fa3e4cbd..10943ac3a1 100644 --- a/src/cli/bgFinalizer.ts +++ b/src/cli/bgFinalizer.ts @@ -1,4 +1,5 @@ import { spawn } from 'node:child_process' +import { resolve } from 'node:path' import { isProcessRunning } from '../utils/genericProcessUtils.js' import { logForDebugging } from '../utils/debug.js' import { registerCleanup } from '../utils/cleanupRegistry.js' @@ -45,7 +46,14 @@ type PrepareBackgroundSessionFinalizerOptions = { onExit?: (listener: (code: number) => void) => void finalize?: typeof recordBackgroundSessionNaturalTermination finalizeSync?: typeof recordBackgroundSessionNaturalTerminationSync - startCleanupWorker?: (ownerPid: number, launcherPid: number) => void + startCleanupWorker?: ( + sessionId: string, + ownerPid: number, + launcherPid: number, + settingsArgs: string[], + ) => void + settingsArgs?: string[] + settingsCwd?: string getObservedSignal?: () => ObservedBackgroundSessionSignal | undefined debug?: (message: string) => void } @@ -169,15 +177,76 @@ async function waitForOwnedSession( } type BackgroundSessionCleanupWorkerStartOptions = { + sessionId: string ownerPid: number launcherPid: number env?: NodeJS.ProcessEnv execPath?: string execArgv?: string[] entrypoint?: string + settingsArgs?: string[] + settingsCwd?: string spawnProcess?: typeof spawn } +function backgroundRetentionSettingsArgs( + args: string[], + settingsCwd: string, +): string[] { + const selected: string[] = [] + for (const flag of ['--settings', '--setting-sources']) { + const inlinePrefix = `${flag}=` + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] + if (arg?.startsWith(inlinePrefix)) { + const value = backgroundRetentionSettingsValue( + flag, + arg.slice(inlinePrefix.length), + settingsCwd, + ) + if (value !== undefined) selected.push(flag, value) + break + } + if (arg === flag && index + 1 < args.length) { + const value = backgroundRetentionSettingsValue( + flag, + args[index + 1]!, + settingsCwd, + ) + if (value !== undefined) selected.push(flag, value) + break + } + } + } + return selected +} + +function backgroundRetentionSettingsValue( + flag: string, + value: string, + settingsCwd: string, +): string | undefined { + if (flag !== '--settings') return value + const trimmed = value.trim() + if (!trimmed.startsWith('{') || !trimmed.endsWith('}')) { + return resolve(settingsCwd, value) + } + try { + const parsed = JSON.parse(trimmed) as unknown + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + return '{' + } + if (!Object.hasOwn(parsed, 'cleanupPeriodDays')) return undefined + return JSON.stringify({ + cleanupPeriodDays: (parsed as { cleanupPeriodDays?: unknown }) + .cleanupPeriodDays, + }) + } catch { + // Preserve fail-closed parsing without forwarding unrelated inline data. + return '{' + } +} + export function startBackgroundSessionCleanupWorker( options: BackgroundSessionCleanupWorkerStartOptions, ): void { @@ -186,7 +255,10 @@ export function startBackgroundSessionCleanupWorker( throw new Error('Background cleanup worker entrypoint is unavailable') } const workerEnv = { ...(options.env ?? process.env) } - delete workerEnv[BACKGROUND_SESSION_ID_ENV] + if (!SAFE_ID_RE.test(options.sessionId)) { + throw new Error('Invalid background cleanup session ID') + } + workerEnv[BACKGROUND_SESSION_ID_ENV] = options.sessionId workerEnv[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] = '1' workerEnv[BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV] = String( options.ownerPid, @@ -194,7 +266,14 @@ export function startBackgroundSessionCleanupWorker( workerEnv[BACKGROUND_SESSION_LAUNCHER_PID_ENV] = String(options.launcherPid) const child = (options.spawnProcess ?? spawn)( options.execPath ?? process.execPath, - [...(options.execArgv ?? process.execArgv), entrypoint], + [ + ...(options.execArgv ?? process.execArgv), + entrypoint, + ...backgroundRetentionSettingsArgs( + options.settingsArgs ?? process.argv.slice(2), + options.settingsCwd ?? process.cwd(), + ), + ], { detached: true, env: workerEnv, @@ -212,17 +291,26 @@ type RunBackgroundSessionCleanupWorkerOptions = { waitMs?: number pollMs?: number cleanup?: ( + sessionId: string, waitForProcessesToExit: () => Promise, + reloadSettings?: () => boolean, ) => Promise + reloadSettings?: () => boolean } async function runDefaultPostFinalizationCleanup( + sessionId: string, waitForProcessesToExit: () => Promise, + reloadSettings?: () => boolean, ): Promise { const { cleanupBackgroundSessionsAfterFinalization } = await import( '../utils/cleanup.js' ) - await cleanupBackgroundSessionsAfterFinalization(waitForProcessesToExit) + await cleanupBackgroundSessionsAfterFinalization( + sessionId, + waitForProcessesToExit, + reloadSettings, + ) } export async function runBackgroundSessionCleanupWorker( @@ -236,11 +324,19 @@ export async function runBackgroundSessionCleanupWorker( const launcherPid = parsePositivePid( env[BACKGROUND_SESSION_LAUNCHER_PID_ENV], ) + const sessionId = env[BACKGROUND_SESSION_ID_ENV] delete env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] delete env[BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV] delete env[BACKGROUND_SESSION_LAUNCHER_PID_ENV] delete env[BACKGROUND_SESSION_ID_ENV] - if (ownerPid === undefined || launcherPid === undefined) return + if ( + ownerPid === undefined || + launcherPid === undefined || + !sessionId || + !SAFE_ID_RE.test(sessionId) + ) { + return + } const isProcessAlive = options.isProcessAlive ?? isBackgroundLauncherAlive const sleep = @@ -258,7 +354,9 @@ export async function runBackgroundSessionCleanupWorker( } await (options.cleanup ?? runDefaultPostFinalizationCleanup)( + sessionId, waitForProcessesToExit, + options.reloadSettings, ) } @@ -307,18 +405,34 @@ export async function prepareBackgroundSessionFinalizer( const getObservedSignal = options.getObservedSignal ?? beginBackgroundSessionSignalTracking() const debug = options.debug ?? defaultDebug + const retentionSettingsArgs = backgroundRetentionSettingsArgs( + options.settingsArgs ?? process.argv.slice(2), + options.settingsCwd ?? process.cwd(), + ) const startCleanupWorker = options.startCleanupWorker ?? - ((cleanupOwnerPid, cleanupLauncherPid) => + (( + cleanupSessionId, + cleanupOwnerPid, + cleanupLauncherPid, + cleanupSettingsArgs, + ) => startBackgroundSessionCleanupWorker({ + sessionId: cleanupSessionId, ownerPid: cleanupOwnerPid, launcherPid: cleanupLauncherPid, + settingsArgs: cleanupSettingsArgs, })) let finalized = false const startPostFinalizationCleanup = () => { try { - startCleanupWorker(ownerPid, launcherPid) + startCleanupWorker( + id, + ownerPid, + launcherPid, + retentionSettingsArgs, + ) } catch (error) { reportPostFinalizationCleanupFailure(debug, error) } diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts index ac197cdd27..24f2670eee 100644 --- a/src/cli/bgRegistry.cleanup.test.ts +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -4,6 +4,7 @@ import { lstat, mkdir, mkdtemp, + readdir, readFile, rm, symlink, @@ -206,6 +207,47 @@ describe('background session retention cleanup', () => { }) } + it('targets prompt cleanup to the requested session generation', async () => { + const target = await createRunning('bg-targeted-cleanup') + const retained = await createRunning('bg-targeted-retained') + await finishNaturally(target, { exitCode: 0 }) + await finishNaturally(retained, { exitCode: 0 }) + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + sessionIds: [target.id], + }), + ).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + expect(await exists(paths(target.id).metadata)).toBe(false) + expect(await exists(paths(retained.id).metadata)).toBe(true) + }) + + it('bounds prompt cleanup before materializing the metadata directory', async () => { + for (const id of [ + 'bg-bounded-cleanup-a', + 'bg-bounded-cleanup-b', + 'bg-bounded-cleanup-c', + ]) { + const session = await createRunning(id) + await finishNaturally(session, { exitCode: 0 }) + } + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + maxDirectoryEntries: 1, + }), + ).toMatchObject({ sessionsRemoved: 1, errors: 0 }) + expect( + (await readdir(join(root, 'sessions'))).filter(name => + name.endsWith('.json'), + ), + ).toHaveLength(2) + }) + it('removes old explicit-kill artifacts', async () => { const session = await createBackgroundSession({ id: 'bg-killed', @@ -853,6 +895,100 @@ describe('background session retention cleanup', () => { } }) + it('bounds orphan recovery when the metadata directory is absent', async () => { + await mkdir(join(root, 'terminal'), { recursive: true }) + for (const id of [ + 'bg-bounded-orphan-a', + 'bg-bounded-orphan-b', + 'bg-bounded-orphan-c', + ]) { + await writeFile( + paths(id).natural, + JSON.stringify({ + version: 1, + id, + pid: nextPid++, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + } + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + maxDirectoryEntries: 1, + }), + ).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 1, + errors: 0, + }) + expect( + (await readdir(join(root, 'terminal'))).filter(name => + name.endsWith('.json'), + ), + ).toHaveLength(2) + }) + + it('does not broaden targeted cleanup into orphan recovery', async () => { + const orphanId = 'bg-unrelated-targeted-orphan' + await mkdir(join(root, 'terminal'), { recursive: true }) + await writeFile( + paths(orphanId).natural, + JSON.stringify({ + version: 1, + id: orphanId, + pid: nextPid++, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + sessionIds: ['bg-missing-target'], + }), + ).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(orphanId).natural)).toBe(true) + }) + + it('skips orphan recovery after inspecting an existing target', async () => { + const target = await createRunning('bg-existing-target') + const orphanId = 'bg-unrelated-existing-target-orphan' + await writeFile( + paths(orphanId).natural, + JSON.stringify({ + version: 1, + id: orphanId, + pid: nextPid++, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + }), + ) + + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + sessionIds: [target.id], + }), + ).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await exists(paths(target.id).metadata)).toBe(true) + expect(await exists(paths(orphanId).natural)).toBe(true) + }) + it('preserves an orphaned fact whose generation does not match its path', async () => { await mkdir(join(root, 'terminal'), { recursive: true }) const id = 'bg-orphan-generation-mismatch' @@ -1277,6 +1413,58 @@ describe('background session retention cleanup', () => { expect(await exists(paths(session.id).metadata)).toBe(false) }) + it('does not remove logs created by a same-ID replacement', async () => { + const session = await createRunning('bg-log-generation-replacement') + await finishNaturally(session, { exitCode: 0 }) + let hookCalls = 0 + let replacement: BackgroundSession | undefined + + const result = await cleanupBackgroundSessionsBefore(CUTOFF, { + _beforeArtifactRemovalForTesting: async id => { + if (id !== session.id || hookCalls++ > 0) return + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + sessionIds: [session.id], + }), + ).toEqual({ + sessionsRemoved: 1, + artifactsRemoved: 4, + errors: 0, + }) + replacement = await createBackgroundSession({ + id: session.id, + pid: nextPid++, + cwd: configDir, + command: ['openclaude', '--print', 'replacement'], + sessionId: 'replacement-log-conversation', + }) + await writeFile(paths(session.id).stdout, 'replacement stdout') + await writeFile(paths(session.id).stderr, 'replacement stderr') + }, + }) + + expect(hookCalls).toBe(1) + expect(replacement).toBeDefined() + expect(result).toEqual({ + sessionsRemoved: 0, + artifactsRemoved: 0, + errors: 0, + }) + expect(await readFile(paths(session.id).stdout, 'utf8')).toBe( + 'replacement stdout', + ) + expect(await readFile(paths(session.id).stderr, 'utf8')).toBe( + 'replacement stderr', + ) + expect( + JSON.parse(await readFile(paths(session.id).metadata, 'utf8')), + ).toMatchObject({ + id: replacement?.id, + pid: replacement?.pid, + sessionId: replacement?.sessionId, + }) + }) + it( 'serializes cleanup with a replacement using the supported name writer', async () => { diff --git a/src/cli/bgRegistry.test.ts b/src/cli/bgRegistry.test.ts index 19dce0dc31..036147cc90 100644 --- a/src/cli/bgRegistry.test.ts +++ b/src/cli/bgRegistry.test.ts @@ -1546,6 +1546,67 @@ describe('background session registry', () => { expect(await Bun.file(nameReservationPath(name)).exists()).toBe(false) }) + it('bounds recurring reconciliation by terminal directory entries', async () => { + const sessions = await Promise.all( + ['bg-bounded-reconcile-a', 'bg-bounded-reconcile-b'].map( + async (id, index) => + await createBackgroundSession({ + id, + pid: 360 + index, + cwd: '/repo', + command: [ + 'openclaude', + backgroundProcessMarkerToken(TEST_PROCESS_MARKER), + '--print', + id, + ], + sessionId: `conversation-${id}`, + processMarker: TEST_PROCESS_MARKER, + }), + ), + ) + for (const session of sessions) { + const metadataPath = join( + configDir, + 'bg-sessions', + 'sessions', + `${session.id}.json`, + ) + const release = await lockfile.lock(metadataPath, { realpath: false }) + try { + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 17 }, + { ownerPid: session.pid, expectedSession: session }, + ) + } finally { + await release() + } + } + + expect( + await reconcileBackgroundSessionTerminalFacts({ + terminalScanLimit: 1, + }), + ).toEqual({ sessionsUpdated: 1, errors: 0 }) + const statuses = await Promise.all( + sessions.map( + async session => + ( + (await Bun.file( + join( + configDir, + 'bg-sessions', + 'sessions', + `${session.id}.json`, + ), + ).json()) as { status: string } + ).status, + ), + ) + expect(statuses.sort()).toEqual(['failed', 'running']) + }) + it('does not bypass a contended metadata lock for a markerless session', async () => { const id = 'bg-contended-sync-legacy' const session = await createBackgroundSession({ diff --git a/src/cli/bgRegistry.ts b/src/cli/bgRegistry.ts index 46b7162424..b9f6315af6 100644 --- a/src/cli/bgRegistry.ts +++ b/src/cli/bgRegistry.ts @@ -3,6 +3,7 @@ import { lstat, mkdir, open, + opendir, readFile, readdir, rename, @@ -182,6 +183,32 @@ function getBackgroundSessionTerminalDir(): string { return join(getBackgroundSessionsRoot(), 'terminal') } +async function readDirectoryEntries( + path: string, + maxEntries?: number, +): Promise[]> { + if (maxEntries === undefined) { + return await readdir(path, { withFileTypes: true }) + } + if (!Number.isSafeInteger(maxEntries) || maxEntries < 1) return [] + + const entries: Dirent[] = [] + const directory = await opendir(path) + try { + for await (const entry of directory) { + entries.push(entry) + if (entries.length >= maxEntries) break + } + } finally { + try { + await directory.close() + } catch { + // The async iterator closes the directory after exhaustion or break. + } + } + return entries +} + function metadataPathForId(id: string): string { assertSafeId(id) return join(getBackgroundSessionMetadataDir(), `${id}.json`) @@ -1080,9 +1107,12 @@ function hasSamePersistedTerminalState( ) } -export async function reconcileBackgroundSessionTerminalFacts(): Promise< - { sessionsUpdated: number; errors: number } -> { +export async function reconcileBackgroundSessionTerminalFacts( + options: { + sessionIds?: readonly string[] + terminalScanLimit?: number + } = {}, +): Promise<{ sessionsUpdated: number; errors: number }> { const result = { sessionsUpdated: 0, errors: 0, @@ -1117,29 +1147,59 @@ export async function reconcileBackgroundSessionTerminalFacts(): Promise< return result } - let entries: Dirent[] - try { - entries = await readdir(getBackgroundSessionMetadataDir(), { - withFileTypes: true, - }) - } catch (error) { - if (!isErrno(error, 'ENOENT')) result.errors++ - return result - } - if ( - (await inspectCleanupDirectory( - metadataDirectory, - fileSystem.lstatFile, - )) !== 'same' - ) { - result.errors++ - return result + let candidateIds: string[] + if (options.sessionIds !== undefined) { + candidateIds = [ + ...new Set(options.sessionIds.filter(id => SAFE_ID_RE.test(id))), + ] + } else { + let entries: Dirent[] + try { + const scanTerminalFacts = options.terminalScanLimit !== undefined + entries = await readDirectoryEntries( + scanTerminalFacts + ? getBackgroundSessionTerminalDir() + : getBackgroundSessionMetadataDir(), + scanTerminalFacts ? options.terminalScanLimit : undefined, + ) + } catch (error) { + if (!isErrno(error, 'ENOENT')) result.errors++ + return result + } + const scannedDirectory = + options.terminalScanLimit === undefined + ? metadataDirectory + : terminalDirectory + if ( + (await inspectCleanupDirectory( + scannedDirectory, + fileSystem.lstatFile, + )) !== 'same' + ) { + result.errors++ + return result + } + candidateIds = [ + ...new Set( + options.terminalScanLimit === undefined + ? entries + .filter( + entry => entry.isFile() && entry.name.endsWith('.json'), + ) + .map(entry => entry.name.slice(0, -'.json'.length)) + .filter(id => SAFE_ID_RE.test(id)) + : entries + .map(terminalFactCandidateFromEntry) + .filter( + (candidate): candidate is NonNullable => + candidate !== null, + ) + .map(candidate => candidate.id), + ), + ] } - for (const entry of entries) { - if (!entry.isFile() || !entry.name.endsWith('.json')) continue - const id = entry.name.slice(0, -'.json'.length) - if (!SAFE_ID_RE.test(id)) continue + for (const id of candidateIds) { const metadataPath = metadataPathForId(id) const candidate = await readCleanupJson( metadataPath, @@ -1337,6 +1397,7 @@ async function cleanupOrphanedTerminalFacts( result: BackgroundCleanupResult, fileSystem: BackgroundCleanupFileSystem, scanStartSeed: number = cutoffMs, + maxDirectoryEntries?: number, ): Promise { const terminalDirectoryState = await inspectCleanupDirectory( terminalDirectory, @@ -1350,9 +1411,10 @@ async function cleanupOrphanedTerminalFacts( let entries: Dirent[] try { - entries = await readdir(getBackgroundSessionTerminalDir(), { - withFileTypes: true, - }) + entries = await readDirectoryEntries( + getBackgroundSessionTerminalDir(), + maxDirectoryEntries, + ) } catch (error) { if (!isErrno(error, 'ENOENT')) result.errors++ return @@ -1459,8 +1521,11 @@ export async function cleanupBackgroundSessionsBefore( readTextFile?: BackgroundCleanupFileSystem['readTextFile'] unlinkFile?: BackgroundCleanupFileSystem['unlinkFile'] _beforeMetadataDirectoryReadForTesting?: () => Promise + _beforeArtifactRemovalForTesting?: (id: string) => Promise _beforeReservationRemovalForTesting?: (path: string) => Promise _orphanedTerminalFactScanStartForTesting?: number + sessionIds?: readonly string[] + maxDirectoryEntries?: number } = {}, ): Promise { const result: BackgroundCleanupResult = { @@ -1518,6 +1583,7 @@ export async function cleanupBackgroundSessionsBefore( return result } if (metadataDirectory.state === 'missing') { + if (options.sessionIds !== undefined) return result await cleanupOrphanedTerminalFacts( cutoffMs, metadataDirectory, @@ -1526,6 +1592,7 @@ export async function cleanupBackgroundSessionsBefore( result, fileSystem, options._orphanedTerminalFactScanStartForTesting, + options.maxDirectoryEntries, ) return result } @@ -1538,36 +1605,41 @@ export async function cleanupBackgroundSessionsBefore( return result } - let entries: Dirent[] - try { - await options._beforeMetadataDirectoryReadForTesting?.() - entries = await readdir(getBackgroundSessionMetadataDir(), { - withFileTypes: true, - }) - } catch (error) { - if (!isErrno(error, 'ENOENT')) result.errors++ - return result - } - const metadataDirectoryAfterRead = await inspectCleanupDirectory( - metadataDirectory, - fileSystem.lstatFile, - ) - if (metadataDirectoryAfterRead !== 'same') { - if (metadataDirectoryAfterRead !== 'missing') result.errors++ - return result - } - - const metadataIds = new Set( - entries + let candidateIds: string[] + if (options.sessionIds !== undefined) { + candidateIds = [ + ...new Set(options.sessionIds.filter(id => SAFE_ID_RE.test(id))), + ] + } else { + let entries: Dirent[] + try { + await options._beforeMetadataDirectoryReadForTesting?.() + entries = await readDirectoryEntries( + getBackgroundSessionMetadataDir(), + options.maxDirectoryEntries, + ) + } catch (error) { + if (!isErrno(error, 'ENOENT')) result.errors++ + return result + } + const metadataDirectoryAfterRead = await inspectCleanupDirectory( + metadataDirectory, + fileSystem.lstatFile, + ) + if (metadataDirectoryAfterRead !== 'same') { + if (metadataDirectoryAfterRead !== 'missing') result.errors++ + return result + } + candidateIds = entries .filter(entry => entry.isFile() && entry.name.endsWith('.json')) .map(entry => basename(entry.name, '.json')) - .filter(id => SAFE_ID_RE.test(id)), - ) + .filter(id => SAFE_ID_RE.test(id)) + } - for (const entry of entries) { - if (!entry.isFile() || !entry.name.endsWith('.json')) continue - const metadataPath = join(getBackgroundSessionMetadataDir(), entry.name) - const expectedId = basename(entry.name, '.json') + const metadataIds = new Set(candidateIds) + + for (const expectedId of candidateIds) { + const metadataPath = metadataPathForId(expectedId) const metadata = await readCleanupJson( metadataPath, (value): value is BackgroundSession => @@ -1656,70 +1728,8 @@ export async function cleanupBackgroundSessionsBefore( } } - const logPaths = getBackgroundSessionLogPaths(session.id) - const stdoutRemoval = await removeCleanupArtifact( - logPaths.stdoutLogPath, - result, - logsDirectory, - fileSystem, - ) - const stderrRemoval = await removeCleanupArtifact( - logPaths.stderrLogPath, - result, - logsDirectory, - fileSystem, - ) - let reservationRemoval: CleanupArtifactRemoval = 'missing' - const sessionName = session.name - if (sessionName && reservation?.state === 'valid' && namesDirectory) { - try { - reservationRemoval = await withNameReservationLock( - sessionName, - async () => { - const latestReservation = await readCleanupJson( - nameReservationPathForName(sessionName), - isBackgroundSessionNameReservation, - namesDirectory, - fileSystem, - ) - if (latestReservation.state === 'error') { - result.errors++ - return 'error' - } - if ( - latestReservation.state !== 'valid' || - latestReservation.value.name !== sessionName || - latestReservation.value.id !== session.id - ) { - return 'missing' - } - const reservationPath = nameReservationPathForName(sessionName) - await options._beforeReservationRemovalForTesting?.( - reservationPath, - ) - return await removeCleanupArtifact( - reservationPath, - result, - namesDirectory, - fileSystem, - latestReservation.identity, - ) - }, - ) - } catch { - result.errors++ - reservationRemoval = 'error' - } - } - if ( - stdoutRemoval === 'error' || - stderrRemoval === 'error' || - reservationRemoval === 'error' - ) { - continue - } - try { + await options._beforeArtifactRemovalForTesting?.(session.id) await withBackgroundSessionIdLock(session.id, async () => { const latestMetadata = await readCleanupJson( metadataPath, @@ -1745,6 +1755,74 @@ export async function cleanupBackgroundSessionsBefore( return } + const logPaths = getBackgroundSessionLogPaths(session.id) + const stdoutRemoval = await removeCleanupArtifact( + logPaths.stdoutLogPath, + result, + logsDirectory, + fileSystem, + ) + const stderrRemoval = await removeCleanupArtifact( + logPaths.stderrLogPath, + result, + logsDirectory, + fileSystem, + ) + let reservationRemoval: CleanupArtifactRemoval = 'missing' + const sessionName = session.name + if ( + sessionName && + reservation?.state === 'valid' && + namesDirectory + ) { + try { + reservationRemoval = await withNameReservationLock( + sessionName, + async () => { + const latestReservation = await readCleanupJson( + nameReservationPathForName(sessionName), + isBackgroundSessionNameReservation, + namesDirectory, + fileSystem, + ) + if (latestReservation.state === 'error') { + result.errors++ + return 'error' + } + if ( + latestReservation.state !== 'valid' || + latestReservation.value.name !== sessionName || + latestReservation.value.id !== session.id + ) { + return 'missing' + } + const reservationPath = + nameReservationPathForName(sessionName) + await options._beforeReservationRemovalForTesting?.( + reservationPath, + ) + return await removeCleanupArtifact( + reservationPath, + result, + namesDirectory, + fileSystem, + latestReservation.identity, + ) + }, + ) + } catch { + result.errors++ + reservationRemoval = 'error' + } + } + if ( + stdoutRemoval === 'error' || + stderrRemoval === 'error' || + reservationRemoval === 'error' + ) { + return + } + const metadataRemoval = await removeCleanupArtifact( metadataPath, result, @@ -1779,15 +1857,18 @@ export async function cleanupBackgroundSessionsBefore( } } - await cleanupOrphanedTerminalFacts( - cutoffMs, - metadataDirectory, - terminalDirectory, - metadataIds, - result, - fileSystem, - options._orphanedTerminalFactScanStartForTesting, - ) + if (options.sessionIds === undefined) { + await cleanupOrphanedTerminalFacts( + cutoffMs, + metadataDirectory, + terminalDirectory, + metadataIds, + result, + fileSystem, + options._orphanedTerminalFactScanStartForTesting, + options.maxDirectoryEntries, + ) + } return result } diff --git a/src/entrypoints/cli.test.ts b/src/entrypoints/cli.test.ts index 4269c54473..af37983bb0 100644 --- a/src/entrypoints/cli.test.ts +++ b/src/entrypoints/cli.test.ts @@ -41,7 +41,12 @@ const mockProfileCheckpoint = mock((_checkpoint: string) => {}) const mockPsHandler = mock(async (_args: string[]) => {}) const mockLogsHandler = mock(async (_args: string[]) => {}) const mockAttachHandler = mock(async (_args: string[]) => {}) -const mockKillHandler = mock(async (_args: string[]) => {}) +const mockKillHandler = mock( + async ( + _args: string[], + _options?: { retentionSettingsReady?: boolean }, + ) => {}, +) const mockHandleBgFlag = mock(async (_args: string[]) => {}) const mockPrepareBackgroundSessionFinalizer = mock(async () => 'installed') const mockRunBackgroundSessionCleanupWorker = mock(async () => {}) @@ -63,6 +68,7 @@ const mockGetProviderValidationError = mock( async (_env: NodeJS.ProcessEnv) => undefined, ) const mockEagerLoadSettingsFromArgs = mock((_args: string[]) => ({ ok: true })) +const mockResetSettingsCache = mock(() => {}) const mockResolveOutOfProcessTeammateProviderFromCliArgs = mock( (_args: string[], _settings: unknown) => undefined, ) @@ -93,6 +99,7 @@ const runtimeMocks = [ mockApplyStartupEnvFromProfile, mockGetProviderValidationError, mockEagerLoadSettingsFromArgs, + mockResetSettingsCache, mockResolveOutOfProcessTeammateProviderFromCliArgs, mockApplyAgentProviderOverrideToEnv, mockGetInitialSettings, @@ -298,18 +305,15 @@ describe('cli.tsx — --provider startup ordering', () => { expect(process.env.GEMINI_MODEL).toBe('gemini-2.0-flash') }) - it('dispatches background session management before config and provider validation', async () => { + it('dispatches background session management before provider validation', async () => { const src = await Bun.file(`${import.meta.dir}/cli.tsx`).text() const bgManagementIndex = src.indexOf("args[0] === 'ps'") - const configEnableIndex = src.indexOf('enableConfigs()', bgManagementIndex) const providerValidationIndex = src.indexOf( 'await validateProviderEnvForStartupOrExit()', ) expect(bgManagementIndex).toBeGreaterThanOrEqual(0) - expect(configEnableIndex).toBeGreaterThanOrEqual(0) expect(providerValidationIndex).toBeGreaterThanOrEqual(0) - expect(bgManagementIndex).toBeLessThan(configEnableIndex) expect(bgManagementIndex).toBeLessThan(providerValidationIndex) }) @@ -369,6 +373,9 @@ const mockImporters = { flagSettings: async () => ({ eagerLoadSettingsFromArgs: mockEagerLoadSettingsFromArgs, }), + settingsCache: async () => ({ + resetSettingsCache: mockResetSettingsCache, + }), agentRouting: async () => ({ applyAgentProviderOverrideToEnv: mockApplyAgentProviderOverrideToEnv, resolveOutOfProcessTeammateProviderFromCliArgs: @@ -439,15 +446,44 @@ describe('cli.tsx — background routing behavior', () => { await runCliEntrypoint([command, ...tail], bgOptions) - expect(handler.mock.calls).toEqual([[tail]]) + if (command === 'kill') { + expect(mockKillHandler.mock.calls).toEqual([ + [tail, { retentionSettingsReady: true }], + ]) + } else { + expect(handler.mock.calls).toEqual([[tail]]) + } expect(mockParseProviderEnvFileArgs).not.toHaveBeenCalled() expect(mockHandleBgFlag).not.toHaveBeenCalled() - expect(mockEnableConfigs).not.toHaveBeenCalled() + expect(mockEnableConfigs).toHaveBeenCalledTimes( + command === 'kill' ? 1 : 0, + ) + expect(mockEagerLoadSettingsFromArgs).toHaveBeenCalledTimes( + command === 'kill' ? 1 : 0, + ) expect(mockValidateProviderEnvForStartupOrExit).not.toHaveBeenCalled() expect(mockCliMain).not.toHaveBeenCalled() } }) + it('keeps kill reachable when retention settings cannot be loaded', async () => { + mockEagerLoadSettingsFromArgs.mockImplementationOnce(() => ({ + ok: false, + message: 'missing retention settings', + })) + + await runCliEntrypoint( + ['kill', 'session-1', '--settings', 'missing.json'], + bgOptions, + ) + + expect(mockKillHandler).toHaveBeenCalledWith( + ['session-1', '--settings', 'missing.json'], + { retentionSettingsReady: false }, + ) + expect(mockValidateProviderEnvForStartupOrExit).not.toHaveBeenCalled() + }) + it('establishes background finalizer ownership before any command path', async () => { process.env[BACKGROUND_SESSION_ID_ENV] = 'bg-entrypoint' mockPrepareBackgroundSessionFinalizer.mockImplementationOnce(async () => { @@ -478,6 +514,22 @@ describe('cli.tsx — background routing behavior', () => { expect(mockPrepareBackgroundSessionFinalizer).not.toHaveBeenCalled() expect(mockPsHandler).not.toHaveBeenCalled() expect(mockEnableConfigs).toHaveBeenCalledTimes(1) + expect(mockResetSettingsCache).toHaveBeenCalledTimes(1) + expect(mockEagerLoadSettingsFromArgs.mock.calls).toEqual([[['ps']]]) + expect(mockRunBackgroundSessionCleanupWorker).toHaveBeenCalledWith({ + reloadSettings: expect.any(Function), + }) + const workerOptions = ( + mockRunBackgroundSessionCleanupWorker.mock.calls as unknown as Array< + [{ reloadSettings?: () => boolean }] + > + )[0]?.[0] + expect(workerOptions?.reloadSettings?.()).toBe(true) + expect(mockResetSettingsCache).toHaveBeenCalledTimes(2) + expect(mockEagerLoadSettingsFromArgs.mock.calls).toEqual([ + [['ps']], + [['ps']], + ]) }) it('routes partial background metadata through the finalizer before dispatch', async () => { @@ -505,10 +557,22 @@ describe('cli.tsx — background routing behavior', () => { await runCliEntrypoint([command, '--bg', 'session-1'], bgOptions) - expect(handler.mock.calls).toEqual([[['--bg', 'session-1']]]) + const tail = ['--bg', 'session-1'] + if (command === 'kill') { + expect(mockKillHandler.mock.calls).toEqual([ + [tail, { retentionSettingsReady: true }], + ]) + } else { + expect(handler.mock.calls).toEqual([[tail]]) + } expect(mockParseProviderEnvFileArgs).not.toHaveBeenCalled() expect(mockHandleBgFlag).not.toHaveBeenCalled() - expect(mockEnableConfigs).not.toHaveBeenCalled() + expect(mockEnableConfigs).toHaveBeenCalledTimes( + command === 'kill' ? 1 : 0, + ) + expect(mockEagerLoadSettingsFromArgs).toHaveBeenCalledTimes( + command === 'kill' ? 1 : 0, + ) expect(mockValidateProviderEnvForStartupOrExit).not.toHaveBeenCalled() expect(mockCliMain).not.toHaveBeenCalled() } diff --git a/src/entrypoints/cli.tsx b/src/entrypoints/cli.tsx index 76df5f828d..fd705419b0 100644 --- a/src/entrypoints/cli.tsx +++ b/src/entrypoints/cli.tsx @@ -266,6 +266,9 @@ type CliEntrypointImporters = { flagSettings: () => Promise< typeof import('../utils/settings/flagSettings.js') > + settingsCache: () => Promise< + typeof import('../utils/settings/settingsCache.js') + > agentRouting: () => Promise< typeof import('../services/api/agentRouting.js') > @@ -290,6 +293,7 @@ const defaultCliEntrypointImporters: CliEntrypointImporters = { providerProfile: () => import('../utils/providerProfile.js'), providerValidation: () => import('../utils/providerValidation.js'), flagSettings: () => import('../utils/settings/flagSettings.js'), + settingsCache: () => import('../utils/settings/settingsCache.js'), agentRouting: () => import('../services/api/agentRouting.js'), settings: () => import('../utils/settings/settings.js'), cliArgs: () => import('../utils/cliArgs.js'), @@ -323,8 +327,15 @@ export async function main( if (process.env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] === '1') { const { enableConfigs } = await importers.config() enableConfigs() + const { eagerLoadSettingsFromArgs } = await importers.flagSettings() + const { resetSettingsCache } = await importers.settingsCache() + const reloadSettings = () => { + resetSettingsCache() + return eagerLoadSettingsFromArgs(args).ok + } + if (!reloadSettings()) return const { runBackgroundSessionCleanupWorker } = await importers.bgFinalizer() - await runBackgroundSessionCleanupWorker() + await runBackgroundSessionCleanupWorker({ reloadSettings }) return } // The detached CLI is the registered background-session PID. Establish @@ -376,7 +387,13 @@ export async function main( await bg.attachHandler(args.slice(1)); break; case 'kill': - await bg.killHandler(args.slice(1)); + { + const { enableConfigs } = await importers.config() + enableConfigs() + const { eagerLoadSettingsFromArgs } = await importers.flagSettings() + const retentionSettingsReady = eagerLoadSettingsFromArgs(args).ok + await bg.killHandler(args.slice(1), { retentionSettingsReady }); + } break; } return; diff --git a/src/utils/backgroundHousekeeping.ts b/src/utils/backgroundHousekeeping.ts index 857ed76019..ade956158f 100644 --- a/src/utils/backgroundHousekeeping.ts +++ b/src/utils/backgroundHousekeeping.ts @@ -26,16 +26,16 @@ import { autoUpdateMarketplacesAndPluginsInBackground } from './plugins/pluginAu // 24 hours in milliseconds const RECURRING_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000 -// Terminal facts are a durable handoff from background children. Polling this -// small registry once a minute gives zero-day retention a prompt later pass -// without rerunning the full message and cache cleanup pipeline. +// Terminal facts are a durable handoff from background children. The cleanup +// coordinator globally throttles this minute trigger and streams only a bounded +// fact/metadata sample, without rerunning the age-based retention sweep. const BACKGROUND_SESSION_RECONCILIATION_INTERVAL_MS = 60 * 1000 // 10 minutes after start. const DELAY_VERY_SLOW_OPERATIONS_THAT_HAPPEN_EVERY_SESSION = 10 * 60 * 1000 type BackgroundSessionReconciliationOptions = { - cleanup?: () => Promise + cleanup?: () => Promise setInterval?: ( callback: () => void, intervalMs: number, diff --git a/src/utils/cleanup.test.ts b/src/utils/cleanup.test.ts index 631658d29d..4e149fea9b 100644 --- a/src/utils/cleanup.test.ts +++ b/src/utils/cleanup.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test } from 'bun:test' -import { mkdir, rm, stat, utimes, writeFile } from 'fs/promises' +import { mkdir, readdir, rm, stat, utimes, writeFile } from 'fs/promises' import { tmpdir } from 'os' import { join } from 'path' @@ -12,6 +12,7 @@ import { cleanupOldSessionFilesInProjectsDir } from './cleanup.js' import { NodeFsOperations } from './fsOperations.js' const tempDirs: string[] = [] +const FIXTURE_OUTPUT_LIMIT_BYTES = 64 * 1024 afterEach(async () => { _setBackgroundSessionsRootForTesting(undefined) @@ -20,12 +21,41 @@ afterEach(async () => { ) }) +async function readBoundedFixtureOutput( + stream: ReadableStream, + label: string, +): Promise { + const reader = stream.getReader() + const decoder = new TextDecoder() + let bytes = 0 + let text = '' + try { + while (true) { + const { done, value } = await reader.read() + if (done) break + bytes += value.byteLength + if (bytes > FIXTURE_OUTPUT_LIMIT_BYTES) { + throw new Error( + `${label} exceeded ${FIXTURE_OUTPUT_LIMIT_BYTES} bytes`, + ) + } + text += decoder.decode(value, { stream: true }) + } + return text + decoder.decode() + } finally { + reader.releaseLock() + } +} + async function runCleanupFixture( configDir: string, mode: | 'once' - | 'post-completion' + | 'marker-failure' + | 'periodic-policy-recheck' + | 'periodic-recovery' | 'post-finalization-gate' + | 'post-finalization-policy-recheck' | 'post-finalization-exact-cutoff' | 'post-finalization-timeout' | 'explicit-kill' = 'once', @@ -52,8 +82,14 @@ async function runCleanupFixture( try { const [exitCode, stdout, stderr] = await Promise.all([ child.exited, - new Response(child.stdout).text(), - new Response(child.stderr).text(), + readBoundedFixtureOutput( + child.stdout as ReadableStream, + 'cleanup fixture stdout', + ), + readBoundedFixtureOutput( + child.stderr as ReadableStream, + 'cleanup fixture stderr', + ), ]) expect(exitCode, stderr).toBe(0) const result = JSON.parse(stdout) as Record @@ -61,6 +97,7 @@ async function runCleanupFixture( return result } finally { clearTimeout(timeout) + controller.abort() } } @@ -88,6 +125,50 @@ async function createCompletedBackgroundSession( return join(root, 'sessions', `${id}.json`) } +async function runConcurrentPeriodicRecovery( + configDir: string, +): Promise { + const fixture = join(import.meta.dir, 'cleanupBackgroundSessions.fixture.ts') + const { NODE_ENV: _nodeEnv, USER_TYPE: _userType, ...inheritedEnv } = + process.env + const runRecovery = async (): Promise => { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 15_000) + const child = Bun.spawn([process.execPath, fixture], { + cwd: configDir, + env: { + ...inheritedEnv, + HOME: configDir, + XDG_CACHE_HOME: join(configDir, 'cache'), + OPENCLAUDE_CONFIG_DIR: configDir, + OPENCLAUDE_CLEANUP_FIXTURE_MODE: 'periodic-recovery', + }, + signal: controller.signal, + stdout: 'pipe', + stderr: 'pipe', + }) + try { + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + readBoundedFixtureOutput( + child.stdout as ReadableStream, + 'periodic recovery stdout', + ), + readBoundedFixtureOutput( + child.stderr as ReadableStream, + 'periodic recovery stderr', + ), + ]) + expect(exitCode, stderr).toBe(0) + return (JSON.parse(stdout) as { result: string }).result + } finally { + clearTimeout(timeout) + controller.abort() + } + } + return await Promise.all([runRecovery(), runRecovery()]) +} + describe('cleanupOldSessionFiles', () => { test('removes old replay sidecars while preserving non-session files', async () => { const projectsDir = join( @@ -122,6 +203,148 @@ describe('cleanupOldSessionFiles', () => { }) describe('cleanupOldMessageFilesInBackground', () => { + test( + 'globally throttles and bounds concurrent prompt-recovery passes', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-bounded-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + const root = join(configDir, 'bg-sessions') + const metadataDir = join(root, 'sessions') + const logsDir = join(root, 'logs') + await Promise.all([ + mkdir(metadataDir, { recursive: true }), + mkdir(logsDir, { recursive: true }), + mkdir(join(root, 'names'), { recursive: true }), + mkdir(join(root, 'terminal'), { recursive: true }), + ]) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const finishedAt = new Date(Date.now() - 60_000).toISOString() + await Promise.all( + Array.from({ length: 300 }, async (_, index) => { + const id = `bg-bounded-${String(index).padStart(3, '0')}` + const stdoutLogPath = join(logsDir, `${id}.out.log`) + const stderrLogPath = join(logsDir, `${id}.err.log`) + await Promise.all([ + writeFile( + join(metadataDir, `${id}.json`), + JSON.stringify({ + id, + pid: 999_999, + cwd: configDir, + status: 'exited', + sessionId: `${id}-conversation`, + startedAt: new Date(Date.now() - 120_000).toISOString(), + updatedAt: finishedAt, + command: ['openclaude', '--print', 'fixture'], + stdoutLogPath, + stderrLogPath, + finishedAt, + exitCode: 0, + terminalReason: 'exit_code', + }), + ), + writeFile(stdoutLogPath, ''), + writeFile(stderrLogPath, ''), + ]) + }), + ) + + expect((await runConcurrentPeriodicRecovery(configDir)).sort()).toEqual([ + 'ran', + 'skipped', + ]) + expect( + (await readdir(metadataDir)).filter(name => name.endsWith('.json')), + ).toHaveLength(44) + }, + 30_000, + ) + + for (const scenario of [ + { label: 'positive', setting: 30, result: 'ran' }, + { label: 'invalid', setting: 'invalid', result: 'invalid-policy' }, + ] as const) { + test( + `keeps periodic recovery non-destructive with ${scenario.label} retention`, + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-periodic-${scenario.label}-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: scenario.setting }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + `bg-periodic-${scenario.label}`, + new Date(Date.now() - 60_000), + ) + + expect( + await runCleanupFixture(configDir, 'periodic-recovery'), + ).toEqual({ result: scenario.result }) + expect(await Bun.file(metadataPath).exists()).toBe(true) + expect( + await Bun.file( + join(configDir, 'bg-sessions', '.recovery-pass'), + ).exists(), + ).toBe(scenario.result === 'ran') + }, + 30_000, + ) + } + + test( + 'rechecks periodic retention policy after acquiring the global lock', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-periodic-recheck-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect( + await runCleanupFixture(configDir, 'periodic-policy-recheck'), + ).toEqual({ result: 'ran', metadataPresent: true }) + }, + 30_000, + ) + + test( + 'continues later cleanup stages when the retention marker cannot be created', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-marker-failure-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect( + await runCleanupFixture(configDir, 'marker-failure'), + ).toEqual({ completed: true, planPresent: false }) + }, + 30_000, + ) + test( 'removes old completed background-session artifacts through the scheduler', async () => { @@ -144,6 +367,11 @@ describe('cleanupOldMessageFilesInBackground', () => { await runCleanupFixture(configDir) expect(await Bun.file(metadataPath).exists()).toBe(false) + expect( + await Bun.file( + join(configDir, 'bg-sessions', '.retention-pass'), + ).exists(), + ).toBe(true) }, 30_000, ) @@ -231,6 +459,35 @@ describe('cleanupOldMessageFilesInBackground', () => { 30_000, ) + test( + 'rechecks the originating settings policy after process handoff', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-policy-recheck-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + + expect( + await runCleanupFixture( + configDir, + 'post-finalization-policy-recheck', + ), + ).toEqual({ + result: 'skipped', + waits: 1, + reloads: 1, + metadataPresent: true, + }) + }, + 30_000, + ) + test( 'reclaims zero-day artifacts after an explicit kill transition', async () => { @@ -277,34 +534,6 @@ describe('cleanupOldMessageFilesInBackground', () => { ) } - test( - 'reclaims a zero-day session that completes after the initial sweep', - async () => { - const configDir = join( - tmpdir(), - `openclaude-cleanup-bg-post-completion-${Date.now()}-${Math.random().toString(16).slice(2)}`, - ) - tempDirs.push(configDir) - await mkdir(configDir, { recursive: true }) - await writeFile( - join(configDir, 'settings.json'), - JSON.stringify({ cleanupPeriodDays: 0 }), - ) - - expect( - await runCleanupFixture(configDir, 'post-completion'), - ).toEqual({ - presentAfterInitialSweep: true, - metadataPresentAfterCompletion: false, - metadataStatusAfterCompletion: null, - reservationPresentAfterCompletion: false, - stdoutPresentAfterCompletion: false, - stderrPresentAfterCompletion: false, - }) - }, - 30_000, - ) - test( 'preserves background artifacts when explicit cleanupPeriodDays is invalid', async () => { @@ -331,34 +560,6 @@ describe('cleanupOldMessageFilesInBackground', () => { 30_000, ) - test( - 'keeps post-completion artifacts when recurring retention is disabled by invalid settings', - async () => { - const configDir = join( - tmpdir(), - `openclaude-cleanup-bg-invalid-post-completion-${Date.now()}-${Math.random().toString(16).slice(2)}`, - ) - tempDirs.push(configDir) - await mkdir(configDir, { recursive: true }) - await writeFile( - join(configDir, 'settings.json'), - JSON.stringify({ cleanupPeriodDays: 'invalid' }), - ) - - expect( - await runCleanupFixture(configDir, 'post-completion'), - ).toEqual({ - presentAfterInitialSweep: true, - metadataPresentAfterCompletion: true, - metadataStatusAfterCompletion: 'running', - reservationPresentAfterCompletion: true, - stdoutPresentAfterCompletion: true, - stderrPresentAfterCompletion: true, - }) - }, - 30_000, - ) - test( 'retains recent background sessions under a nonzero cleanup period', async () => { diff --git a/src/utils/cleanup.ts b/src/utils/cleanup.ts index 02da4498dd..998337bfa1 100644 --- a/src/utils/cleanup.ts +++ b/src/utils/cleanup.ts @@ -1,6 +1,6 @@ import * as fs from 'fs/promises' import { homedir } from 'os' -import { join } from 'path' +import { dirname, join } from 'path' import { cleanupBackgroundSessionsBefore, reconcileBackgroundSessionTerminalFacts, @@ -19,12 +19,45 @@ import { getDefaultPlansDirectory } from './plans.js' import { getSettingsWithAllErrors } from './settings/allErrors.js' import { getSettings_DEPRECATED, + getSettingsWithErrors, rawSettingsContainsKey, } from './settings/settings.js' +import { resetSettingsCache } from './settings/settingsCache.js' import { TOOL_RESULTS_SUBDIR } from './toolResultStorage.js' import { cleanupStaleAgentWorktrees } from './worktree.js' const DEFAULT_CLEANUP_PERIOD_DAYS = 30 +const BACKGROUND_RECOVERY_INTERVAL_MS = 60 * 1000 +const BACKGROUND_RETENTION_SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000 +const BACKGROUND_RECOVERY_ENTRY_LIMIT = 256 + +export type BackgroundSessionRetentionTrigger = + | 'natural-finalization' + | 'explicit-kill' + | 'periodic-recovery' + | 'periodic-retention' + +export type BackgroundSessionRetentionResult = + | 'ran' + | 'skipped' + | 'invalid-policy' + +type BackgroundSessionRetentionRequest = + | { + trigger: 'natural-finalization' + sessionId: string + waitForProcessesToExit: () => Promise + reloadSettings?: () => boolean + } + | { trigger: 'explicit-kill'; sessionId: string } + | { + trigger: 'periodic-recovery' + _afterThrottleLockForTesting?: () => Promise + } + | { + trigger: 'periodic-retention' + _afterThrottleLockForTesting?: () => Promise + } function getCutoffDate(): Date { const settings = getSettings_DEPRECATED() || {} @@ -601,18 +634,7 @@ export async function cleanupOldMessageFilesInBackground(): Promise { await cleanupOldMessageFiles() await cleanupOldSessionFiles() - const backgroundResult = await cleanupBackgroundSessionsBefore( - getCutoffDate(), - ) - if ( - backgroundResult.sessionsRemoved > 0 || - backgroundResult.artifactsRemoved > 0 || - backgroundResult.errors > 0 - ) { - logForDebugging( - `Background session cleanup: ${backgroundResult.sessionsRemoved} sessions, ${backgroundResult.artifactsRemoved} artifacts, ${backgroundResult.errors} errors`, - ) - } + await runBackgroundSessionRetention({ trigger: 'periodic-retention' }) await cleanupOldPlanFiles() await cleanupOldFileHistoryBackups() await cleanupOldSessionEnvDirs() @@ -628,28 +650,45 @@ export async function cleanupOldMessageFilesInBackground(): Promise { } } -function cleanupPeriodSettingIsInvalid(): boolean { - const { errors } = getSettingsWithAllErrors() - return errors.length > 0 && rawSettingsContainsKey('cleanupPeriodDays') -} - -export async function cleanupBackgroundSessionsInBackground(): Promise { - if (cleanupPeriodSettingIsInvalid()) { - return +function readBackgroundRetentionPeriod( + options: { fresh?: boolean } = {}, +): + | { state: 'valid'; days: number } + | { state: 'invalid' } { + try { + if (options.fresh) resetSettingsCache() + const { errors } = getSettingsWithErrors() + if (errors.length > 0 && rawSettingsContainsKey('cleanupPeriodDays')) { + return { state: 'invalid' } + } + const days = + getSettings_DEPRECATED()?.cleanupPeriodDays ?? + DEFAULT_CLEANUP_PERIOD_DAYS + return Number.isFinite(days) && days >= 0 + ? { state: 'valid', days } + : { state: 'invalid' } + } catch { + return { state: 'invalid' } } - await cleanupBackgroundSessionsAtCutoff(getCutoffDate()) } -async function cleanupBackgroundSessionsAtCutoff(cutoff: Date): Promise { - const reconciliation = await reconcileBackgroundSessionTerminalFacts() +function logBackgroundReconciliation( + reconciliation: Awaited< + ReturnType + >, +): void { if (reconciliation.errors > 0) { logForDebugging( `Background session reconciliation: ${reconciliation.sessionsUpdated} sessions, ${reconciliation.errors} errors`, ) } - const backgroundResult = await cleanupBackgroundSessionsBefore( - cutoff, - ) +} + +function logBackgroundCleanup( + backgroundResult: Awaited< + ReturnType + >, +): void { if ( backgroundResult.sessionsRemoved > 0 || backgroundResult.artifactsRemoved > 0 || @@ -661,16 +700,167 @@ async function cleanupBackgroundSessionsAtCutoff(cutoff: Date): Promise { } } +async function runWithBackgroundRetentionThrottle( + markerName: string, + intervalMs: number, + operation: () => Promise, + afterLock?: () => Promise, +): Promise { + const markerPath = join( + getClaudeConfigHomeDir(), + 'bg-sessions', + markerName, + ) + const markerIsFresh = async (): Promise => { + try { + return Date.now() - (await fs.stat(markerPath)).mtimeMs < intervalMs + } catch { + return false + } + } + let locked = false + try { + if (await markerIsFresh()) return false + await fs.mkdir(dirname(markerPath), { recursive: true }) + await lockfile.lock(markerPath, { retries: 0, realpath: false }) + locked = true + } catch { + return false + } + try { + if (await markerIsFresh()) return false + await afterLock?.() + await operation() + try { + await fs.writeFile(markerPath, new Date().toISOString()) + } catch (error) { + logError(error instanceof Error ? error : new Error(String(error))) + } + return true + } catch (error) { + logError(error instanceof Error ? error : new Error(String(error))) + return false + } finally { + if (locked) { + await lockfile.unlock(markerPath, { realpath: false }).catch(() => {}) + } + } +} + +export async function runBackgroundSessionRetention( + request: BackgroundSessionRetentionRequest, +): Promise { + let policy = readBackgroundRetentionPeriod() + if (policy.state === 'invalid') return 'invalid-policy' + + if (request.trigger === 'natural-finalization') { + if (policy.days !== 0) return 'skipped' + if (!(await request.waitForProcessesToExit())) return 'skipped' + if (request.reloadSettings && !request.reloadSettings()) { + return 'invalid-policy' + } + policy = readBackgroundRetentionPeriod() + if (policy.state === 'invalid') return 'invalid-policy' + if (policy.days !== 0) return 'skipped' + } + + if ( + request.trigger === 'natural-finalization' || + request.trigger === 'explicit-kill' + ) { + const reconciliation = await reconcileBackgroundSessionTerminalFacts({ + sessionIds: [request.sessionId], + }) + logBackgroundReconciliation(reconciliation) + if (policy.days === 0) { + const cleanup = await cleanupBackgroundSessionsBefore( + new Date(Date.now() + 1), + { sessionIds: [request.sessionId] }, + ) + logBackgroundCleanup(cleanup) + } + return 'ran' + } + + if (request.trigger === 'periodic-recovery') { + let passResult: BackgroundSessionRetentionResult = 'ran' + const ran = await runWithBackgroundRetentionThrottle( + '.recovery-pass', + BACKGROUND_RECOVERY_INTERVAL_MS, + async () => { + const currentPolicy = readBackgroundRetentionPeriod({ fresh: true }) + if (currentPolicy.state === 'invalid') { + passResult = 'invalid-policy' + return + } + if (currentPolicy.days !== 0) return + const reconciliation = + await reconcileBackgroundSessionTerminalFacts({ + terminalScanLimit: BACKGROUND_RECOVERY_ENTRY_LIMIT, + }) + logBackgroundReconciliation(reconciliation) + const cleanup = await cleanupBackgroundSessionsBefore( + new Date(Date.now() + 1), + { maxDirectoryEntries: BACKGROUND_RECOVERY_ENTRY_LIMIT }, + ) + logBackgroundCleanup(cleanup) + }, + request._afterThrottleLockForTesting, + ) + return ran ? passResult : 'skipped' + } + + let passResult: BackgroundSessionRetentionResult = 'ran' + const ran = await runWithBackgroundRetentionThrottle( + '.retention-pass', + BACKGROUND_RETENTION_SWEEP_INTERVAL_MS, + async () => { + const currentPolicy = readBackgroundRetentionPeriod({ fresh: true }) + if (currentPolicy.state === 'invalid') { + passResult = 'invalid-policy' + return + } + const cutoff = new Date( + Date.now() - currentPolicy.days * 24 * 60 * 60 * 1000, + ) + logBackgroundReconciliation( + await reconcileBackgroundSessionTerminalFacts(), + ) + logBackgroundCleanup( + await cleanupBackgroundSessionsBefore(cutoff), + ) + }, + request._afterThrottleLockForTesting, + ) + return ran ? passResult : 'skipped' +} + +function cleanupPeriodSettingIsInvalid(): boolean { + try { + const { errors } = getSettingsWithAllErrors() + return errors.length > 0 && rawSettingsContainsKey('cleanupPeriodDays') + } catch { + return true + } +} + +export async function cleanupBackgroundSessionsInBackground(): Promise< + BackgroundSessionRetentionResult +> { + return await runBackgroundSessionRetention({ + trigger: 'periodic-recovery', + }) +} + export async function cleanupBackgroundSessionsAfterFinalization( + sessionId: string, waitForProcessesToExit: () => Promise, -): Promise { - if (cleanupPeriodSettingIsInvalid()) return - const cleanupPeriodDays = - getSettings_DEPRECATED()?.cleanupPeriodDays ?? DEFAULT_CLEANUP_PERIOD_DAYS - if (cleanupPeriodDays !== 0) return - - if (!(await waitForProcessesToExit())) return - if (cleanupPeriodSettingIsInvalid()) return - if (getSettings_DEPRECATED()?.cleanupPeriodDays !== 0) return - await cleanupBackgroundSessionsAtCutoff(new Date(Date.now() + 1)) + reloadSettings?: () => boolean, +): Promise { + return await runBackgroundSessionRetention({ + trigger: 'natural-finalization', + sessionId, + waitForProcessesToExit, + ...(reloadSettings ? { reloadSettings } : {}), + }) } diff --git a/src/utils/cleanupBackgroundSessions.fixture.ts b/src/utils/cleanupBackgroundSessions.fixture.ts index d88fa1188f..6567918129 100644 --- a/src/utils/cleanupBackgroundSessions.fixture.ts +++ b/src/utils/cleanupBackgroundSessions.fixture.ts @@ -1,112 +1,171 @@ import { createHash } from 'node:crypto' +import { mkdir, utimes } from 'node:fs/promises' import { dirname, join } from 'node:path' -import * as lockfile from './lockfile.js' import { createBackgroundSession, recordBackgroundSessionNaturalTerminationSync, } from '../cli/bgRegistry.js' import { killHandler } from '../cli/bg.js' -import { startBackgroundHousekeeping } from './backgroundHousekeeping.js' +import { enableConfigs } from './config.js' +import { eagerLoadSettingsFromArgs } from './settings/flagSettings.js' import { cleanupBackgroundSessionsAfterFinalization, + cleanupBackgroundSessionsInBackground, cleanupOldMessageFilesInBackground, + runBackgroundSessionRetention, } from './cleanup.js' try { const mode = process.env.OPENCLAUDE_CLEANUP_FIXTURE_MODE - if (mode === 'post-completion') { - const processMarker = 'c'.repeat(64) + if (mode === 'periodic-recovery') { + enableConfigs() + const settingsResult = eagerLoadSettingsFromArgs([ + '--settings', + join(process.env.OPENCLAUDE_CONFIG_DIR!, 'settings.json'), + ]) + if (!settingsResult.ok) throw new Error(settingsResult.message) + process.stdout.write( + `${JSON.stringify({ + result: await cleanupBackgroundSessionsInBackground(), + })}\n`, + ) + } else if (mode === 'periodic-policy-recheck') { + enableConfigs() + const settingsPath = join( + process.env.OPENCLAUDE_CONFIG_DIR!, + 'settings.json', + ) + const settingsResult = eagerLoadSettingsFromArgs([ + '--settings', + settingsPath, + ]) + if (!settingsResult.ok) throw new Error(settingsResult.message) const session = await createBackgroundSession({ - id: 'bg-post-completion', - name: 'post-completion', + id: 'bg-periodic-policy-recheck', pid: process.pid, cwd: process.cwd(), - command: [ - 'openclaude', - `--openclaude-bg-session-marker=${processMarker}`, - '--print', - 'fixture', - ], - sessionId: 'bg-post-completion-conversation', - processMarker, + command: ['openclaude', '--print', 'fixture'], + sessionId: 'bg-periodic-policy-recheck-conversation', }) - const root = dirname(dirname(session.stdoutLogPath)) - const metadataPath = join(root, 'sessions', `${session.id}.json`) - const reservationPath = join( - root, - 'names', - `${createHash('sha256').update(session.name!).digest('hex')}.json`, + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { + ownerPid: process.pid, + expectedSession: session, + now: new Date(Date.now() - 1_000), + }, ) - let scheduledCallback: (() => void) | undefined - let finishPass: (() => void) | undefined - startBackgroundHousekeeping({ - backgroundSessionReconciliation: { - setInterval: callback => { - scheduledCallback = callback - return { unref: () => {} } - }, - _onPassFinishedForTesting: () => finishPass?.(), + const result = await runBackgroundSessionRetention({ + trigger: 'periodic-recovery', + _afterThrottleLockForTesting: async () => { + await Bun.write( + settingsPath, + JSON.stringify({ cleanupPeriodDays: 30 }), + ) }, - _reconciliationOnlyForTesting: true, }) - const runScheduledPass = async (): Promise => { - if (!scheduledCallback) { - throw new Error('background reconciliation was not scheduled') - } - await new Promise(resolve => { - finishPass = resolve - scheduledCallback?.() - }) - finishPass = undefined - } - - await runScheduledPass() - const presentAfterInitialSweep = await Bun.file( - session.stdoutLogPath, - ).exists() - const release = await lockfile.lock(metadataPath, { realpath: false }) - try { - recordBackgroundSessionNaturalTerminationSync( - session.id, - { exitCode: 0 }, - { - ownerPid: process.pid, - expectedSession: session, - now: new Date(Date.now() - 1_000), - }, - ) - } finally { - await release() - } - await runScheduledPass() - const metadataPresentAfterCompletion = await Bun.file( - metadataPath, - ).exists() - const storedMetadata = metadataPresentAfterCompletion - ? ((await Bun.file(metadataPath).json()) as { status?: unknown }) - : undefined process.stdout.write( `${JSON.stringify({ - presentAfterInitialSweep, - metadataPresentAfterCompletion, - metadataStatusAfterCompletion: storedMetadata?.status ?? null, - reservationPresentAfterCompletion: await Bun.file( - reservationPath, - ).exists(), - stdoutPresentAfterCompletion: await Bun.file( - session.stdoutLogPath, + result, + metadataPresent: await Bun.file( + join( + dirname(dirname(session.stdoutLogPath)), + 'sessions', + `${session.id}.json`, + ), ).exists(), - stderrPresentAfterCompletion: await Bun.file( - session.stderrLogPath, + })}\n`, + ) + } else if (mode === 'marker-failure') { + enableConfigs() + const settingsPath = join( + process.env.OPENCLAUDE_CONFIG_DIR!, + 'settings.json', + ) + const settingsResult = eagerLoadSettingsFromArgs([ + '--settings', + settingsPath, + ]) + if (!settingsResult.ok) throw new Error(settingsResult.message) + const plansDir = join(process.env.OPENCLAUDE_CONFIG_DIR!, 'plans') + const planPath = join(plansDir, 'old-plan.md') + await mkdir(plansDir, { recursive: true }) + await Bun.write(planPath, 'old plan') + const old = new Date('2000-01-01T00:00:00.000Z') + await utimes(planPath, old, old) + await Bun.write( + join(process.env.OPENCLAUDE_CONFIG_DIR!, 'bg-sessions'), + 'not a directory', + ) + await cleanupOldMessageFilesInBackground() + process.stdout.write( + `${JSON.stringify({ + completed: true, + planPresent: await Bun.file(planPath).exists(), + })}\n`, + ) + } else if (mode === 'post-finalization-policy-recheck') { + const session = await createBackgroundSession({ + id: 'bg-post-finalization-policy-recheck', + pid: process.pid, + cwd: process.cwd(), + command: ['openclaude', '--print', 'fixture'], + sessionId: 'bg-post-finalization-policy-recheck-conversation', + }) + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { + ownerPid: process.pid, + expectedSession: session, + now: new Date(Date.now() - 1_000), + }, + ) + const settingsPath = join( + process.env.OPENCLAUDE_CONFIG_DIR!, + 'settings.json', + ) + let waits = 0 + let reloads = 0 + const result = await cleanupBackgroundSessionsAfterFinalization( + session.id, + async () => { + waits++ + await Bun.write( + settingsPath, + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + return true + }, + () => { + reloads++ + return eagerLoadSettingsFromArgs(['--settings', settingsPath]).ok + }, + ) + process.stdout.write( + `${JSON.stringify({ + result, + waits, + reloads, + metadataPresent: await Bun.file( + join( + dirname(dirname(session.stdoutLogPath)), + 'sessions', + `${session.id}.json`, + ), ).exists(), })}\n`, ) } else if (mode === 'post-finalization-gate') { let waits = 0 - await cleanupBackgroundSessionsAfterFinalization(async () => { - waits++ - return true - }) + await cleanupBackgroundSessionsAfterFinalization( + 'bg-post-finalization-gate', + async () => { + waits++ + return true + }, + ) process.stdout.write(`${JSON.stringify({ waits })}\n`) } else if ( mode === 'post-finalization-exact-cutoff' || @@ -149,10 +208,13 @@ try { let waits = 0 try { Date.now = () => exactNow - await cleanupBackgroundSessionsAfterFinalization(async () => { - waits++ - return mode === 'post-finalization-exact-cutoff' - }) + await cleanupBackgroundSessionsAfterFinalization( + session.id, + async () => { + waits++ + return mode === 'post-finalization-exact-cutoff' + }, + ) } finally { Date.now = originalDateNow } From c1de846915f893402aa19934bc08a4ae941c2382 Mon Sep 17 00:00:00 2001 From: chioarub Date: Wed, 2 Sep 2026 14:01:19 +0300 Subject: [PATCH 11/13] fix(bg): harden retention coordination --- src/cli/bgFinalizer.fixture.ts | 28 +- src/cli/bgFinalizer.test.ts | 331 +++++++++---- src/cli/bgFinalizer.ts | 101 +++- src/cli/bgRegistry.cleanup.test.ts | 20 +- src/cli/bgRegistry.test.ts | 187 ++++++- src/cli/bgRegistry.ts | 456 ++++++++++++++++-- src/utils/backgroundHousekeeping.test.ts | 35 ++ src/utils/backgroundHousekeeping.ts | 38 +- src/utils/cleanup.test.ts | 179 ++++++- src/utils/cleanup.ts | 30 +- .../cleanupBackgroundSessions.fixture.ts | 43 +- 11 files changed, 1265 insertions(+), 183 deletions(-) diff --git a/src/cli/bgFinalizer.fixture.ts b/src/cli/bgFinalizer.fixture.ts index 5c56e6b0eb..d2a158b77d 100644 --- a/src/cli/bgFinalizer.fixture.ts +++ b/src/cli/bgFinalizer.fixture.ts @@ -17,11 +17,24 @@ if (process.env[cleanupWorkerEnv] === '1') { await runCleanupWorker() } else if (invocation[0] === 'launcher') { const name = invocation[2] - await handleBgFlag([ - '--bg', - ...(name ? ['--name', name] : []), - invocation[1] ?? 'success', - ]) + const originalLog = console.log + if ( + process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_OUTPUT === 'omit-logs' + ) { + console.log = (...args: unknown[]) => { + if (String(args[0]).startsWith('Logs:')) return + originalLog(...args) + } + } + try { + await handleBgFlag([ + '--bg', + ...(name ? ['--name', name] : []), + invocation[1] ?? 'success', + ]) + } finally { + console.log = originalLog + } } else { const mode = invocation.at(-1) await backgroundFinalizer.prepareBackgroundSessionFinalizer() @@ -64,7 +77,10 @@ if (process.env[cleanupWorkerEnv] === '1') { queueMicrotask(() => { throw new Error('intentional handled fixture failure') }) - } else if (readyPath) { + } else if ( + readyPath && + process.env.OPENCLAUDE_BG_FINALIZER_FIXTURE_SKIP_READY !== '1' + ) { await writeFile(readyPath, 'ready') } if (mode === 'controlled' || mode === 'controlled-exit') { diff --git a/src/cli/bgFinalizer.test.ts b/src/cli/bgFinalizer.test.ts index a89125aa34..87f89fc7c3 100644 --- a/src/cli/bgFinalizer.test.ts +++ b/src/cli/bgFinalizer.test.ts @@ -243,7 +243,9 @@ describe('background session finalizer', () => { it('starts a detached cleanup owner after awaited finalization', async () => { let beforeExitListener: (() => void | Promise) | undefined const order: string[] = [] - let workerSettingsArgs: string[] | undefined + let spawnedArgs: readonly string[] | undefined + let spawnedEnv: NodeJS.ProcessEnv | undefined + let unrefCalls = 0 await prepareBackgroundSessionFinalizer({ env: { @@ -261,28 +263,36 @@ describe('background session finalizer', () => { order.push('finalize') return ownedSession('bg-launcher-handoff', 500) }, - settingsArgs: ['--settings', 'retention.json'], + settingsArgs: [ + '--settings', + JSON.stringify({ + cleanupPeriodDays: 0, + permissions: { defaultMode: 'bogus' }, + }), + ], settingsCwd: configDir, - startCleanupWorker: ( - sessionId, - ownerPid, - launcherPid, - settingsArgs, - ) => { - order.push(`worker:${sessionId}:${ownerPid}:${launcherPid}`) - workerSettingsArgs = settingsArgs - }, + spawnCleanupWorkerProcess: ((_command, args, options) => { + order.push('worker') + spawnedArgs = args + spawnedEnv = options.env + return { + once: () => {}, + unref: () => { + unrefCalls++ + }, + } + }) as unknown as typeof spawn, }) await beforeExitListener?.() - expect(order).toEqual([ - 'finalize', - 'worker:bg-launcher-handoff:500:123', - ]) - expect(workerSettingsArgs).toEqual([ - '--settings', - join(configDir, 'retention.json'), - ]) + expect(order).toEqual(['finalize', 'worker']) + expect(spawnedArgs).not.toContain('--settings') + expect( + spawnedEnv?.[ + 'OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE' + ], + ).toBe('invalid') + expect(unrefCalls).toBe(1) }) it('passes only settings-source inputs to the detached cleanup worker', () => { @@ -326,9 +336,45 @@ describe('background session finalizer', () => { expect(spawnedEnv?.[BACKGROUND_SESSION_ID_ENV]).toBe( 'bg-worker-settings', ) + expect( + spawnedEnv?.[ + 'OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE' + ], + ).toBe('preserved') expect(unrefCalls).toBe(1) }) + it('marks a projected inline settings handoff invalid when a sibling setting fails validation', () => { + let spawnedArgs: readonly string[] | undefined + let spawnedEnv: NodeJS.ProcessEnv | undefined + startBackgroundSessionCleanupWorker({ + sessionId: 'bg-worker-invalid-settings', + ownerPid: 500, + launcherPid: 123, + execPath: 'node', + entrypoint: '/openclaude/cli.mjs', + settingsArgs: [ + '--settings', + JSON.stringify({ + cleanupPeriodDays: 0, + permissions: { defaultMode: 'bogus' }, + }), + ], + spawnProcess: ((_command, args, options) => { + spawnedArgs = args + spawnedEnv = options.env + return { once: () => {}, unref: () => {} } + }) as unknown as typeof spawn, + }) + + expect(spawnedArgs).toEqual(['/openclaude/cli.mjs']) + expect( + spawnedEnv?.[ + 'OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE' + ], + ).toBe('invalid') + }) + it('retains cleanup when the detached worker cannot observe process exit', async () => { const { runBackgroundSessionCleanupWorker } = await import( './bgFinalizer.js' @@ -358,6 +404,32 @@ describe('background session finalizer', () => { expect(env[BACKGROUND_SESSION_LAUNCHER_PID_ENV]).toBeUndefined() }) + it('does not run detached cleanup for an invalid originating policy', async () => { + const { runBackgroundSessionCleanupWorker } = await import( + './bgFinalizer.js' + ) + const env = { + [BACKGROUND_SESSION_CLEANUP_WORKER_ENV]: '1', + [BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV]: '500', + [BACKGROUND_SESSION_ID_ENV]: 'bg-worker-invalid-policy', + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: '123', + OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE: 'invalid', + } + let cleanupRuns = 0 + + await runBackgroundSessionCleanupWorker({ + env, + cleanup: async () => { + cleanupRuns++ + }, + }) + + expect(cleanupRuns).toBe(0) + expect( + env.OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE, + ).toBeUndefined() + }) + it('passes the registered generation to the synchronous exit fallback', async () => { let cleanup: (() => void | Promise) | undefined let exitListener: ((code: number) => void) | undefined @@ -577,6 +649,26 @@ describe('background session finalizer', () => { return !isProcessRunning(pid) } + async function stopDetachedFixtureChild( + childPid: number | undefined, + ): Promise { + if ( + childPid === undefined || + !Number.isSafeInteger(childPid) || + !isProcessRunning(childPid) + ) { + return + } + try { + process.kill(childPid, 'SIGTERM') + } catch {} + if (await waitForProcessStop(childPid)) return + try { + process.kill(childPid, 'SIGKILL') + } catch {} + await waitForProcessStop(childPid) + } + function sessionArtifactPaths(id: string, name: string): string[] { const reservationDigest = createHash('sha256').update(name).digest('hex') return [ @@ -744,39 +836,42 @@ describe('background session finalizer', () => { launcher.stderr, 'built background launcher stderr', ) - let launcherCode: number | null + let childPid: number | undefined + let ownershipTransferred = false try { - launcherCode = await waitForChildClose( + const launcherCode = await waitForChildClose( launcher, 'built background launcher', ) stdoutCapture.assertWithinLimit() stderrCapture.assertWithinLimit() - } catch (error) { const stdout = stdoutCapture.read() - const childPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) - if (Number.isSafeInteger(childPid) && isProcessRunning(childPid)) { - try { - process.kill(childPid, 'SIGKILL') - } catch {} + const parsedChildPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + if (Number.isSafeInteger(parsedChildPid)) childPid = parsedChildPid + const stderr = stderrCapture.read() + expect(launcherCode).toBe(0) + expect(stderr).toBe('') + const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] + const id = stdoutLogPath + ? basename(stdoutLogPath, '.out.log') + : undefined + expect(id).toBeDefined() + expect(childPid).toBeDefined() + if (!id || childPid === undefined) { + throw new Error('built launch output omitted its session identity') + } + ownershipTransferred = true + return { id, childPid } + } finally { + if (!ownershipTransferred) { + if (childPid === undefined) { + const stdout = stdoutCapture.read() + const parsedChildPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + if (Number.isSafeInteger(parsedChildPid)) childPid = parsedChildPid + } + await stopDetachedFixtureChild(childPid) } - throw error - } - const stdout = stdoutCapture.read() - const stderr = stderrCapture.read() - expect(launcherCode).toBe(0) - expect(stderr).toBe('') - const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] - const id = stdoutLogPath - ? basename(stdoutLogPath, '.out.log') - : undefined - const childPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) - expect(id).toBeDefined() - expect(Number.isSafeInteger(childPid)).toBe(true) - if (!id || !Number.isSafeInteger(childPid)) { - throw new Error('built launch output omitted its session identity') } - return { id, childPid } } async function runBuiltCliSession( @@ -820,7 +915,7 @@ describe('background session finalizer', () => { } async function waitForTerminalSession(): Promise { - for (let attempt = 0; attempt < 200; attempt += 1) { + for (let attempt = 0; attempt < 500; attempt += 1) { const [session] = await listBackgroundSessions() if ( session && @@ -828,7 +923,7 @@ describe('background session finalizer', () => { ) { return session } - await new Promise(resolve => setTimeout(resolve, 5)) + await new Promise(resolve => setTimeout(resolve, 10)) } throw new Error('detached fixture did not persist a terminal outcome') } @@ -865,6 +960,7 @@ describe('background session finalizer', () => { async function runWaitingDetachedLaunch( name: string, + fixtureEnv: NodeJS.ProcessEnv = {}, ): Promise<{ id: string; childPid: number; readyPath: string }> { const readyPath = join(configDir, `${name}.ready`) const launcher = spawn( @@ -876,6 +972,7 @@ describe('background session finalizer', () => { ...process.env, OPENCLAUDE_CONFIG_DIR: configDir, OPENCLAUDE_BG_FINALIZER_FIXTURE_READY: readyPath, + ...fixtureEnv, }, stdio: ['ignore', 'pipe', 'pipe'], }, @@ -890,40 +987,76 @@ describe('background session finalizer', () => { launcher.stderr, 'detached wait launcher stderr', ) - let launcherCode: number | null + let childPid: number | undefined + let ownershipTransferred = false try { - launcherCode = await waitForChildClose( + const launcherCode = await waitForChildClose( launcher, 'detached wait launcher', ) stdoutCapture.assertWithinLimit() stderrCapture.assertWithinLimit() - } catch (error) { const stdout = stdoutCapture.read() - const childPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) - if (Number.isSafeInteger(childPid) && isProcessRunning(childPid)) { - try { - process.kill(childPid, 'SIGKILL') - } catch {} + const parsedChildPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + if (Number.isSafeInteger(parsedChildPid)) childPid = parsedChildPid + const stderr = stderrCapture.read() + expect(launcherCode).toBe(0) + expect(stderr).toBe('') + const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] + const id = stdoutLogPath + ? basename(stdoutLogPath, '.out.log') + : undefined + expect(id).toBeDefined() + expect(childPid).toBeDefined() + if (!id || childPid === undefined) { + throw new Error('detached wait launch omitted its session identity') + } + await waitForFile(readyPath) + ownershipTransferred = true + return { id, childPid, readyPath } + } finally { + if (!ownershipTransferred) { + if (childPid === undefined) { + const stdout = stdoutCapture.read() + const parsedChildPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) + if (Number.isSafeInteger(parsedChildPid)) childPid = parsedChildPid + } + await stopDetachedFixtureChild(childPid) } - throw error - } - const stdout = stdoutCapture.read() - const stderr = stderrCapture.read() - expect(launcherCode).toBe(0) - expect(stderr).toBe('') - const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] - const id = stdoutLogPath - ? basename(stdoutLogPath, '.out.log') - : undefined - const childPid = Number(stdout.match(/^PID: (\d+)$/m)?.[1]) - expect(id).toBeDefined() - expect(Number.isSafeInteger(childPid)).toBe(true) - if (!id || !Number.isSafeInteger(childPid)) { - throw new Error('detached wait launch omitted its session identity') } - await waitForFile(readyPath) - return { id, childPid, readyPath } + } + + for (const scenario of [ + { + label: 'launcher output omits Logs', + env: { + OPENCLAUDE_BG_FINALIZER_FIXTURE_OUTPUT: 'omit-logs', + }, + }, + { + label: 'readiness is never published', + env: { + OPENCLAUDE_BG_FINALIZER_FIXTURE_SKIP_READY: '1', + }, + }, + ]) { + it( + `reaps the detached child when ${scenario.label}`, + async () => { + const name = `failed-handoff-${scenario.label.replaceAll(' ', '-')}` + await expect( + runWaitingDetachedLaunch(name, scenario.env), + ).rejects.toThrow() + const session = (await listBackgroundSessions()).find( + candidate => candidate.name === name, + ) + expect(session).toBeDefined() + expect( + session ? isProcessRunning(session.pid) : true, + ).toBe(false) + }, + 30_000, + ) } async function runBuiltKill( @@ -1139,27 +1272,33 @@ describe('background session finalizer', () => { { mode: 'success' as const, status: 'exited', exitCode: 0 }, { mode: 'fail' as const, status: 'failed', exitCode: 23 }, ]) { - it(`finalizes a detached ${expectation.mode} launch through handleBgFlag`, async () => { - const { stdout, session } = await runDetachedLaunch(expectation.mode) + it( + `finalizes a detached ${expectation.mode} launch through handleBgFlag`, + async () => { + const { stdout, session } = await runDetachedLaunch(expectation.mode) - expect(stdout).toMatch( - new RegExp( - `(?:Started background session ${session.id}\\.|Background session ${session.id} finished with status ${expectation.status}\\.)`, - ), - ) - expect(session).toMatchObject({ - status: expectation.status, - exitCode: expectation.exitCode, - terminalReason: 'exit_code', - }) - expect(isValidBackgroundProcessMarker(session.processMarker)).toBe(true) - expect(session.command).toContain( - backgroundProcessMarkerToken(session.processMarker!), - ) - expect(stdout).not.toContain(BACKGROUND_PROCESS_MARKER_FLAG) - expect(await Bun.file(session.stdoutLogPath).exists()).toBe(true) - expect(await Bun.file(session.stderrLogPath).exists()).toBe(true) - }) + expect(stdout).toMatch( + new RegExp( + `(?:Started background session ${session.id}\\.|Background session ${session.id} finished with status ${expectation.status}\\.)`, + ), + ) + expect(session).toMatchObject({ + status: expectation.status, + exitCode: expectation.exitCode, + terminalReason: 'exit_code', + }) + expect(isValidBackgroundProcessMarker(session.processMarker)).toBe( + true, + ) + expect(session.command).toContain( + backgroundProcessMarkerToken(session.processMarker!), + ) + expect(stdout).not.toContain(BACKGROUND_PROCESS_MARKER_FLAG) + expect(await Bun.file(session.stdoutLogPath).exists()).toBe(true) + expect(await Bun.file(session.stderrLogPath).exists()).toBe(true) + }, + 30_000, + ) } async function runZeroRetentionFinalizerScenario( @@ -1302,6 +1441,18 @@ describe('background session finalizer', () => { ], reclaimed: true, }, + { + label: 'invalid inline sibling retains against global zero-day policy', + globalDays: 0, + prepare: async () => [ + '--settings', + JSON.stringify({ + cleanupPeriodDays: 0, + permissions: { defaultMode: 'bogus' }, + }), + ], + reclaimed: false, + }, { label: 'project-only sources retain against global zero-day policy', globalDays: 0, diff --git a/src/cli/bgFinalizer.ts b/src/cli/bgFinalizer.ts index 10943ac3a1..a832186203 100644 --- a/src/cli/bgFinalizer.ts +++ b/src/cli/bgFinalizer.ts @@ -3,6 +3,11 @@ import { resolve } from 'node:path' import { isProcessRunning } from '../utils/genericProcessUtils.js' import { logForDebugging } from '../utils/debug.js' import { registerCleanup } from '../utils/cleanupRegistry.js' +import { SettingsSchema } from '../utils/settings/types.js' +import { + filterInvalidModelPricing, + filterInvalidPermissionRules, +} from '../utils/settings/validation.js' import { beginBackgroundSessionSignalTracking, type ObservedBackgroundSessionSignal, @@ -32,6 +37,15 @@ const DEFAULT_REGISTRATION_WAIT_MS = 5_000 const DEFAULT_REGISTRATION_POLL_MS = 10 const DEFAULT_CLEANUP_WORKER_WAIT_MS = 10_000 const DEFAULT_CLEANUP_WORKER_POLL_MS = 25 +const BACKGROUND_SESSION_RETENTION_POLICY_STATE_ENV = + 'OPENCLAUDE_INTERNAL_BACKGROUND_RETENTION_POLICY_STATE' + +type BackgroundRetentionPolicyState = 'preserved' | 'invalid' + +type BackgroundRetentionSettingsHandoff = { + args: string[] + policyState: BackgroundRetentionPolicyState +} type PrepareBackgroundSessionFinalizerOptions = { env?: NodeJS.ProcessEnv @@ -51,7 +65,9 @@ type PrepareBackgroundSessionFinalizerOptions = { ownerPid: number, launcherPid: number, settingsArgs: string[], + policyState: BackgroundRetentionPolicyState, ) => void + spawnCleanupWorkerProcess?: typeof spawn settingsArgs?: string[] settingsCwd?: string getObservedSignal?: () => ObservedBackgroundSessionSignal | undefined @@ -186,64 +202,87 @@ type BackgroundSessionCleanupWorkerStartOptions = { entrypoint?: string settingsArgs?: string[] settingsCwd?: string + policyState?: BackgroundRetentionPolicyState spawnProcess?: typeof spawn } -function backgroundRetentionSettingsArgs( +function backgroundRetentionSettingsHandoff( args: string[], settingsCwd: string, -): string[] { +): BackgroundRetentionSettingsHandoff { const selected: string[] = [] + let policyState: BackgroundRetentionPolicyState = 'preserved' for (const flag of ['--settings', '--setting-sources']) { const inlinePrefix = `${flag}=` for (let index = 0; index < args.length; index += 1) { const arg = args[index] if (arg?.startsWith(inlinePrefix)) { - const value = backgroundRetentionSettingsValue( + const handoff = backgroundRetentionSettingsValue( flag, arg.slice(inlinePrefix.length), settingsCwd, ) - if (value !== undefined) selected.push(flag, value) + if (handoff.policyState === 'invalid') policyState = 'invalid' + if (handoff.value !== undefined) selected.push(flag, handoff.value) break } if (arg === flag && index + 1 < args.length) { - const value = backgroundRetentionSettingsValue( + const handoff = backgroundRetentionSettingsValue( flag, args[index + 1]!, settingsCwd, ) - if (value !== undefined) selected.push(flag, value) + if (handoff.policyState === 'invalid') policyState = 'invalid' + if (handoff.value !== undefined) selected.push(flag, handoff.value) break } } } - return selected + return { args: selected, policyState } } function backgroundRetentionSettingsValue( flag: string, value: string, settingsCwd: string, -): string | undefined { - if (flag !== '--settings') return value +): { value?: string; policyState: BackgroundRetentionPolicyState } { + if (flag !== '--settings') { + return { value, policyState: 'preserved' } + } const trimmed = value.trim() if (!trimmed.startsWith('{') || !trimmed.endsWith('}')) { - return resolve(settingsCwd, value) + return { + value: resolve(settingsCwd, value), + policyState: 'preserved', + } } try { const parsed = JSON.parse(trimmed) as unknown - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - return '{' + const inlineErrors = [ + ...filterInvalidPermissionRules(parsed, '--settings'), + ...filterInvalidModelPricing(parsed, '--settings'), + ] + if ( + !parsed || + typeof parsed !== 'object' || + Array.isArray(parsed) || + inlineErrors.length > 0 || + !SettingsSchema().safeParse(parsed).success + ) { + return { policyState: 'invalid' } + } + if (!Object.hasOwn(parsed, 'cleanupPeriodDays')) { + return { policyState: 'preserved' } + } + return { + value: JSON.stringify({ + cleanupPeriodDays: (parsed as { cleanupPeriodDays?: unknown }) + .cleanupPeriodDays, + }), + policyState: 'preserved', } - if (!Object.hasOwn(parsed, 'cleanupPeriodDays')) return undefined - return JSON.stringify({ - cleanupPeriodDays: (parsed as { cleanupPeriodDays?: unknown }) - .cleanupPeriodDays, - }) } catch { - // Preserve fail-closed parsing without forwarding unrelated inline data. - return '{' + return { policyState: 'invalid' } } } @@ -264,15 +303,20 @@ export function startBackgroundSessionCleanupWorker( options.ownerPid, ) workerEnv[BACKGROUND_SESSION_LAUNCHER_PID_ENV] = String(options.launcherPid) + const settingsHandoff = backgroundRetentionSettingsHandoff( + options.settingsArgs ?? process.argv.slice(2), + options.settingsCwd ?? process.cwd(), + ) + workerEnv[BACKGROUND_SESSION_RETENTION_POLICY_STATE_ENV] = + options.policyState === 'invalid' + ? 'invalid' + : settingsHandoff.policyState const child = (options.spawnProcess ?? spawn)( options.execPath ?? process.execPath, [ ...(options.execArgv ?? process.execArgv), entrypoint, - ...backgroundRetentionSettingsArgs( - options.settingsArgs ?? process.argv.slice(2), - options.settingsCwd ?? process.cwd(), - ), + ...settingsHandoff.args, ], { detached: true, @@ -325,10 +369,12 @@ export async function runBackgroundSessionCleanupWorker( env[BACKGROUND_SESSION_LAUNCHER_PID_ENV], ) const sessionId = env[BACKGROUND_SESSION_ID_ENV] + const policyState = env[BACKGROUND_SESSION_RETENTION_POLICY_STATE_ENV] delete env[BACKGROUND_SESSION_CLEANUP_WORKER_ENV] delete env[BACKGROUND_SESSION_CLEANUP_OWNER_PID_ENV] delete env[BACKGROUND_SESSION_LAUNCHER_PID_ENV] delete env[BACKGROUND_SESSION_ID_ENV] + delete env[BACKGROUND_SESSION_RETENTION_POLICY_STATE_ENV] if ( ownerPid === undefined || launcherPid === undefined || @@ -337,6 +383,7 @@ export async function runBackgroundSessionCleanupWorker( ) { return } + if (policyState !== undefined && policyState !== 'preserved') return const isProcessAlive = options.isProcessAlive ?? isBackgroundLauncherAlive const sleep = @@ -405,7 +452,7 @@ export async function prepareBackgroundSessionFinalizer( const getObservedSignal = options.getObservedSignal ?? beginBackgroundSessionSignalTracking() const debug = options.debug ?? defaultDebug - const retentionSettingsArgs = backgroundRetentionSettingsArgs( + const retentionSettingsHandoff = backgroundRetentionSettingsHandoff( options.settingsArgs ?? process.argv.slice(2), options.settingsCwd ?? process.cwd(), ) @@ -416,12 +463,15 @@ export async function prepareBackgroundSessionFinalizer( cleanupOwnerPid, cleanupLauncherPid, cleanupSettingsArgs, + cleanupPolicyState, ) => startBackgroundSessionCleanupWorker({ sessionId: cleanupSessionId, ownerPid: cleanupOwnerPid, launcherPid: cleanupLauncherPid, settingsArgs: cleanupSettingsArgs, + policyState: cleanupPolicyState, + spawnProcess: options.spawnCleanupWorkerProcess, })) let finalized = false @@ -431,7 +481,8 @@ export async function prepareBackgroundSessionFinalizer( id, ownerPid, launcherPid, - retentionSettingsArgs, + retentionSettingsHandoff.args, + retentionSettingsHandoff.policyState, ) } catch (error) { reportPostFinalizationCleanupFailure(debug, error) diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts index 24f2670eee..e554d595dc 100644 --- a/src/cli/bgRegistry.cleanup.test.ts +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -1313,7 +1313,7 @@ describe('background session retention cleanup', () => { expect(await exists(paths(malformed.id).natural)).toBe(true) }) - it('advances the bounded scan past preserved orphaned facts', async () => { + it('reclaims eligible orphaned facts beyond a retained prefix in one full sweep', async () => { const terminalDir = join(root, 'terminal') await mkdir(terminalDir, { recursive: true }) const preservedFacts = Array.from({ length: 256 }, (_, index) => { @@ -1345,22 +1345,12 @@ describe('background session retention cleanup', () => { exitCode: 0, }), ) - - // The fixed seed selects the initial index; later calls resume from the module-level offset. - const scanOptions = { - _orphanedTerminalFactScanStartForTesting: 0, - } - expect( - await cleanupBackgroundSessionsBefore(CUTOFF, scanOptions), - ).toEqual({ - sessionsRemoved: 0, - artifactsRemoved: 0, - errors: 0, - }) - expect(await exists(paths(oldId).natural)).toBe(true) + const deterministicCutoff = new Date( + CUTOFF.getTime() - (CUTOFF.getTime() % 257), + ) expect( - await cleanupBackgroundSessionsBefore(CUTOFF, scanOptions), + await cleanupBackgroundSessionsBefore(deterministicCutoff), ).toEqual({ sessionsRemoved: 0, artifactsRemoved: 1, diff --git a/src/cli/bgRegistry.test.ts b/src/cli/bgRegistry.test.ts index 036147cc90..dc23efd29a 100644 --- a/src/cli/bgRegistry.test.ts +++ b/src/cli/bgRegistry.test.ts @@ -1,6 +1,13 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import { createHash } from 'node:crypto' -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { + mkdir, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import * as lockfile from '../utils/lockfile.js' @@ -16,6 +23,8 @@ import { readBackgroundSessionForOwner, refreshBackgroundSessionStatuses, resolveBackgroundSession, + snapshotBackgroundSessionRecoveryJournal, + takeBackgroundSessionRecoveryBatch, verifyBackgroundSessionProcessIdentity, type BackgroundSession, } from './bgRegistry.js' @@ -1031,6 +1040,182 @@ describe('background session registry', () => { }) }) + it('does not follow a symlinked recovery journal after terminal persistence', async () => { + const session = await createBackgroundSession({ + id: 'bg-symlinked-recovery-journal', + pid: 451, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: 'conversation-symlinked-recovery-journal', + }) + const outside = join(configDir, 'outside-recovery-journal') + await writeFile(outside, 'keep') + await symlink( + outside, + join(configDir, 'bg-sessions', '.recovery-journal'), + ) + + expect( + await recordBackgroundSessionNaturalTermination( + session.id, + { exitCode: 0 }, + { ownerPid: session.pid }, + ), + ).toMatchObject({ status: 'exited' }) + expect(await readFile(outside, 'utf8')).toBe('keep') + }) + + it('queues async terminal persistence for bounded recovery', async () => { + const session = await createBackgroundSession({ + id: 'bg-async-recovery-queue', + pid: 452, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: 'conversation-async-recovery-queue', + }) + await recordBackgroundSessionNaturalTermination( + session.id, + { exitCode: 0 }, + { ownerPid: session.pid }, + ) + + const batch = await takeBackgroundSessionRecoveryBatch(1) + expect(batch.sessionIds).toEqual([session.id]) + await batch.commit() + expect( + (await takeBackgroundSessionRecoveryBatch(1)).sessionIds, + ).toEqual([]) + }) + + it('caps recovery batches at 256 journal records', async () => { + const root = join(configDir, 'bg-sessions') + await mkdir(root, { recursive: true }) + await writeFile( + join(root, '.recovery-journal'), + `${Array.from({ length: 300 }, (_, index) => `bg-batch-cap-${index}`).join('\n')}\n`, + ) + + expect( + (await takeBackgroundSessionRecoveryBatch(1_000)).sessionIds, + ).toHaveLength(256) + }) + + it('queues sync terminal persistence for bounded recovery', async () => { + const session = await createBackgroundSession({ + id: 'bg-sync-recovery-queue', + pid: 453, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: 'conversation-sync-recovery-queue', + }) + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { ownerPid: session.pid, expectedSession: session }, + ) + + const batch = await takeBackgroundSessionRecoveryBatch(1) + expect(batch.sessionIds).toEqual([session.id]) + }) + + it('preserves records appended after a full-sweep journal snapshot', async () => { + const first = await createBackgroundSession({ + id: 'bg-recovery-snapshot-first', + pid: 455, + cwd: '/repo', + command: ['openclaude', '--print', 'first'], + sessionId: 'conversation-recovery-snapshot-first', + }) + await recordBackgroundSessionNaturalTermination( + first.id, + { exitCode: 0 }, + { ownerPid: first.pid }, + ) + const snapshot = + await snapshotBackgroundSessionRecoveryJournal() + + const second = await createBackgroundSession({ + id: 'bg-recovery-snapshot-second', + pid: 456, + cwd: '/repo', + command: ['openclaude', '--print', 'second'], + sessionId: 'conversation-recovery-snapshot-second', + }) + await recordBackgroundSessionNaturalTermination( + second.id, + { exitCode: 0 }, + { ownerPid: second.pid }, + ) + + expect(await snapshot.commit()).toBe(true) + expect( + (await takeBackgroundSessionRecoveryBatch(2)).sessionIds, + ).toEqual([second.id]) + }) + + it('ignores a stale batch commit after journal rotation', async () => { + const first = await createBackgroundSession({ + id: 'bg-stale-recovery-batch-first', + pid: 457, + cwd: '/repo', + command: ['openclaude', '--print', 'first'], + sessionId: 'conversation-stale-recovery-batch-first', + }) + await recordBackgroundSessionNaturalTermination( + first.id, + { exitCode: 0 }, + { ownerPid: first.pid }, + ) + const staleBatch = await takeBackgroundSessionRecoveryBatch(1) + const snapshot = + await snapshotBackgroundSessionRecoveryJournal() + expect(await snapshot.commit()).toBe(true) + + const second = await createBackgroundSession({ + id: 'bg-stale-recovery-batch-second', + pid: 458, + cwd: '/repo', + command: ['openclaude', '--print', 'second'], + sessionId: 'conversation-stale-recovery-batch-second', + }) + await recordBackgroundSessionNaturalTermination( + second.id, + { exitCode: 0 }, + { ownerPid: second.pid }, + ) + await staleBatch.commit() + + expect( + (await takeBackgroundSessionRecoveryBatch(1)).sessionIds, + ).toEqual([second.id]) + }) + + it('does not follow a symlinked recovery journal during sync persistence', async () => { + const session = await createBackgroundSession({ + id: 'bg-sync-symlinked-recovery-journal', + pid: 454, + cwd: '/repo', + command: ['openclaude', '--print', 'work'], + sessionId: 'conversation-sync-symlinked-recovery-journal', + }) + const outside = join(configDir, 'outside-sync-recovery-journal') + await writeFile(outside, 'keep') + await symlink( + outside, + join(configDir, 'bg-sessions', '.recovery-journal'), + ) + + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { ownerPid: session.pid, expectedSession: session }, + ) + expect((await resolveBackgroundSession(session.id)).status).toBe( + 'exited', + ) + expect(await readFile(outside, 'utf8')).toBe('keep') + }) + it('does not let a late natural finalizer replace the first valid fact', async () => { await createBackgroundSession({ id: 'bg-first-fact', diff --git a/src/cli/bgRegistry.ts b/src/cli/bgRegistry.ts index b9f6315af6..63cfdb3072 100644 --- a/src/cli/bgRegistry.ts +++ b/src/cli/bgRegistry.ts @@ -13,8 +13,10 @@ import { } from 'node:fs/promises' import { closeSync, + fstatSync, fsyncSync, linkSync, + lstatSync, mkdirSync, openSync, readFileSync, @@ -135,7 +137,9 @@ const COMPLETED_STATUSES = new Set([ ]) const SAFE_ID_RE = /^[A-Za-z0-9._-]+$/ const SAFE_SIGNAL_RE = /^SIG[A-Z0-9]{1,24}$/ -const ORPHANED_TERMINAL_FACT_SCAN_LIMIT = 256 +const BACKGROUND_RECOVERY_JOURNAL_ENTRY_LIMIT = 256 +const BACKGROUND_RECOVERY_JOURNAL_RECORD_MAX_BYTES = 512 +const BACKGROUND_RECOVERY_JOURNAL_VERSION = 1 const NAME_RESERVATION_LOCK_OPTIONS = { realpath: false, retries: { @@ -151,13 +155,30 @@ const NAME_RESERVATION_SYNC_LOCK_OPTIONS = { retries: 0, } satisfies NonNullable[1]> let backgroundSessionsRootForTesting: string | undefined -let orphanedTerminalFactScanOffset = 0 + +type BackgroundSessionRecoveryJournalIdentity = { + dev: number + ino: number +} + +type BackgroundSessionRecoveryCursor = { + version: typeof BACKGROUND_RECOVERY_JOURNAL_VERSION + offset: number +} & BackgroundSessionRecoveryJournalIdentity + +export type BackgroundSessionRecoveryBatch = { + sessionIds: string[] + commit: () => Promise +} + +export type BackgroundSessionRecoverySnapshot = { + commit: () => Promise +} export function _setBackgroundSessionsRootForTesting( root: string | undefined, ): void { backgroundSessionsRootForTesting = root?.normalize('NFC') - orphanedTerminalFactScanOffset = 0 } function getBackgroundSessionsRoot(): string { @@ -183,6 +204,14 @@ function getBackgroundSessionTerminalDir(): string { return join(getBackgroundSessionsRoot(), 'terminal') } +function getBackgroundSessionRecoveryJournalPath(): string { + return join(getBackgroundSessionsRoot(), '.recovery-journal') +} + +function getBackgroundSessionRecoveryCursorPath(): string { + return join(getBackgroundSessionsRoot(), '.recovery-cursor.json') +} + async function readDirectoryEntries( path: string, maxEntries?: number, @@ -209,6 +238,370 @@ async function readDirectoryEntries( return entries } +async function withBackgroundRecoveryJournalLock( + operation: () => Promise, +): Promise { + await mkdir(getBackgroundSessionsRoot(), { recursive: true, mode: 0o700 }) + const journalPath = getBackgroundSessionRecoveryJournalPath() + const handle = await openVerifiedBackgroundRecoveryJournal('a') + await handle.close() + const release = await lockfile.lock( + journalPath, + NAME_RESERVATION_LOCK_OPTIONS, + ) + try { + return await operation() + } finally { + await release().catch(() => {}) + } +} + +function withBackgroundRecoveryJournalLockSync(operation: () => T): T { + mkdirSync(getBackgroundSessionsRoot(), { recursive: true, mode: 0o700 }) + const journalPath = getBackgroundSessionRecoveryJournalPath() + const journalFd = openVerifiedBackgroundRecoveryJournalSync('a') + closeSync(journalFd) + const release = lockfile.lockSync( + journalPath, + NAME_RESERVATION_SYNC_LOCK_OPTIONS, + ) + try { + return operation() + } finally { + try { + release() + } catch {} + } +} + +async function openVerifiedBackgroundRecoveryJournal( + flags: 'a' | 'r' | 'r+', +): Promise>> { + const path = getBackgroundSessionRecoveryJournalPath() + const handle = await open(path, flags, 0o600) + try { + const [pathIdentity, handleIdentity] = await Promise.all([ + lstat(path), + handle.stat(), + ]) + if ( + !pathIdentity.isFile() || + !handleIdentity.isFile() || + pathIdentity.dev !== handleIdentity.dev || + pathIdentity.ino !== handleIdentity.ino + ) { + throw new Error('Invalid background recovery journal') + } + return handle + } catch (error) { + await handle.close().catch(() => {}) + throw error + } +} + +function openVerifiedBackgroundRecoveryJournalSync( + flags: 'a' | 'r' | 'r+', +): number { + const path = getBackgroundSessionRecoveryJournalPath() + const fd = openSync(path, flags, 0o600) + try { + const pathIdentity = lstatSync(path) + const handleIdentity = fstatSync(fd) + if ( + !pathIdentity.isFile() || + !handleIdentity.isFile() || + pathIdentity.dev !== handleIdentity.dev || + pathIdentity.ino !== handleIdentity.ino + ) { + throw new Error('Invalid background recovery journal') + } + return fd + } catch (error) { + closeSync(fd) + throw error + } +} + +function backgroundRecoveryJournalRecord(id: string): string | undefined { + if (!SAFE_ID_RE.test(id)) return undefined + const record = `${id}\n` + return Buffer.byteLength(record) <= + BACKGROUND_RECOVERY_JOURNAL_RECORD_MAX_BYTES + ? record + : undefined +} + +async function enqueueBackgroundSessionRecovery(id: string): Promise { + const record = backgroundRecoveryJournalRecord(id) + if (!record) return + await withBackgroundRecoveryJournalLock(async () => { + const handle = await openVerifiedBackgroundRecoveryJournal('a') + try { + await handle.writeFile(record) + await handle.sync() + } finally { + await handle.close().catch(() => {}) + } + }) +} + +function enqueueBackgroundSessionRecoverySync(id: string): void { + const record = backgroundRecoveryJournalRecord(id) + if (!record) return + withBackgroundRecoveryJournalLockSync(() => { + const fd = openVerifiedBackgroundRecoveryJournalSync('a') + try { + writeFileSync(fd, record) + fsyncSync(fd) + } finally { + closeSync(fd) + } + }) +} + +function sameBackgroundRecoveryJournalIdentity( + left: Partial, + right: BackgroundSessionRecoveryJournalIdentity, +): boolean { + return ( + typeof left.dev === 'number' && + typeof left.ino === 'number' && + left.dev === right.dev && + left.ino === right.ino + ) +} + +async function readBackgroundSessionRecoveryCursor( + identity: BackgroundSessionRecoveryJournalIdentity, +): Promise { + try { + if (!(await lstat(getBackgroundSessionRecoveryCursorPath())).isFile()) { + return 0 + } + const parsed = jsonParse( + await readFile(getBackgroundSessionRecoveryCursorPath(), 'utf8'), + ) as Partial + return parsed.version === BACKGROUND_RECOVERY_JOURNAL_VERSION && + sameBackgroundRecoveryJournalIdentity(parsed, identity) && + Number.isSafeInteger(parsed.offset) && + parsed.offset! >= 0 + ? parsed.offset! + : 0 + } catch { + return 0 + } +} + +async function writeBackgroundSessionRecoveryCursor( + offset: number, + identity: BackgroundSessionRecoveryJournalIdentity, +): Promise { + const path = getBackgroundSessionRecoveryCursorPath() + const tmp = join( + getBackgroundSessionsRoot(), + `.recovery-cursor.${process.pid}.${randomUUID()}.tmp`, + ) + const handle = await open(tmp, 'wx', 0o600) + try { + await handle.writeFile( + jsonStringify({ + version: BACKGROUND_RECOVERY_JOURNAL_VERSION, + offset, + ...identity, + } satisfies BackgroundSessionRecoveryCursor), + ) + await handle.sync() + await handle.close() + await rename(tmp, path) + } finally { + await handle.close().catch(() => {}) + await unlink(tmp).catch(() => {}) + } +} + +async function rotateBackgroundSessionRecoveryJournal( + startOffset: number, + expectedIdentity: BackgroundSessionRecoveryJournalIdentity, +): Promise { + const journalPath = getBackgroundSessionRecoveryJournalPath() + const source = await openVerifiedBackgroundRecoveryJournal('r') + const sourceIdentity = await source.stat() + if ( + !sameBackgroundRecoveryJournalIdentity( + sourceIdentity, + expectedIdentity, + ) + ) { + await source.close() + return false + } + const tmp = join( + getBackgroundSessionsRoot(), + `.recovery-journal.${process.pid}.${randomUUID()}.tmp`, + ) + let target: Awaited> | undefined + try { + target = await open(tmp, 'wx', 0o600) + const buffer = Buffer.alloc(64 * 1024) + let position = startOffset + while (true) { + const { bytesRead } = await source.read( + buffer, + 0, + buffer.byteLength, + position, + ) + if (bytesRead === 0) break + await target.writeFile(buffer.subarray(0, bytesRead)) + position += bytesRead + } + await target.sync() + await source.close() + await target.close() + target = undefined + await rename(tmp, journalPath) + await unlink(getBackgroundSessionRecoveryCursorPath()).catch(() => {}) + return true + } finally { + await source.close().catch(() => {}) + await target?.close().catch(() => {}) + await unlink(tmp).catch(() => {}) + } +} + +export async function snapshotBackgroundSessionRecoveryJournal(): Promise< + BackgroundSessionRecoverySnapshot +> { + let snapshotSize = 0 + let snapshotIdentity: BackgroundSessionRecoveryJournalIdentity = { + dev: 0, + ino: 0, + } + await withBackgroundRecoveryJournalLock(async () => { + const handle = await openVerifiedBackgroundRecoveryJournal('r') + try { + const identity = await handle.stat() + snapshotSize = identity.size + snapshotIdentity = { dev: identity.dev, ino: identity.ino } + } finally { + await handle.close().catch(() => {}) + } + }) + return { + commit: async () => + await withBackgroundRecoveryJournalLock( + async () => + await rotateBackgroundSessionRecoveryJournal( + snapshotSize, + snapshotIdentity, + ), + ), + } +} + +export async function takeBackgroundSessionRecoveryBatch( + maxEntries: number, +): Promise { + if (!Number.isSafeInteger(maxEntries) || maxEntries < 1) { + return { sessionIds: [], commit: async () => {} } + } + const boundedMaxEntries = Math.min( + maxEntries, + BACKGROUND_RECOVERY_JOURNAL_ENTRY_LIMIT, + ) + + let startOffset = 0 + let nextOffset = 0 + let snapshotSize = 0 + let snapshotIdentity: BackgroundSessionRecoveryJournalIdentity = { + dev: 0, + ino: 0, + } + let sessionIds: string[] = [] + await withBackgroundRecoveryJournalLock(async () => { + const handle = await openVerifiedBackgroundRecoveryJournal('r') + try { + const identity = await handle.stat() + snapshotSize = identity.size + snapshotIdentity = { dev: identity.dev, ino: identity.ino } + startOffset = await readBackgroundSessionRecoveryCursor( + snapshotIdentity, + ) + if (startOffset > snapshotSize) startOffset = 0 + nextOffset = startOffset + const maxBytes = + boundedMaxEntries * BACKGROUND_RECOVERY_JOURNAL_RECORD_MAX_BYTES + const buffer = Buffer.alloc(maxBytes) + const { bytesRead } = await handle.read( + buffer, + 0, + maxBytes, + startOffset, + ) + const contents = buffer.subarray(0, bytesRead) + let lineStart = 0 + let linesRead = 0 + const ids = new Set() + while (linesRead < boundedMaxEntries) { + const newline = contents.indexOf(0x0a, lineStart) + if (newline < 0) break + const id = contents.subarray(lineStart, newline).toString('utf8') + if (SAFE_ID_RE.test(id)) ids.add(id) + nextOffset = startOffset + newline + 1 + lineStart = newline + 1 + linesRead++ + } + if (linesRead === 0 && bytesRead === maxBytes) { + nextOffset = startOffset + bytesRead + } + sessionIds = [...ids] + } finally { + await handle.close().catch(() => {}) + } + }) + + return { + sessionIds, + commit: async () => { + if (nextOffset === startOffset) return + await withBackgroundRecoveryJournalLock(async () => { + const handle = await openVerifiedBackgroundRecoveryJournal('r') + let identity: Stats + try { + identity = await handle.stat() + } finally { + await handle.close().catch(() => {}) + } + const currentIdentity = { dev: identity.dev, ino: identity.ino } + if ( + !sameBackgroundRecoveryJournalIdentity( + currentIdentity, + snapshotIdentity, + ) + ) { + return + } + const currentOffset = await readBackgroundSessionRecoveryCursor( + currentIdentity, + ) + if (currentOffset !== startOffset) return + const currentSize = identity.size + if (currentSize === snapshotSize && nextOffset >= currentSize) { + await rotateBackgroundSessionRecoveryJournal( + currentSize, + currentIdentity, + ) + return + } + await writeBackgroundSessionRecoveryCursor( + nextOffset, + currentIdentity, + ) + }) + }, + } +} + function metadataPathForId(id: string): string { assertSafeId(id) return join(getBackgroundSessionMetadataDir(), `${id}.json`) @@ -978,6 +1371,7 @@ async function installTerminalFact( const target = terminalFactPathForId(fact.id, kind, fact.generation) const tmp = terminalFactTempPath(fact.id) let handle: Awaited> | undefined + let installed: BackgroundSessionTerminalFact try { handle = await open(tmp, 'wx', 0o600) await handle.writeFile(jsonStringify(fact)) @@ -985,18 +1379,24 @@ async function installTerminalFact( await handle.close() handle = undefined await link(tmp, target) - return fact + installed = fact } catch (error) { if (isErrno(error, 'EEXIST')) { const existing = await readTerminalFact(fact.id, kind, fact.generation) - if (existing) return existing - throw new Error(`Invalid background session ${kind} terminal fact`) + if (existing) { + installed = existing + } else { + throw new Error(`Invalid background session ${kind} terminal fact`) + } + } else { + throw error } - throw error } finally { await handle?.close().catch(() => {}) await unlink(tmp).catch(() => {}) } + await enqueueBackgroundSessionRecovery(installed.id).catch(() => {}) + return installed } function installTerminalFactSync( @@ -1010,6 +1410,7 @@ function installTerminalFactSync( const target = terminalFactPathForId(fact.id, kind, fact.generation) const tmp = terminalFactTempPath(fact.id) let fd: number | undefined + let installed: BackgroundSessionTerminalFact try { fd = openSync(tmp, 'wx', 0o600) writeFileSync(fd, jsonStringify(fact)) @@ -1017,14 +1418,18 @@ function installTerminalFactSync( closeSync(fd) fd = undefined linkSync(tmp, target) - return fact + installed = fact } catch (error) { if (isErrno(error, 'EEXIST')) { const existing = readTerminalFactSync(fact.id, kind, fact.generation) - if (existing) return existing - throw new Error(`Invalid background session ${kind} terminal fact`) + if (existing) { + installed = existing + } else { + throw new Error(`Invalid background session ${kind} terminal fact`) + } + } else { + throw error } - throw error } finally { if (fd !== undefined) { try { @@ -1035,6 +1440,12 @@ function installTerminalFactSync( unlinkSync(tmp) } catch {} } + try { + enqueueBackgroundSessionRecoverySync(installed.id) + } catch { + // The terminal fact remains the durable fallback for the daily sweep. + } + return installed } type BackgroundSessionRecord = { @@ -1396,7 +1807,6 @@ async function cleanupOrphanedTerminalFacts( metadataIds: ReadonlySet, result: BackgroundCleanupResult, fileSystem: BackgroundCleanupFileSystem, - scanStartSeed: number = cutoffMs, maxDirectoryEntries?: number, ): Promise { const terminalDirectoryState = await inspectCleanupDirectory( @@ -1444,24 +1854,7 @@ async function cleanupOrphanedTerminalFacts( }) if (candidates.length === 0) return - const scanCount = Math.min( - candidates.length, - ORPHANED_TERMINAL_FACT_SCAN_LIMIT, - ) - const effectiveScanStartSeed = Number.isFinite(scanStartSeed) - ? scanStartSeed - : cutoffMs - const normalizedSeed = - ((Math.trunc(effectiveScanStartSeed) % candidates.length) + - candidates.length) % - candidates.length - const startIndex = - (normalizedSeed + orphanedTerminalFactScanOffset) % candidates.length - orphanedTerminalFactScanOffset = - (orphanedTerminalFactScanOffset + scanCount) % candidates.length - for (let index = 0; index < scanCount; index++) { - const candidate = candidates[(startIndex + index) % candidates.length] - if (!candidate) continue + for (const candidate of candidates) { const factPath = terminalFactPathForId( candidate.id, candidate.kind, @@ -1523,7 +1916,6 @@ export async function cleanupBackgroundSessionsBefore( _beforeMetadataDirectoryReadForTesting?: () => Promise _beforeArtifactRemovalForTesting?: (id: string) => Promise _beforeReservationRemovalForTesting?: (path: string) => Promise - _orphanedTerminalFactScanStartForTesting?: number sessionIds?: readonly string[] maxDirectoryEntries?: number } = {}, @@ -1591,7 +1983,6 @@ export async function cleanupBackgroundSessionsBefore( new Set(), result, fileSystem, - options._orphanedTerminalFactScanStartForTesting, options.maxDirectoryEntries, ) return result @@ -1865,7 +2256,6 @@ export async function cleanupBackgroundSessionsBefore( metadataIds, result, fileSystem, - options._orphanedTerminalFactScanStartForTesting, options.maxDirectoryEntries, ) } diff --git a/src/utils/backgroundHousekeeping.test.ts b/src/utils/backgroundHousekeeping.test.ts index 7614ae738e..53c9753102 100644 --- a/src/utils/backgroundHousekeeping.test.ts +++ b/src/utils/backgroundHousekeeping.test.ts @@ -8,6 +8,7 @@ describe('background session housekeeping', () => { it('starts recurring background-session reconciliation', async () => { let callback: (() => void) | undefined let runs = 0 + let retentionSchedules = 0 startBackgroundHousekeeping({ backgroundSessionReconciliation: { cleanup: async () => { @@ -18,12 +19,46 @@ describe('background session housekeeping', () => { return { unref: () => {} } }, }, + backgroundSessionRetention: { + setInterval: () => { + retentionSchedules++ + return { unref: () => {} } + }, + }, _reconciliationOnlyForTesting: true, }) callback?.() await new Promise(resolve => setTimeout(resolve, 0)) expect(runs).toBe(1) + expect(retentionSchedules).toBe(0) + }) + + it('schedules recurring positive-retention cleanup every 24 hours', async () => { + let callback: (() => void) | undefined + let intervalMs: number | undefined + let runs = 0 + startBackgroundHousekeeping({ + backgroundSessionReconciliation: { + setInterval: () => ({ unref: () => {} }), + }, + backgroundSessionRetention: { + cleanup: async () => { + runs++ + }, + setInterval: (scheduledCallback, scheduledIntervalMs) => { + callback = scheduledCallback + intervalMs = scheduledIntervalMs + return { unref: () => {} } + }, + }, + _backgroundSessionTimersOnlyForTesting: true, + }) + + expect(intervalMs).toBe(24 * 60 * 60 * 1000) + callback?.() + await new Promise(resolve => setTimeout(resolve, 0)) + expect(runs).toBe(1) }) it('runs recurring reconciliation without overlapping passes', async () => { diff --git a/src/utils/backgroundHousekeeping.ts b/src/utils/backgroundHousekeeping.ts index ade956158f..da2b7bf513 100644 --- a/src/utils/backgroundHousekeeping.ts +++ b/src/utils/backgroundHousekeeping.ts @@ -15,6 +15,7 @@ const registerProtocolModule = feature('LODESTONE') import { getIsInteractive, getLastInteractionTime } from '../bootstrap/state.js' import { + cleanupBackgroundSessionRetentionInBackground, cleanupBackgroundSessionsInBackground, cleanupNpmCacheForAnthropicPackages, cleanupOldMessageFilesInBackground, @@ -34,7 +35,7 @@ const BACKGROUND_SESSION_RECONCILIATION_INTERVAL_MS = 60 * 1000 // 10 minutes after start. const DELAY_VERY_SLOW_OPERATIONS_THAT_HAPPEN_EVERY_SESSION = 10 * 60 * 1000 -type BackgroundSessionReconciliationOptions = { +type BackgroundSessionScheduleOptions = { cleanup?: () => Promise setInterval?: ( callback: () => void, @@ -44,14 +45,17 @@ type BackgroundSessionReconciliationOptions = { } type BackgroundHousekeepingOptions = { - backgroundSessionReconciliation?: BackgroundSessionReconciliationOptions + backgroundSessionReconciliation?: BackgroundSessionScheduleOptions + backgroundSessionRetention?: BackgroundSessionScheduleOptions _reconciliationOnlyForTesting?: boolean + _backgroundSessionTimersOnlyForTesting?: boolean } -export function startBackgroundSessionReconciliation( - options: BackgroundSessionReconciliationOptions = {}, +function scheduleNonOverlappingBackgroundCleanup( + cleanup: () => Promise, + intervalMs: number, + options: BackgroundSessionScheduleOptions, ): void { - const cleanup = options.cleanup ?? cleanupBackgroundSessionsInBackground const scheduleInterval = options.setInterval ?? ((callback, intervalMs) => setInterval(callback, intervalMs)) @@ -66,10 +70,30 @@ export function startBackgroundSessionReconciliation( running = false options._onPassFinishedForTesting?.() }) - }, BACKGROUND_SESSION_RECONCILIATION_INTERVAL_MS) + }, intervalMs) interval.unref() } +function startBackgroundSessionRetention( + options: BackgroundSessionScheduleOptions = {}, +): void { + scheduleNonOverlappingBackgroundCleanup( + options.cleanup ?? cleanupBackgroundSessionRetentionInBackground, + RECURRING_CLEANUP_INTERVAL_MS, + options, + ) +} + +export function startBackgroundSessionReconciliation( + options: BackgroundSessionScheduleOptions = {}, +): void { + scheduleNonOverlappingBackgroundCleanup( + options.cleanup ?? cleanupBackgroundSessionsInBackground, + BACKGROUND_SESSION_RECONCILIATION_INTERVAL_MS, + options, + ) +} + export function startBackgroundHousekeeping( options: BackgroundHousekeepingOptions = {}, ): void { @@ -77,6 +101,8 @@ export function startBackgroundHousekeeping( options.backgroundSessionReconciliation, ) if (options._reconciliationOnlyForTesting) return + startBackgroundSessionRetention(options.backgroundSessionRetention) + if (options._backgroundSessionTimersOnlyForTesting) return void initMagicDocs() void initSkillImprovement() diff --git a/src/utils/cleanup.test.ts b/src/utils/cleanup.test.ts index 4e149fea9b..73ed4ab213 100644 --- a/src/utils/cleanup.test.ts +++ b/src/utils/cleanup.test.ts @@ -1,5 +1,13 @@ import { afterEach, describe, expect, test } from 'bun:test' -import { mkdir, readdir, rm, stat, utimes, writeFile } from 'fs/promises' +import { + mkdir, + readdir, + readFile, + rm, + stat, + utimes, + writeFile, +} from 'fs/promises' import { tmpdir } from 'os' import { join } from 'path' @@ -54,11 +62,15 @@ async function runCleanupFixture( | 'marker-failure' | 'periodic-policy-recheck' | 'periodic-recovery' + | 'periodic-recovery-bounded' + | 'periodic-retention' + | 'periodic-retention-policy-recheck' | 'post-finalization-gate' | 'post-finalization-policy-recheck' | 'post-finalization-exact-cutoff' | 'post-finalization-timeout' | 'explicit-kill' = 'once', + fixtureEnv: NodeJS.ProcessEnv = {}, ): Promise> { const fixture = join(import.meta.dir, 'cleanupBackgroundSessions.fixture.ts') const { USER_TYPE: _userType, ...inheritedEnv } = process.env @@ -73,6 +85,7 @@ async function runCleanupFixture( OPENCLAUDE_CONFIG_DIR: configDir, OPENCLAUDE_CLEANUP_FIXTURE_MODE: mode, NODE_ENV: 'test', + ...fixtureEnv, }, signal: controller.signal, stdout: 'pipe', @@ -225,9 +238,12 @@ describe('cleanupOldMessageFilesInBackground', () => { JSON.stringify({ cleanupPeriodDays: 0 }), ) const finishedAt = new Date(Date.now() - 60_000).toISOString() + const ids = Array.from( + { length: 300 }, + (_, index) => `bg-bounded-${String(index).padStart(3, '0')}`, + ) await Promise.all( - Array.from({ length: 300 }, async (_, index) => { - const id = `bg-bounded-${String(index).padStart(3, '0')}` + ids.map(async id => { const stdoutLogPath = join(logsDir, `${id}.out.log`) const stderrLogPath = join(logsDir, `${id}.err.log`) await Promise.all([ @@ -254,6 +270,7 @@ describe('cleanupOldMessageFilesInBackground', () => { ]) }), ) + await writeFile(join(root, '.recovery-journal'), `${ids.join('\n')}\n`) expect((await runConcurrentPeriodicRecovery(configDir)).sort()).toEqual([ 'ran', @@ -266,6 +283,65 @@ describe('cleanupOldMessageFilesInBackground', () => { 30_000, ) + test( + 'advances bounded recovery past erroring retained entries across process restarts', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-progress-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + const root = join(configDir, 'bg-sessions') + const metadataDir = join(root, 'sessions') + const logsDir = join(root, 'logs') + await Promise.all([ + mkdir(metadataDir, { recursive: true }), + mkdir(logsDir, { recursive: true }), + mkdir(join(root, 'names'), { recursive: true }), + mkdir(join(root, 'terminal'), { recursive: true }), + ]) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + for (const id of ['bg-retained-a', 'bg-retained-b']) { + await writeFile(join(metadataDir, `${id}.json`), '{') + } + const targetId = 'bg-progress-target' + const targetPath = await createCompletedBackgroundSession( + configDir, + targetId, + new Date(Date.now() - 60_000), + ) + await writeFile( + join(root, '.recovery-journal'), + `bg-retained-a\nbg-retained-b\n${targetId}\n`, + ) + await rm(join(root, 'names'), { recursive: true, force: true }) + await writeFile(join(root, 'names'), 'not a directory') + + expect( + await runCleanupFixture( + configDir, + 'periodic-recovery-bounded', + { OPENCLAUDE_CLEANUP_RECOVERY_LIMIT: '2' }, + ), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(targetPath).exists()).toBe(true) + await rm(join(root, '.recovery-pass'), { force: true }) + + expect( + await runCleanupFixture( + configDir, + 'periodic-recovery-bounded', + { OPENCLAUDE_CLEANUP_RECOVERY_LIMIT: '2' }, + ), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(targetPath).exists()).toBe(false) + }, + 30_000, + ) + for (const scenario of [ { label: 'positive', setting: 30, result: 'ran' }, { label: 'invalid', setting: 'invalid', result: 'invalid-policy' }, @@ -303,6 +379,103 @@ describe('cleanupOldMessageFilesInBackground', () => { ) } + test( + 'compacts the recovery journal after a positive-retention full sweep', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-journal-compact-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-journal-compact', + new Date(Date.now() - 60_000), + ) + const journalPath = join( + configDir, + 'bg-sessions', + '.recovery-journal', + ) + expect((await readFile(journalPath, 'utf8')).length).toBeGreaterThan(0) + + expect( + await runCleanupFixture(configDir, 'periodic-retention'), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(metadataPath).exists()).toBe(true) + expect(await readFile(journalPath, 'utf8')).toBe('') + }, + 30_000, + ) + + test( + 'fails the recurring retention owner closed on project MCP errors', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-daily-invalid-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + await writeFile(join(configDir, '.mcp.json'), '{invalid') + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-daily-invalid-policy', + new Date(Date.now() - 31 * 24 * 60 * 60 * 1000), + ) + + expect( + await runCleanupFixture(configDir, 'periodic-retention'), + ).toEqual({ result: 'invalid-policy' }) + expect(await Bun.file(metadataPath).exists()).toBe(true) + expect( + await Bun.file( + join(configDir, 'bg-sessions', '.retention-pass'), + ).exists(), + ).toBe(false) + }, + 30_000, + ) + + test( + 'rechecks project MCP errors after acquiring the daily retention lock', + async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-bg-daily-recheck-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const metadataPath = await createCompletedBackgroundSession( + configDir, + 'bg-daily-policy-recheck', + new Date(Date.now() - 31 * 24 * 60 * 60 * 1000), + ) + + expect( + await runCleanupFixture( + configDir, + 'periodic-retention-policy-recheck', + ), + ).toEqual({ result: 'invalid-policy' }) + expect(await Bun.file(metadataPath).exists()).toBe(true) + }, + 30_000, + ) + test( 'rechecks periodic retention policy after acquiring the global lock', async () => { diff --git a/src/utils/cleanup.ts b/src/utils/cleanup.ts index 998337bfa1..956710f05c 100644 --- a/src/utils/cleanup.ts +++ b/src/utils/cleanup.ts @@ -4,6 +4,8 @@ import { dirname, join } from 'path' import { cleanupBackgroundSessionsBefore, reconcileBackgroundSessionTerminalFacts, + snapshotBackgroundSessionRecoveryJournal, + takeBackgroundSessionRecoveryBatch, } from '../cli/bgRegistry.js' import { logEvent } from '../services/analytics/index.js' import { CACHE_PATHS } from './cachePaths.js' @@ -53,6 +55,7 @@ type BackgroundSessionRetentionRequest = | { trigger: 'periodic-recovery' _afterThrottleLockForTesting?: () => Promise + _recoveryBatchLimitForTesting?: number } | { trigger: 'periodic-retention' @@ -794,16 +797,21 @@ export async function runBackgroundSessionRetention( return } if (currentPolicy.days !== 0) return + const batch = await takeBackgroundSessionRecoveryBatch( + request._recoveryBatchLimitForTesting ?? + BACKGROUND_RECOVERY_ENTRY_LIMIT, + ) const reconciliation = await reconcileBackgroundSessionTerminalFacts({ - terminalScanLimit: BACKGROUND_RECOVERY_ENTRY_LIMIT, + sessionIds: batch.sessionIds, }) logBackgroundReconciliation(reconciliation) const cleanup = await cleanupBackgroundSessionsBefore( new Date(Date.now() + 1), - { maxDirectoryEntries: BACKGROUND_RECOVERY_ENTRY_LIMIT }, + { sessionIds: batch.sessionIds }, ) logBackgroundCleanup(cleanup) + await batch.commit() }, request._afterThrottleLockForTesting, ) @@ -815,6 +823,11 @@ export async function runBackgroundSessionRetention( '.retention-pass', BACKGROUND_RETENTION_SWEEP_INTERVAL_MS, async () => { + resetSettingsCache() + if (cleanupPeriodSettingIsInvalid()) { + passResult = 'invalid-policy' + return + } const currentPolicy = readBackgroundRetentionPeriod({ fresh: true }) if (currentPolicy.state === 'invalid') { passResult = 'invalid-policy' @@ -823,12 +836,15 @@ export async function runBackgroundSessionRetention( const cutoff = new Date( Date.now() - currentPolicy.days * 24 * 60 * 60 * 1000, ) + const journal = + await snapshotBackgroundSessionRecoveryJournal() logBackgroundReconciliation( await reconcileBackgroundSessionTerminalFacts(), ) logBackgroundCleanup( await cleanupBackgroundSessionsBefore(cutoff), ) + await journal.commit() }, request._afterThrottleLockForTesting, ) @@ -852,6 +868,16 @@ export async function cleanupBackgroundSessionsInBackground(): Promise< }) } +export async function cleanupBackgroundSessionRetentionInBackground(): Promise< + BackgroundSessionRetentionResult +> { + resetSettingsCache() + if (cleanupPeriodSettingIsInvalid()) return 'invalid-policy' + return await runBackgroundSessionRetention({ + trigger: 'periodic-retention', + }) +} + export async function cleanupBackgroundSessionsAfterFinalization( sessionId: string, waitForProcessesToExit: () => Promise, diff --git a/src/utils/cleanupBackgroundSessions.fixture.ts b/src/utils/cleanupBackgroundSessions.fixture.ts index 6567918129..2947e684b1 100644 --- a/src/utils/cleanupBackgroundSessions.fixture.ts +++ b/src/utils/cleanupBackgroundSessions.fixture.ts @@ -17,7 +17,38 @@ import { try { const mode = process.env.OPENCLAUDE_CLEANUP_FIXTURE_MODE - if (mode === 'periodic-recovery') { + if ( + mode === 'periodic-retention' || + mode === 'periodic-retention-policy-recheck' + ) { + enableConfigs() + const settingsResult = eagerLoadSettingsFromArgs([ + '--settings', + join(process.env.OPENCLAUDE_CONFIG_DIR!, 'settings.json'), + ]) + if (!settingsResult.ok) throw new Error(settingsResult.message) + const { cleanupBackgroundSessionRetentionInBackground } = + await import('./cleanup.js') + process.stdout.write( + `${JSON.stringify({ + result: + mode === 'periodic-retention' + ? await cleanupBackgroundSessionRetentionInBackground() + : await runBackgroundSessionRetention({ + trigger: 'periodic-retention', + _afterThrottleLockForTesting: async () => { + await Bun.write( + join(process.cwd(), '.mcp.json'), + '{invalid', + ) + }, + }), + })}\n`, + ) + } else if ( + mode === 'periodic-recovery' || + mode === 'periodic-recovery-bounded' + ) { enableConfigs() const settingsResult = eagerLoadSettingsFromArgs([ '--settings', @@ -26,7 +57,15 @@ try { if (!settingsResult.ok) throw new Error(settingsResult.message) process.stdout.write( `${JSON.stringify({ - result: await cleanupBackgroundSessionsInBackground(), + result: + mode === 'periodic-recovery' + ? await cleanupBackgroundSessionsInBackground() + : await runBackgroundSessionRetention({ + trigger: 'periodic-recovery', + _recoveryBatchLimitForTesting: Number( + process.env.OPENCLAUDE_CLEANUP_RECOVERY_LIMIT, + ), + }), })}\n`, ) } else if (mode === 'periodic-policy-recheck') { From 7824266947b6079ffd68b1d18d9be3f75f349a26 Mon Sep 17 00:00:00 2001 From: chioarub Date: Sat, 5 Sep 2026 04:10:28 +0300 Subject: [PATCH 12/13] fix(bg): preserve retention pass outcomes --- src/cli/bgFinalizer.test.ts | 87 ++ src/cli/bgFinalizer.ts | 10 +- src/cli/bgRegistry.cleanup.test.ts | 153 ++- src/cli/bgRegistry.test.ts | 114 ++- src/cli/bgRegistry.ts | 953 +++++++++++------- src/utils/cleanup.test.ts | 402 +++++++- src/utils/cleanup.ts | 120 ++- .../cleanupBackgroundSessions.fixture.ts | 60 +- 8 files changed, 1447 insertions(+), 452 deletions(-) diff --git a/src/cli/bgFinalizer.test.ts b/src/cli/bgFinalizer.test.ts index 87f89fc7c3..f357208f80 100644 --- a/src/cli/bgFinalizer.test.ts +++ b/src/cli/bgFinalizer.test.ts @@ -1098,6 +1098,93 @@ describe('background session finalizer', () => { return stdout } + for (const scenario of [ + { label: 'null input', input: 'process.exit(null)', expected: 0 }, + { + label: 'cleared exitCode', + input: 'process.exitCode = null; process.exit()', + expected: 0, + }, + { label: 'omitted input', input: 'process.exit()', expected: 23 }, + { + label: 'undefined input', + input: 'process.exit(undefined)', + expected: 0, + }, + ]) { + it(`records Node's actual explicit exit with ${scenario.label}`, async () => { + const id = 'bg-node-explicit-exit' + const preload = join(configDir, 'explicit-exit.cjs') + // Run the input as soon as the built CLI installs its exit wrapper. + // Node owns conversion of the input and the observed child status. + await writeFile( + preload, + ` + let exit = process.exit + Object.defineProperty(process, 'exit', { + configurable: true, + get: () => exit, + set: value => { + exit = value + queueMicrotask(() => { + require('node:fs').writeFileSync(${JSON.stringify(join(configDir, 'exit-input-ran'))}, 'ran') + process.exitCode = 23 + ${scenario.input} + }) + }, + }) + `, + ) + const child = spawn( + 'node', + ['--require', preload, installedLauncherPath, '--print', 'fixture'], + { + cwd: configDir, + env: { + ...process.env, + NODE_ENV: '', + OPENCLAUDE_DISABLE_HEAP_RELAUNCH: '1', + OPENCLAUDE_DISABLE_CLI_ENTRYPOINT_AUTO_RUN: '0', + OPENCLAUDE_CONFIG_DIR: configDir, + [BACKGROUND_SESSION_ID_ENV]: id, + [BACKGROUND_SESSION_LAUNCHER_PID_ENV]: String(process.pid), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + const closed = waitForChildClose(child, 'Node explicit exit') + const stderr = captureBoundedOutput( + child, + child.stderr, + 'Node explicit exit stderr', + ) + try { + if (!child.pid) + throw new Error('Node explicit-exit fixture did not start') + await mkdir(join(sessionsRoot, 'sessions'), { recursive: true }) + await writeFile( + join(sessionsRoot, 'sessions', `${id}.json`), + JSON.stringify(ownedSession(id, child.pid)), + ) + const code = await closed + expect( + await Bun.file(join(configDir, 'exit-input-ran')).exists(), + ).toBe(true) + expect(code, stderr.read()).toBe(scenario.expected) + stderr.assertWithinLimit() + expect((await listBackgroundSessions())[0]).toMatchObject({ + id, + status: scenario.expected === 0 ? 'exited' : 'failed', + exitCode: scenario.expected, + }) + } finally { + if (child.exitCode === null && child.signalCode === null) + child.kill('SIGKILL') + await closed.catch(() => {}) + } + }, 30_000) + } + for (const expectation of [ { mode: 'success' as const, status: 'exited', exitCode: 0 }, { mode: 'fail' as const, status: 'failed', exitCode: 23 }, diff --git a/src/cli/bgFinalizer.ts b/src/cli/bgFinalizer.ts index a832186203..875e166829 100644 --- a/src/cli/bgFinalizer.ts +++ b/src/cli/bgFinalizer.ts @@ -98,7 +98,7 @@ function defaultDebug(message: string): void { function normalizeProcessExitCode( value: string | number | null | undefined, ): number { - if (value === undefined) return 0 + if (value === undefined || value === null) return 0 const parsed = typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : value return typeof parsed === 'number' && @@ -539,9 +539,11 @@ export async function prepareBackgroundSessionFinalizer( options.onExit(onExit) } else { const originalExit = process.exit - process.exit = ((code?: string | number | null) => { - onExit(normalizeProcessExitCode(code ?? process.exitCode)) - return originalExit(code) + process.exit = ((...args: Parameters) => { + onExit( + normalizeProcessExitCode(args.length === 0 ? process.exitCode : args[0]), + ) + return originalExit(...args) }) as typeof process.exit process.once('exit', onExit) } diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts index e554d595dc..76b41a7b95 100644 --- a/src/cli/bgRegistry.cleanup.test.ts +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -1,7 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, spyOn } from 'bun:test' import { createHash } from 'node:crypto' +import * as fsPromises from 'node:fs/promises' import { lstat, + chmod, + rename, mkdir, mkdtemp, readdir, @@ -207,6 +210,146 @@ describe('background session retention cleanup', () => { }) } + for (const consumer of ['reconciliation', 'cleanup'] as const) { + for (const fault of [ + 'invalid', + 'unreadable', + 'replaced', + 'replaced-before-read', + ] as const) { + if (consumer === 'reconciliation' && fault === 'replaced-before-read') + continue + it(`reports ${fault} ${consumer} inventory as unacknowledged responsibility`, async () => { + const session = await writeRawSession({ + id: 'bg-inventory-report', + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + const directory = join(root, 'sessions') + const saved = join(root, 'saved-sessions') + const retries: unknown[] = [] + const onRetry = (target?: unknown) => { + retries.push(target) + } + let readSpy: ReturnType | undefined + let replaced = false + let metadataSnapshotRead = false + const lstatForTesting = async (path: string) => { + if (path === directory) { + if (metadataSnapshotRead && !replaced) { + replaced = true + await rename(directory, saved) + await mkdir(directory) + } + metadataSnapshotRead = true + } + return await lstat(path) + } + try { + if (fault === 'invalid') { + await rename(directory, saved) + await writeFile(directory, 'not a directory') + } else if (fault === 'unreadable') { + await chmod(directory, 0o000) + } else if (fault === 'replaced') { + const original = fsPromises.readdir + const replacement = async ( + ...args: Parameters + ) => { + const entries = await original(...args) + if (String(args[0]) === directory && !replaced) { + replaced = true + await rename(directory, saved) + await mkdir(directory) + } + return entries + } + readSpy = spyOn(fsPromises, 'readdir').mockImplementation( + replacement as typeof fsPromises.readdir, + ) + } + const result = + consumer === 'reconciliation' + ? await reconcileBackgroundSessionTerminalFacts({ onRetry }) + : await cleanupBackgroundSessionsBefore(CUTOFF, { + onRetry, + lstatFile: + fault === 'replaced-before-read' + ? lstatForTesting + : undefined, + }) + expect(result.errors).toBeGreaterThan(0) + expect(retries).toHaveLength(1) + expect(retries[0]).toBeUndefined() + if (fault.startsWith('replaced')) expect(replaced).toBe(true) + expect( + await exists( + fault === 'unreadable' + ? paths(session.id).metadata + : join(saved, `${session.id}.json`), + ), + ).toBe(fault !== 'unreadable') + } finally { + readSpy?.mockRestore() + if (fault === 'unreadable') await chmod(directory, 0o755) + } + if (fault === 'unreadable') + expect(await exists(paths(session.id).metadata)).toBe(true) + }) + } + } + + for (const missingDirectory of [false, true]) { + it(`reclaims exact generation orphans without scanning directories, missing metadata directory ${missingDirectory}`, async () => { + const id = 'bg-targeted-orphan' + await writeRawSession({ + id, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + }) + const factPath = markedTerminalFactPath( + id, + 'natural', + OLD_PROCESS_MARKER, + ) + const fact = { + version: 1, + id, + pid: nextPid++, + generation: OLD_PROCESS_MARKER, + status: 'exited', + finishedAt: OLD_FINISH.toISOString(), + terminalReason: 'exit_code', + exitCode: 0, + } + await writeFile(factPath, JSON.stringify(fact)) + const unrelated = markedTerminalFactPath( + 'bg-unrelated-orphan', + 'natural', + OLD_PROCESS_MARKER, + ) + await writeFile( + unrelated, + JSON.stringify({ ...fact, id: 'bg-unrelated-orphan' }), + ) + if (missingDirectory) + await rm(join(root, 'sessions'), { recursive: true }) + else await unlink(paths(id).metadata) + expect( + await cleanupBackgroundSessionsBefore(CUTOFF, { + sessionIds: [id], + orphanedTerminalFacts: [ + { id, generation: OLD_PROCESS_MARKER }, + { id, generation: '../outside' }, + ], + maxDirectoryEntries: 0, + }), + ).toEqual({ sessionsRemoved: 0, artifactsRemoved: 1, errors: 0 }) + expect(await exists(factPath)).toBe(false) + expect(await exists(unrelated)).toBe(true) + }) + } + it('targets prompt cleanup to the requested session generation', async () => { const target = await createRunning('bg-targeted-cleanup') const retained = await createRunning('bg-targeted-retained') @@ -1345,13 +1488,7 @@ describe('background session retention cleanup', () => { exitCode: 0, }), ) - const deterministicCutoff = new Date( - CUTOFF.getTime() - (CUTOFF.getTime() % 257), - ) - - expect( - await cleanupBackgroundSessionsBefore(deterministicCutoff), - ).toEqual({ + expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ sessionsRemoved: 0, artifactsRemoved: 1, errors: 0, @@ -1715,7 +1852,7 @@ describe('background session retention cleanup', () => { expect(await cleanupBackgroundSessionsBefore(CUTOFF)).toEqual({ sessionsRemoved: 1, artifactsRemoved: 3, - errors: 1, + errors: 0, }) expect(await exists(paths(session.id).metadata)).toBe(false) }) diff --git a/src/cli/bgRegistry.test.ts b/src/cli/bgRegistry.test.ts index dc23efd29a..b9a6069a9b 100644 --- a/src/cli/bgRegistry.test.ts +++ b/src/cli/bgRegistry.test.ts @@ -1087,6 +1087,118 @@ describe('background session registry', () => { ).toEqual([]) }) + it('persists only owned retries behind new work before acknowledging a batch', async () => { + const root = join(configDir, 'bg-sessions') + await mkdir(root, { recursive: true }) + const journal = join(root, '.recovery-journal') + await writeFile(journal, 'bg-retry\nbg-done\nbg-next\n') + const batch = await takeBackgroundSessionRecoveryBatch(2) + await batch.commit(['bg-retry', 'bg-foreign', 'bg-retry', '../outside']) + const contents = await readFile(journal, 'utf8') + expect(contents).toBe('bg-retry\nbg-done\nbg-next\nbg-retry\n') + const cursor = JSON.parse( + await readFile(join(root, '.recovery-cursor.json'), 'utf8'), + ) + expect(cursor.offset).toBe(Buffer.byteLength('bg-retry\nbg-done\n')) + await batch.commit(['bg-retry']) + expect(await readFile(journal, 'utf8')).toBe(contents) + + const next = await takeBackgroundSessionRecoveryBatch(2) + expect(next.sessionIds).toEqual(['bg-next', 'bg-retry']) + await next.commit(['bg-retry']) + // Repeated failures rotate the consumed prefix instead of growing forever. + expect(await readFile(journal, 'utf8')).toBe('bg-retry\n') + const retry = await takeBackgroundSessionRecoveryBatch(2) + expect(retry.sessionIds).toEqual(['bg-retry']) + await retry.commit() + expect(await readFile(journal, 'utf8')).toBe('') + }) + + it('retains validated generations for terminal retries after metadata removal', async () => { + const root = join(configDir, 'bg-sessions') + await mkdir(root, { recursive: true }) + const journal = join(root, '.recovery-journal') + const generation = 'a'.repeat(64) + await writeFile(journal, 'bg-fact-retry\n') + const batch = await takeBackgroundSessionRecoveryBatch(1) + await batch.commit( + ['bg-fact-retry'], + [ + { id: 'bg-fact-retry', generation }, + { id: 'bg-foreign', generation }, + { id: 'bg-fact-retry', generation: '../outside' }, + ], + ) + const retry = await takeBackgroundSessionRecoveryBatch(256) + expect(retry.sessionIds).toEqual(['bg-fact-retry']) + expect(retry.terminalFacts).toEqual([ + { id: 'bg-fact-retry', generation: undefined }, + { id: 'bg-fact-retry', generation }, + ]) + await retry.commit(['bg-fact-retry']) + expect( + (await takeBackgroundSessionRecoveryBatch(256)).terminalFacts, + ).toEqual(retry.terminalFacts) + await writeFile( + journal, + `bg-bad~invalid\nbg-bad~${generation}~extra\nbg-good\n`, + ) + expect( + (await takeBackgroundSessionRecoveryBatch(256)).sessionIds, + ).toEqual(['bg-good']) + }) + + for (const synchronous of [false, true]) { + it(`journals marked ownership before interrupted cleanup, sync ${synchronous}`, async () => { + const id = 'bg-marked-interruption' + const session = await createBackgroundSession({ + id, + pid: 452, + cwd: '/repo', + command: ['openclaude', '--print', 'fixture'], + sessionId: 'marked-interruption', + processMarker: TEST_PROCESS_MARKER, + }) + const options = { + ownerPid: session.pid, + expectedSession: session, + now: new Date(0), + } + if (synchronous) + recordBackgroundSessionNaturalTerminationSync( + id, + { exitCode: 0 }, + options, + ) + else + await recordBackgroundSessionNaturalTermination( + id, + { exitCode: 0 }, + options, + ) + const batch = await takeBackgroundSessionRecoveryBatch(256) + expect(batch.terminalFacts).toEqual([ + { id, generation: TEST_PROCESS_MARKER }, + ]) + const root = join(configDir, 'bg-sessions') + // Simulate interruption after logs/metadata were removed, before the fact unlink. + await rm(join(root, 'sessions', `${id}.json`)) + await rm(session.stdoutLogPath, { force: true }) + await rm(session.stderrLogPath, { force: true }) + expect( + await cleanupBackgroundSessionsBefore(new Date(), { + sessionIds: batch.sessionIds, + orphanedTerminalFacts: batch.terminalFacts, + maxDirectoryEntries: 0, + }), + ).toEqual({ sessionsRemoved: 0, artifactsRemoved: 1, errors: 0 }) + await batch.commit() + expect( + (await takeBackgroundSessionRecoveryBatch(256)).sessionIds, + ).toEqual([]) + }) + } + it('caps recovery batches at 256 journal records', async () => { const root = join(configDir, 'bg-sessions') await mkdir(root, { recursive: true }) @@ -1183,7 +1295,7 @@ describe('background session registry', () => { { exitCode: 0 }, { ownerPid: second.pid }, ) - await staleBatch.commit() + await staleBatch.commit([first.id]) expect( (await takeBackgroundSessionRecoveryBatch(1)).sessionIds, diff --git a/src/cli/bgRegistry.ts b/src/cli/bgRegistry.ts index 63cfdb3072..7a5962e208 100644 --- a/src/cli/bgRegistry.ts +++ b/src/cli/bgRegistry.ts @@ -166,12 +166,22 @@ type BackgroundSessionRecoveryCursor = { offset: number } & BackgroundSessionRecoveryJournalIdentity +export type BackgroundSessionRecoveryTarget = { + id: string + generation?: string +} + export type BackgroundSessionRecoveryBatch = { sessionIds: string[] - commit: () => Promise + terminalFacts: BackgroundSessionRecoveryTarget[] + commit: ( + retrySessionIds?: readonly string[], + retryTerminalFacts?: readonly BackgroundSessionRecoveryTarget[], + ) => Promise } export type BackgroundSessionRecoverySnapshot = { + retry: (targets: readonly BackgroundSessionRecoveryTarget[]) => Promise commit: () => Promise } @@ -322,22 +332,48 @@ function openVerifiedBackgroundRecoveryJournalSync( } } -function backgroundRecoveryJournalRecord(id: string): string | undefined { +function backgroundRecoveryJournalRecord( + id: string, + generation?: string, +): string | undefined { if (!SAFE_ID_RE.test(id)) return undefined - const record = `${id}\n` + if ( + generation !== undefined && + !isValidBackgroundProcessMarker(generation) + ) { + return undefined + } + const record = `${id}${generation === undefined ? '' : `~${generation}`}\n` return Buffer.byteLength(record) <= BACKGROUND_RECOVERY_JOURNAL_RECORD_MAX_BYTES ? record : undefined } -async function enqueueBackgroundSessionRecovery(id: string): Promise { - const record = backgroundRecoveryJournalRecord(id) - if (!record) return +async function enqueueBackgroundSessionRecovery( + id: string, + generation?: string, +): Promise { + await enqueueBackgroundSessionRecoveryTargets([{ id, generation }]) +} + +async function enqueueBackgroundSessionRecoveryTargets( + targets: readonly BackgroundSessionRecoveryTarget[], +): Promise { + const records = [ + ...new Set( + targets + .map(target => + backgroundRecoveryJournalRecord(target.id, target.generation), + ) + .filter((record): record is string => record !== undefined), + ), + ].join('') + if (!records) return await withBackgroundRecoveryJournalLock(async () => { const handle = await openVerifiedBackgroundRecoveryJournal('a') try { - await handle.writeFile(record) + await handle.writeFile(records) await handle.sync() } finally { await handle.close().catch(() => {}) @@ -345,8 +381,11 @@ async function enqueueBackgroundSessionRecovery(id: string): Promise { }) } -function enqueueBackgroundSessionRecoverySync(id: string): void { - const record = backgroundRecoveryJournalRecord(id) +function enqueueBackgroundSessionRecoverySync( + id: string, + generation?: string, +): void { + const record = backgroundRecoveryJournalRecord(id, generation) if (!record) return withBackgroundRecoveryJournalLockSync(() => { const fd = openVerifiedBackgroundRecoveryJournalSync('a') @@ -488,6 +527,7 @@ export async function snapshotBackgroundSessionRecoveryJournal(): Promise< } }) return { + retry: enqueueBackgroundSessionRecoveryTargets, commit: async () => await withBackgroundRecoveryJournalLock( async () => @@ -503,7 +543,7 @@ export async function takeBackgroundSessionRecoveryBatch( maxEntries: number, ): Promise { if (!Number.isSafeInteger(maxEntries) || maxEntries < 1) { - return { sessionIds: [], commit: async () => {} } + return { sessionIds: [], terminalFacts: [], commit: async () => {} } } const boundedMaxEntries = Math.min( maxEntries, @@ -518,6 +558,7 @@ export async function takeBackgroundSessionRecoveryBatch( ino: 0, } let sessionIds: string[] = [] + const terminalFacts: BackgroundSessionRecoveryTarget[] = [] await withBackgroundRecoveryJournalLock(async () => { const handle = await openVerifiedBackgroundRecoveryJournal('r') try { @@ -545,8 +586,18 @@ export async function takeBackgroundSessionRecoveryBatch( while (linesRead < boundedMaxEntries) { const newline = contents.indexOf(0x0a, lineStart) if (newline < 0) break - const id = contents.subarray(lineStart, newline).toString('utf8') - if (SAFE_ID_RE.test(id)) ids.add(id) + const record = contents + .subarray(lineStart, newline) + .toString('utf8') + const [id, generation, extra] = record.split('~') + if ( + id && + extra === undefined && + backgroundRecoveryJournalRecord(id, generation) !== undefined + ) { + ids.add(id) + terminalFacts.push({ id, generation }) + } nextOffset = startOffset + newline + 1 lineStart = newline + 1 linesRead++ @@ -562,7 +613,8 @@ export async function takeBackgroundSessionRecoveryBatch( return { sessionIds, - commit: async () => { + terminalFacts, + commit: async (retrySessionIds = [], retryTerminalFacts = []) => { if (nextOffset === startOffset) return await withBackgroundRecoveryJournalLock(async () => { const handle = await openVerifiedBackgroundRecoveryJournal('r') @@ -585,10 +637,46 @@ export async function takeBackgroundSessionRecoveryBatch( currentIdentity, ) if (currentOffset !== startOffset) return - const currentSize = identity.size - if (currentSize === snapshotSize && nextOffset >= currentSize) { + const retryIds = new Set(retrySessionIds) + const retryRecords = [ + ...new Set( + [ + ...terminalFacts.filter(target => retryIds.has(target.id)), + ...retryTerminalFacts, + ] + .filter(target => sessionIds.includes(target.id)) + .map(target => + backgroundRecoveryJournalRecord( + target.id, + target.generation, + ), + ) + .filter((record): record is string => record !== undefined), + ), + ].join('') + if (retryRecords) { + const retryHandle = + await openVerifiedBackgroundRecoveryJournal('a') + try { + if ( + !sameBackgroundRecoveryJournalIdentity( + await retryHandle.stat(), + currentIdentity, + ) + ) { + throw new Error( + 'Background recovery journal changed before retry', + ) + } + await retryHandle.writeFile(retryRecords) + await retryHandle.sync() + } finally { + await retryHandle.close().catch(() => {}) + } + } + if (identity.size === snapshotSize && nextOffset >= snapshotSize) { await rotateBackgroundSessionRecoveryJournal( - currentSize, + nextOffset, currentIdentity, ) return @@ -1395,7 +1483,7 @@ async function installTerminalFact( await handle?.close().catch(() => {}) await unlink(tmp).catch(() => {}) } - await enqueueBackgroundSessionRecovery(installed.id).catch(() => {}) + await enqueueBackgroundSessionRecovery(installed.id, installed.generation).catch(() => {}) return installed } @@ -1441,7 +1529,7 @@ function installTerminalFactSync( } catch {} } try { - enqueueBackgroundSessionRecoverySync(installed.id) + enqueueBackgroundSessionRecoverySync(installed.id, installed.generation) } catch { // The terminal fact remains the durable fallback for the daily sweep. } @@ -1521,6 +1609,7 @@ function hasSamePersistedTerminalState( export async function reconcileBackgroundSessionTerminalFacts( options: { sessionIds?: readonly string[] + onRetry?: (target?: BackgroundSessionRecoveryTarget) => void terminalScanLimit?: number } = {}, ): Promise<{ sessionsUpdated: number; errors: number }> { @@ -1547,6 +1636,7 @@ export async function reconcileBackgroundSessionTerminalFacts( if (metadataRead.state !== 'valid' || terminalRead.state !== 'valid') { result.errors += Number(metadataRead.state !== 'valid') result.errors += Number(terminalRead.state !== 'valid') + if (result.errors > 0) options.onRetry?.() return result } const metadataDirectory = metadataRead.value @@ -1555,6 +1645,7 @@ export async function reconcileBackgroundSessionTerminalFacts( metadataDirectory.state === 'missing' || terminalDirectory.state === 'missing' ) { + if (result.errors > 0) options.onRetry?.() return result } @@ -1575,6 +1666,7 @@ export async function reconcileBackgroundSessionTerminalFacts( ) } catch (error) { if (!isErrno(error, 'ENOENT')) result.errors++ + if (result.errors > 0) options.onRetry?.() return result } const scannedDirectory = @@ -1588,6 +1680,7 @@ export async function reconcileBackgroundSessionTerminalFacts( )) !== 'same' ) { result.errors++ + if (result.errors > 0) options.onRetry?.() return result } candidateIds = [ @@ -1611,104 +1704,118 @@ export async function reconcileBackgroundSessionTerminalFacts( } for (const id of candidateIds) { - const metadataPath = metadataPathForId(id) - const candidate = await readCleanupJson( - metadataPath, - (value): value is BackgroundSession => isBackgroundSession(value, id), - metadataDirectory, - fileSystem, - ) - if (candidate.state === 'error') { - result.errors++ - continue - } - if (candidate.state !== 'valid' || !candidate.identity) continue - + const errorsBefore = result.errors + let retryGeneration: string | undefined try { - await withBackgroundSessionIdLock(id, async () => { - const natural = await readReconciliationTerminalFact( - candidate.value, - 'natural', - terminalDirectory, - fileSystem, - ) - const killed = await readReconciliationTerminalFact( - candidate.value, - 'killed', - terminalDirectory, - fileSystem, - ) - if (natural.state === 'error' || killed.state === 'error') { - result.errors++ - return - } - - const latest = await readCleanupJson( - metadataPath, - (value): value is BackgroundSession => - isBackgroundSession(value, id), - metadataDirectory, - fileSystem, - ) - if (latest.state === 'error') { - result.errors++ - return - } - if ( - latest.state !== 'valid' || - !latest.identity || - latest.identity.dev !== candidate.identity?.dev || - latest.identity.ino !== candidate.identity?.ino || - !isSameBackgroundSessionGeneration(latest.value, candidate.value) - ) { - return - } + const metadataPath = metadataPathForId(id) + const candidate = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => + isBackgroundSession(value, id), + metadataDirectory, + fileSystem, + ) + if (candidate.state === 'error') { + result.errors++ + continue + } + if (candidate.state !== 'valid' || !candidate.identity) continue + retryGeneration = candidate.value.terminalFactGeneration - const effective = applyTerminalFacts( - latest.value, - natural.state === 'valid' ? natural.value : null, - killed.state === 'valid' ? killed.value : null, - ) - if (!COMPLETED_STATUSES.has(effective.status)) return + try { + await withBackgroundSessionIdLock(id, async () => { + const natural = await readReconciliationTerminalFact( + candidate.value, + 'natural', + terminalDirectory, + fileSystem, + ) + const killed = await readReconciliationTerminalFact( + candidate.value, + 'killed', + terminalDirectory, + fileSystem, + ) + if (natural.state === 'error' || killed.state === 'error') { + result.errors++ + return + } - if (!hasSamePersistedTerminalState(latest.value, effective)) { + const latest = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => + isBackgroundSession(value, id), + metadataDirectory, + fileSystem, + ) + if (latest.state === 'error') { + result.errors++ + return + } if ( - (await inspectCleanupDirectory( - metadataDirectory, - fileSystem.lstatFile, - )) !== 'same' + latest.state !== 'valid' || + !latest.identity || + latest.identity.dev !== candidate.identity?.dev || + latest.identity.ino !== candidate.identity?.ino || + !isSameBackgroundSessionGeneration( + latest.value, + candidate.value, + ) ) { - result.errors++ return } - try { - const beforeWrite = await fileSystem.lstatFile(metadataPath) + + const effective = applyTerminalFacts( + latest.value, + natural.state === 'valid' ? natural.value : null, + killed.state === 'valid' ? killed.value : null, + ) + if (!COMPLETED_STATUSES.has(effective.status)) return + + if (!hasSamePersistedTerminalState(latest.value, effective)) { if ( - !beforeWrite.isFile() || - beforeWrite.isSymbolicLink() || - beforeWrite.dev !== latest.identity.dev || - beforeWrite.ino !== latest.identity.ino + (await inspectCleanupDirectory( + metadataDirectory, + fileSystem.lstatFile, + )) !== 'same' ) { + result.errors++ return } - } catch (error) { - if (!isErrno(error, 'ENOENT')) result.errors++ - return + try { + const beforeWrite = await fileSystem.lstatFile(metadataPath) + if ( + !beforeWrite.isFile() || + beforeWrite.isSymbolicLink() || + beforeWrite.dev !== latest.identity.dev || + beforeWrite.ino !== latest.identity.ino + ) { + return + } + } catch (error) { + if (!isErrno(error, 'ENOENT')) result.errors++ + return + } + await replaceSessionFile(effective, metadataPath) + result.sessionsUpdated++ } - await replaceSessionFile(effective, metadataPath) - result.sessionsUpdated++ - } - if ( - effective.name && - !(await tryReleaseNameReservation(effective.name, effective.id)) - ) { - result.errors++ - } - }) - } catch { - result.errors++ + if ( + effective.name && + !(await tryReleaseNameReservation(effective.name, effective.id)) + ) { + result.errors++ + } + }) + } catch { + result.errors++ + } + } finally { + if (result.errors > errorsBefore) { + options.onRetry?.({ id: id, generation: retryGeneration }) + } } } + return result } @@ -1808,6 +1915,8 @@ async function cleanupOrphanedTerminalFacts( result: BackgroundCleanupResult, fileSystem: BackgroundCleanupFileSystem, maxDirectoryEntries?: number, + targets?: readonly BackgroundSessionRecoveryTarget[], + onRetry?: (target?: BackgroundSessionRecoveryTarget) => void, ): Promise { const terminalDirectoryState = await inspectCleanupDirectory( terminalDirectory, @@ -1816,94 +1925,124 @@ async function cleanupOrphanedTerminalFacts( if (terminalDirectoryState === 'missing') return if (terminalDirectoryState !== 'same') { result.errors++ + onRetry?.() return } - let entries: Dirent[] - try { - entries = await readDirectoryEntries( - getBackgroundSessionTerminalDir(), - maxDirectoryEntries, + let candidates: NonNullable< + ReturnType + >[] + if (targets !== undefined) { + candidates = targets + .filter( + target => + backgroundRecoveryJournalRecord(target.id, target.generation) !== + undefined, + ) + .flatMap(target => [ + { ...target, kind: 'natural' as const }, + { ...target, kind: 'killed' as const }, + ]) + } else { + let entries: Dirent[] + try { + entries = await readDirectoryEntries( + getBackgroundSessionTerminalDir(), + maxDirectoryEntries, + ) + } catch (error) { + if (!isErrno(error, 'ENOENT')) { + result.errors++ + onRetry?.() + } + return + } + const terminalDirectoryAfterRead = await inspectCleanupDirectory( + terminalDirectory, + fileSystem.lstatFile, ) - } catch (error) { - if (!isErrno(error, 'ENOENT')) result.errors++ - return - } - const terminalDirectoryAfterRead = await inspectCleanupDirectory( - terminalDirectory, - fileSystem.lstatFile, - ) - if (terminalDirectoryAfterRead !== 'same') { - if (terminalDirectoryAfterRead !== 'missing') result.errors++ - return - } + if (terminalDirectoryAfterRead !== 'same') { + if (terminalDirectoryAfterRead !== 'missing') { + result.errors++ + onRetry?.() + } + return + } - const candidates = entries - .map(terminalFactCandidateFromEntry) - .filter( - (candidate): candidate is NonNullable => - candidate !== null && !metadataIds.has(candidate.id), - ) - .sort((a, b) => { - const idOrder = a.id.localeCompare(b.id) - if (idOrder !== 0) return idOrder - const markerOrder = (a.generation ?? '').localeCompare( - b.generation ?? '', + candidates = entries + .map(terminalFactCandidateFromEntry) + .filter( + (candidate): candidate is NonNullable => + candidate !== null && !metadataIds.has(candidate.id), ) - return markerOrder !== 0 ? markerOrder : a.kind.localeCompare(b.kind) - }) + .sort((a, b) => { + const idOrder = a.id.localeCompare(b.id) + if (idOrder !== 0) return idOrder + const markerOrder = (a.generation ?? '').localeCompare( + b.generation ?? '', + ) + return markerOrder !== 0 + ? markerOrder + : a.kind.localeCompare(b.kind) + }) + } if (candidates.length === 0) return for (const candidate of candidates) { - const factPath = terminalFactPathForId( - candidate.id, - candidate.kind, - candidate.generation, - ) - const fact = await readCleanupJson( - factPath, - (value): value is BackgroundSessionTerminalFact => - isBackgroundSessionTerminalFact( - value, - candidate.id, - candidate.kind, - ) && - value.generation === candidate.generation && - parseCanonicalCompletionTimestamp(value.finishedAt) !== null, - terminalDirectory, - fileSystem, - ) - if (fact.state === 'error') { - result.errors++ - continue - } - if ( - fact.state !== 'valid' || - Date.parse(fact.value.finishedAt) >= cutoffMs - ) { - continue - } + const errorsBefore = result.errors + try { + const factPath = terminalFactPathForId( + candidate.id, + candidate.kind, + candidate.generation, + ) + const fact = await readCleanupJson( + factPath, + (value): value is BackgroundSessionTerminalFact => + isBackgroundSessionTerminalFact( + value, + candidate.id, + candidate.kind, + ) && + value.generation === candidate.generation && + parseCanonicalCompletionTimestamp(value.finishedAt) !== null, + terminalDirectory, + fileSystem, + ) + if (fact.state === 'error') { + result.errors++ + continue + } + if ( + fact.state !== 'valid' || + Date.parse(fact.value.finishedAt) >= cutoffMs + ) { + continue + } - const metadata = await readCleanupJson( - metadataPathForId(candidate.id), - (value): value is BackgroundSession => - isBackgroundSession(value, candidate.id), - metadataDirectory, - fileSystem, - ) - if (metadata.state === 'error') { - result.errors++ - continue - } - if (metadata.state !== 'missing') continue + const metadata = await readCleanupJson( + metadataPathForId(candidate.id), + (value): value is BackgroundSession => + isBackgroundSession(value, candidate.id), + metadataDirectory, + fileSystem, + ) + if (metadata.state === 'error') { + result.errors++ + continue + } + if (metadata.state !== 'missing') continue - await removeCleanupArtifact( - factPath, - result, - terminalDirectory, - fileSystem, - fact.identity, - ) + await removeCleanupArtifact( + factPath, + result, + terminalDirectory, + fileSystem, + fact.identity, + ) + } finally { + if (result.errors > errorsBefore) onRetry?.(candidate) + } } } @@ -1917,6 +2056,8 @@ export async function cleanupBackgroundSessionsBefore( _beforeArtifactRemovalForTesting?: (id: string) => Promise _beforeReservationRemovalForTesting?: (path: string) => Promise sessionIds?: readonly string[] + onRetry?: (target?: BackgroundSessionRecoveryTarget) => void + orphanedTerminalFacts?: readonly BackgroundSessionRecoveryTarget[] maxDirectoryEntries?: number } = {}, ): Promise { @@ -1928,6 +2069,7 @@ export async function cleanupBackgroundSessionsBefore( const cutoffMs = cutoff.getTime() if (!Number.isFinite(cutoffMs)) { result.errors++ + options.onRetry?.() return result } @@ -1943,8 +2085,9 @@ export async function cleanupBackgroundSessionsBefore( getBackgroundSessionLogsDir(), getBackgroundSessionNamesDir(), getBackgroundSessionTerminalDir(), - ].map(async path => - await snapshotCleanupDirectory(path, fileSystem.lstatFile), + ].map( + async path => + await snapshotCleanupDirectory(path, fileSystem.lstatFile), ), ) const [metadataRead, logsRead, namesRead, terminalRead] = directoryReads @@ -1954,11 +2097,9 @@ export async function cleanupBackgroundSessionsBefore( ).length if (requiredDirectoryErrors > 0) { result.errors += requiredDirectoryErrors + options.onRetry?.() return result } - if (namesRead.state === 'error' || namesRead.state === 'invalid') { - result.errors++ - } const metadataDirectory = metadataRead.state === 'valid' ? metadataRead.value : undefined const logsDirectory = @@ -1967,15 +2108,13 @@ export async function cleanupBackgroundSessionsBefore( namesRead.state === 'valid' ? namesRead.value : undefined const terminalDirectory = terminalRead.state === 'valid' ? terminalRead.value : undefined - if ( - !metadataDirectory || - !logsDirectory || - !terminalDirectory - ) { + if (!metadataDirectory || !logsDirectory || !terminalDirectory) { + if (result.errors > 0) options.onRetry?.() return result } if (metadataDirectory.state === 'missing') { - if (options.sessionIds !== undefined) return result + if (options.sessionIds !== undefined && !options.orphanedTerminalFacts) + return result await cleanupOrphanedTerminalFacts( cutoffMs, metadataDirectory, @@ -1984,6 +2123,8 @@ export async function cleanupBackgroundSessionsBefore( result, fileSystem, options.maxDirectoryEntries, + options.orphanedTerminalFacts, + options.onRetry, ) return result } @@ -1993,6 +2134,7 @@ export async function cleanupBackgroundSessionsBefore( ) if (metadataDirectoryState !== 'same') { if (metadataDirectoryState !== 'missing') result.errors++ + if (result.errors > 0) options.onRetry?.() return result } @@ -2011,6 +2153,7 @@ export async function cleanupBackgroundSessionsBefore( ) } catch (error) { if (!isErrno(error, 'ENOENT')) result.errors++ + if (result.errors > 0) options.onRetry?.() return result } const metadataDirectoryAfterRead = await inspectCleanupDirectory( @@ -2019,6 +2162,7 @@ export async function cleanupBackgroundSessionsBefore( ) if (metadataDirectoryAfterRead !== 'same') { if (metadataDirectoryAfterRead !== 'missing') result.errors++ + if (result.errors > 0) options.onRetry?.() return result } candidateIds = entries @@ -2030,233 +2174,268 @@ export async function cleanupBackgroundSessionsBefore( const metadataIds = new Set(candidateIds) for (const expectedId of candidateIds) { - const metadataPath = metadataPathForId(expectedId) - const metadata = await readCleanupJson( - metadataPath, - (value): value is BackgroundSession => - isBackgroundSession(value, expectedId) && - (value.finishedAt === undefined || - parseCanonicalCompletionTimestamp(value.finishedAt) !== null), - metadataDirectory, - fileSystem, - ) - if (metadata.state === 'error') { - result.errors++ - continue - } - if (metadata.state !== 'valid') continue - - const session = metadata.value - const naturalPath = terminalFactPathForId( - session.id, - 'natural', - session.terminalFactGeneration, - ) - const killedPath = terminalFactPathForId( - session.id, - 'killed', - session.terminalFactGeneration, - ) - const natural = await readCleanupJson( - naturalPath, - (value): value is BackgroundSessionTerminalFact => - isBackgroundSessionTerminalFact(value, session.id, 'natural') && - value.generation === session.terminalFactGeneration && - parseCanonicalCompletionTimestamp(value.finishedAt) !== null, - terminalDirectory, - fileSystem, - ) - const killed = await readCleanupJson( - killedPath, - (value): value is BackgroundSessionTerminalFact => - isBackgroundSessionTerminalFact(value, session.id, 'killed') && - value.generation === session.terminalFactGeneration && - parseCanonicalCompletionTimestamp(value.finishedAt) !== null, - terminalDirectory, - fileSystem, - ) - const naturalBlocksRemoval = cleanupReadBlocksRemoval(natural, result) - const killedBlocksRemoval = cleanupReadBlocksRemoval(killed, result) - if ( - naturalBlocksRemoval || - killedBlocksRemoval || - (natural.state === 'valid' && natural.value.pid !== session.pid) || - (killed.state === 'valid' && killed.value.pid !== session.pid) - ) { - continue - } - - const effective = applyTerminalFacts( - session, - natural.state === 'valid' ? natural.value : null, - killed.state === 'valid' ? killed.value : null, - ) - if ( - effective.status !== 'exited' && - effective.status !== 'failed' && - effective.status !== 'killed' - ) { - continue - } - if (effective.finishedAt === undefined) continue - const finishedAtMs = Date.parse(effective.finishedAt) - if (!Number.isFinite(finishedAtMs) || finishedAtMs >= cutoffMs) continue - - let reservation: - | CleanupJsonRead - | undefined - if (session.name) { - if (!namesDirectory) continue - reservation = await readCleanupJson( - nameReservationPathForName(session.name), - isBackgroundSessionNameReservation, - namesDirectory, + const errorsBefore = result.errors + let retryGeneration: string | undefined + try { + const metadataPath = metadataPathForId(expectedId) + const metadata = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => + isBackgroundSession(value, expectedId) && + (value.finishedAt === undefined || + parseCanonicalCompletionTimestamp(value.finishedAt) !== null), + metadataDirectory, fileSystem, ) - if (reservation.state === 'error') { + if (metadata.state === 'error') { result.errors++ continue } - } + if (metadata.state !== 'valid') continue - try { - await options._beforeArtifactRemovalForTesting?.(session.id) - await withBackgroundSessionIdLock(session.id, async () => { - const latestMetadata = await readCleanupJson( - metadataPath, - (value): value is BackgroundSession => - isBackgroundSession(value, session.id) && - (value.finishedAt === undefined || - parseCanonicalCompletionTimestamp(value.finishedAt) !== null), - metadataDirectory, + const session = metadata.value + retryGeneration = session.terminalFactGeneration + const naturalPath = terminalFactPathForId( + session.id, + 'natural', + session.terminalFactGeneration, + ) + const killedPath = terminalFactPathForId( + session.id, + 'killed', + session.terminalFactGeneration, + ) + const natural = await readCleanupJson( + naturalPath, + (value): value is BackgroundSessionTerminalFact => + isBackgroundSessionTerminalFact(value, session.id, 'natural') && + value.generation === session.terminalFactGeneration && + parseCanonicalCompletionTimestamp(value.finishedAt) !== null, + terminalDirectory, + fileSystem, + ) + const killed = await readCleanupJson( + killedPath, + (value): value is BackgroundSessionTerminalFact => + isBackgroundSessionTerminalFact(value, session.id, 'killed') && + value.generation === session.terminalFactGeneration && + parseCanonicalCompletionTimestamp(value.finishedAt) !== null, + terminalDirectory, + fileSystem, + ) + const naturalBlocksRemoval = cleanupReadBlocksRemoval(natural, result) + const killedBlocksRemoval = cleanupReadBlocksRemoval(killed, result) + if ( + naturalBlocksRemoval || + killedBlocksRemoval || + (natural.state === 'valid' && natural.value.pid !== session.pid) || + (killed.state === 'valid' && killed.value.pid !== session.pid) + ) { + continue + } + + const effective = applyTerminalFacts( + session, + natural.state === 'valid' ? natural.value : null, + killed.state === 'valid' ? killed.value : null, + ) + if ( + effective.status !== 'exited' && + effective.status !== 'failed' && + effective.status !== 'killed' + ) { + continue + } + if (effective.finishedAt === undefined) continue + const finishedAtMs = Date.parse(effective.finishedAt) + if (!Number.isFinite(finishedAtMs) || finishedAtMs >= cutoffMs) + continue + + let reservation: + | CleanupJsonRead + | undefined + if (session.name) { + if (!namesDirectory) { + result.errors++ + continue + } + reservation = await readCleanupJson( + nameReservationPathForName(session.name), + isBackgroundSessionNameReservation, + namesDirectory, fileSystem, ) - if (latestMetadata.state === 'error') { + if (reservation.state === 'error') { result.errors++ - return - } - if ( - latestMetadata.state !== 'valid' || - !latestMetadata.identity || - !metadata.identity || - latestMetadata.identity.dev !== metadata.identity.dev || - latestMetadata.identity.ino !== metadata.identity.ino || - !isSameBackgroundSessionGeneration(latestMetadata.value, session) - ) { - return + continue } + } - const logPaths = getBackgroundSessionLogPaths(session.id) - const stdoutRemoval = await removeCleanupArtifact( - logPaths.stdoutLogPath, - result, - logsDirectory, - fileSystem, - ) - const stderrRemoval = await removeCleanupArtifact( - logPaths.stderrLogPath, - result, - logsDirectory, - fileSystem, - ) - let reservationRemoval: CleanupArtifactRemoval = 'missing' - const sessionName = session.name - if ( - sessionName && - reservation?.state === 'valid' && - namesDirectory - ) { - try { - reservationRemoval = await withNameReservationLock( - sessionName, - async () => { - const latestReservation = await readCleanupJson( - nameReservationPathForName(sessionName), - isBackgroundSessionNameReservation, - namesDirectory, - fileSystem, - ) - if (latestReservation.state === 'error') { - result.errors++ - return 'error' - } - if ( - latestReservation.state !== 'valid' || - latestReservation.value.name !== sessionName || - latestReservation.value.id !== session.id - ) { - return 'missing' - } - const reservationPath = - nameReservationPathForName(sessionName) - await options._beforeReservationRemovalForTesting?.( - reservationPath, - ) - return await removeCleanupArtifact( - reservationPath, - result, - namesDirectory, - fileSystem, - latestReservation.identity, - ) - }, - ) - } catch { + try { + await options._beforeArtifactRemovalForTesting?.(session.id) + await withBackgroundSessionIdLock(session.id, async () => { + const latestMetadata = await readCleanupJson( + metadataPath, + (value): value is BackgroundSession => + isBackgroundSession(value, session.id) && + (value.finishedAt === undefined || + parseCanonicalCompletionTimestamp(value.finishedAt) !== + null), + metadataDirectory, + fileSystem, + ) + if (latestMetadata.state === 'error') { result.errors++ - reservationRemoval = 'error' + return + } + if ( + latestMetadata.state !== 'valid' || + !latestMetadata.identity || + !metadata.identity || + latestMetadata.identity.dev !== metadata.identity.dev || + latestMetadata.identity.ino !== metadata.identity.ino || + !isSameBackgroundSessionGeneration( + latestMetadata.value, + session, + ) + ) { + return } - } - if ( - stdoutRemoval === 'error' || - stderrRemoval === 'error' || - reservationRemoval === 'error' - ) { - return - } - - const metadataRemoval = await removeCleanupArtifact( - metadataPath, - result, - metadataDirectory, - fileSystem, - latestMetadata.identity, - ) - if (metadataRemoval === 'error') return - if (metadataRemoval === 'removed') result.sessionsRemoved++ - if (natural.state === 'valid') { - await removeCleanupArtifact( - naturalPath, + const logPaths = getBackgroundSessionLogPaths(session.id) + const stdoutRemoval = await removeCleanupArtifact( + logPaths.stdoutLogPath, result, - terminalDirectory, + logsDirectory, fileSystem, - natural.identity, ) - } - if (killed.state === 'valid') { - await removeCleanupArtifact( - killedPath, + const stderrRemoval = await removeCleanupArtifact( + logPaths.stderrLogPath, result, - terminalDirectory, + logsDirectory, fileSystem, - killed.identity, ) - } - }) - } catch { - result.errors++ + let reservationRemoval: CleanupArtifactRemoval = 'missing' + const sessionName = session.name + if ( + sessionName && + reservation?.state === 'valid' && + namesDirectory + ) { + try { + reservationRemoval = await withNameReservationLock( + sessionName, + async () => { + const latestReservation = await readCleanupJson( + nameReservationPathForName(sessionName), + isBackgroundSessionNameReservation, + namesDirectory, + fileSystem, + ) + if (latestReservation.state === 'error') { + result.errors++ + return 'error' + } + if ( + latestReservation.state !== 'valid' || + latestReservation.value.name !== sessionName || + latestReservation.value.id !== session.id + ) { + return 'missing' + } + const reservationPath = + nameReservationPathForName(sessionName) + await options._beforeReservationRemovalForTesting?.( + reservationPath, + ) + return await removeCleanupArtifact( + reservationPath, + result, + namesDirectory, + fileSystem, + latestReservation.identity, + ) + }, + ) + } catch { + result.errors++ + reservationRemoval = 'error' + } + } + if ( + stdoutRemoval === 'error' || + stderrRemoval === 'error' || + reservationRemoval === 'error' + ) { + return + } + + const metadataRemoval = await removeCleanupArtifact( + metadataPath, + result, + metadataDirectory, + fileSystem, + latestMetadata.identity, + ) + if (metadataRemoval === 'error') return + if (metadataRemoval === 'removed') result.sessionsRemoved++ + + const errorsBeforeFacts = result.errors + if (natural.state === 'valid') { + await removeCleanupArtifact( + naturalPath, + result, + terminalDirectory, + fileSystem, + natural.identity, + ) + } + if (killed.state === 'valid') { + await removeCleanupArtifact( + killedPath, + result, + terminalDirectory, + fileSystem, + killed.identity, + ) + } + if (result.errors > errorsBeforeFacts) { + const target = { + id: session.id, + generation: session.terminalFactGeneration, + } + if (!options.onRetry) { + await enqueueBackgroundSessionRecovery( + target.id, + target.generation, + ) + } + } + }) + } catch { + result.errors++ + } + } finally { + if (result.errors > errorsBefore) { + options.onRetry?.({ id: expectedId, generation: retryGeneration }) + } } } - if (options.sessionIds === undefined) { + if ( + options.sessionIds === undefined || + options.orphanedTerminalFacts !== undefined + ) { await cleanupOrphanedTerminalFacts( cutoffMs, metadataDirectory, terminalDirectory, - metadataIds, + options.sessionIds === undefined ? metadataIds : new Set(), result, fileSystem, options.maxDirectoryEntries, + options.orphanedTerminalFacts, + options.onRetry, ) } diff --git a/src/utils/cleanup.test.ts b/src/utils/cleanup.test.ts index 73ed4ab213..c128ab9ca7 100644 --- a/src/utils/cleanup.test.ts +++ b/src/utils/cleanup.test.ts @@ -1,8 +1,10 @@ import { afterEach, describe, expect, test } from 'bun:test' import { mkdir, + chmod, readdir, readFile, + rename, rm, stat, utimes, @@ -25,7 +27,10 @@ const FIXTURE_OUTPUT_LIMIT_BYTES = 64 * 1024 afterEach(async () => { _setBackgroundSessionsRootForTesting(undefined) await Promise.all( - tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })), + tempDirs.splice(0).map(async dir => { + await chmod(join(dir, 'bg-sessions', 'terminal'), 0o755).catch(() => {}) + await rm(dir, { recursive: true, force: true }) + }), ) }) @@ -59,6 +64,7 @@ async function runCleanupFixture( configDir: string, mode: | 'once' + | 'pass-outcome' | 'marker-failure' | 'periodic-policy-recheck' | 'periodic-recovery' @@ -118,11 +124,15 @@ async function createCompletedBackgroundSession( configDir: string, id: string, finishedAt: Date, + name?: string, + processMarker?: string, ): Promise { const root = join(configDir, 'bg-sessions') _setBackgroundSessionsRootForTesting(root) await createBackgroundSession({ id, + name, + processMarker, pid: process.pid, cwd: configDir, command: ['openclaude', '--print', 'fixture'], @@ -284,7 +294,7 @@ describe('cleanupOldMessageFilesInBackground', () => { ) test( - 'advances bounded recovery past erroring retained entries across process restarts', + 'advances bounded recovery past malformed entries across process restarts', async () => { const configDir = join( tmpdir(), @@ -317,9 +327,6 @@ describe('cleanupOldMessageFilesInBackground', () => { join(root, '.recovery-journal'), `bg-retained-a\nbg-retained-b\n${targetId}\n`, ) - await rm(join(root, 'names'), { recursive: true, force: true }) - await writeFile(join(root, 'names'), 'not a directory') - expect( await runCleanupFixture( configDir, @@ -342,6 +349,387 @@ describe('cleanupOldMessageFilesInBackground', () => { 30_000, ) + for (const trigger of [ + 'periodic-recovery', + 'periodic-retention', + ] as const) { + for (const timing of ['before', 'after'] as const) { + test(`does not acknowledge policy rejected ${timing} the ${trigger} lock`, async () => { + const configDir = join( + tmpdir(), + `openclaude-policy-outcome-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + const settingsPath = join(configDir, 'settings.json') + const validPolicy = JSON.stringify({ cleanupPeriodDays: 0 }) + await writeFile( + settingsPath, + timing === 'before' + ? JSON.stringify({ cleanupPeriodDays: 'invalid' }) + : validPolicy, + ) + const id = 'bg-rejected-policy' + const metadataPath = await createCompletedBackgroundSession( + configDir, + id, + new Date(0), + ) + const root = join(configDir, 'bg-sessions') + const marker = join( + root, + trigger === 'periodic-recovery' + ? '.recovery-pass' + : '.retention-pass', + ) + const journal = join(root, '.recovery-journal') + const before = await readFile(journal, 'utf8') + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + OPENCLAUDE_CLEANUP_REJECT_AFTER_LOCK: + timing === 'after' ? '1' : '0', + }), + ).toEqual({ result: 'invalid-policy' }) + expect(await Bun.file(metadataPath).exists()).toBe(true) + expect(await Bun.file(marker).exists()).toBe(false) + expect( + await Bun.file(join(root, '.recovery-cursor.json')).exists(), + ).toBe(false) + expect(await readFile(journal, 'utf8')).toBe(before) + + await writeFile(settingsPath, validPolicy) + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(metadataPath).exists()).toBe(false) + expect(await readFile(journal, 'utf8')).toBe('') + expect(await Bun.file(marker).exists()).toBe(true) + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'skipped' }) + await rm(marker) + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(marker).exists()).toBe(true) + }, 30_000) + } + + for (const failure of [ + 'names-directory', + 'log-unlink', + 'terminal-unlink', + 'marked-terminal-unlink', + ] as const) { + test(`retries ${failure} after a partial ${trigger} pass in a new process`, async () => { + const configDir = join( + tmpdir(), + `openclaude-retry-outcome-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const root = join(configDir, 'bg-sessions') + const id = 'bg-retryable' + const terminalFailure = failure.endsWith('terminal-unlink') + const metadata = await createCompletedBackgroundSession( + configDir, + id, + new Date(0), + 'retryable', + failure === 'marked-terminal-unlink' ? 'a'.repeat(64) : undefined, + ) + const eligible = await createCompletedBackgroundSession( + configDir, + 'bg-independent', + new Date(0), + ) + const malformed = join(root, 'sessions', 'bg-malformed.json') + await writeFile(malformed, '{invalid') + const journal = join(root, '.recovery-journal') + // A full sweep must also recover work whose original journal append failed. + await writeFile( + journal, + trigger === 'periodic-retention' + ? '' + : `bg-malformed\n${id}\nbg-independent\n`, + ) + const brokenPath = terminalFailure + ? join(root, 'terminal') + : failure === 'names-directory' + ? join(root, 'names') + : join(root, 'logs', `${id}.out.log`) + if (terminalFailure) { + await chmod(brokenPath, 0o555) + } else { + await rm(brokenPath, { recursive: true, force: true }) + if (failure === 'names-directory') + await writeFile(brokenPath, 'not a directory') + else await mkdir(brokenPath) + } + const marker = join( + root, + trigger === 'periodic-recovery' + ? '.recovery-pass' + : '.retention-pass', + ) + + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'partial' }) + expect(await Bun.file(metadata).exists()).toBe(!terminalFailure) + expect(await Bun.file(eligible).exists()).toBe(false) + expect(await readFile(malformed, 'utf8')).toBe('{invalid') + expect(await readFile(journal, 'utf8')).toContain(id) + if ( + failure === 'names-directory' && + trigger === 'periodic-recovery' + ) { + expect(await readFile(journal, 'utf8')).toBe(`${id}\n`) + } + expect(await Bun.file(marker).exists()).toBe(true) + + if (terminalFailure) await chmod(brokenPath, 0o755) + else { + await rm(brokenPath, { recursive: true, force: true }) + if (failure === 'names-directory') await mkdir(brokenPath) + } + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'skipped' }) + await rm(marker) + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: terminalFailure + ? 'periodic-recovery' + : trigger, + }), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(metadata).exists()).toBe(false) + expect(await readdir(join(root, 'terminal'))).toEqual([]) + expect(await readFile(journal, 'utf8')).toBe('') + expect( + await Bun.file(join(root, '.recovery-cursor.json')).exists(), + ).toBe(false) + expect( + await Bun.file( + terminalFailure ? join(root, '.recovery-pass') : marker, + ).exists(), + ).toBe(true) + }, 30_000) + } + } + + for (const trigger of [ + 'periodic-recovery', + 'periodic-retention', + ] as const) { + test(`acknowledges unnamed work despite a broken names directory during ${trigger}`, async () => { + const configDir = join( + tmpdir(), + `openclaude-unnamed-retry-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const metadata = await createCompletedBackgroundSession( + configDir, + 'bg-unnamed', + new Date(0), + ) + const root = join(configDir, 'bg-sessions') + await rm(join(root, 'names'), { recursive: true, force: true }) + await writeFile(join(root, 'names'), 'not a directory') + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: trigger, + }), + ).toEqual({ result: 'ran' }) + expect(await Bun.file(metadata).exists()).toBe(false) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe('') + }, 30_000) + } + + test('queues recent reconciliation failures that were absent from the journal', async () => { + const configDir = join( + tmpdir(), + `openclaude-reconcile-retry-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const id = 'bg-recent-reconcile' + const generation = 'b'.repeat(64) + const metadata = await createCompletedBackgroundSession( + configDir, + id, + new Date(), + 'recent-reconcile', + generation, + ) + const root = join(configDir, 'bg-sessions') + await writeFile(join(root, '.recovery-journal'), '') + await rm(join(root, 'names'), { recursive: true, force: true }) + await writeFile(join(root, 'names'), 'not a directory') + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ + result: 'partial', + }) + expect(await Bun.file(metadata).exists()).toBe(true) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe( + `${id}~${generation}\n`, + ) + await rm(join(root, 'names')) + await mkdir(join(root, 'names')) + await rm(join(root, '.retention-pass')) + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ + result: 'ran', + }) + expect(await Bun.file(metadata).exists()).toBe(true) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe('') + }, 30_000) + + test('reports an interrupted full sweep without acknowledging its snapshot', async () => { + const configDir = join( + tmpdir(), + `openclaude-failed-outcome-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const metadata = await createCompletedBackgroundSession( + configDir, + 'bg-before-interruption', + new Date(0), + ) + const root = join(configDir, 'bg-sessions') + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: 'periodic-retention', + OPENCLAUDE_CLEANUP_APPEND_AFTER_SNAPSHOT: '1', + OPENCLAUDE_CLEANUP_THROW_AFTER_SNAPSHOT: '1', + }), + ).toEqual({ result: 'failed' }) + expect(await Bun.file(metadata).exists()).toBe(true) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe( + 'bg-before-interruption\nbg-snapshot-appended\n', + ) + expect(await Bun.file(join(root, '.retention-pass')).exists()).toBe(false) + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ + result: 'ran', + }) + expect(await Bun.file(metadata).exists()).toBe(false) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe('') + expect(await readdir(join(root, 'terminal'))).toEqual([]) + }, 30_000) + + test('does not throttle a full sweep when its retry inventory is unreadable', async () => { + const configDir = join( + tmpdir(), + `openclaude-failed-inventory-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const metadata = await createCompletedBackgroundSession( + configDir, + 'bg-unreadable-inventory', + new Date(0), + ) + const root = join(configDir, 'bg-sessions') + const directory = join(root, 'sessions') + const saved = join(root, 'saved-sessions') + await rename(directory, saved) + await writeFile(directory, 'not a directory') + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ + result: 'failed', + }) + expect(await Bun.file(join(root, '.retention-pass')).exists()).toBe(false) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe( + 'bg-unreadable-inventory\n', + ) + await rm(directory) + await rename(saved, directory) + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ + result: 'ran', + }) + expect(await Bun.file(metadata).exists()).toBe(false) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe('') + }, 30_000) + + test('acknowledges only the full-sweep snapshot while preserving a concurrent append', async () => { + const configDir = join( + tmpdir(), + `openclaude-snapshot-outcome-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 30 }), + ) + const metadata = await createCompletedBackgroundSession( + configDir, + 'bg-snapshot-old', + new Date(0), + ) + expect( + await runCleanupFixture(configDir, 'pass-outcome', { + OPENCLAUDE_CLEANUP_TRIGGER: 'periodic-retention', + OPENCLAUDE_CLEANUP_APPEND_AFTER_SNAPSHOT: '1', + }), + ).toEqual({ result: 'ran' }) + const root = join(configDir, 'bg-sessions') + expect(await Bun.file(metadata).exists()).toBe(false) + expect( + await Bun.file( + join(root, 'sessions', 'bg-snapshot-appended.json'), + ).exists(), + ).toBe(true) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe( + 'bg-snapshot-appended\n', + ) + expect(await Bun.file(join(root, '.retention-pass')).exists()).toBe(true) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + expect(await runCleanupFixture(configDir, 'periodic-recovery')).toEqual({ + result: 'ran', + }) + expect( + await Bun.file( + join(root, 'sessions', 'bg-snapshot-appended.json'), + ).exists(), + ).toBe(false) + expect(await readFile(join(root, '.recovery-journal'), 'utf8')).toBe('') + }, 30_000) + for (const scenario of [ { label: 'positive', setting: 30, result: 'ran' }, { label: 'invalid', setting: 'invalid', result: 'invalid-policy' }, @@ -472,6 +860,10 @@ describe('cleanupOldMessageFilesInBackground', () => { ), ).toEqual({ result: 'invalid-policy' }) expect(await Bun.file(metadataPath).exists()).toBe(true) + expect(await Bun.file(join(configDir, 'bg-sessions', '.retention-pass')).exists()).toBe(false) + await rm(join(configDir, '.mcp.json')) + expect(await runCleanupFixture(configDir, 'periodic-retention')).toEqual({ result: 'ran' }) + expect(await Bun.file(metadataPath).exists()).toBe(false) }, 30_000, ) diff --git a/src/utils/cleanup.ts b/src/utils/cleanup.ts index 956710f05c..b4c3edb47d 100644 --- a/src/utils/cleanup.ts +++ b/src/utils/cleanup.ts @@ -6,6 +6,7 @@ import { reconcileBackgroundSessionTerminalFacts, snapshotBackgroundSessionRecoveryJournal, takeBackgroundSessionRecoveryBatch, + type BackgroundSessionRecoveryTarget, } from '../cli/bgRegistry.js' import { logEvent } from '../services/analytics/index.js' import { CACHE_PATHS } from './cachePaths.js' @@ -41,6 +42,8 @@ export type BackgroundSessionRetentionTrigger = export type BackgroundSessionRetentionResult = | 'ran' + | 'partial' + | 'failed' | 'skipped' | 'invalid-policy' @@ -60,6 +63,7 @@ type BackgroundSessionRetentionRequest = | { trigger: 'periodic-retention' _afterThrottleLockForTesting?: () => Promise + _afterJournalSnapshotForTesting?: () => Promise } function getCutoffDate(): Date { @@ -706,9 +710,11 @@ function logBackgroundCleanup( async function runWithBackgroundRetentionThrottle( markerName: string, intervalMs: number, - operation: () => Promise, + operation: () => Promise< + Exclude + >, afterLock?: () => Promise, -): Promise { +): Promise { const markerPath = join( getClaudeConfigHomeDir(), 'bg-sessions', @@ -723,26 +729,27 @@ async function runWithBackgroundRetentionThrottle( } let locked = false try { - if (await markerIsFresh()) return false + if (await markerIsFresh()) return 'skipped' await fs.mkdir(dirname(markerPath), { recursive: true }) await lockfile.lock(markerPath, { retries: 0, realpath: false }) locked = true } catch { - return false + return 'skipped' } try { - if (await markerIsFresh()) return false + if (await markerIsFresh()) return 'skipped' await afterLock?.() - await operation() + const outcome = await operation() + if (outcome !== 'ran' && outcome !== 'partial') return outcome try { await fs.writeFile(markerPath, new Date().toISOString()) } catch (error) { logError(error instanceof Error ? error : new Error(String(error))) } - return true + return outcome } catch (error) { logError(error instanceof Error ? error : new Error(String(error))) - return false + return 'failed' } finally { if (locked) { await lockfile.unlock(markerPath, { realpath: false }).catch(() => {}) @@ -786,69 +793,90 @@ export async function runBackgroundSessionRetention( } if (request.trigger === 'periodic-recovery') { - let passResult: BackgroundSessionRetentionResult = 'ran' - const ran = await runWithBackgroundRetentionThrottle( + return await runWithBackgroundRetentionThrottle( '.recovery-pass', BACKGROUND_RECOVERY_INTERVAL_MS, async () => { const currentPolicy = readBackgroundRetentionPeriod({ fresh: true }) - if (currentPolicy.state === 'invalid') { - passResult = 'invalid-policy' - return - } - if (currentPolicy.days !== 0) return + if (currentPolicy.state === 'invalid') return 'invalid-policy' + if (currentPolicy.days !== 0) return 'ran' const batch = await takeBackgroundSessionRecoveryBatch( request._recoveryBatchLimitForTesting ?? BACKGROUND_RECOVERY_ENTRY_LIMIT, ) - const reconciliation = - await reconcileBackgroundSessionTerminalFacts({ - sessionIds: batch.sessionIds, - }) - logBackgroundReconciliation(reconciliation) - const cleanup = await cleanupBackgroundSessionsBefore( - new Date(Date.now() + 1), - { sessionIds: batch.sessionIds }, - ) - logBackgroundCleanup(cleanup) - await batch.commit() + const retrySessionIds: string[] = [] + const retryTerminalFacts: typeof batch.terminalFacts = [] + for (const id of batch.sessionIds) { + // Attribute retryable errors to their ID, so retained entries cannot + // hold unrelated work at the front of the bounded journal. + const reconciliation = + await reconcileBackgroundSessionTerminalFacts({ + sessionIds: [id], + }) + logBackgroundReconciliation(reconciliation) + const cleanup = await cleanupBackgroundSessionsBefore( + new Date(Date.now() + 1), + { + sessionIds: [id], + orphanedTerminalFacts: batch.terminalFacts.filter( + target => target.id === id, + ), + onRetry: target => { + if (target) retryTerminalFacts.push(target) + }, + }, + ) + logBackgroundCleanup(cleanup) + if (reconciliation.errors > 0 || cleanup.errors > 0) { + retrySessionIds.push(id) + } + } + // Save retries before publishing the cursor that acknowledges this batch. + await batch.commit(retrySessionIds, retryTerminalFacts) + return retrySessionIds.length > 0 ? 'partial' : 'ran' }, request._afterThrottleLockForTesting, ) - return ran ? passResult : 'skipped' } - let passResult: BackgroundSessionRetentionResult = 'ran' - const ran = await runWithBackgroundRetentionThrottle( + return await runWithBackgroundRetentionThrottle( '.retention-pass', BACKGROUND_RETENTION_SWEEP_INTERVAL_MS, async () => { resetSettingsCache() - if (cleanupPeriodSettingIsInvalid()) { - passResult = 'invalid-policy' - return - } + if (cleanupPeriodSettingIsInvalid()) return 'invalid-policy' const currentPolicy = readBackgroundRetentionPeriod({ fresh: true }) - if (currentPolicy.state === 'invalid') { - passResult = 'invalid-policy' - return - } + if (currentPolicy.state === 'invalid') return 'invalid-policy' const cutoff = new Date( Date.now() - currentPolicy.days * 24 * 60 * 60 * 1000, ) - const journal = - await snapshotBackgroundSessionRecoveryJournal() - logBackgroundReconciliation( - await reconcileBackgroundSessionTerminalFacts(), - ) - logBackgroundCleanup( - await cleanupBackgroundSessionsBefore(cutoff), - ) + const journal = await snapshotBackgroundSessionRecoveryJournal() + await request._afterJournalSnapshotForTesting?.() + const retryTargets: BackgroundSessionRecoveryTarget[] = [] + let retryInventoryComplete = true + const onRetry = (target?: BackgroundSessionRecoveryTarget) => { + if (target) retryTargets.push(target) + else retryInventoryComplete = false + } + const reconciliation = await reconcileBackgroundSessionTerminalFacts({ + onRetry, + }) + logBackgroundReconciliation(reconciliation) + const cleanup = await cleanupBackgroundSessionsBefore(cutoff, { + onRetry, + }) + logBackgroundCleanup(cleanup) + // A full sweep owns the snapshot only when all its work succeeded. + // On partial failure leave it discoverable by a later process. + if (reconciliation.errors > 0 || cleanup.errors > 0) { + await journal.retry(retryTargets) + return retryInventoryComplete ? 'partial' : 'failed' + } await journal.commit() + return 'ran' }, request._afterThrottleLockForTesting, ) - return ran ? passResult : 'skipped' } function cleanupPeriodSettingIsInvalid(): boolean { diff --git a/src/utils/cleanupBackgroundSessions.fixture.ts b/src/utils/cleanupBackgroundSessions.fixture.ts index 2947e684b1..12292abbec 100644 --- a/src/utils/cleanupBackgroundSessions.fixture.ts +++ b/src/utils/cleanupBackgroundSessions.fixture.ts @@ -17,7 +17,65 @@ import { try { const mode = process.env.OPENCLAUDE_CLEANUP_FIXTURE_MODE - if ( + if (mode === 'pass-outcome') { + enableConfigs() + const settingsPath = join( + process.env.OPENCLAUDE_CONFIG_DIR!, + 'settings.json', + ) + const settingsResult = eagerLoadSettingsFromArgs([ + '--settings', + settingsPath, + ]) + if (!settingsResult.ok) throw new Error(settingsResult.message) + const trigger = + process.env.OPENCLAUDE_CLEANUP_TRIGGER === 'periodic-retention' + ? 'periodic-retention' + : 'periodic-recovery' + const afterLock = async () => { + if (process.env.OPENCLAUDE_CLEANUP_REJECT_AFTER_LOCK === '1') { + await Bun.write( + settingsPath, + JSON.stringify({ cleanupPeriodDays: 'invalid' }), + ) + } + } + const result = await runBackgroundSessionRetention( + trigger === 'periodic-recovery' + ? { trigger, _afterThrottleLockForTesting: afterLock } + : { + trigger, + _afterThrottleLockForTesting: afterLock, + _afterJournalSnapshotForTesting: async () => { + if ( + process.env.OPENCLAUDE_CLEANUP_APPEND_AFTER_SNAPSHOT !== '1' + ) + return + const session = await createBackgroundSession({ + id: 'bg-snapshot-appended', + pid: process.pid, + cwd: process.cwd(), + command: ['openclaude', '--print', 'fixture'], + sessionId: 'snapshot-appended-conversation', + }) + recordBackgroundSessionNaturalTerminationSync( + session.id, + { exitCode: 0 }, + { + ownerPid: process.pid, + expectedSession: session, + }, + ) + if ( + process.env.OPENCLAUDE_CLEANUP_THROW_AFTER_SNAPSHOT === '1' + ) { + throw new Error('intentional full-sweep interruption') + } + }, + }, + ) + process.stdout.write(`${JSON.stringify({ result })}\n`) + } else if ( mode === 'periodic-retention' || mode === 'periodic-retention-policy-recheck' ) { From 11e6c9eca64c56a172a595e5624bc8aef73f1f82 Mon Sep 17 00:00:00 2001 From: chioarub Date: Sat, 5 Sep 2026 07:47:42 +0300 Subject: [PATCH 13/13] test(bg): harden cleanup fixtures and launcher checks --- src/cli/bgFinalizer.test.ts | 28 ++++---- src/cli/bgRegistry.cleanup.test.ts | 5 +- src/utils/cleanup.test.ts | 72 ++++++++++++++++++- .../cleanupBackgroundSessions.fixture.ts | 22 ++++-- 4 files changed, 106 insertions(+), 21 deletions(-) diff --git a/src/cli/bgFinalizer.test.ts b/src/cli/bgFinalizer.test.ts index f357208f80..a8f46538d4 100644 --- a/src/cli/bgFinalizer.test.ts +++ b/src/cli/bgFinalizer.test.ts @@ -1418,20 +1418,24 @@ describe('background session finalizer', () => { stdio: ['ignore', 'pipe', 'pipe'], }, ) - let stdout = '' - let stderr = '' - launcher.stdout?.setEncoding('utf8') - launcher.stderr?.setEncoding('utf8') - launcher.stdout?.on('data', chunk => { - stdout += chunk - }) - launcher.stderr?.on('data', chunk => { - stderr += chunk - }) - const launcherClose = once(launcher, 'close') + const stdoutCapture = captureBoundedOutput( + launcher, + launcher.stdout, + 'zero-retention launcher stdout', + ) + const stderrCapture = captureBoundedOutput( + launcher, + launcher.stderr, + 'zero-retention launcher stderr', + ) + const launcherClose = waitForChildClose(launcher, 'zero-retention launcher') let childPid: number | undefined try { - const [launcherCode] = (await launcherClose) as [number] + const launcherCode = await launcherClose + stdoutCapture.assertWithinLimit() + stderrCapture.assertWithinLimit() + const stdout = stdoutCapture.read() + const stderr = stderrCapture.read() expect(launcherCode).toBe(0) expect(stderr).toBe('') const stdoutLogPath = stdout.match(/^Logs: (.+)$/m)?.[1] diff --git a/src/cli/bgRegistry.cleanup.test.ts b/src/cli/bgRegistry.cleanup.test.ts index 76b41a7b95..6ea0b1be08 100644 --- a/src/cli/bgRegistry.cleanup.test.ts +++ b/src/cli/bgRegistry.cleanup.test.ts @@ -219,7 +219,10 @@ describe('background session retention cleanup', () => { ] as const) { if (consumer === 'reconciliation' && fault === 'replaced-before-read') continue - it(`reports ${fault} ${consumer} inventory as unacknowledged responsibility`, async () => { + it.skipIf( + fault === 'unreadable' && + (process.platform === 'win32' || process.geteuid?.() === 0), + )(`reports ${fault} ${consumer} inventory as unacknowledged responsibility`, async () => { const session = await writeRawSession({ id: 'bg-inventory-report', status: 'exited', diff --git a/src/utils/cleanup.test.ts b/src/utils/cleanup.test.ts index c128ab9ca7..3306f63453 100644 --- a/src/utils/cleanup.test.ts +++ b/src/utils/cleanup.test.ts @@ -23,6 +23,8 @@ import { NodeFsOperations } from './fsOperations.js' const tempDirs: string[] = [] const FIXTURE_OUTPUT_LIMIT_BYTES = 64 * 1024 +// The production minute-recovery contract caps each pass at 256 records. +const BACKGROUND_RECOVERY_ENTRY_LIMIT = 256 afterEach(async () => { _setBackgroundSessionsRootForTesting(undefined) @@ -77,6 +79,7 @@ async function runCleanupFixture( | 'post-finalization-timeout' | 'explicit-kill' = 'once', fixtureEnv: NodeJS.ProcessEnv = {}, + expectedError?: string, ): Promise> { const fixture = join(import.meta.dir, 'cleanupBackgroundSessions.fixture.ts') const { USER_TYPE: _userType, ...inheritedEnv } = process.env @@ -110,6 +113,17 @@ async function runCleanupFixture( 'cleanup fixture stderr', ), ]) + if (expectedError !== undefined) { + expect(exitCode, stderr).toBe(1) + expect(stdout).toBe('') + expect(stderr).toContain( + `background cleanup fixture failed: Error: ${expectedError}`, + ) + expect(stderr).toMatch( + /\bat .*cleanupBackgroundSessions\.fixture\.ts:\d+:\d+/, + ) + return {} + } expect(exitCode, stderr).toBe(0) const result = JSON.parse(stdout) as Record if (mode === 'once') expect(result).toEqual({ completed: true }) @@ -226,6 +240,55 @@ describe('cleanupOldSessionFiles', () => { }) describe('cleanupOldMessageFilesInBackground', () => { + for (const limit of [ + undefined, + '', + '0', + '-1', + '1.5', + 'invalid', + '9007199254740992', + ]) { + test(`rejects invalid bounded recovery fixture limit ${JSON.stringify(limit)}`, async () => { + const configDir = join( + tmpdir(), + `openclaude-cleanup-invalid-limit-${Date.now()}-${Math.random()}`, + ) + tempDirs.push(configDir) + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'settings.json'), + JSON.stringify({ cleanupPeriodDays: 0 }), + ) + const id = 'bg-invalid-limit' + const metadata = await createCompletedBackgroundSession( + configDir, + id, + new Date(0), + ) + const root = join(configDir, 'bg-sessions') + const journal = join(root, '.recovery-journal') + const before = await readFile(journal, 'utf8') + + await runCleanupFixture( + configDir, + 'periodic-recovery-bounded', + { OPENCLAUDE_CLEANUP_RECOVERY_LIMIT: limit }, + 'OPENCLAUDE_CLEANUP_RECOVERY_LIMIT must be a positive safe integer', + ) + + expect(await Bun.file(metadata).exists()).toBe(true) + expect( + await Bun.file(join(root, 'logs', `${id}.out.log`)).exists(), + ).toBe(true) + expect(await readFile(journal, 'utf8')).toBe(before) + expect(await Bun.file(join(root, '.recovery-pass')).exists()).toBe(false) + expect( + await Bun.file(join(root, '.recovery-cursor.json')).exists(), + ).toBe(false) + }, 30_000) + } + test( 'globally throttles and bounds concurrent prompt-recovery passes', async () => { @@ -288,7 +351,7 @@ describe('cleanupOldMessageFilesInBackground', () => { ]) expect( (await readdir(metadataDir)).filter(name => name.endsWith('.json')), - ).toHaveLength(44) + ).toHaveLength(ids.length - BACKGROUND_RECOVERY_ENTRY_LIMIT) }, 30_000, ) @@ -428,7 +491,11 @@ describe('cleanupOldMessageFilesInBackground', () => { 'terminal-unlink', 'marked-terminal-unlink', ] as const) { - test(`retries ${failure} after a partial ${trigger} pass in a new process`, async () => { + const terminalFailure = failure.endsWith('terminal-unlink') + test.skipIf( + terminalFailure && + (process.platform === 'win32' || process.geteuid?.() === 0), + )(`retries ${failure} after a partial ${trigger} pass in a new process`, async () => { const configDir = join( tmpdir(), `openclaude-retry-outcome-${Date.now()}-${Math.random()}`, @@ -441,7 +508,6 @@ describe('cleanupOldMessageFilesInBackground', () => { ) const root = join(configDir, 'bg-sessions') const id = 'bg-retryable' - const terminalFailure = failure.endsWith('terminal-unlink') const metadata = await createCompletedBackgroundSession( configDir, id, diff --git a/src/utils/cleanupBackgroundSessions.fixture.ts b/src/utils/cleanupBackgroundSessions.fixture.ts index 12292abbec..a9b0a838cc 100644 --- a/src/utils/cleanupBackgroundSessions.fixture.ts +++ b/src/utils/cleanupBackgroundSessions.fixture.ts @@ -120,9 +120,17 @@ try { ? await cleanupBackgroundSessionsInBackground() : await runBackgroundSessionRetention({ trigger: 'periodic-recovery', - _recoveryBatchLimitForTesting: Number( - process.env.OPENCLAUDE_CLEANUP_RECOVERY_LIMIT, - ), + _recoveryBatchLimitForTesting: (() => { + const limit = Number( + process.env.OPENCLAUDE_CLEANUP_RECOVERY_LIMIT, + ) + if (!Number.isSafeInteger(limit) || limit < 1) { + throw new Error( + 'OPENCLAUDE_CLEANUP_RECOVERY_LIMIT must be a positive safe integer', + ) + } + return limit + })(), }), })}\n`, ) @@ -362,7 +370,11 @@ try { await cleanupOldMessageFilesInBackground() process.stdout.write(`${JSON.stringify({ completed: true })}\n`) } -} catch { - process.stderr.write('background cleanup fixture failed\n') +} catch (error) { + process.stderr.write( + `background cleanup fixture failed: ${ + error instanceof Error ? (error.stack ?? error.message) : String(error) + }\n`, + ) process.exitCode = 1 }