diff --git a/.changeset/6261-stale-lifecycle-ready-resume.md b/.changeset/6261-stale-lifecycle-ready-resume.md new file mode 100644 index 00000000000..6d781fd7b5d --- /dev/null +++ b/.changeset/6261-stale-lifecycle-ready-resume.md @@ -0,0 +1,3 @@ +### Fixed + +- `session.resume` of a session whose detached-idle host already exited no longer fails with `EEXIST` on `.lifecycle.ready.json`: the host revokes its own ready marker on graceful exit, including the global SIGTERM path, and a launch retires the same id's leftover ready/marker pair when the recorded owner is proven exited (a live or unknown owner still wins) (#6261). diff --git a/packages/coding-agent/changelog.d/6261-stale-lifecycle-ready-resume.md b/packages/coding-agent/changelog.d/6261-stale-lifecycle-ready-resume.md new file mode 100644 index 00000000000..bc9c8d925c8 --- /dev/null +++ b/packages/coding-agent/changelog.d/6261-stale-lifecycle-ready-resume.md @@ -0,0 +1,3 @@ +### Fixed + +- `session.resume` of a session whose detached-idle host already exited no longer fails with `EEXIST` on `.lifecycle.ready.json`: the host revokes its own ready marker on graceful exit, and a launch retires the same id's leftover ready/marker pair when the recorded owner is proven exited (a live or unknown owner still wins) (#6261). diff --git a/packages/coding-agent/src/commands/sdk.ts b/packages/coding-agent/src/commands/sdk.ts index 6b68e03245d..562cf5da6f8 100644 --- a/packages/coding-agent/src/commands/sdk.ts +++ b/packages/coding-agent/src/commands/sdk.ts @@ -1,10 +1,15 @@ import { type ChildProcess, spawn } from "node:child_process"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import * as nodeFs from "node:fs"; import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; -import { exactRemoveDirectoryTree, type NativeDirectoryTreeSnapshot, snapshotDirectoryTree } from "@gajae-code/natives"; +import { + exactRemoveDirectoryTree, + exactUnlinkDirect, + type NativeDirectoryTreeSnapshot, + snapshotDirectoryTree, +} from "@gajae-code/natives"; import { logger, postmortem } from "@gajae-code/utils"; import { CliParseError, Command } from "@gajae-code/utils/cli"; import type { Args as ParsedArgs } from "../cli/args"; @@ -24,6 +29,7 @@ import { writeBrokerStartupExitRecordBounded, } from "../sdk/broker/broker-exit"; import { readBrokerDiscovery } from "../sdk/broker/discovery"; +import { type EndpointFileRead, readEndpointFile } from "../sdk/broker/endpoint-authority"; import { emitBrokerStartupTestSignal, reconcileBrokerGenerationForStartup, @@ -653,6 +659,7 @@ export async function runSessionHost( let startupComplete = false; let readinessPublicationCleanupComplete = true; let revokePendingReadinessMarker: (() => Promise) | undefined; + let revokePublishedReadinessMarker: (() => Promise) | undefined; let readinessRevocation: Promise | undefined; let startupInterruption: SdkStartupFailure | undefined; const interrupted = Promise.withResolvers(); @@ -1267,20 +1274,76 @@ export async function runSessionHost( return failureRollback; }; const sessionEndpointPath = path.join(request.stateRoot, "sdk", `${request.sessionId}.json`); - const exitAfterSessionDisposal = async (reason?: "detached_idle"): Promise => { - await disposeSession(reason); + const removeOwnedSessionEndpoint = async (): Promise => { + try { + await fs.stat(sessionEndpointPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return true; + return false; + } + const endpoint: EndpointFileRead | undefined = await readEndpointFile(sessionEndpointPath); + if (!endpoint) return false; + let parsed: { pid?: unknown; sessionId?: unknown }; + try { + parsed = JSON.parse(endpoint.source) as { pid?: unknown; sessionId?: unknown }; + } catch { + return false; + } + if (parsed.pid !== process.pid || parsed.sessionId !== request.sessionId) { + return false; + } + const removal = (() => { + try { + return exactUnlinkDirect(sessionEndpointPath, { + dev: endpoint.dev, + ino: endpoint.ino, + nlink: endpoint.nlink, + size: endpoint.size, + mtimeNs: endpoint.mtimeNs, + quarantineName: `.gjc-delete-endpoint-${request.sessionId}-${randomUUID()}`, + sha256: createHash("sha256").update(endpoint.source).digest("hex"), + allowHardLink: true, + }); + } catch (error) { + logger.warn("SDK host endpoint cleanup failed", { error: String(error) }); + return undefined; + } + })(); + if (!removal) return false; + return ( + removal.ok || + removal.code === "not_found" || + (removal.code === "cleanup_pending" && + removal.payloadDurable === true && + !removal.retainedSuccessorPath && + !removal.retainedUnknownPath) + ); + }; + let unregisterHostPostmortem: (() => void) | undefined; + let sessionDisposalExit: Promise | undefined; + const exitAfterSessionDisposal = async (reason?: "detached_idle", exitProcess = true): Promise => { let failure: SdkStartupFailure | undefined; + let ownsEndpoint = false; + const endpointCleanup = (async (): Promise => { + let complete = await removeOwnedSessionEndpoint(); + for (let attempt = 0; !complete && attempt < 20; attempt += 1) { + await Bun.sleep(25); + complete = await removeOwnedSessionEndpoint(); + } + return complete; + })(); + let [endpointCleanupComplete, readinessRevoked] = await Promise.all([ + endpointCleanup, + revokePublishedReadinessMarker?.() ?? Promise.resolve(true), + ]); + await disposeSession(reason); + endpointCleanupComplete = await removeOwnedSessionEndpoint(); try { const endpoint = JSON.parse(await fs.readFile(sessionEndpointPath, "utf8")) as { pid?: unknown; sessionId?: unknown; }; - if (endpoint.pid === process.pid && endpoint.sessionId === request.sessionId) - failure = { - phase: "startup", - reason: "failed", - message: `SDK host endpoint remained after graceful shutdown: ${request.sessionId}`, - }; + ownsEndpoint = endpoint.pid === process.pid && endpoint.sessionId === request.sessionId; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") { failure = { @@ -1288,27 +1351,57 @@ export async function runSessionHost( reason: "failed", message: `SDK host endpoint cleanup could not be verified: ${request.sessionId}`, }; - } + } else endpointCleanupComplete = true; } + if (!ownsEndpoint && !failure && !readinessRevoked && revokePublishedReadinessMarker) { + failure = { + phase: "startup", + reason: "failed", + message: `SDK host readiness marker cleanup could not be verified: ${request.sessionId}`, + }; + } + if (ownsEndpoint && !endpointCleanupComplete && !failure) + failure = { + phase: "startup", + reason: "failed", + message: `SDK host endpoint remained after graceful shutdown: ${request.sessionId}`, + }; if (failure) { process.exitCode = 1; process.stderr.write(`${failure.message}\n`); await writeFailure(failure, rollback.result).catch(() => {}); } - process.exit(process.exitCode ?? 0); + unregisterHostPostmortem?.(); + unregisterHostPostmortem = undefined; + if (exitProcess) process.exit(process.exitCode ?? 0); }; let stopping = false; + const requestSessionStop = (reason?: "detached_idle", exitProcess = true): Promise => { + if (stopping) return sessionDisposalExit ?? Promise.resolve(); + stopping = true; + sessionDisposalExit = exitAfterSessionDisposal(reason, exitProcess); + return sessionDisposalExit; + }; const stop = (reason?: "detached_idle") => { if (startupComplete) { - if (stopping) return; - stopping = true; - void exitAfterSessionDisposal(reason); + void requestSessionStop(reason); return; } const failure = capability.normalizeFailure("startup", "failed", "SDK lifecycle host terminated."); interruptStartup(failure); }; - const onReadySignal = (): void => stop(); + const registerHostPostmortem = (): void => { + if (unregisterHostPostmortem) return; + unregisterHostPostmortem = postmortem.register("sdk-session-host:exit", async reason => { + if (reason !== postmortem.Reason.SIGTERM && reason !== postmortem.Reason.SIGINT) return; + // The postmortem signal authority defaults to 143/130; a session host + // signal is graceful and must preserve its cleanup result instead. + await requestSessionStop(undefined, false); + process.exitCode = 0; + process.exit(0); + }); + }; + registerHostPostmortem(); try { const startupThinkingLevel = request.modelId ? parseModelString(request.modelId)?.thinkingLevel : undefined; @@ -1356,6 +1449,7 @@ export async function runSessionHost( effectMarker, () => startupInterruption === undefined && now() < request.semanticReadyDeadlineAt, revoke => { + revokePublishedReadinessMarker = revoke; revokePendingReadinessMarker = revoke; if (startupInterruption !== undefined) startReadinessRevocation(revoke); else if (now() >= request.semanticReadyDeadlineAt) { @@ -1390,8 +1484,6 @@ export async function runSessionHost( revokePendingReadinessMarker = undefined; readinessRevocation = undefined; removeStartupSignalHandlers(); - process.once("SIGTERM", onReadySignal); - process.once("SIGINT", onReadySignal); } catch (error) { if (error instanceof LifecycleReadinessCleanupError) constructionCleanupComplete = false; const failure = @@ -1404,8 +1496,6 @@ export async function runSessionHost( await failAfterRollback(durableFailure); } finally { removeStartupSignalHandlers(); - process.removeListener("SIGTERM", onReadySignal); - process.removeListener("SIGINT", onReadySignal); } throw error; } @@ -1655,7 +1745,8 @@ export default class Sdk extends Command { child.unref(); return; } - if (internal.action === "broker-internal") usePostmortemSignalExitAuthority(); + if (internal.action === "broker-internal" || internal.action === "session-host-internal") + usePostmortemSignalExitAuthority(); if (internal.action === "session-host-internal") { await runSessionHost(); return; diff --git a/packages/coding-agent/src/sdk/broker/lifecycle.ts b/packages/coding-agent/src/sdk/broker/lifecycle.ts index caeb14e9bd8..dc9393c4039 100644 --- a/packages/coding-agent/src/sdk/broker/lifecycle.ts +++ b/packages/coding-agent/src/sdk/broker/lifecycle.ts @@ -1667,9 +1667,9 @@ export async function reapDeadLifecycleMarkers( let inspected = 0; for await (const entry of directoryHandle) { if (inspected >= inspectionLimit) break; - inspected += 1; const name = entry.name; if (!entry.isFile() || !name.endsWith(".lifecycle.json")) continue; + inspected += 1; const id = name.slice(0, -".lifecycle.json".length); if (!isCanonicalSessionId(id)) continue; const markerPath = path.join(directory, name); @@ -1741,6 +1741,108 @@ export async function reapDeadLifecycleMarkers( return reaped; } +/** + * Retires one launch id's lifecycle marker pair after its recorded owner has + * exited. Unlike the bounded background sweep, this launch-local repair is + * not age-gated and does not require the ready sibling to contain the same + * effect marker: the primary marker is the owner authority for this id. + */ +type RetireExitedLifecycleMarkerPairHook = () => void | Promise; + +async function retireExitedLifecycleMarkerPairImpl( + root: string, + id: string, + afterObservation?: RetireExitedLifecycleMarkerPairHook, +): Promise { + if (!isCanonicalSessionId(id)) return false; + let directory: string; + let directoryIdentity: { dev: bigint; ino: bigint }; + try { + const canonicalRoot = fsSync.realpathSync(root); + directory = path.join(canonicalRoot, "sdk"); + const directoryStat = fsSync.lstatSync(directory, { bigint: true }); + if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink()) return false; + directoryIdentity = { dev: directoryStat.dev, ino: directoryStat.ino }; + } catch { + return false; + } + const markerPath = lifecycleMarkerPath(path.dirname(directory), id); + const primary = captureLifecycleFile(markerPath, true, true); + if (!primary) return false; + let marker: EffectMarker; + try { + const parsed = parseLifecycleJson(primary.bytes); + if (!isExactEffectMarker(parsed)) return false; + marker = parsed; + } catch { + return false; + } + if (observeProcess(marker.pid, marker.incarnation) !== "exited") return false; + await afterObservation?.(); + const readyPath = lifecycleReadyPath(path.dirname(directory), id); + const ready = captureLifecycleFile(readyPath, true, true); + try { + const currentParent = lifecycleParentIdentity(directory); + if ( + !currentParent || + BigInt(currentParent.dev) !== directoryIdentity.dev || + BigInt(currentParent.ino) !== directoryIdentity.ino + ) + return false; + const currentPrimary = captureLifecycleFile(markerPath, true, true); + await afterObservation?.(); + if ( + !currentPrimary || + !sameLifecycleCleanupIdentity( + currentPrimary.identity, + serializeCleanupIdentity({ ...primary.identity, size: Number(primary.identity.size) }), + ) + ) + return false; + const currentMarker = parseLifecycleJson(currentPrimary.bytes); + if (!isExactEffectMarker(currentMarker) || !sameEffectMarker(currentMarker, marker)) return false; + if (ready) { + const currentReady = captureLifecycleFile(readyPath, true, true); + if ( + !currentReady || + !sameLifecycleCleanupIdentity( + currentReady.identity, + serializeCleanupIdentity({ ...ready.identity, size: Number(ready.identity.size) }), + ) + ) + return false; + const removedReady = nativeLifecycle().exactUnlinkDirect(readyPath, { + ...ready.identity, + parentDev: directoryIdentity.dev, + parentIno: directoryIdentity.ino, + quarantineName: `.gjc-reap-${randomUUID()}-${path.basename(readyPath)}`, + }); + if (!removedReady.ok && removedReady.code !== "not_found") return false; + } + const removedMarker = nativeLifecycle().exactUnlinkDirect(markerPath, { + ...primary.identity, + parentDev: directoryIdentity.dev, + parentIno: directoryIdentity.ino, + quarantineName: `.gjc-reap-${randomUUID()}-${path.basename(markerPath)}`, + }); + return removedMarker.ok || removedMarker.code === "not_found"; + } catch { + return false; + } +} + +export async function retireExitedLifecycleMarkerPair(root: string, id: string): Promise { + return retireExitedLifecycleMarkerPairImpl(root, id); +} + +export async function retireExitedLifecycleMarkerPairForTest( + root: string, + id: string, + afterObservation: RetireExitedLifecycleMarkerPairHook, +): Promise { + return retireExitedLifecycleMarkerPairImpl(root, id, afterObservation); +} + export async function writeEffectMarker(root: string, id: string, marker: EffectMarker): Promise { const directory = path.join(root, "sdk"); await fs.mkdir(directory, { recursive: true, mode: 0o700 }); @@ -6650,6 +6752,7 @@ async function executeLifecycleResponse( terminationStartDeadline = childDeadlines.terminationStartDeadlineAt; lifecycleDeadline = childDeadlines.lifecycleCleanupDeadlineAt; } + await retireExitedLifecycleMarkerPair(launch.root, launch.id); await reapDeadLifecycleMarkers(launch.root); } catch (error) { if (launch.worktreePlan && worktreeReceipt?.created && !worktreeReceipt.reused) { diff --git a/packages/coding-agent/test/sdk-broker-lifecycle-e2e.test.ts b/packages/coding-agent/test/sdk-broker-lifecycle-e2e.test.ts index df92ae17b19..ccdd058ae96 100644 --- a/packages/coding-agent/test/sdk-broker-lifecycle-e2e.test.ts +++ b/packages/coding-agent/test/sdk-broker-lifecycle-e2e.test.ts @@ -813,13 +813,14 @@ test("shipped session host exits promptly after publishing a cutoff receipt", as }); if (!child.pid) throw new Error("session host has no pid"); const childIncarnation = await incarnation(child.pid); - const cutoffStartedAt = performance.now(); await fs.writeFile( path.join(stateRoot, "sdk", `${sessionId}.lifecycle.json`), JSON.stringify({ pid: child.pid, effectMarker, incarnation: childIncarnation }), ); + await waitFor(async () => ((await Bun.file(failurePath).exists()) ? true : undefined), "cutoff receipt"); + const receiptPublishedAt = performance.now(); const outcome = await Promise.race([ - child.exited.then(code => ({ code, latencyMs: performance.now() - cutoffStartedAt })), + child.exited.then(code => ({ code, latencyMs: performance.now() - receiptPublishedAt })), Bun.sleep(1_500).then(() => undefined), ]); expect(outcome).toBeDefined(); @@ -4387,7 +4388,7 @@ test("broker directly resumes and forks a canonical cold saved session with scop () => true, () => false, ), - ).toBe(true); + ).toBe(false); expect(await broker.handleRequest("session.get_endpoint", { sessionId: sourceId })).toMatchObject({ ok: false, error: { code: "resource_gone" }, @@ -4455,7 +4456,7 @@ test("broker directly resumes and forks a canonical cold saved session with scop () => true, () => false, ), - ).toBe(true); + ).toBe(false); expect(await broker.handleRequest("session.get_endpoint", { sessionId: forkId })).toMatchObject({ ok: false, error: { code: "resource_gone" }, @@ -4588,7 +4589,7 @@ test("broker replays one identity-bound lifecycle metadata cleanup plan after th const markerPath = path.join(stateRoot, "sdk", `${sessionId}.lifecycle.json`); const readyPath = path.join(stateRoot, "sdk", `${sessionId}.lifecycle.ready.json`); await expect(fs.stat(markerPath)).resolves.toBeDefined(); - await expect(fs.stat(readyPath)).resolves.toBeDefined(); + await expect(fs.stat(readyPath)).rejects.toThrow(); setLifecycleCleanupHookForTest(crashing, () => {}); const deleteInput = { cwd: root, stateRoot, sessionId, sessionPath }; await expect( diff --git a/packages/coding-agent/test/sdk-broker-stale-ready-regression.test.ts b/packages/coding-agent/test/sdk-broker-stale-ready-regression.test.ts new file mode 100644 index 00000000000..f521c93dcbf --- /dev/null +++ b/packages/coding-agent/test/sdk-broker-stale-ready-regression.test.ts @@ -0,0 +1,300 @@ +import { expect, test } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { Broker } from "../src/sdk/broker/broker"; +import * as lifecycle from "../src/sdk/broker/lifecycle"; +import { SessionManager } from "../src/session/session-manager"; + +async function makePair(label: string) { + const root = await fs.mkdtemp(path.join(process.env.TMPDIR ?? "/tmp", `gjc-${label}-`)); + const sdk = path.join(root, "sdk"); + await fs.mkdir(sdk, { recursive: true }); + const id = "race-resume"; + return { + root, + sdk, + id, + markerPath: path.join(sdk, `${id}.lifecycle.json`), + readyPath: path.join(sdk, `${id}.lifecycle.ready.json`), + }; +} + +const deadMarker = { pid: 999_999_999, effectMarker: "dead", incarnation: "dead" }; + +test("retire binds cleanup to the observed marker when a live pair is republished", async () => { + const pair = await makePair("retire-live-race"); + try { + await fs.writeFile(pair.markerPath, JSON.stringify(deadMarker)); + await fs.writeFile(pair.readyPath, JSON.stringify(deadMarker)); + const incarnation = lifecycle.processIncarnation(process.pid); + const live = { pid: process.pid, effectMarker: "live", incarnation }; + let calls = 0; + const result = await lifecycle.retireExitedLifecycleMarkerPairForTest(pair.root, pair.id, async () => { + calls += 1; + if (calls === 1) { + await fs.writeFile(pair.markerPath, JSON.stringify(live)); + await fs.writeFile(pair.readyPath, JSON.stringify(live)); + } + }); + expect(result).toBe(false); + expect(await fs.readFile(pair.markerPath, "utf8")).toBe(JSON.stringify(live)); + expect(await fs.readFile(pair.readyPath, "utf8")).toBe(JSON.stringify(live)); + } finally { + await fs.rm(pair.root, { recursive: true, force: true }); + } +}); + +test("retire binds cleanup to an atomically republished live pair", async () => { + const pair = await makePair("retire-live-atomic-race"); + try { + await fs.writeFile(pair.markerPath, JSON.stringify(deadMarker)); + await fs.writeFile(pair.readyPath, JSON.stringify(deadMarker)); + const incarnation = lifecycle.processIncarnation(process.pid); + const live = { pid: process.pid, effectMarker: "atomic-live", incarnation }; + let calls = 0; + const result = await lifecycle.retireExitedLifecycleMarkerPairForTest(pair.root, pair.id, async () => { + calls += 1; + if (calls !== 1) return; + for (const [target, suffix] of [ + [pair.markerPath, "marker"], + [pair.readyPath, "ready"], + ] as const) { + await fs.rm(target, { force: true }); + const temporary = path.join(pair.root, `${suffix}.tmp`); + await fs.writeFile(temporary, JSON.stringify(live)); + await fs.rename(temporary, target); + } + }); + expect(result).toBe(false); + expect(await fs.readFile(pair.markerPath, "utf8")).toBe(JSON.stringify(live)); + expect(await fs.readFile(pair.readyPath, "utf8")).toBe(JSON.stringify(live)); + } finally { + await fs.rm(pair.root, { recursive: true, force: true }); + } +}); + +test("retire keeps a ready marker replaced after observation", async () => { + const pair = await makePair("retire-ready-race"); + try { + await fs.writeFile(pair.markerPath, JSON.stringify(deadMarker)); + await fs.writeFile(pair.readyPath, JSON.stringify(deadMarker)); + const live = { + pid: process.pid, + effectMarker: "ready-live", + incarnation: lifecycle.processIncarnation(process.pid), + }; + let calls = 0; + const result = await lifecycle.retireExitedLifecycleMarkerPairForTest(pair.root, pair.id, async () => { + calls += 1; + if (calls === 2) await fs.writeFile(pair.readyPath, JSON.stringify(live)); + }); + expect(result).toBe(false); + expect(await fs.readFile(pair.markerPath, "utf8")).toBe(JSON.stringify(deadMarker)); + expect(await fs.readFile(pair.readyPath, "utf8")).toBe(JSON.stringify(live)); + } finally { + await fs.rm(pair.root, { recursive: true, force: true }); + } +}); + +test("retire does not remove a different exited owner", async () => { + const pair = await makePair("retire-other-dead"); + try { + await fs.writeFile(pair.markerPath, JSON.stringify(deadMarker)); + await fs.writeFile(pair.readyPath, JSON.stringify(deadMarker)); + const other = { pid: 999_999_998, effectMarker: "other", incarnation: "other" }; + const result = await lifecycle.retireExitedLifecycleMarkerPairForTest(pair.root, pair.id, async () => { + await fs.writeFile(pair.markerPath, JSON.stringify(other)); + }); + expect(result).toBe(false); + expect(await fs.readFile(pair.markerPath, "utf8")).toBe(JSON.stringify(other)); + } finally { + await fs.rm(pair.root, { recursive: true, force: true }); + } +}); + +test("retire tolerates primary removal during observation race", async () => { + const pair = await makePair("retire-removed"); + try { + await fs.writeFile(pair.markerPath, JSON.stringify(deadMarker)); + await fs.writeFile(pair.readyPath, JSON.stringify(deadMarker)); + const result = await lifecycle.retireExitedLifecycleMarkerPairForTest(pair.root, pair.id, async () => { + await fs.rm(pair.markerPath); + }); + expect(result).toBe(false); + expect(await fs.readFile(pair.readyPath, "utf8")).toBe(JSON.stringify(deadMarker)); + } finally { + await fs.rm(pair.root, { recursive: true, force: true }); + } +}); + +test("retire tolerates malformed primary during observation race", async () => { + const pair = await makePair("retire-malformed"); + try { + await fs.writeFile(pair.markerPath, JSON.stringify(deadMarker)); + await fs.writeFile(pair.readyPath, JSON.stringify(deadMarker)); + const result = await lifecycle.retireExitedLifecycleMarkerPairForTest(pair.root, pair.id, async () => { + await fs.writeFile(pair.markerPath, "{}"); + }); + expect(result).toBe(false); + expect(await fs.readFile(pair.markerPath, "utf8")).toBe("{}"); + } finally { + await fs.rm(pair.root, { recursive: true, force: true }); + } +}); + +test("launch cleanup retires an exited id pair regardless of age or ready marker contents", async () => { + const root = await fs.mkdtemp(path.join(process.env.TMPDIR ?? "/tmp", "gjc-stale-ready-launch-")); + const sdk = path.join(root, "sdk"); + const id = "stale-resume"; + const markerPath = path.join(sdk, `${id}.lifecycle.json`); + const readyPath = path.join(sdk, `${id}.lifecycle.ready.json`); + try { + await fs.mkdir(sdk, { recursive: true }); + await fs.writeFile(markerPath, JSON.stringify({ pid: 999_999_999, effectMarker: "old", incarnation: "old" })); + await fs.writeFile( + readyPath, + JSON.stringify({ pid: 999_999_998, effectMarker: "different", incarnation: "different" }), + ); + + const retire = ( + lifecycle as typeof lifecycle & { + retireExitedLifecycleMarkerPair?: (root: string, id: string) => Promise; + } + ).retireExitedLifecycleMarkerPair; + expect(retire).toBeFunction(); + await expect(retire?.(root, id)).resolves.toBe(true); + await expect(fs.stat(markerPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.stat(readyPath)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } +}); + +test("launch cleanup keeps a pair owned by the current live process", async () => { + const root = await fs.mkdtemp(path.join(process.env.TMPDIR ?? "/tmp", "gjc-stale-ready-live-")); + const sdk = path.join(root, "sdk"); + const id = "live-resume"; + const markerPath = path.join(sdk, `${id}.lifecycle.json`); + const readyPath = path.join(sdk, `${id}.lifecycle.ready.json`); + try { + await fs.mkdir(sdk, { recursive: true }); + const incarnation = lifecycle.processIncarnation(process.pid); + expect(incarnation).toBeString(); + if (!incarnation) throw new Error("Expected the current process incarnation."); + const marker = { pid: process.pid, effectMarker: "live", incarnation }; + await fs.writeFile(markerPath, JSON.stringify(marker)); + await fs.writeFile(readyPath, JSON.stringify(marker)); + + await expect(lifecycle.retireExitedLifecycleMarkerPair(root, id)).resolves.toBe(false); + await expect(fs.stat(markerPath)).resolves.toBeDefined(); + await expect(fs.stat(readyPath)).resolves.toBeDefined(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } +}); + +test("marker sweep counts only lifecycle marker candidates against its inspection limit", async () => { + const root = await fs.mkdtemp(path.join(process.env.TMPDIR ?? "/tmp", "gjc-stale-ready-limit-")); + const sdk = path.join(root, "sdk"); + const id = "expired-marker"; + const markerPath = path.join(sdk, `${id}.lifecycle.json`); + try { + await fs.mkdir(sdk, { recursive: true }); + const marker = { pid: 999_999_999, effectMarker: "old", incarnation: "old" }; + for (let index = 0; index < 70; index += 1) { + await fs.writeFile(path.join(sdk, `unrelated-${index}.lifecycle.ready.json`), "unrelated"); + } + await fs.writeFile(path.join(sdk, `${id}.lifecycle.ready.json`), JSON.stringify(marker)); + await fs.writeFile(markerPath, JSON.stringify(marker)); + const expiredAt = new Date(Date.now() - 2 * 60 * 60 * 1000); + await fs.utimes(markerPath, expiredAt, expiredAt); + + await expect(lifecycle.reapDeadLifecycleMarkers(root)).resolves.toBe(1); + await expect(fs.stat(markerPath)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } +}); + +test("published ready marker exposes revocation for detached host shutdown", async () => { + const root = await fs.mkdtemp(path.join(process.env.TMPDIR ?? "/tmp", "gjc-stale-ready-revoke-")); + const id = "detached-ready"; + const effectMarker = "detached-ready-effect"; + let revoke: (() => Promise) | undefined; + try { + await lifecycle.writeSessionLifecycleReady( + root, + id, + effectMarker, + () => true, + callback => { + revoke = callback; + }, + ); + const readyPath = path.join(root, "sdk", `${id}.lifecycle.ready.json`); + await expect(fs.stat(readyPath)).resolves.toBeDefined(); + expect(revoke).toBeFunction(); + await expect(revoke?.()).resolves.toBe(true); + await expect(fs.stat(readyPath)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } +}); + +test("real session host removes ready and endpoint markers after SIGTERM", async () => { + const root = await fs.mkdtemp(path.join(process.env.TMPDIR ?? "/tmp", "gjc-stale-ready-host-exit-")); + const agentDir = path.join(root, "agent"); + const stateRoot = path.join(root, ".gjc", "state"); + const broker = new Broker({ agentDir }); + let hostPid: number | undefined; + try { + await fs.mkdir(agentDir, { recursive: true, mode: 0o700 }); + const source = SessionManager.create(root, SessionManager.managedDestination(root, agentDir)); + await source.ensureOnDisk(); + const sessionId = source.getSessionId(); + const sessionPath = source.getSessionFile(); + await broker.start(); + const result = await broker.handleRequest( + "session.resume", + { cwd: root, stateRoot, sessionId, sessionPath, readinessTimeoutMs: 20_000 }, + "stale-ready-host-exit", + ); + expect(result.ok).toBe(true); + const endpointPath = path.join(stateRoot, "sdk", `${sessionId}.json`); + const readyPath = path.join(stateRoot, "sdk", `${sessionId}.lifecycle.ready.json`); + const endpoint = JSON.parse(await fs.readFile(endpointPath, "utf8")) as { pid?: unknown }; + hostPid = typeof endpoint.pid === "number" ? endpoint.pid : undefined; + if (!hostPid) throw new Error("Session endpoint did not publish a host pid."); + process.kill(hostPid, "SIGTERM"); + const exitDeadline = Date.now() + 20_000; + let exited = false; + while (Date.now() < exitDeadline) { + try { + process.kill(hostPid, 0); + } catch { + exited = true; + break; + } + await Bun.sleep(25); + } + expect(exited).toBe(true); + const cleanupDeadline = Date.now() + 5_000; + while (Date.now() < cleanupDeadline) { + if (!(await Bun.file(readyPath).exists()) && !(await Bun.file(endpointPath).exists())) break; + await Bun.sleep(25); + } + expect(await Bun.file(readyPath).exists()).toBe(false); + expect(await Bun.file(endpointPath).exists()).toBe(false); + await broker.stop(); + + expect(await Bun.file(endpointPath).exists()).toBe(false); + } finally { + if (hostPid) { + try { + process.kill(hostPid, "SIGTERM"); + } catch {} + } + await broker.stop().catch(() => {}); + await fs.rm(root, { recursive: true, force: true }); + } +}, 35_000);