fix: settle Fast turns during API shutdown

Author: daniel-lxs

apps/api/src/__tests__/graceful-shutdown.test.ts

@@ -0,0 +1,61 @@
+import type { ServerType } from '@hono/node-server';
+import {
+  abortActiveFastAgentTurns,
+  FastAgentProcessShutdownError,
+} from '@roomote/cloud-agents/server';
+
+type ApiShutdownOptions = {
+  abortTurns?: typeof abortActiveFastAgentTurns;
+  exitProcess?: (code?: number) => never;
+  flushSentry?: () => Promise<unknown>;
+  logError?: (...args: Parameters<typeof console.error>) => void;
+};
+
+export async function gracefullyShutdownApi(
+  server: ServerType,
+  signal: NodeJS.Signals,
+  {
+    abortTurns = abortActiveFastAgentTurns,
+    exitProcess = process.exit,
+    flushSentry = async () => undefined,
+    logError = (...args) => console.error(...args),
+  }: ApiShutdownOptions = {},
+): Promise<void> {
+  const abortPromise = abortTurns(new FastAgentProcessShutdownError(signal));
+  const closeError = await new Promise<Error | null>((resolve) => {
+    server.close((error) => resolve(error ?? null));
+  });
+  await abortPromise;
+  if (closeError) {
+    logError('[api] Graceful shutdown failed', closeError);
+  }
+  await flushSentry();
+  exitProcess(closeError ? 1 : 0);
+}
+
+export function installApiGracefulShutdown(
+  server: ServerType,
+  options: ApiShutdownOptions = {},
+): () => void {
+  let shuttingDown = false;
+  const handlers = new Map<NodeJS.Signals, () => void>();
+
+  for (const signal of ['SIGTERM', 'SIGINT'] as const) {
+    const handler = () => {
+      if (shuttingDown) {
+        (options.exitProcess ?? process.exit)(1);
+        return;
+      }
+      shuttingDown = true;
+      void gracefullyShutdownApi(server, signal, options);
+    };
+    handlers.set(signal, handler);
+    process.on(signal, handler);
+  }
+
+  return () => {
+    for (const [signal, handler] of handlers) {
+      process.off(signal, handler);
+    }
+  };
+}

apps/api/src/graceful-shutdown.ts

@@ -25,7 +25,8 @@ import {
 import type { Variables } from './types';
 import { resolveApiCorsOrigin } from './cors';
 import { createSingleLineWarnLogger } from './logging';
-import { captureApiException } from './monitoring/sentry';
+import { captureApiException, flushApiSentry } from './monitoring/sentry';
+import { installApiGracefulShutdown } from './graceful-shutdown';
 import {
   requestObservabilityMiddleware,
   routePolicyMiddleware,
@@ -280,6 +281,7 @@ export async function startApiServer({
   const app = createApiApp();
   const server = createAdaptorServer({ fetch: app.fetch });
   const address = await listen(server, { port, hostname });
+  installApiGracefulShutdown(server, { flushSentry: flushApiSentry });
 
   if (Env.NODE_ENV === 'development') {
     showRoutes(app);

apps/api/src/server.ts

@@ -104,6 +104,7 @@ import {
 } from './fast-agent-tasks';
 import { getFastAgentUserIdentity } from './fast-agent-user-identity';
 import { FastAgentTurnDiagnostics } from './fast-agent-turn-diagnostics';
+import { FastAgentProcessShutdownError } from './fast-agent-turn-lock';
 import {
   captureFastAgentInferenceAttemptOutcome,
   captureFastAgentInferenceContext,
@@ -2553,6 +2554,29 @@ export async function answerFastAgentQuestion({
           },
           true,
         );
+      } else if (
+        terminalError instanceof FastAgentProcessShutdownError &&
+        !closed
+      ) {
+        const reply = {
+          purpose: 'closeout' as const,
+          message: INTERRUPTED_INFERENCE_RETRY_MESSAGE,
+        };
+        try {
+          const posted = await adapter.postReply(reply);
+          diagnostics.recordVisibleReply();
+          await persistAssistantReply({
+            reply,
+            event: allocateCanonicalEvent(
+              `assistant:${nextAssistantOrdinal++}`,
+            ),
+            platformMessageId: posted?.messageId,
+          });
+        } catch (postError) {
+          console.error(
+            `[Fast Agent] Failed to post shutdown closeout: ${formatErrorForLog(postError)}`,
+          );
+        }
       }
       throw signal.reason instanceof Error ? signal.reason : error;
     }