Skip to content

Commit 01236ed

Browse files
fix: make file-producing task retries idempotent (#45)
* fix: make file-producing task retries idempotent Reuse operation-scoped file identities across task and import checkpoint gaps. Publish complete PDFs without overwriting earlier results and recover metadata through insert-if-absent. Cover interrupted document/model/import flows and concurrent replay. Assisted-by: OpenAI Codex * fix: make keyed file IDs UUID-shaped --------- Co-authored-by: Jerel Velarde <85066839+jerelvelarde@users.noreply.github.com>
1 parent d344dc8 commit 01236ed

7 files changed

Lines changed: 382 additions & 15 deletions

File tree

‎apps/server/src/engine/model.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,8 +150,15 @@ export async function executeModelTask(
150150
}),
151151
async (args) =>
152152
cached("fill_pdf", args, async () => {
153-
const file = await service.files.fill(owner, args.fileId, args.fields);
154-
task = await ctx.checkpoint({ artifactIds: [...task.artifactIds, file.id] });
153+
const file = await service.files.fill(
154+
owner,
155+
args.fileId,
156+
args.fields,
157+
`model:${task.id}:fill_pdf`,
158+
);
159+
task = await ctx.checkpoint({
160+
artifactIds: [...new Set([...task.artifactIds, file.id])],
161+
});
155162
return { id: file.id, name: file.name, fields: file.fields };
156163
}),
157164
),

‎apps/server/src/engine/service.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -986,7 +986,7 @@ export class AgentService {
986986
let filledId = typeof task.state.filledId === "string" ? task.state.filledId : undefined;
987987
if (!filledId) {
988988
await ctx.guard();
989-
const filled = await this.files.fill(owner, source.fileId, fields);
989+
const filled = await this.files.fill(owner, source.fileId, fields, `document:${task.id}`);
990990
filledId = filled.id;
991991
task = await ctx.checkpoint({
992992
state: { ...task.state, source, filledId },

‎apps/server/src/files.ts‎

Lines changed: 66 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { randomUUID } from "node:crypto";
2-
import { mkdir, readFile, writeFile } from "node:fs/promises";
1+
import { createHash, randomUUID } from "node:crypto";
2+
import { link, mkdir, readFile, rm, writeFile } from "node:fs/promises";
33
import { join, resolve, sep } from "node:path";
44
import type { Artifact } from "../../../packages/domain/src/index.ts";
55
import { fillPdf, inspectPdf } from "../../../packages/integrations/src/pdf.ts";
@@ -23,6 +23,21 @@ function assertFileId(id: string): void {
2323
if (!FILE_ID.test(id)) throw new AppError("File not found", 404);
2424
}
2525

26+
// Keyed operations need the same ID on every retry, so the ID is derived from the
27+
// owner and operation key instead of random. It is shaped as a version 5 UUID
28+
// (first 128 bits of a SHA-256 hash, version and variant bits set) so it is
29+
// accepted anywhere a random file UUID is.
30+
export function deterministicFileId(owner: string, operationKey: string): string {
31+
const bytes = createHash("sha256")
32+
.update(JSON.stringify([owner, operationKey]))
33+
.digest()
34+
.subarray(0, 16);
35+
bytes[6] = (bytes[6] & 0x0f) | 0x50;
36+
bytes[8] = (bytes[8] & 0x3f) | 0x80;
37+
const hex = bytes.toString("hex");
38+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
39+
}
40+
2641
export class Files {
2742
constructor(
2843
private readonly db: Store,
@@ -35,11 +50,36 @@ export class Files {
3550
bytes: Uint8Array,
3651
source: string,
3752
parentId?: string,
53+
operationKey?: string,
3854
): Promise<Artifact> {
55+
const id = operationKey === undefined ? randomUUID() : deterministicFileId(owner, operationKey);
56+
if (operationKey !== undefined) {
57+
const existing = await this.db.get<Artifact>(owner, "files", id);
58+
if (existing) return this.signed(owner, existing);
59+
}
3960
if (bytes.length > 10 * 1024 * 1024) throw new AppError("PDFs must be 10 MB or smaller", 413);
40-
const metadata = await inspectPdf(bytes);
61+
let metadata = await inspectPdf(bytes);
4162
if (metadata.pageCount > 500) throw new AppError("PDFs must have 500 pages or fewer", 422);
42-
const id = randomUUID();
63+
const directory = join(this.config.dataDir, "files");
64+
await mkdir(directory, { recursive: true, mode: 0o700 });
65+
const path = this.pathFor(id);
66+
if (operationKey === undefined) {
67+
await writeFile(path, bytes, { mode: 0o600, flag: "wx" });
68+
} else {
69+
const temporary = `${path}.${randomUUID()}.tmp`;
70+
try {
71+
await writeFile(temporary, bytes, { mode: 0o600, flag: "wx" });
72+
try {
73+
await link(temporary, path);
74+
} catch (error) {
75+
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
76+
bytes = await readFile(path);
77+
metadata = await inspectPdf(bytes);
78+
}
79+
} finally {
80+
await rm(temporary, { force: true });
81+
}
82+
}
4383
const safeName = sanitizeFileName(name);
4484
const artifact: Artifact = {
4585
id,
@@ -53,11 +93,12 @@ export class Files {
5393
source,
5494
parentId,
5595
};
56-
const directory = join(this.config.dataDir, "files");
57-
await mkdir(directory, { recursive: true, mode: 0o700 });
58-
await writeFile(join(directory, `${id}.pdf`), bytes, { mode: 0o600, flag: "wx" });
59-
await this.db.put(owner, "files", artifact);
60-
return this.signed(owner, artifact);
96+
if (operationKey === undefined) {
97+
await this.db.put(owner, "files", artifact);
98+
return this.signed(owner, artifact);
99+
}
100+
const persisted = await this.db.insertIfAbsent(owner, "files", artifact);
101+
return this.signed(owner, persisted ?? (await this.get(owner, id)));
61102
}
62103
signed(owner: string, file: Artifact): Artifact {
63104
return { ...file, url: this.auth.sign(owner, `/api/files/${file.id}/content`) };
@@ -82,7 +123,12 @@ export class Files {
82123
await this.get(owner, id);
83124
return readFile(this.pathFor(id));
84125
}
85-
async fill(owner: string, id: string, values: Record<string, string | boolean>) {
126+
async fill(
127+
owner: string,
128+
id: string,
129+
values: Record<string, string | boolean>,
130+
operationKey?: string,
131+
) {
86132
const file = await this.get(owner, id);
87133
const bytes = await this.bytes(owner, id);
88134
const output = await fillPdf(bytes, values);
@@ -92,6 +138,16 @@ export class Files {
92138
output,
93139
`Filled from ${file.name}`,
94140
id,
141+
operationKey === undefined
142+
? undefined
143+
: JSON.stringify([
144+
"fill",
145+
operationKey,
146+
id,
147+
Object.keys(values)
148+
.sort()
149+
.map((name) => [name, values[name]]),
150+
]),
95151
);
96152
}
97153
}

‎apps/server/src/workspace.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -474,6 +474,8 @@ export class WorkspaceService {
474474
decodeURIComponent(filename),
475475
await this.google(owner, connection.id).getAttachment(messageId, attachmentId),
476476
`Gmail · ${message.subject}`,
477+
undefined,
478+
JSON.stringify(["attachment", connection.id, reference]),
477479
);
478480
await this.db.put(owner, "imports", {
479481
id: reference,

‎tests/api.test.ts‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,14 @@ import { after, before, test } from "node:test";
66
import { EventType } from "@ag-ui/core";
77
import { lastValueFrom, toArray } from "rxjs";
88
import { createApp } from "../apps/server/src/app.ts";
9+
import { Auth } from "../apps/server/src/auth.ts";
910
import type { Config } from "../apps/server/src/config.ts";
1011
import { createStore, type Store } from "../apps/server/src/db.ts";
1112
import { ConversationAgent } from "../apps/server/src/engine/conversation.ts";
1213
import type { AgentService } from "../apps/server/src/engine/service.ts";
14+
import { deterministicFileId, Files } from "../apps/server/src/files.ts";
1315
import type { ActionProposal, Artifact, Workspace } from "../packages/domain/src/index.ts";
16+
import { createSamplePdf } from "../packages/integrations/src/pdf.ts";
1417

1518
let db: Store,
1619
app: Awaited<ReturnType<typeof createApp>>["app"],
@@ -58,6 +61,127 @@ test("API protects private data and rejects unrelated web origins", async () =>
5861
403,
5962
);
6063
});
64+
test("file operations recover a published PDF when metadata persistence fails", async (t) => {
65+
const owner = "file-recovery";
66+
const bytes = await createSamplePdf();
67+
const insert = db.insertIfAbsent.bind(db);
68+
let publishedId = "";
69+
t.mock.method(db, "insertIfAbsent", async (...args: Parameters<Store["insertIfAbsent"]>) => {
70+
if (args[0] === owner && args[1] === "files" && !publishedId) {
71+
publishedId = args[2].id;
72+
throw new Error("metadata unavailable");
73+
}
74+
return insert(...args);
75+
});
76+
await assert.rejects(
77+
agent.files.import(owner, "sample.pdf", bytes, "fixture", undefined, "import-once"),
78+
/metadata unavailable/,
79+
);
80+
assert.ok(publishedId);
81+
assert.equal((await agent.files.list(owner)).length, 0);
82+
const restarted = new Files(db, config, new Auth(db, config, "sample-signing-key"));
83+
const recovered = await restarted.import(
84+
owner,
85+
"sample.pdf",
86+
bytes,
87+
"fixture",
88+
undefined,
89+
"import-once",
90+
);
91+
assert.equal(recovered.id, publishedId);
92+
assert.deepEqual(new Uint8Array(await restarted.bytes(owner, recovered.id)), bytes);
93+
assert.equal((await restarted.list(owner)).length, 1);
94+
});
95+
96+
test("file operation identity isolates owners, inputs and independent requests", async () => {
97+
const owner = "file-identities";
98+
const bytes = await createSamplePdf();
99+
const source = await agent.files.import(owner, "sample.pdf", bytes, "fixture");
100+
const fields = { participant_name: "Sample Student", permission_granted: true };
101+
const [first, concurrent] = await Promise.all([
102+
agent.files.fill(owner, source.id, fields, "task-fill"),
103+
agent.files.fill(owner, source.id, fields, "task-fill"),
104+
]);
105+
assert.equal(first.id, concurrent.id);
106+
assert.equal(first.createdAt, concurrent.createdAt);
107+
assert.equal((await agent.files.list(owner)).filter((file) => file.parentId).length, 1);
108+
const reordered = await agent.files.fill(
109+
owner,
110+
source.id,
111+
{
112+
permission_granted: true,
113+
participant_name: "Sample Student",
114+
},
115+
"task-fill",
116+
);
117+
assert.equal(reordered.id, first.id);
118+
assert.notEqual((await agent.files.fill(owner, source.id, fields, "another-task")).id, first.id);
119+
assert.notEqual(
120+
(await agent.files.fill(owner, source.id, { participant_name: "Another Student" }, "task-fill"))
121+
.id,
122+
first.id,
123+
);
124+
const manual = await agent.files.fill(owner, source.id, fields);
125+
assert.notEqual((await agent.files.fill(owner, source.id, fields)).id, manual.id);
126+
const otherOwner = await agent.files.import(
127+
"other-file-owner",
128+
"sample.pdf",
129+
bytes,
130+
"fixture",
131+
undefined,
132+
"owner-scoped",
133+
);
134+
assert.notEqual(
135+
(await agent.files.import(owner, "sample.pdf", bytes, "fixture", undefined, "owner-scoped")).id,
136+
otherOwner.id,
137+
);
138+
});
139+
140+
test("keyed file IDs are UUIDs that work through lookup, content reads and attachments", async () => {
141+
const owner = "local-user";
142+
const sample = await createSamplePdf();
143+
const source = await agent.files.import(owner, "form.pdf", sample, "test");
144+
const fields = { participant_name: "Sample Student", permission_granted: true };
145+
const filled = await agent.files.fill(owner, source.id, fields, "task-1:fill_pdf");
146+
const imported = await agent.files.import(
147+
owner,
148+
"a.pdf",
149+
sample,
150+
"test",
151+
undefined,
152+
"attachment:1",
153+
);
154+
assert.equal((await agent.files.fill(owner, source.id, fields, "task-1:fill_pdf")).id, filled.id);
155+
for (const id of [filled.id, imported.id]) {
156+
assert.match(id, /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/);
157+
assert.equal((await agent.files.get(owner, id)).id, id);
158+
const content = await app.request(`/api/files/${id}/content`, { headers: headers() });
159+
assert.equal(content.status, 200);
160+
assert.equal(
161+
Buffer.from(await content.arrayBuffer())
162+
.subarray(0, 5)
163+
.toString(),
164+
"%PDF-",
165+
);
166+
const proposal = await app.request("/api/actions", {
167+
method: "POST",
168+
headers: headers(),
169+
body: JSON.stringify({
170+
kind: "email.send",
171+
data: {
172+
to: ["sample@example.com"],
173+
subject: "Attached",
174+
body: "See attached.",
175+
attachmentIds: [id],
176+
},
177+
}),
178+
});
179+
assert.equal(proposal.status, 201);
180+
}
181+
assert.equal(deterministicFileId(owner, "x"), deterministicFileId(owner, "x"));
182+
assert.notEqual(deterministicFileId(owner, "x"), deterministicFileId("other", "x"));
183+
});
184+
61185
test("sample workspace serves a real PDF and filling creates a new version", async () => {
62186
const response = await app.request("/api/workspace", { headers: headers() });
63187
assert.equal(response.status, 200);

‎tests/model-worker.test.ts‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,96 @@ import { tmpdir } from "node:os";
66
import { join } from "node:path";
77
import { test } from "node:test";
88
import { createApp } from "../apps/server/src/app.ts";
9-
import { createStore } from "../apps/server/src/db.ts";
9+
import { createStore, type Store } from "../apps/server/src/db.ts";
1010
import { createDemoModel, demoModel } from "../apps/server/src/demo/model.ts";
11+
import type { AgentTask } from "../packages/domain/src/agent.ts";
1112
import type { ActionProposal } from "../packages/domain/src/index.ts";
1213
import { browserFixture } from "./helpers/browser.ts";
1314
import { fixture as computerFixture } from "./helpers/computer.ts";
1415
import { modelFixture } from "./helpers/model.ts";
1516

17+
for (const checkpoint of ["artifact", "cache"] as const) {
18+
test(`model fill reuses its PDF after a failed ${checkpoint} checkpoint`, async (t) => {
19+
const directory = await mkdtemp(join(tmpdir(), "openmuse-model-fill-"));
20+
const db = await createStore();
21+
const calls: { name: string; arguments: object }[] = [];
22+
const { requests } = await modelFixture(t, (index) => calls[index]);
23+
const server = await createApp(db, {
24+
mode: "sample",
25+
port: 8787,
26+
host: "127.0.0.1",
27+
publicUrl: "http://localhost:8787",
28+
dataDir: directory,
29+
agentBackend: "model",
30+
intelligenceApiKey: "test-project-key-never-sent",
31+
model: "openai/fixture",
32+
googleRedirectUri: "http://localhost:8787/api/google/callback",
33+
allowedOrigins: [],
34+
});
35+
try {
36+
const owner = "model-fill-replay";
37+
await server.workspace.ensureSample(owner, server.actions);
38+
const source = (await server.files.list(owner))[0];
39+
const fill = {
40+
name: "fill_pdf",
41+
arguments: { fileId: source.id, fields: { participant_name: "Sample Student" } },
42+
};
43+
calls.push(fill, {
44+
name: "ask_user",
45+
arguments: { question: "Retry the interrupted operation?" },
46+
});
47+
const task = await server.agent.createTask(owner, { prompt: "Fill the sample form" });
48+
const compareAndSwap = db.compareAndSwap.bind(db);
49+
let interrupted = false;
50+
t.mock.method(db, "compareAndSwap", async (...args: Parameters<Store["compareAndSwap"]>) => {
51+
const patch = args[4] as Partial<AgentTask>;
52+
if (
53+
args[0] === owner &&
54+
args[1] === "tasks" &&
55+
args[2] === task.id &&
56+
(checkpoint === "artifact" ? patch.artifactIds?.length : patch.state?.operations) &&
57+
!interrupted
58+
) {
59+
interrupted = true;
60+
return null;
61+
}
62+
return compareAndSwap(...args);
63+
});
64+
await server.agent.worker.tick();
65+
assert.ok(interrupted);
66+
const waiting = await server.agent.getTask(owner, task.id);
67+
assert.equal(waiting.status, "waiting_input", waiting.error ?? waiting.question);
68+
const outputs = (await server.files.list(owner)).filter(
69+
(file) => file.parentId === source.id,
70+
);
71+
assert.equal(outputs.length, 1);
72+
assert.deepEqual(waiting.artifactIds, checkpoint === "artifact" ? [] : [outputs[0].id]);
73+
assert.equal(waiting.state.operations, undefined);
74+
requests.length = 0;
75+
calls.splice(0, calls.length, fill, {
76+
name: "finish_task",
77+
arguments: { summary: "Saved the sample form." },
78+
});
79+
await server.agent.answer(owner, task.id, "Retry");
80+
await server.agent.worker.tick();
81+
const completed = await server.agent.getTask(owner, task.id);
82+
assert.equal(completed.status, "succeeded", completed.error ?? completed.question);
83+
assert.deepEqual(
84+
completed.artifactIds.filter((id) => id === outputs[0].id),
85+
[outputs[0].id],
86+
);
87+
assert.equal(
88+
(await server.files.list(owner)).filter((file) => file.parentId === source.id).length,
89+
1,
90+
);
91+
} finally {
92+
await server.agent.stop();
93+
await db.close();
94+
await rm(directory, { recursive: true, force: true });
95+
}
96+
});
97+
}
98+
1699
test("CopilotKit model worker executes server tools and persists the confirmed outcome", async (t) => {
17100
const directory = await mkdtemp(join(tmpdir(), "openmuse-model-"));
18101
const db = await createStore();

0 commit comments

Comments
 (0)