Skip to content

Commit c6434a6

Browse files
charan-rathoreIron1c-delayer
authored andcommitted
fix: make file-producing task retries idempotent
Reuse operation-scoped file identities across task and import checkpoint gaps. Publish complete PDFs without overwriting earlier results and recover metadata through insert-if-absent. Cover interrupted document/model/import flows and concurrent replay. Assisted-by: OpenAI Codex
1 parent f5534c7 commit c6434a6

7 files changed

Lines changed: 327 additions & 15 deletions

File tree

‎apps/server/src/engine/model.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,8 +158,15 @@ export async function executeModelTask(
158158
}),
159159
async (args) =>
160160
cached("fill_pdf", args, async () => {
161-
const file = await service.files.fill(owner, args.fileId, args.fields);
162-
task = await ctx.checkpoint({ artifactIds: [...task.artifactIds, file.id] });
161+
const file = await service.files.fill(
162+
owner,
163+
args.fileId,
164+
args.fields,
165+
`model:${task.id}:fill_pdf`,
166+
);
167+
task = await ctx.checkpoint({
168+
artifactIds: [...new Set([...task.artifactIds, file.id])],
169+
});
163170
return { id: file.id, name: file.name, fields: file.fields };
164171
}),
165172
),

‎apps/server/src/engine/service.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -860,7 +860,7 @@ export class AgentService {
860860
let filledId = typeof task.state.filledId === "string" ? task.state.filledId : undefined;
861861
if (!filledId) {
862862
await ctx.guard();
863-
const filled = await this.files.fill(owner, source.fileId, fields);
863+
const filled = await this.files.fill(owner, source.fileId, fields, `document:${task.id}`);
864864
filledId = filled.id;
865865
task = await ctx.checkpoint({
866866
state: { ...task.state, source, filledId },

‎apps/server/src/files.ts‎

Lines changed: 56 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { randomUUID } from "node:crypto";
2-
import { mkdir, readFile, writeFile } from "node:fs/promises";
1+
import { createHash, randomUUID } from "node:crypto";
2+
import { link, mkdir, readFile, rm, writeFile } from "node:fs/promises";
33
import { join } from "node:path";
44
import type { Artifact } from "../../../packages/domain/src/index.ts";
55
import { fillPdf, inspectPdf } from "../../../packages/integrations/src/pdf.ts";
@@ -20,11 +20,41 @@ export class Files {
2020
bytes: Uint8Array,
2121
source: string,
2222
parentId?: string,
23+
operationKey?: string,
2324
): Promise<Artifact> {
25+
const id =
26+
operationKey === undefined
27+
? randomUUID()
28+
: createHash("sha256")
29+
.update(JSON.stringify([owner, operationKey]))
30+
.digest("hex");
31+
if (operationKey !== undefined) {
32+
const existing = await this.db.get<Artifact>(owner, "files", id);
33+
if (existing) return this.signed(owner, existing);
34+
}
2435
if (bytes.length > 10 * 1024 * 1024) throw new AppError("PDFs must be 10 MB or smaller", 413);
25-
const metadata = await inspectPdf(bytes);
36+
let metadata = await inspectPdf(bytes);
2637
if (metadata.pageCount > 500) throw new AppError("PDFs must have 500 pages or fewer", 422);
27-
const id = randomUUID();
38+
const directory = join(this.config.dataDir, "files");
39+
await mkdir(directory, { recursive: true, mode: 0o700 });
40+
const path = join(directory, `${id}.pdf`);
41+
if (operationKey === undefined) {
42+
await writeFile(path, bytes, { mode: 0o600, flag: "wx" });
43+
} else {
44+
const temporary = `${path}.${randomUUID()}.tmp`;
45+
try {
46+
await writeFile(temporary, bytes, { mode: 0o600, flag: "wx" });
47+
try {
48+
await link(temporary, path);
49+
} catch (error) {
50+
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
51+
bytes = await readFile(path);
52+
metadata = await inspectPdf(bytes);
53+
}
54+
} finally {
55+
await rm(temporary, { force: true });
56+
}
57+
}
2858
const safeName = Array.from(name.split(/[\\/]/).at(-1) ?? "document.pdf")
2959
.filter((character) => character.charCodeAt(0) >= 32 && character.charCodeAt(0) !== 127)
3060
.join("")
@@ -41,11 +71,12 @@ export class Files {
4171
source,
4272
parentId,
4373
};
44-
const directory = join(this.config.dataDir, "files");
45-
await mkdir(directory, { recursive: true, mode: 0o700 });
46-
await writeFile(join(directory, `${id}.pdf`), bytes, { mode: 0o600, flag: "wx" });
47-
await this.db.put(owner, "files", artifact);
48-
return this.signed(owner, artifact);
74+
if (operationKey === undefined) {
75+
await this.db.put(owner, "files", artifact);
76+
return this.signed(owner, artifact);
77+
}
78+
const persisted = await this.db.insertIfAbsent(owner, "files", artifact);
79+
return this.signed(owner, persisted ?? (await this.get(owner, id)));
4980
}
5081
signed(owner: string, file: Artifact): Artifact {
5182
return { ...file, url: this.auth.sign(owner, `/api/files/${file.id}/content`) };
@@ -62,7 +93,12 @@ export class Files {
6293
await this.get(owner, id);
6394
return readFile(join(this.config.dataDir, "files", `${id}.pdf`));
6495
}
65-
async fill(owner: string, id: string, values: Record<string, string | boolean>) {
96+
async fill(
97+
owner: string,
98+
id: string,
99+
values: Record<string, string | boolean>,
100+
operationKey?: string,
101+
) {
66102
const file = await this.get(owner, id);
67103
const bytes = await this.bytes(owner, id);
68104
const output = await fillPdf(bytes, values);
@@ -72,6 +108,16 @@ export class Files {
72108
output,
73109
`Filled from ${file.name}`,
74110
id,
111+
operationKey === undefined
112+
? undefined
113+
: JSON.stringify([
114+
"fill",
115+
operationKey,
116+
id,
117+
Object.keys(values)
118+
.sort()
119+
.map((name) => [name, values[name]]),
120+
]),
75121
);
76122
}
77123
}

‎apps/server/src/workspace.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,8 @@ export class WorkspaceService {
438438
decodeURIComponent(filename),
439439
await this.google(owner, connection.id).getAttachment(messageId, attachmentId),
440440
`Gmail · ${message.subject}`,
441+
undefined,
442+
JSON.stringify(["attachment", connection.id, reference]),
441443
);
442444
await this.db.put(owner, "imports", {
443445
id: reference,

‎tests/api.test.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,14 @@ import { after, before, test } from "node:test";
66
import { EventType } from "@ag-ui/core";
77
import { lastValueFrom, toArray } from "rxjs";
88
import { createApp } from "../apps/server/src/app.ts";
9+
import { Auth } from "../apps/server/src/auth.ts";
910
import type { Config } from "../apps/server/src/config.ts";
1011
import { createStore, type Store } from "../apps/server/src/db.ts";
1112
import { ConversationAgent } from "../apps/server/src/engine/conversation.ts";
1213
import type { AgentService } from "../apps/server/src/engine/service.ts";
14+
import { Files } from "../apps/server/src/files.ts";
1315
import type { ActionProposal, Artifact, Workspace } from "../packages/domain/src/index.ts";
16+
import { createSamplePdf } from "../packages/integrations/src/pdf.ts";
1417

1518
let db: Store,
1619
app: Awaited<ReturnType<typeof createApp>>["app"],
@@ -58,6 +61,82 @@ test("API protects private data and rejects unrelated web origins", async () =>
5861
403,
5962
);
6063
});
64+
test("file operations recover a published PDF when metadata persistence fails", async (t) => {
65+
const owner = "file-recovery";
66+
const bytes = await createSamplePdf();
67+
const insert = db.insertIfAbsent.bind(db);
68+
let publishedId = "";
69+
t.mock.method(db, "insertIfAbsent", async (...args: Parameters<Store["insertIfAbsent"]>) => {
70+
if (args[0] === owner && args[1] === "files" && !publishedId) {
71+
publishedId = args[2].id;
72+
throw new Error("metadata unavailable");
73+
}
74+
return insert(...args);
75+
});
76+
await assert.rejects(
77+
agent.files.import(owner, "sample.pdf", bytes, "fixture", undefined, "import-once"),
78+
/metadata unavailable/,
79+
);
80+
assert.ok(publishedId);
81+
assert.equal((await agent.files.list(owner)).length, 0);
82+
const restarted = new Files(db, config, new Auth(db, config, "sample-signing-key"));
83+
const recovered = await restarted.import(
84+
owner,
85+
"sample.pdf",
86+
bytes,
87+
"fixture",
88+
undefined,
89+
"import-once",
90+
);
91+
assert.equal(recovered.id, publishedId);
92+
assert.deepEqual(new Uint8Array(await restarted.bytes(owner, recovered.id)), bytes);
93+
assert.equal((await restarted.list(owner)).length, 1);
94+
});
95+
96+
test("file operation identity isolates owners, inputs and independent requests", async () => {
97+
const owner = "file-identities";
98+
const bytes = await createSamplePdf();
99+
const source = await agent.files.import(owner, "sample.pdf", bytes, "fixture");
100+
const fields = { participant_name: "Sample Student", permission_granted: true };
101+
const [first, concurrent] = await Promise.all([
102+
agent.files.fill(owner, source.id, fields, "task-fill"),
103+
agent.files.fill(owner, source.id, fields, "task-fill"),
104+
]);
105+
assert.equal(first.id, concurrent.id);
106+
assert.equal(first.createdAt, concurrent.createdAt);
107+
assert.equal((await agent.files.list(owner)).filter((file) => file.parentId).length, 1);
108+
const reordered = await agent.files.fill(
109+
owner,
110+
source.id,
111+
{
112+
permission_granted: true,
113+
participant_name: "Sample Student",
114+
},
115+
"task-fill",
116+
);
117+
assert.equal(reordered.id, first.id);
118+
assert.notEqual((await agent.files.fill(owner, source.id, fields, "another-task")).id, first.id);
119+
assert.notEqual(
120+
(await agent.files.fill(owner, source.id, { participant_name: "Another Student" }, "task-fill"))
121+
.id,
122+
first.id,
123+
);
124+
const manual = await agent.files.fill(owner, source.id, fields);
125+
assert.notEqual((await agent.files.fill(owner, source.id, fields)).id, manual.id);
126+
const otherOwner = await agent.files.import(
127+
"other-file-owner",
128+
"sample.pdf",
129+
bytes,
130+
"fixture",
131+
undefined,
132+
"owner-scoped",
133+
);
134+
assert.notEqual(
135+
(await agent.files.import(owner, "sample.pdf", bytes, "fixture", undefined, "owner-scoped")).id,
136+
otherOwner.id,
137+
);
138+
});
139+
61140
test("sample workspace serves a real PDF and filling creates a new version", async () => {
62141
const response = await app.request("/api/workspace", { headers: headers() });
63142
assert.equal(response.status, 200);

‎tests/model-worker.test.ts‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,94 @@ import { tmpdir } from "node:os";
44
import { join } from "node:path";
55
import { test } from "node:test";
66
import { createApp } from "../apps/server/src/app.ts";
7-
import { createStore } from "../apps/server/src/db.ts";
7+
import { createStore, type Store } from "../apps/server/src/db.ts";
8+
import type { AgentTask } from "../packages/domain/src/agent.ts";
89
import type { ActionProposal } from "../packages/domain/src/index.ts";
910
import { fixture as computerFixture } from "./helpers/computer.ts";
1011
import { modelFixture } from "./helpers/model.ts";
1112

13+
for (const checkpoint of ["artifact", "cache"] as const) {
14+
test(`model fill reuses its PDF after a failed ${checkpoint} checkpoint`, async (t) => {
15+
const directory = await mkdtemp(join(tmpdir(), "openmuse-model-fill-"));
16+
const db = await createStore();
17+
const calls: { name: string; arguments: object }[] = [];
18+
const { requests } = await modelFixture(t, (index) => calls[index]);
19+
const server = await createApp(db, {
20+
mode: "sample",
21+
port: 8787,
22+
host: "127.0.0.1",
23+
publicUrl: "http://localhost:8787",
24+
dataDir: directory,
25+
agentBackend: "model",
26+
intelligenceApiKey: "test-project-key-never-sent",
27+
model: "openai/fixture",
28+
googleRedirectUri: "http://localhost:8787/api/google/callback",
29+
allowedOrigins: [],
30+
});
31+
try {
32+
const owner = "model-fill-replay";
33+
await server.workspace.ensureSample(owner, server.actions);
34+
const source = (await server.files.list(owner))[0];
35+
const fill = {
36+
name: "fill_pdf",
37+
arguments: { fileId: source.id, fields: { participant_name: "Sample Student" } },
38+
};
39+
calls.push(fill, {
40+
name: "ask_user",
41+
arguments: { question: "Retry the interrupted operation?" },
42+
});
43+
const task = await server.agent.createTask(owner, { prompt: "Fill the sample form" });
44+
const compareAndSwap = db.compareAndSwap.bind(db);
45+
let interrupted = false;
46+
t.mock.method(db, "compareAndSwap", async (...args: Parameters<Store["compareAndSwap"]>) => {
47+
const patch = args[4] as Partial<AgentTask>;
48+
if (
49+
args[0] === owner &&
50+
args[1] === "tasks" &&
51+
args[2] === task.id &&
52+
(checkpoint === "artifact" ? patch.artifactIds?.length : patch.state?.operations) &&
53+
!interrupted
54+
) {
55+
interrupted = true;
56+
return null;
57+
}
58+
return compareAndSwap(...args);
59+
});
60+
await server.agent.worker.tick();
61+
assert.ok(interrupted);
62+
const waiting = await server.agent.getTask(owner, task.id);
63+
assert.equal(waiting.status, "waiting_input", waiting.error ?? waiting.question);
64+
const outputs = (await server.files.list(owner)).filter(
65+
(file) => file.parentId === source.id,
66+
);
67+
assert.equal(outputs.length, 1);
68+
assert.deepEqual(waiting.artifactIds, checkpoint === "artifact" ? [] : [outputs[0].id]);
69+
assert.equal(waiting.state.operations, undefined);
70+
requests.length = 0;
71+
calls.splice(0, calls.length, fill, {
72+
name: "finish_task",
73+
arguments: { summary: "Saved the sample form." },
74+
});
75+
await server.agent.answer(owner, task.id, "Retry");
76+
await server.agent.worker.tick();
77+
const completed = await server.agent.getTask(owner, task.id);
78+
assert.equal(completed.status, "succeeded", completed.error ?? completed.question);
79+
assert.deepEqual(
80+
completed.artifactIds.filter((id) => id === outputs[0].id),
81+
[outputs[0].id],
82+
);
83+
assert.equal(
84+
(await server.files.list(owner)).filter((file) => file.parentId === source.id).length,
85+
1,
86+
);
87+
} finally {
88+
await server.agent.stop();
89+
await db.close();
90+
await rm(directory, { recursive: true, force: true });
91+
}
92+
});
93+
}
94+
1295
test("CopilotKit model worker executes server tools and persists the confirmed outcome", async (t) => {
1396
const directory = await mkdtemp(join(tmpdir(), "openmuse-model-"));
1497
const db = await createStore();

0 commit comments

Comments
 (0)