From 36b49fc4c9128f7e8e83155d3c124beab58887e7 Mon Sep 17 00:00:00 2001 From: Eliezer Steinbock <3090527+elie222@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:02:07 -0400 Subject: [PATCH 1/6] fix: Delete a user's Tinybird data when their account is deleted Account deletion only tried to delete AI usage rows, and that request was rejected because the ingest token cannot delete. Deletion now uses a dedicated delete token and removes AI usage rows plus every per-mailbox datasource, both when a user is deleted and when one email account is removed. Adds a script to purge rows left by earlier deletions. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/web/.env.example | 2 + apps/web/env.ts | 1 + .../scripts/purge-orphaned-tinybird-data.ts | 97 ++++++++++++++ apps/web/utils/actions/user.test.ts | 8 ++ apps/web/utils/actions/user.ts | 11 ++ apps/web/utils/user/delete.test.ts | 9 +- apps/web/utils/user/delete.ts | 19 +-- packages/tinybird-ai-analytics/src/delete.ts | 41 ------ packages/tinybird-ai-analytics/src/index.ts | 1 - packages/tinybird/package.json | 6 +- packages/tinybird/src/delete.test.ts | 83 ++++++++++++ packages/tinybird/src/delete.ts | 119 ++++++++++-------- pnpm-lock.yaml | 3 + turbo.json | 1 + 14 files changed, 294 insertions(+), 107 deletions(-) create mode 100644 apps/web/scripts/purge-orphaned-tinybird-data.ts delete mode 100644 packages/tinybird-ai-analytics/src/delete.ts create mode 100644 packages/tinybird/src/delete.test.ts diff --git a/apps/web/.env.example b/apps/web/.env.example index 44e21ad65d..2e414e4145 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -248,6 +248,8 @@ LLM_API_KEY= # Tinybird TINYBIRD_TOKEN= +# Token with DATASOURCES:CREATE scope, used only to delete user data on account deletion +TINYBIRD_DELETE_TOKEN= TINYBIRD_BASE_URL=https://api.us-east.tinybird.co/ # Stripe AI overage billing (optional; unset = unlimited/no overage charges) diff --git a/apps/web/env.ts b/apps/web/env.ts index 2f0c49fe53..718562f68f 100644 --- a/apps/web/env.ts +++ b/apps/web/env.ts @@ -291,6 +291,7 @@ const parsedEnv = createEnv({ APNS_TRANSPORT: z.enum(["apns", "fake"]).optional(), TINYBIRD_TOKEN: z.string().optional(), + TINYBIRD_DELETE_TOKEN: z.string().optional(), TINYBIRD_BASE_URL: z.string().default("https://api.us-east.tinybird.co/"), API_KEY_SALT: z.string().optional(), diff --git a/apps/web/scripts/purge-orphaned-tinybird-data.ts b/apps/web/scripts/purge-orphaned-tinybird-data.ts new file mode 100644 index 0000000000..19e63c2337 --- /dev/null +++ b/apps/web/scripts/purge-orphaned-tinybird-data.ts @@ -0,0 +1,97 @@ +// Deletes Tinybird rows that belong to users or mailboxes that no longer exist, +// left behind before account deletion cleaned up Tinybird. +// +// Needs TINYBIRD_DELETE_TOKEN with read and delete access to the datasources. +// Dry run (counts only): `pnpm --filter inbox-zero-ai exec tsx scripts/purge-orphaned-tinybird-data.ts` +// Delete: `pnpm --filter inbox-zero-ai exec tsx scripts/purge-orphaned-tinybird-data.ts --apply` + +import "dotenv/config"; +import chunk from "lodash/chunk"; +import { deleteTinybirdData } from "@inboxzero/tinybird"; +import prisma from "@/utils/prisma"; + +const EMAIL_DATASOURCES = [ + "email_action", + "email", + "last_and_oldest_emails_mv", +]; +const BATCH_SIZE = 100; + +async function main() { + const apply = process.argv.includes("--apply"); + + const [emailAccounts, users] = await Promise.all([ + prisma.emailAccount.findMany({ select: { email: true } }), + prisma.user.findMany({ select: { id: true } }), + ]); + const liveEmails = new Set( + emailAccounts.map(({ email }) => email.toLowerCase()), + ); + const liveUserIds = new Set(users.map(({ id }) => id)); + + const orphanedEmails = new Set(); + for (const datasource of EMAIL_DATASOURCES) { + const owners = await getDistinctValues(datasource, "ownerEmail"); + if (!owners) { + console.log(`${datasource}: datasource not found, skipping`); + continue; + } + const orphaned = owners.filter( + (email) => !liveEmails.has(email.toLowerCase()), + ); + for (const email of orphaned) orphanedEmails.add(email); + console.log( + `${datasource}: ${owners.length} mailboxes, ${orphaned.length} orphaned`, + ); + } + + const aiCallUsers = (await getDistinctValues("aiCall", "userId")) ?? []; + const orphanedUserIds = aiCallUsers.filter((id) => !liveUserIds.has(id)); + console.log( + `aiCall: ${aiCallUsers.length} users, ${orphanedUserIds.length} orphaned`, + ); + + if (!apply) { + console.log("Dry run. Re-run with --apply to delete."); + return; + } + + for (const emails of chunk([...orphanedEmails], BATCH_SIZE)) { + await deleteTinybirdData({ emails }); + } + for (const userIds of chunk(orphanedUserIds, BATCH_SIZE)) { + await deleteTinybirdData({ userIds }); + } + console.log("Delete jobs submitted."); +} + +async function getDistinctValues(datasource: string, column: string) { + const url = new URL( + "/v0/sql", + process.env.TINYBIRD_BASE_URL || "https://api.us-east.tinybird.co/", + ); + url.searchParams.set( + "q", + `SELECT DISTINCT ${column} AS value FROM ${datasource} FORMAT JSON`, + ); + + const response = await fetch(url, { + headers: { Authorization: `Bearer ${process.env.TINYBIRD_DELETE_TOKEN}` }, + }); + if (response.status === 404) return null; + if (!response.ok) { + throw new Error( + `Tinybird query on ${datasource} failed: [${response.status}] ${await response.text()}`, + ); + } + + const body = (await response.json()) as { data: { value: string }[] }; + return body.data.map(({ value }) => value); +} + +main() + .catch((error) => { + console.error(error); + process.exitCode = 1; + }) + .finally(() => prisma.$disconnect()); diff --git a/apps/web/utils/actions/user.test.ts b/apps/web/utils/actions/user.test.ts index d8d519e924..345663419e 100644 --- a/apps/web/utils/actions/user.test.ts +++ b/apps/web/utils/actions/user.test.ts @@ -9,6 +9,7 @@ import { deleteAccountUploadDirectory } from "@/utils/mail-api/upload-blobs"; import { deleteUser } from "@/utils/user/delete"; import { clearLastEmailAccountCookie } from "@/utils/cookies.server"; import { LAST_EMAIL_ACCOUNT_COOKIE } from "@/utils/cookies"; +import { deleteTinybirdData } from "@inboxzero/tinybird"; import { deleteAccountAction, deleteEmailAccountAction } from "./user"; vi.mock("@/utils/prisma"); @@ -40,6 +41,9 @@ vi.mock("@/utils/cookies.server", () => ({ vi.mock("@/utils/mail-api/upload-blobs", () => ({ deleteAccountUploadDirectory: vi.fn(() => Promise.resolve()), })); +vi.mock("@inboxzero/tinybird", () => ({ + deleteTinybirdData: vi.fn(() => Promise.resolve()), +})); vi.mock("@/utils/user/delete", () => ({ deleteUser: vi.fn(), })); @@ -221,6 +225,10 @@ describe("deleteEmailAccountAction", () => { expect(prisma.$transaction.mock.invocationCallOrder[0]).toBeLessThan( vi.mocked(deleteAccountUploadDirectory).mock.invocationCallOrder[0], ); + expect(deleteTinybirdData).toHaveBeenCalledWith({ + emailAccountIds: ["secondary-account"], + emails: ["secondary@example.com"], + }); expect(clearLastEmailAccountCookie).not.toHaveBeenCalled(); }); diff --git a/apps/web/utils/actions/user.ts b/apps/web/utils/actions/user.ts index dd1ed82ca4..6bd0c07483 100644 --- a/apps/web/utils/actions/user.ts +++ b/apps/web/utils/actions/user.ts @@ -23,6 +23,7 @@ import { } from "@/utils/actions/user.validation"; import { clearLastEmailAccountCookie } from "@/utils/cookies.server"; import { deleteAccountUploadDirectory } from "@/utils/mail-api/upload-blobs"; +import { deleteTinybirdData } from "@inboxzero/tinybird"; import { aliasPosthogUser } from "@/utils/posthog"; import { cleanupAIDraftsForAccount, @@ -242,6 +243,16 @@ export const deleteEmailAccountAction = actionClientUser ); } + after(() => + deleteTinybirdData({ + emailAccountIds: [emailAccountId], + emails: [emailAccount.email], + }).catch((error) => { + logger.error("Error deleting Tinybird data", { error }); + captureException(error); + }), + ); + await deleteAccountUploadDirectory(emailAccountId).catch((error) => { logger.error("Failed to delete account mail uploads", { error, diff --git a/apps/web/utils/user/delete.test.ts b/apps/web/utils/user/delete.test.ts index be7810165f..80c28234d4 100644 --- a/apps/web/utils/user/delete.test.ts +++ b/apps/web/utils/user/delete.test.ts @@ -5,6 +5,7 @@ import { createTestLogger } from "@/__tests__/helpers"; import prisma from "@/utils/__mocks__/prisma"; import { deleteAccountUploadDirectory } from "@/utils/mail-api/upload-blobs"; import { deleteUser } from "@/utils/user/delete"; +import { deleteTinybirdData } from "@inboxzero/tinybird"; vi.mock("@/utils/prisma"); vi.mock("@/utils/mail-api/upload-blobs", () => ({ @@ -19,8 +20,8 @@ vi.mock("@inboxzero/loops", () => ({ vi.mock("@inboxzero/transactional-email", () => ({ deleteContact: vi.fn(), })); -vi.mock("@inboxzero/tinybird-ai-analytics", () => ({ - deleteTinybirdAiCalls: vi.fn(() => Promise.resolve()), +vi.mock("@inboxzero/tinybird", () => ({ + deleteTinybirdData: vi.fn(() => Promise.resolve()), })); vi.mock("@/utils/posthog", () => ({ deletePosthogUser: vi.fn(() => Promise.resolve()), @@ -110,6 +111,10 @@ describe("deleteUser", () => { prisma.user.deleteMany.mockResolvedValue({ count: 1 } as any); await deleteUser({ userId: "user-1", logger }); + expect(deleteTinybirdData).toHaveBeenCalledWith({ + userIds: ["user-1"], + emails: ["owner@example.com"], + }); expect(withThreadPageBufferDeletion).toHaveBeenCalledWith( ["email-account-1"], expect.any(Function), diff --git a/apps/web/utils/user/delete.ts b/apps/web/utils/user/delete.ts index f145da26c3..e54819cf17 100644 --- a/apps/web/utils/user/delete.ts +++ b/apps/web/utils/user/delete.ts @@ -2,7 +2,8 @@ import { deleteContact as deleteLoopsContact } from "@inboxzero/loops"; import { deleteContact as deleteResendContact } from "@inboxzero/transactional-email"; import { withThreadPageBufferDeletion } from "@/utils/redis/thread-page-buffer"; import prisma from "@/utils/prisma"; -import { deleteTinybirdAiCalls } from "@inboxzero/tinybird-ai-analytics"; +import { deleteTinybirdData } from "@inboxzero/tinybird"; +import { after } from "next/server"; import { deletePosthogUser, trackUserDeleted, @@ -69,13 +70,15 @@ export async function deleteUser({ captureException(error); }); - deleteTinybirdAiCalls({ userId }).catch((error) => { - logger.error("Error deleting Tinybird AI calls", { - error, - userId, - }); - captureException(error); - }); + const emails = accounts + .map((account) => account.emailAccount?.email) + .filter((email): email is string => Boolean(email)); + after(() => + deleteTinybirdData({ userIds: [userId], emails }).catch((error) => { + logger.error("Error deleting Tinybird data", { error }); + captureException(error); + }), + ); clearCachedResearchForUser(userId).catch((error) => { logger.error("Error clearing cached research", { error }); diff --git a/packages/tinybird-ai-analytics/src/delete.ts b/packages/tinybird-ai-analytics/src/delete.ts deleted file mode 100644 index 3a0851aa20..0000000000 --- a/packages/tinybird-ai-analytics/src/delete.ts +++ /dev/null @@ -1,41 +0,0 @@ -const TINYBIRD_BASE_URL = process.env.TINYBIRD_BASE_URL; -const TINYBIRD_TOKEN = process.env.TINYBIRD_TOKEN; - -async function deleteFromDatasource( - datasource: string, - deleteCondition: string, // e.g. "userId='abc@example.com'" -): Promise { - if (!TINYBIRD_BASE_URL || !TINYBIRD_TOKEN) { - console.warn("TINYBIRD_BASE_URL or TINYBIRD_TOKEN missing"); - return; - } - - const url = new URL( - `/v0/datasources/${datasource}/delete`, - TINYBIRD_BASE_URL, - ); - const res = await fetch(url, { - method: "POST", - body: `delete_condition=(${deleteCondition})`, - headers: { - Authorization: `Bearer ${TINYBIRD_TOKEN}`, - "Content-Type": "application/x-www-form-urlencoded", - }, - }); - - if (!res.ok) { - throw new Error( - `Unable to delete for datasource ${datasource}: [${ - res.status - }] ${await res.text()}`, - ); - } - - return await res.json(); -} - -export async function deleteTinybirdAiCalls(options: { - userId: string; -}): Promise { - return await deleteFromDatasource("aiCall", `userId='${options.userId}'`); -} diff --git a/packages/tinybird-ai-analytics/src/index.ts b/packages/tinybird-ai-analytics/src/index.ts index db54bd397a..a086c3bcf8 100644 --- a/packages/tinybird-ai-analytics/src/index.ts +++ b/packages/tinybird-ai-analytics/src/index.ts @@ -1,5 +1,4 @@ // biome-ignore lint/performance/noBarrelFile: fix later export * from "./client"; export * from "./publish"; -export * from "./delete"; export * from "./query"; diff --git a/packages/tinybird/package.json b/packages/tinybird/package.json index 109f4ccb3d..fbd21d330f 100644 --- a/packages/tinybird/package.json +++ b/packages/tinybird/package.json @@ -2,6 +2,9 @@ "name": "@inboxzero/tinybird", "version": "0.0.0", "main": "src/index.ts", + "scripts": { + "test": "vitest run" + }, "dependencies": { "@chronark/zod-bird": "1.0.0", "p-retry": "8.0.1", @@ -10,6 +13,7 @@ "devDependencies": { "@types/node": "24.10.1", "tsconfig": "workspace:*", - "typescript": "6.0.3" + "typescript": "6.0.3", + "vitest": "4.1.10" } } diff --git a/packages/tinybird/src/delete.test.ts b/packages/tinybird/src/delete.test.ts new file mode 100644 index 0000000000..a448eb14eb --- /dev/null +++ b/packages/tinybird/src/delete.test.ts @@ -0,0 +1,83 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { deleteTinybirdData } from "./delete"; + +const fetchMock = vi.fn(); + +describe("deleteTinybirdData", () => { + beforeEach(() => { + vi.stubGlobal("fetch", fetchMock); + vi.stubEnv("TINYBIRD_TOKEN", "append-token"); + vi.stubEnv("TINYBIRD_DELETE_TOKEN", "delete-token"); + vi.stubEnv("TINYBIRD_BASE_URL", "https://tinybird.test/"); + fetchMock.mockReset(); + fetchMock.mockResolvedValue(new Response("{}", { status: 200 })); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + }); + + it("deletes AI calls and every mailbox datasource with the delete token", async () => { + await deleteTinybirdData({ + userIds: ["user-1"], + emails: ["a@example.com", "b@example.com"], + }); + + const calls = fetchMock.mock.calls.map(([url, init]) => ({ + path: new URL(url).pathname, + condition: new URLSearchParams(init.body).get("delete_condition"), + auth: init.headers.Authorization, + })); + + expect(calls).toEqual([ + { + path: "/v0/datasources/aiCall/delete", + condition: "userId IN ('user-1')", + auth: "Bearer delete-token", + }, + ...["email_action", "email", "last_and_oldest_emails_mv"].map( + (datasource) => ({ + path: `/v0/datasources/${datasource}/delete`, + condition: "ownerEmail IN ('a@example.com', 'b@example.com')", + auth: "Bearer delete-token", + }), + ), + ]); + }); + + it("escapes quotes in values", async () => { + await deleteTinybirdData({ emails: ["o'brien@example.com"] }); + + expect( + new URLSearchParams(fetchMock.mock.calls[0][1].body).get( + "delete_condition", + ), + ).toBe("ownerEmail IN ('o\\'brien@example.com')"); + }); + + it("skips datasources that do not exist", async () => { + fetchMock.mockResolvedValueOnce(new Response("not found", { status: 404 })); + + await expect( + deleteTinybirdData({ emails: ["a@example.com"] }), + ).resolves.toBeUndefined(); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it("fails loudly when Tinybird is enabled without a delete token", async () => { + vi.stubEnv("TINYBIRD_DELETE_TOKEN", ""); + + await expect(deleteTinybirdData({ userIds: ["user-1"] })).rejects.toThrow( + "TINYBIRD_DELETE_TOKEN is not set", + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("does nothing when Tinybird is not configured", async () => { + vi.stubEnv("TINYBIRD_TOKEN", ""); + + await deleteTinybirdData({ userIds: ["user-1"] }); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/tinybird/src/delete.ts b/packages/tinybird/src/delete.ts index 2bfc8aea0d..ccba3d9e5a 100644 --- a/packages/tinybird/src/delete.ts +++ b/packages/tinybird/src/delete.ts @@ -1,67 +1,78 @@ import pRetry, { AbortError } from "p-retry"; -import { isTinybirdEnabled } from "./client"; -const TINYBIRD_BASE_URL = process.env.TINYBIRD_BASE_URL; -const TINYBIRD_TOKEN = process.env.TINYBIRD_TOKEN; +// Datasources that store rows per mailbox, keyed by the mailbox address. +const EMAIL_DATASOURCES = [ + "email_action", + "email", + "last_and_oldest_emails_mv", +] as const; -async function deleteFromDatasource( - datasource: string, - deleteCondition: string, // e.g. "email='abc@example.com'" -): Promise { - if (!isTinybirdEnabled()) return; +export async function deleteTinybirdData({ + userIds = [], + emailAccountIds = [], + emails = [], +}: { + userIds?: string[]; + emailAccountIds?: string[]; + emails?: string[]; +}) { + if (!process.env.TINYBIRD_TOKEN) return; - const url = new URL( - `/v0/datasources/${datasource}/delete`, - TINYBIRD_BASE_URL, - ); - const res = await fetch(url, { - method: "POST", - body: `delete_condition=(${deleteCondition})`, - headers: { - Authorization: `Bearer ${TINYBIRD_TOKEN}`, - "Content-Type": "application/x-www-form-urlencoded", - }, - }); + const aiCallConditions = [ + userIds.length ? `userId IN (${userIds.map(quote).join(", ")})` : null, + emailAccountIds.length + ? `emailAccountId IN (${emailAccountIds.map(quote).join(", ")})` + : null, + ].filter(Boolean); + if (aiCallConditions.length) { + await deleteRows("aiCall", aiCallConditions.join(" OR ")); + } - if (!res.ok) { + if (emails.length) { + const condition = `ownerEmail IN (${emails.map(quote).join(", ")})`; + for (const datasource of EMAIL_DATASOURCES) { + await deleteRows(datasource, condition); + } + } +} + +// Tinybird runs one delete job at a time and answers 429 while one is running. +async function deleteRows(datasource: string, deleteCondition: string) { + const token = process.env.TINYBIRD_DELETE_TOKEN; + if (!token) { throw new Error( - `Unable to delete for datasource ${datasource}: [${ - res.status - }] ${await res.text()}`, + "TINYBIRD_DELETE_TOKEN is not set, so Tinybird data cannot be deleted", ); } - return await res.json(); -} - -// Tinybird only allows 1 delete at a time -async function _deleteFromDatasourceWithRetry( - datasource: string, - deleteCondition: string, -): Promise { - return pRetry( + await pRetry( async () => { - try { - return await deleteFromDatasource(datasource, deleteCondition); - } catch (error) { - // Only retry on rate limit errors - if (error instanceof Error && error.message.includes("429")) { - throw error; // pRetry will handle this - } - throw new AbortError(error as Error); // Don't retry other errors - } - }, - { - retries: 5, - factor: 2, - minTimeout: 1000, - maxTimeout: 30_000, - randomize: true, - onFailedAttempt: (error) => { - console.log( - `Rate limited when deleting from ${datasource}. Attempt ${error.attemptNumber} failed. ${error.retriesLeft} retries left.`, - ); - }, + const response = await fetch( + new URL( + `/v0/datasources/${datasource}/delete`, + process.env.TINYBIRD_BASE_URL || "https://api.us-east.tinybird.co/", + ), + { + method: "POST", + body: new URLSearchParams({ delete_condition: deleteCondition }), + headers: { Authorization: `Bearer ${token}` }, + }, + ); + + if (response.ok) return; + // The datasource does not exist in this workspace, so there is nothing to delete. + if (response.status === 404) return; + + const error = new Error( + `Unable to delete from Tinybird datasource ${datasource}: [${response.status}] ${await response.text()}`, + ); + if (response.status === 429) throw error; + throw new AbortError(error); }, + { retries: 6, factor: 2, minTimeout: 1000, maxTimeout: 15_000 }, ); } + +function quote(value: string) { + return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e118755821..869ae68b1e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1129,6 +1129,9 @@ importers: typescript: specifier: 6.0.3 version: 6.0.3 + vitest: + specifier: 4.1.10 + version: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@24.10.1)(@vitest/coverage-v8@4.1.10)(@vitest/ui@4.1.10)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@7.3.6(@types/node@24.10.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.47.0)(tsx@4.23.13)(yaml@2.9.0)) packages/tinybird-ai-analytics: dependencies: diff --git a/turbo.json b/turbo.json index 50a3aff151..b5f612fee1 100644 --- a/turbo.json +++ b/turbo.json @@ -157,6 +157,7 @@ "LEMON_SQUEEZY_API_KEY", "TINYBIRD_TOKEN", + "TINYBIRD_DELETE_TOKEN", "TINYBIRD_BASE_URL", "POSTHOG_API_SECRET", From dadf2b23f4c9963f31e5817c8b56d4409355b243 Mon Sep 17 00:00:00 2001 From: Eliezer Steinbock <3090527+elie222@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:08:04 -0400 Subject: [PATCH 2/6] fix: Delete Tinybird data only after the user is deleted, with a request timeout Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scripts/purge-orphaned-tinybird-data.ts | 12 +++++++--- apps/web/utils/user/delete.test.ts | 23 +++++++++++++++++++ apps/web/utils/user/delete.ts | 20 ++++++++-------- packages/tinybird/src/delete.ts | 1 + 4 files changed, 43 insertions(+), 13 deletions(-) diff --git a/apps/web/scripts/purge-orphaned-tinybird-data.ts b/apps/web/scripts/purge-orphaned-tinybird-data.ts index 19e63c2337..a38b9fc44e 100644 --- a/apps/web/scripts/purge-orphaned-tinybird-data.ts +++ b/apps/web/scripts/purge-orphaned-tinybird-data.ts @@ -19,6 +19,9 @@ const BATCH_SIZE = 100; async function main() { const apply = process.argv.includes("--apply"); + if (!process.env.TINYBIRD_DELETE_TOKEN) { + throw new Error("TINYBIRD_DELETE_TOKEN is not set"); + } const [emailAccounts, users] = await Promise.all([ prisma.emailAccount.findMany({ select: { email: true } }), @@ -45,10 +48,13 @@ async function main() { ); } - const aiCallUsers = (await getDistinctValues("aiCall", "userId")) ?? []; - const orphanedUserIds = aiCallUsers.filter((id) => !liveUserIds.has(id)); + const aiCallUsers = await getDistinctValues("aiCall", "userId"); + const orphanedUserIds = + aiCallUsers?.filter((id) => !liveUserIds.has(id)) ?? []; console.log( - `aiCall: ${aiCallUsers.length} users, ${orphanedUserIds.length} orphaned`, + aiCallUsers + ? `aiCall: ${aiCallUsers.length} users, ${orphanedUserIds.length} orphaned` + : "aiCall: datasource not found, skipping", ); if (!apply) { diff --git a/apps/web/utils/user/delete.test.ts b/apps/web/utils/user/delete.test.ts index 80c28234d4..29412621b8 100644 --- a/apps/web/utils/user/delete.test.ts +++ b/apps/web/utils/user/delete.test.ts @@ -83,6 +83,29 @@ describe("deleteUser", () => { expect(deleteAccountUploadDirectory).not.toHaveBeenCalled(); }); + it("keeps Tinybird data when deleting the user fails", async () => { + prisma.account.findMany.mockResolvedValue([ + { + provider: "google", + access_token: null, + refresh_token: null, + expires_at: null, + emailAccount: { + id: "email-account-1", + email: "owner@example.com", + watchEmailsSubscriptionId: null, + }, + }, + ] as Awaited>); + prisma.executedRule.findMany.mockResolvedValue([]); + prisma.user.deleteMany.mockRejectedValue(new Error("database unavailable")); + + await expect(deleteUser({ userId: "user-1", logger })).rejects.toThrow( + "database unavailable", + ); + expect(deleteTinybirdData).not.toHaveBeenCalled(); + }); + it("deletes solo organizations before deleting the user", async () => { prisma.account.findMany.mockResolvedValue([ { diff --git a/apps/web/utils/user/delete.ts b/apps/web/utils/user/delete.ts index e54819cf17..3792dd7fba 100644 --- a/apps/web/utils/user/delete.ts +++ b/apps/web/utils/user/delete.ts @@ -70,16 +70,6 @@ export async function deleteUser({ captureException(error); }); - const emails = accounts - .map((account) => account.emailAccount?.email) - .filter((email): email is string => Boolean(email)); - after(() => - deleteTinybirdData({ userIds: [userId], emails }).catch((error) => { - logger.error("Error deleting Tinybird data", { error }); - captureException(error); - }), - ); - clearCachedResearchForUser(userId).catch((error) => { logger.error("Error clearing cached research", { error }); captureException(error); @@ -133,6 +123,16 @@ export async function deleteUser({ throw originalError; } }); + + const emails = accounts + .map((account) => account.emailAccount?.email) + .filter((email): email is string => Boolean(email)); + after(() => + deleteTinybirdData({ userIds: [userId], emails }).catch((error) => { + logger.error("Error deleting Tinybird data", { error }); + captureException(error); + }), + ); } catch (error) { logger.error("Error during user resources deletion process", { error, diff --git a/packages/tinybird/src/delete.ts b/packages/tinybird/src/delete.ts index ccba3d9e5a..c46b2a0068 100644 --- a/packages/tinybird/src/delete.ts +++ b/packages/tinybird/src/delete.ts @@ -56,6 +56,7 @@ async function deleteRows(datasource: string, deleteCondition: string) { method: "POST", body: new URLSearchParams({ delete_condition: deleteCondition }), headers: { Authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(30_000), }, ); From d54925120a7cdd5540c8ca894ee7801253e078f9 Mon Sep 17 00:00:00 2001 From: Eliezer Steinbock <3090527+elie222@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:18:38 -0400 Subject: [PATCH 3/6] fix: Use the existing Tinybird token for deletes Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/web/.env.example | 2 -- apps/web/env.ts | 1 - .../scripts/purge-orphaned-tinybird-data.ts | 8 ++++---- packages/tinybird/src/delete.test.ts | 18 ++++-------------- packages/tinybird/src/delete.ts | 9 +-------- turbo.json | 1 - 6 files changed, 9 insertions(+), 30 deletions(-) diff --git a/apps/web/.env.example b/apps/web/.env.example index 2e414e4145..44e21ad65d 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -248,8 +248,6 @@ LLM_API_KEY= # Tinybird TINYBIRD_TOKEN= -# Token with DATASOURCES:CREATE scope, used only to delete user data on account deletion -TINYBIRD_DELETE_TOKEN= TINYBIRD_BASE_URL=https://api.us-east.tinybird.co/ # Stripe AI overage billing (optional; unset = unlimited/no overage charges) diff --git a/apps/web/env.ts b/apps/web/env.ts index 718562f68f..2f0c49fe53 100644 --- a/apps/web/env.ts +++ b/apps/web/env.ts @@ -291,7 +291,6 @@ const parsedEnv = createEnv({ APNS_TRANSPORT: z.enum(["apns", "fake"]).optional(), TINYBIRD_TOKEN: z.string().optional(), - TINYBIRD_DELETE_TOKEN: z.string().optional(), TINYBIRD_BASE_URL: z.string().default("https://api.us-east.tinybird.co/"), API_KEY_SALT: z.string().optional(), diff --git a/apps/web/scripts/purge-orphaned-tinybird-data.ts b/apps/web/scripts/purge-orphaned-tinybird-data.ts index a38b9fc44e..c81846ea66 100644 --- a/apps/web/scripts/purge-orphaned-tinybird-data.ts +++ b/apps/web/scripts/purge-orphaned-tinybird-data.ts @@ -1,7 +1,7 @@ // Deletes Tinybird rows that belong to users or mailboxes that no longer exist, // left behind before account deletion cleaned up Tinybird. // -// Needs TINYBIRD_DELETE_TOKEN with read and delete access to the datasources. +// TINYBIRD_TOKEN needs read and delete (DATASOURCES:CREATE) access to the datasources. // Dry run (counts only): `pnpm --filter inbox-zero-ai exec tsx scripts/purge-orphaned-tinybird-data.ts` // Delete: `pnpm --filter inbox-zero-ai exec tsx scripts/purge-orphaned-tinybird-data.ts --apply` @@ -19,8 +19,8 @@ const BATCH_SIZE = 100; async function main() { const apply = process.argv.includes("--apply"); - if (!process.env.TINYBIRD_DELETE_TOKEN) { - throw new Error("TINYBIRD_DELETE_TOKEN is not set"); + if (!process.env.TINYBIRD_TOKEN) { + throw new Error("TINYBIRD_TOKEN is not set"); } const [emailAccounts, users] = await Promise.all([ @@ -82,7 +82,7 @@ async function getDistinctValues(datasource: string, column: string) { ); const response = await fetch(url, { - headers: { Authorization: `Bearer ${process.env.TINYBIRD_DELETE_TOKEN}` }, + headers: { Authorization: `Bearer ${process.env.TINYBIRD_TOKEN}` }, }); if (response.status === 404) return null; if (!response.ok) { diff --git a/packages/tinybird/src/delete.test.ts b/packages/tinybird/src/delete.test.ts index a448eb14eb..6926e0eaa2 100644 --- a/packages/tinybird/src/delete.test.ts +++ b/packages/tinybird/src/delete.test.ts @@ -6,8 +6,7 @@ const fetchMock = vi.fn(); describe("deleteTinybirdData", () => { beforeEach(() => { vi.stubGlobal("fetch", fetchMock); - vi.stubEnv("TINYBIRD_TOKEN", "append-token"); - vi.stubEnv("TINYBIRD_DELETE_TOKEN", "delete-token"); + vi.stubEnv("TINYBIRD_TOKEN", "tinybird-token"); vi.stubEnv("TINYBIRD_BASE_URL", "https://tinybird.test/"); fetchMock.mockReset(); fetchMock.mockResolvedValue(new Response("{}", { status: 200 })); @@ -18,7 +17,7 @@ describe("deleteTinybirdData", () => { vi.unstubAllEnvs(); }); - it("deletes AI calls and every mailbox datasource with the delete token", async () => { + it("deletes AI calls and every mailbox datasource", async () => { await deleteTinybirdData({ userIds: ["user-1"], emails: ["a@example.com", "b@example.com"], @@ -34,13 +33,13 @@ describe("deleteTinybirdData", () => { { path: "/v0/datasources/aiCall/delete", condition: "userId IN ('user-1')", - auth: "Bearer delete-token", + auth: "Bearer tinybird-token", }, ...["email_action", "email", "last_and_oldest_emails_mv"].map( (datasource) => ({ path: `/v0/datasources/${datasource}/delete`, condition: "ownerEmail IN ('a@example.com', 'b@example.com')", - auth: "Bearer delete-token", + auth: "Bearer tinybird-token", }), ), ]); @@ -65,15 +64,6 @@ describe("deleteTinybirdData", () => { expect(fetchMock).toHaveBeenCalledTimes(3); }); - it("fails loudly when Tinybird is enabled without a delete token", async () => { - vi.stubEnv("TINYBIRD_DELETE_TOKEN", ""); - - await expect(deleteTinybirdData({ userIds: ["user-1"] })).rejects.toThrow( - "TINYBIRD_DELETE_TOKEN is not set", - ); - expect(fetchMock).not.toHaveBeenCalled(); - }); - it("does nothing when Tinybird is not configured", async () => { vi.stubEnv("TINYBIRD_TOKEN", ""); diff --git a/packages/tinybird/src/delete.ts b/packages/tinybird/src/delete.ts index c46b2a0068..b3a125e89e 100644 --- a/packages/tinybird/src/delete.ts +++ b/packages/tinybird/src/delete.ts @@ -38,13 +38,6 @@ export async function deleteTinybirdData({ // Tinybird runs one delete job at a time and answers 429 while one is running. async function deleteRows(datasource: string, deleteCondition: string) { - const token = process.env.TINYBIRD_DELETE_TOKEN; - if (!token) { - throw new Error( - "TINYBIRD_DELETE_TOKEN is not set, so Tinybird data cannot be deleted", - ); - } - await pRetry( async () => { const response = await fetch( @@ -55,7 +48,7 @@ async function deleteRows(datasource: string, deleteCondition: string) { { method: "POST", body: new URLSearchParams({ delete_condition: deleteCondition }), - headers: { Authorization: `Bearer ${token}` }, + headers: { Authorization: `Bearer ${process.env.TINYBIRD_TOKEN}` }, signal: AbortSignal.timeout(30_000), }, ); diff --git a/turbo.json b/turbo.json index b5f612fee1..50a3aff151 100644 --- a/turbo.json +++ b/turbo.json @@ -157,7 +157,6 @@ "LEMON_SQUEEZY_API_KEY", "TINYBIRD_TOKEN", - "TINYBIRD_DELETE_TOKEN", "TINYBIRD_BASE_URL", "POSTHOG_API_SECRET", From 63b1fb4586f8b6ab38ea4dee89648fe47aa2891e Mon Sep 17 00:00:00 2001 From: Eliezer Steinbock <3090527+elie222@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:00:49 -0400 Subject: [PATCH 4/6] fix: Include email account IDs in full-user Tinybird cleanup Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/web/utils/user/delete.test.ts | 1 + apps/web/utils/user/delete.ts | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/web/utils/user/delete.test.ts b/apps/web/utils/user/delete.test.ts index c40daf3155..90b8976212 100644 --- a/apps/web/utils/user/delete.test.ts +++ b/apps/web/utils/user/delete.test.ts @@ -137,6 +137,7 @@ describe("deleteUser", () => { await deleteUser({ userId: "user-1", logger }); expect(deleteTinybirdData).toHaveBeenCalledWith({ userIds: ["user-1"], + emailAccountIds: ["email-account-1"], emails: ["owner@example.com"], }); expect(withThreadPageBufferDeletion).toHaveBeenCalledWith( diff --git a/apps/web/utils/user/delete.ts b/apps/web/utils/user/delete.ts index 21e76611bc..436555edb3 100644 --- a/apps/web/utils/user/delete.ts +++ b/apps/web/utils/user/delete.ts @@ -138,10 +138,12 @@ export async function deleteUser({ .map((account) => account.emailAccount?.email) .filter((email): email is string => Boolean(email)); after(() => - deleteTinybirdData({ userIds: [userId], emails }).catch((error) => { - logger.error("Error deleting Tinybird data", { error }); - captureException(error); - }), + deleteTinybirdData({ userIds: [userId], emailAccountIds, emails }).catch( + (error) => { + logger.error("Error deleting Tinybird data", { error }); + captureException(error); + }, + ), ); } catch (error) { logger.error("Error during user resources deletion process", { From a21edc8d4f1f0eab32c9188de897e39f83f45cf7 Mon Sep 17 00:00:00 2001 From: Eliezer Steinbock <3090527+elie222@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:59:49 -0400 Subject: [PATCH 5/6] fix: Keep AI usage rows and delete only email-keyed Tinybird data AI usage rows are keyed by user id and needed for cost reporting, so account deletion now keeps them and only deletes rows that identify a mailbox by its address, including usage rows that fell back to the email. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scripts/purge-orphaned-tinybird-data.ts | 28 +++++++-------- apps/web/utils/actions/user.test.ts | 11 +++--- apps/web/utils/actions/user.ts | 7 ++-- apps/web/utils/user/delete.test.ts | 12 +++---- apps/web/utils/user/delete.ts | 12 +++---- packages/tinybird/src/delete.test.ts | 23 ++++++------ packages/tinybird/src/delete.ts | 36 ++++++------------- 7 files changed, 50 insertions(+), 79 deletions(-) diff --git a/apps/web/scripts/purge-orphaned-tinybird-data.ts b/apps/web/scripts/purge-orphaned-tinybird-data.ts index c81846ea66..49e37a182b 100644 --- a/apps/web/scripts/purge-orphaned-tinybird-data.ts +++ b/apps/web/scripts/purge-orphaned-tinybird-data.ts @@ -1,4 +1,4 @@ -// Deletes Tinybird rows that belong to users or mailboxes that no longer exist, +// Deletes Tinybird rows that identify mailboxes that no longer exist, // left behind before account deletion cleaned up Tinybird. // // TINYBIRD_TOKEN needs read and delete (DATASOURCES:CREATE) access to the datasources. @@ -7,7 +7,7 @@ import "dotenv/config"; import chunk from "lodash/chunk"; -import { deleteTinybirdData } from "@inboxzero/tinybird"; +import { deleteTinybirdEmailData } from "@inboxzero/tinybird"; import prisma from "@/utils/prisma"; const EMAIL_DATASOURCES = [ @@ -23,14 +23,12 @@ async function main() { throw new Error("TINYBIRD_TOKEN is not set"); } - const [emailAccounts, users] = await Promise.all([ - prisma.emailAccount.findMany({ select: { email: true } }), - prisma.user.findMany({ select: { id: true } }), - ]); + const emailAccounts = await prisma.emailAccount.findMany({ + select: { email: true }, + }); const liveEmails = new Set( emailAccounts.map(({ email }) => email.toLowerCase()), ); - const liveUserIds = new Set(users.map(({ id }) => id)); const orphanedEmails = new Set(); for (const datasource of EMAIL_DATASOURCES) { @@ -48,12 +46,17 @@ async function main() { ); } + // AI usage rows are kept; only rows that fell back to an email address as + // the user id identify a person. const aiCallUsers = await getDistinctValues("aiCall", "userId"); - const orphanedUserIds = - aiCallUsers?.filter((id) => !liveUserIds.has(id)) ?? []; + const orphanedAiCallEmails = + aiCallUsers?.filter( + (value) => value.includes("@") && !liveEmails.has(value.toLowerCase()), + ) ?? []; + for (const email of orphanedAiCallEmails) orphanedEmails.add(email); console.log( aiCallUsers - ? `aiCall: ${aiCallUsers.length} users, ${orphanedUserIds.length} orphaned` + ? `aiCall: ${orphanedAiCallEmails.length} orphaned email-keyed users` : "aiCall: datasource not found, skipping", ); @@ -63,10 +66,7 @@ async function main() { } for (const emails of chunk([...orphanedEmails], BATCH_SIZE)) { - await deleteTinybirdData({ emails }); - } - for (const userIds of chunk(orphanedUserIds, BATCH_SIZE)) { - await deleteTinybirdData({ userIds }); + await deleteTinybirdEmailData(emails); } console.log("Delete jobs submitted."); } diff --git a/apps/web/utils/actions/user.test.ts b/apps/web/utils/actions/user.test.ts index 345663419e..f9a680bcd8 100644 --- a/apps/web/utils/actions/user.test.ts +++ b/apps/web/utils/actions/user.test.ts @@ -9,7 +9,7 @@ import { deleteAccountUploadDirectory } from "@/utils/mail-api/upload-blobs"; import { deleteUser } from "@/utils/user/delete"; import { clearLastEmailAccountCookie } from "@/utils/cookies.server"; import { LAST_EMAIL_ACCOUNT_COOKIE } from "@/utils/cookies"; -import { deleteTinybirdData } from "@inboxzero/tinybird"; +import { deleteTinybirdEmailData } from "@inboxzero/tinybird"; import { deleteAccountAction, deleteEmailAccountAction } from "./user"; vi.mock("@/utils/prisma"); @@ -42,7 +42,7 @@ vi.mock("@/utils/mail-api/upload-blobs", () => ({ deleteAccountUploadDirectory: vi.fn(() => Promise.resolve()), })); vi.mock("@inboxzero/tinybird", () => ({ - deleteTinybirdData: vi.fn(() => Promise.resolve()), + deleteTinybirdEmailData: vi.fn(() => Promise.resolve()), })); vi.mock("@/utils/user/delete", () => ({ deleteUser: vi.fn(), @@ -225,10 +225,9 @@ describe("deleteEmailAccountAction", () => { expect(prisma.$transaction.mock.invocationCallOrder[0]).toBeLessThan( vi.mocked(deleteAccountUploadDirectory).mock.invocationCallOrder[0], ); - expect(deleteTinybirdData).toHaveBeenCalledWith({ - emailAccountIds: ["secondary-account"], - emails: ["secondary@example.com"], - }); + expect(deleteTinybirdEmailData).toHaveBeenCalledWith([ + "secondary@example.com", + ]); expect(clearLastEmailAccountCookie).not.toHaveBeenCalled(); }); diff --git a/apps/web/utils/actions/user.ts b/apps/web/utils/actions/user.ts index 6bd0c07483..3f6fb284a3 100644 --- a/apps/web/utils/actions/user.ts +++ b/apps/web/utils/actions/user.ts @@ -23,7 +23,7 @@ import { } from "@/utils/actions/user.validation"; import { clearLastEmailAccountCookie } from "@/utils/cookies.server"; import { deleteAccountUploadDirectory } from "@/utils/mail-api/upload-blobs"; -import { deleteTinybirdData } from "@inboxzero/tinybird"; +import { deleteTinybirdEmailData } from "@inboxzero/tinybird"; import { aliasPosthogUser } from "@/utils/posthog"; import { cleanupAIDraftsForAccount, @@ -244,10 +244,7 @@ export const deleteEmailAccountAction = actionClientUser } after(() => - deleteTinybirdData({ - emailAccountIds: [emailAccountId], - emails: [emailAccount.email], - }).catch((error) => { + deleteTinybirdEmailData([emailAccount.email]).catch((error) => { logger.error("Error deleting Tinybird data", { error }); captureException(error); }), diff --git a/apps/web/utils/user/delete.test.ts b/apps/web/utils/user/delete.test.ts index 90b8976212..85a3d38b9a 100644 --- a/apps/web/utils/user/delete.test.ts +++ b/apps/web/utils/user/delete.test.ts @@ -5,7 +5,7 @@ import { createTestLogger } from "@/__tests__/helpers"; import prisma from "@/utils/__mocks__/prisma"; import { deleteAccountUploadDirectory } from "@/utils/mail-api/upload-blobs"; import { deleteUser } from "@/utils/user/delete"; -import { deleteTinybirdData } from "@inboxzero/tinybird"; +import { deleteTinybirdEmailData } from "@inboxzero/tinybird"; import { createEmailProvider } from "@/utils/email/provider"; vi.mock("@/utils/prisma"); @@ -22,7 +22,7 @@ vi.mock("@inboxzero/transactional-email", () => ({ deleteContact: vi.fn(), })); vi.mock("@inboxzero/tinybird", () => ({ - deleteTinybirdData: vi.fn(() => Promise.resolve()), + deleteTinybirdEmailData: vi.fn(() => Promise.resolve()), })); vi.mock("@/utils/posthog", () => ({ deletePosthogUser: vi.fn(() => Promise.resolve()), @@ -104,7 +104,7 @@ describe("deleteUser", () => { await expect(deleteUser({ userId: "user-1", logger })).rejects.toThrow( "database unavailable", ); - expect(deleteTinybirdData).not.toHaveBeenCalled(); + expect(deleteTinybirdEmailData).not.toHaveBeenCalled(); }); it("deletes solo organizations before deleting the user", async () => { @@ -135,11 +135,7 @@ describe("deleteUser", () => { prisma.user.deleteMany.mockResolvedValue({ count: 1 } as any); await deleteUser({ userId: "user-1", logger }); - expect(deleteTinybirdData).toHaveBeenCalledWith({ - userIds: ["user-1"], - emailAccountIds: ["email-account-1"], - emails: ["owner@example.com"], - }); + expect(deleteTinybirdEmailData).toHaveBeenCalledWith(["owner@example.com"]); expect(withThreadPageBufferDeletion).toHaveBeenCalledWith( ["email-account-1"], expect.any(Function), diff --git a/apps/web/utils/user/delete.ts b/apps/web/utils/user/delete.ts index 436555edb3..e2cc8a5b46 100644 --- a/apps/web/utils/user/delete.ts +++ b/apps/web/utils/user/delete.ts @@ -2,7 +2,7 @@ import { deleteContact as deleteLoopsContact } from "@inboxzero/loops"; import { deleteContact as deleteResendContact } from "@inboxzero/transactional-email"; import { withThreadPageBufferDeletion } from "@/utils/redis/thread-page-buffer"; import prisma from "@/utils/prisma"; -import { deleteTinybirdData } from "@inboxzero/tinybird"; +import { deleteTinybirdEmailData } from "@inboxzero/tinybird"; import { after } from "next/server"; import { deletePosthogUser, @@ -138,12 +138,10 @@ export async function deleteUser({ .map((account) => account.emailAccount?.email) .filter((email): email is string => Boolean(email)); after(() => - deleteTinybirdData({ userIds: [userId], emailAccountIds, emails }).catch( - (error) => { - logger.error("Error deleting Tinybird data", { error }); - captureException(error); - }, - ), + deleteTinybirdEmailData(emails).catch((error) => { + logger.error("Error deleting Tinybird data", { error }); + captureException(error); + }), ); } catch (error) { logger.error("Error during user resources deletion process", { diff --git a/packages/tinybird/src/delete.test.ts b/packages/tinybird/src/delete.test.ts index 6926e0eaa2..149f2fc307 100644 --- a/packages/tinybird/src/delete.test.ts +++ b/packages/tinybird/src/delete.test.ts @@ -1,9 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { deleteTinybirdData } from "./delete"; +import { deleteTinybirdEmailData } from "./delete"; const fetchMock = vi.fn(); -describe("deleteTinybirdData", () => { +describe("deleteTinybirdEmailData", () => { beforeEach(() => { vi.stubGlobal("fetch", fetchMock); vi.stubEnv("TINYBIRD_TOKEN", "tinybird-token"); @@ -17,11 +17,8 @@ describe("deleteTinybirdData", () => { vi.unstubAllEnvs(); }); - it("deletes AI calls and every mailbox datasource", async () => { - await deleteTinybirdData({ - userIds: ["user-1"], - emails: ["a@example.com", "b@example.com"], - }); + it("deletes mailbox rows and only AI usage rows keyed by the email address", async () => { + await deleteTinybirdEmailData(["a@example.com", "b@example.com"]); const calls = fetchMock.mock.calls.map(([url, init]) => ({ path: new URL(url).pathname, @@ -32,7 +29,7 @@ describe("deleteTinybirdData", () => { expect(calls).toEqual([ { path: "/v0/datasources/aiCall/delete", - condition: "userId IN ('user-1')", + condition: "userId IN ('a@example.com', 'b@example.com')", auth: "Bearer tinybird-token", }, ...["email_action", "email", "last_and_oldest_emails_mv"].map( @@ -46,28 +43,28 @@ describe("deleteTinybirdData", () => { }); it("escapes quotes in values", async () => { - await deleteTinybirdData({ emails: ["o'brien@example.com"] }); + await deleteTinybirdEmailData(["o'brien@example.com"]); expect( new URLSearchParams(fetchMock.mock.calls[0][1].body).get( "delete_condition", ), - ).toBe("ownerEmail IN ('o\\'brien@example.com')"); + ).toBe("userId IN ('o\\'brien@example.com')"); }); it("skips datasources that do not exist", async () => { fetchMock.mockResolvedValueOnce(new Response("not found", { status: 404 })); await expect( - deleteTinybirdData({ emails: ["a@example.com"] }), + deleteTinybirdEmailData(["a@example.com"]), ).resolves.toBeUndefined(); - expect(fetchMock).toHaveBeenCalledTimes(3); + expect(fetchMock).toHaveBeenCalledTimes(4); }); it("does nothing when Tinybird is not configured", async () => { vi.stubEnv("TINYBIRD_TOKEN", ""); - await deleteTinybirdData({ userIds: ["user-1"] }); + await deleteTinybirdEmailData(["a@example.com"]); expect(fetchMock).not.toHaveBeenCalled(); }); }); diff --git a/packages/tinybird/src/delete.ts b/packages/tinybird/src/delete.ts index b3a125e89e..082bbfb73f 100644 --- a/packages/tinybird/src/delete.ts +++ b/packages/tinybird/src/delete.ts @@ -7,32 +7,16 @@ const EMAIL_DATASOURCES = [ "last_and_oldest_emails_mv", ] as const; -export async function deleteTinybirdData({ - userIds = [], - emailAccountIds = [], - emails = [], -}: { - userIds?: string[]; - emailAccountIds?: string[]; - emails?: string[]; -}) { - if (!process.env.TINYBIRD_TOKEN) return; - - const aiCallConditions = [ - userIds.length ? `userId IN (${userIds.map(quote).join(", ")})` : null, - emailAccountIds.length - ? `emailAccountId IN (${emailAccountIds.map(quote).join(", ")})` - : null, - ].filter(Boolean); - if (aiCallConditions.length) { - await deleteRows("aiCall", aiCallConditions.join(" OR ")); - } - - if (emails.length) { - const condition = `ownerEmail IN (${emails.map(quote).join(", ")})`; - for (const datasource of EMAIL_DATASOURCES) { - await deleteRows(datasource, condition); - } +// Deletes rows that identify a mailbox by its address. AI usage rows are kept +// for cost reporting; they are keyed by user id except when usage tracking +// fell back to the email address, and those rows are deleted too. +export async function deleteTinybirdEmailData(emails: string[]) { + if (!process.env.TINYBIRD_TOKEN || !emails.length) return; + + const quotedEmails = emails.map(quote).join(", "); + await deleteRows("aiCall", `userId IN (${quotedEmails})`); + for (const datasource of EMAIL_DATASOURCES) { + await deleteRows(datasource, `ownerEmail IN (${quotedEmails})`); } } From efb3b30b6d25a7a3bdc45682fd42dfa16dfb5a05 Mon Sep 17 00:00:00 2001 From: Eliezer Steinbock <3090527+elie222@users.noreply.github.com> Date: Wed, 30 Sep 2026 09:58:27 -0400 Subject: [PATCH 6/6] fix: Use a dedicated Tinybird token for deletes The ingest token can only append, so deletes use TINYBIRD_DELETE_TOKEN, a token with DATASOURCES:CREATE. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/web/.env.example | 2 ++ apps/web/env.ts | 1 + apps/web/scripts/purge-orphaned-tinybird-data.ts | 9 +++++---- packages/tinybird/src/delete.test.ts | 16 +++++++++++++--- packages/tinybird/src/delete.ts | 10 +++++++++- turbo.json | 1 + 6 files changed, 31 insertions(+), 8 deletions(-) diff --git a/apps/web/.env.example b/apps/web/.env.example index 44e21ad65d..0c4a901542 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -248,6 +248,8 @@ LLM_API_KEY= # Tinybird TINYBIRD_TOKEN= +# Token with DATASOURCES:CREATE, used to delete a user's data on account deletion +TINYBIRD_DELETE_TOKEN= TINYBIRD_BASE_URL=https://api.us-east.tinybird.co/ # Stripe AI overage billing (optional; unset = unlimited/no overage charges) diff --git a/apps/web/env.ts b/apps/web/env.ts index 2f0c49fe53..718562f68f 100644 --- a/apps/web/env.ts +++ b/apps/web/env.ts @@ -291,6 +291,7 @@ const parsedEnv = createEnv({ APNS_TRANSPORT: z.enum(["apns", "fake"]).optional(), TINYBIRD_TOKEN: z.string().optional(), + TINYBIRD_DELETE_TOKEN: z.string().optional(), TINYBIRD_BASE_URL: z.string().default("https://api.us-east.tinybird.co/"), API_KEY_SALT: z.string().optional(), diff --git a/apps/web/scripts/purge-orphaned-tinybird-data.ts b/apps/web/scripts/purge-orphaned-tinybird-data.ts index 49e37a182b..995f44676c 100644 --- a/apps/web/scripts/purge-orphaned-tinybird-data.ts +++ b/apps/web/scripts/purge-orphaned-tinybird-data.ts @@ -1,7 +1,8 @@ // Deletes Tinybird rows that identify mailboxes that no longer exist, // left behind before account deletion cleaned up Tinybird. // -// TINYBIRD_TOKEN needs read and delete (DATASOURCES:CREATE) access to the datasources. +// Run with TINYBIRD_DELETE_TOKEN set to a token that can also read the datasources +// (for example the workspace admin token). // Dry run (counts only): `pnpm --filter inbox-zero-ai exec tsx scripts/purge-orphaned-tinybird-data.ts` // Delete: `pnpm --filter inbox-zero-ai exec tsx scripts/purge-orphaned-tinybird-data.ts --apply` @@ -19,8 +20,8 @@ const BATCH_SIZE = 100; async function main() { const apply = process.argv.includes("--apply"); - if (!process.env.TINYBIRD_TOKEN) { - throw new Error("TINYBIRD_TOKEN is not set"); + if (!process.env.TINYBIRD_TOKEN || !process.env.TINYBIRD_DELETE_TOKEN) { + throw new Error("TINYBIRD_TOKEN and TINYBIRD_DELETE_TOKEN must be set"); } const emailAccounts = await prisma.emailAccount.findMany({ @@ -82,7 +83,7 @@ async function getDistinctValues(datasource: string, column: string) { ); const response = await fetch(url, { - headers: { Authorization: `Bearer ${process.env.TINYBIRD_TOKEN}` }, + headers: { Authorization: `Bearer ${process.env.TINYBIRD_DELETE_TOKEN}` }, }); if (response.status === 404) return null; if (!response.ok) { diff --git a/packages/tinybird/src/delete.test.ts b/packages/tinybird/src/delete.test.ts index 149f2fc307..41f7873bbc 100644 --- a/packages/tinybird/src/delete.test.ts +++ b/packages/tinybird/src/delete.test.ts @@ -6,7 +6,8 @@ const fetchMock = vi.fn(); describe("deleteTinybirdEmailData", () => { beforeEach(() => { vi.stubGlobal("fetch", fetchMock); - vi.stubEnv("TINYBIRD_TOKEN", "tinybird-token"); + vi.stubEnv("TINYBIRD_TOKEN", "ingest-token"); + vi.stubEnv("TINYBIRD_DELETE_TOKEN", "delete-token"); vi.stubEnv("TINYBIRD_BASE_URL", "https://tinybird.test/"); fetchMock.mockReset(); fetchMock.mockResolvedValue(new Response("{}", { status: 200 })); @@ -30,13 +31,13 @@ describe("deleteTinybirdEmailData", () => { { path: "/v0/datasources/aiCall/delete", condition: "userId IN ('a@example.com', 'b@example.com')", - auth: "Bearer tinybird-token", + auth: "Bearer delete-token", }, ...["email_action", "email", "last_and_oldest_emails_mv"].map( (datasource) => ({ path: `/v0/datasources/${datasource}/delete`, condition: "ownerEmail IN ('a@example.com', 'b@example.com')", - auth: "Bearer tinybird-token", + auth: "Bearer delete-token", }), ), ]); @@ -61,6 +62,15 @@ describe("deleteTinybirdEmailData", () => { expect(fetchMock).toHaveBeenCalledTimes(4); }); + it("fails loudly when Tinybird is enabled without a delete token", async () => { + vi.stubEnv("TINYBIRD_DELETE_TOKEN", ""); + + await expect(deleteTinybirdEmailData(["a@example.com"])).rejects.toThrow( + "TINYBIRD_DELETE_TOKEN is not set", + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); + it("does nothing when Tinybird is not configured", async () => { vi.stubEnv("TINYBIRD_TOKEN", ""); diff --git a/packages/tinybird/src/delete.ts b/packages/tinybird/src/delete.ts index 082bbfb73f..74edbe4225 100644 --- a/packages/tinybird/src/delete.ts +++ b/packages/tinybird/src/delete.ts @@ -22,6 +22,14 @@ export async function deleteTinybirdEmailData(emails: string[]) { // Tinybird runs one delete job at a time and answers 429 while one is running. async function deleteRows(datasource: string, deleteCondition: string) { + // The ingest token can only append, so deletes need a token with DATASOURCES:CREATE. + const token = process.env.TINYBIRD_DELETE_TOKEN; + if (!token) { + throw new Error( + "TINYBIRD_DELETE_TOKEN is not set, so Tinybird data cannot be deleted", + ); + } + await pRetry( async () => { const response = await fetch( @@ -32,7 +40,7 @@ async function deleteRows(datasource: string, deleteCondition: string) { { method: "POST", body: new URLSearchParams({ delete_condition: deleteCondition }), - headers: { Authorization: `Bearer ${process.env.TINYBIRD_TOKEN}` }, + headers: { Authorization: `Bearer ${token}` }, signal: AbortSignal.timeout(30_000), }, ); diff --git a/turbo.json b/turbo.json index 50a3aff151..b5f612fee1 100644 --- a/turbo.json +++ b/turbo.json @@ -157,6 +157,7 @@ "LEMON_SQUEEZY_API_KEY", "TINYBIRD_TOKEN", + "TINYBIRD_DELETE_TOKEN", "TINYBIRD_BASE_URL", "POSTHOG_API_SECRET",