diff --git a/.changeset/sharp-wasms-opfs.md b/.changeset/sharp-wasms-opfs.md new file mode 100644 index 0000000..14fc428 --- /dev/null +++ b/.changeset/sharp-wasms-opfs.md @@ -0,0 +1,8 @@ +--- +"@mandel59/mojidata-api-sqlite-wasm": minor +"@mandel59/mojidata-api-runtime": patch +--- + +Add the initial SQLite wasm OPFS backend package for `mojidata-api`, including +an OO1 `SqlExecutor` adapter, OPFS SAH pool materialization helpers, and a +browser worker entrypoint. diff --git a/.pnp.cjs b/.pnp.cjs index 0c2b8df..961b446 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -70,6 +70,10 @@ const RAW_RUNTIME_STATE = "name": "@mandel59/mojidata-api-runtime",\ "reference": "workspace:packages/mojidata-api-runtime"\ },\ + {\ + "name": "@mandel59/mojidata-api-sqlite-wasm",\ + "reference": "workspace:packages/mojidata-api-sqlite-wasm"\ + },\ {\ "name": "@mandel59/mojidata-api-sqljs",\ "reference": "workspace:packages/mojidata-api-sqljs"\ @@ -101,6 +105,7 @@ const RAW_RUNTIME_STATE = ["@mandel59/mojidata-api-hono", ["workspace:packages/mojidata-api-hono"]],\ ["@mandel59/mojidata-api-node-sqlite", ["workspace:packages/mojidata-api-node-sqlite"]],\ ["@mandel59/mojidata-api-runtime", ["workspace:packages/mojidata-api-runtime"]],\ + ["@mandel59/mojidata-api-sqlite-wasm", ["workspace:packages/mojidata-api-sqlite-wasm"]],\ ["@mandel59/mojidata-api-sqljs", ["workspace:packages/mojidata-api-sqljs"]],\ ["@mandel59/mojidata-cli", ["workspace:packages/mojidata-cli"]],\ ["@mandel59/react-mojidata-api", ["workspace:packages/react-mojidata-api"]],\ @@ -1307,6 +1312,21 @@ const RAW_RUNTIME_STATE = "linkType": "SOFT"\ }]\ ]],\ + ["@mandel59/mojidata-api-sqlite-wasm", [\ + ["workspace:packages/mojidata-api-sqlite-wasm", {\ + "packageLocation": "./packages/mojidata-api-sqlite-wasm/",\ + "packageDependencies": [\ + ["@mandel59/mojidata-api-core", "workspace:packages/mojidata-api-core"],\ + ["@mandel59/mojidata-api-runtime", "workspace:packages/mojidata-api-runtime"],\ + ["@mandel59/mojidata-api-sqlite-wasm", "workspace:packages/mojidata-api-sqlite-wasm"],\ + ["@sqlite.org/sqlite-wasm", "npm:3.53.0-build1"],\ + ["@types/node", "npm:24.10.4"],\ + ["tsx", "npm:4.21.0"],\ + ["typescript", "patch:typescript@npm%3A5.9.3#optional!builtin::version=5.9.3&hash=5786d5"]\ + ],\ + "linkType": "SOFT"\ + }]\ + ]],\ ["@mandel59/mojidata-api-sqljs", [\ ["workspace:packages/mojidata-api-sqljs", {\ "packageLocation": "./packages/mojidata-api-sqljs/",\ @@ -1674,6 +1694,15 @@ const RAW_RUNTIME_STATE = "linkType": "HARD"\ }]\ ]],\ + ["@sqlite.org/sqlite-wasm", [\ + ["npm:3.53.0-build1", {\ + "packageLocation": "./.yarn/cache/@sqlite.org-sqlite-wasm-npm-3.53.0-build1-38c03d882c-7e8aedffb6.zip/node_modules/@sqlite.org/sqlite-wasm/",\ + "packageDependencies": [\ + ["@sqlite.org/sqlite-wasm", "npm:3.53.0-build1"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["@szmarczak/http-timer", [\ ["npm:1.1.2", {\ "packageLocation": "./.yarn/cache/@szmarczak-http-timer-npm-1.1.2-ea82ca2d55-9b63853bd5.zip/node_modules/@szmarczak/http-timer/",\ diff --git a/.yarn/cache/@sqlite.org-sqlite-wasm-npm-3.53.0-build1-38c03d882c-7e8aedffb6.zip b/.yarn/cache/@sqlite.org-sqlite-wasm-npm-3.53.0-build1-38c03d882c-7e8aedffb6.zip new file mode 100644 index 0000000..5745afc Binary files /dev/null and b/.yarn/cache/@sqlite.org-sqlite-wasm-npm-3.53.0-build1-38c03d882c-7e8aedffb6.zip differ diff --git a/README.md b/README.md index 0db598b..d4dfb70 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ - [Mojidata API compatibility facade](packages/mojidata-api) - [Mojidata API runtime](packages/mojidata-api-runtime) - [Mojidata API sql.js backend](packages/mojidata-api-sqljs) +- [Mojidata API SQLite wasm OPFS backend](packages/mojidata-api-sqlite-wasm) - [Mojidata API better-sqlite3 backend](packages/mojidata-api-better-sqlite3) - [Mojidata API node:sqlite backend](packages/mojidata-api-node-sqlite) - [Mojidata API D1 backend](packages/mojidata-api-d1) diff --git a/docs/releasing.md b/docs/releasing.md index 5228441..79b93cd 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -24,6 +24,7 @@ For the split `mojidata-api` packages, use this bootstrap flow for: - `@mandel59/mojidata-api-core` - `@mandel59/mojidata-api-sqljs` +- `@mandel59/mojidata-api-sqlite-wasm` - `@mandel59/mojidata-api-hono` - `@mandel59/mojidata-api-runtime` - `@mandel59/mojidata-api-better-sqlite3` diff --git a/package.json b/package.json index c3b2ce8..02ec8b6 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "scripts": { "changeset": "changeset", "build:idsdb-stack": "(cd packages/mojidata && yarn prepare) && (cd packages/idsdb-utils && yarn prepare) && (cd packages/idsdb && yarn prepare)", - "build:mojidata-api-split": "(cd packages/mojidata-api-core && yarn prepare) && (cd packages/mojidata-api-hono && yarn prepare) && (cd packages/mojidata-api-runtime && yarn prepare) && (cd packages/mojidata-api-sqljs && yarn prepare) && (cd packages/mojidata-api-better-sqlite3 && yarn prepare) && (cd packages/mojidata-api-node-sqlite && yarn prepare) && (cd packages/mojidata-api-d1 && yarn prepare)", + "build:mojidata-api-split": "(cd packages/mojidata-api-core && yarn prepare) && (cd packages/mojidata-api-hono && yarn prepare) && (cd packages/mojidata-api-runtime && yarn prepare) && (cd packages/mojidata-api-sqljs && yarn prepare) && (cd packages/mojidata-api-sqlite-wasm && yarn prepare) && (cd packages/mojidata-api-better-sqlite3 && yarn prepare) && (cd packages/mojidata-api-node-sqlite && yarn prepare) && (cd packages/mojidata-api-d1 && yarn prepare)", "ci:build:affected": "node ./scripts/run-affected-workspaces.mjs build", "ci:build": "yarn build", "ci:pack:affected": "node ./scripts/run-affected-workspaces.mjs pack", @@ -26,7 +26,7 @@ "mojidata-api:bench:remote": "yarn workspace @mandel59/mojidata-api-bench bench:remote", "mojidata-api:bench:prepare": "yarn workspace @mandel59/mojidata-api-bench bench:prepare", "release": "yarn ci:build && changeset publish", - "test": "yarn build:idsdb-stack && yarn build:mojidata-api-split && (cd packages/mojidata && yarn test) && (cd packages/mojidata-api && yarn test) && (cd packages/mojidata-api-sqljs && yarn test) && (cd packages/mojidata-api-better-sqlite3 && yarn test) && (cd packages/mojidata-api-node-sqlite && yarn test) && (cd packages/mojidata-api-d1 && yarn test) && (cd packages/react-mojidata-api && yarn test:unit) && (cd packages/idstool && yarn test) && (cd packages/mojidata-cli && yarn test)", + "test": "yarn build:idsdb-stack && yarn build:mojidata-api-split && (cd packages/mojidata && yarn test) && (cd packages/mojidata-api && yarn test) && (cd packages/mojidata-api-sqljs && yarn test) && (cd packages/mojidata-api-sqlite-wasm && yarn test) && (cd packages/mojidata-api-better-sqlite3 && yarn test) && (cd packages/mojidata-api-node-sqlite && yarn test) && (cd packages/mojidata-api-d1 && yarn test) && (cd packages/react-mojidata-api && yarn test:unit) && (cd packages/idstool && yarn test) && (cd packages/mojidata-cli && yarn test)", "version-packages": "changeset version", "build": "yarn build:idsdb-stack && yarn build:mojidata-api-split && (cd packages/mojidata-api && yarn prepare) && (cd packages/react-mojidata-api && yarn prepare) && (cd packages/idstool && yarn prepare) && (cd packages/mojidata-cli && yarn prepare)" }, diff --git a/packages/mojidata-api-runtime/lib/worker-protocol.ts b/packages/mojidata-api-runtime/lib/worker-protocol.ts index f838f57..f7e2298 100644 --- a/packages/mojidata-api-runtime/lib/worker-protocol.ts +++ b/packages/mojidata-api-runtime/lib/worker-protocol.ts @@ -4,6 +4,21 @@ export type WorkerInit = { sqlWasmUrl: string mojidataDbUrl: string idsfindDbUrl: string + sqliteWasm?: { + wasmUrl?: string + wasmBinary?: ArrayBuffer | Uint8Array + opfsName?: string + opfsDirectory?: string + initialCapacity?: number + clearOnInit?: boolean + manifestDirectory?: string + mojidataDbName?: string + idsfindDbName?: string + mojidataDbVersion?: string + idsfindDbVersion?: string + mojidataDbByteLength?: number + idsfindDbByteLength?: number + } } export type WorkerCall = { diff --git a/packages/mojidata-api-sqlite-wasm/.gitignore b/packages/mojidata-api-sqlite-wasm/.gitignore new file mode 100644 index 0000000..1521c8b --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/.gitignore @@ -0,0 +1 @@ +dist diff --git a/packages/mojidata-api-sqlite-wasm/CHANGELOG.md b/packages/mojidata-api-sqlite-wasm/CHANGELOG.md new file mode 100644 index 0000000..def31a4 --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/CHANGELOG.md @@ -0,0 +1,16 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [1.8.0] - 2026-05-03 + +### Added + +- Publish the initial SQLite wasm OPFS backend package for `mojidata-api`. +- Add a `SqlExecutor` adapter for the SQLite wasm OO1 API. +- Add `opfs-sahpool` initialization, asset import, manifest, and worker entrypoint helpers. + +[1.8.0]: https://github.com/mandel59/mojidata/releases/tag/v1.8.0 diff --git a/packages/mojidata-api-sqlite-wasm/LICENSE.md b/packages/mojidata-api-sqlite-wasm/LICENSE.md new file mode 100644 index 0000000..f0f8426 --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/LICENSE.md @@ -0,0 +1,19 @@ +Copyright (c) 2026 Ryusei Yamaguchi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/mojidata-api-sqlite-wasm/README.md b/packages/mojidata-api-sqlite-wasm/README.md new file mode 100644 index 0000000..7e96fcd --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/README.md @@ -0,0 +1,62 @@ +# mojidata-api-sqlite-wasm + +`@mandel59/mojidata-api-sqlite-wasm` provides a browser-oriented SQLite wasm +backend for `mojidata-api`. + +The package targets `@sqlite.org/sqlite-wasm` and its `opfs-sahpool` VFS. It +keeps the default `@mandel59/mojidata-api` and `@mandel59/mojidata-api-sqljs` +paths unchanged, so applications can opt in to the larger SQLite wasm backend +only when they need persistent OPFS storage. + +It includes: + +- `createSqliteWasmExecutor()`: an OO1 API adapter for `SqlExecutor` +- `installMojidataSqliteWasmFunctions()`: registers mojidata UDFs on SQLite wasm +- `installOpfsSAHPool()`: initializes `opfs-sahpool` in a Worker context +- `tryInstallOpfsSAHPool()`: reports pool initialization failures for fallback handling +- `ensureOpfsSAHPoolDatabase()`: imports an asset DB into OPFS with a manifest +- `createSqliteWasmDbFromOpfsSAHPool()`: wires OPFS DB handles to `createSqlApiDb()` +- `@mandel59/mojidata-api-sqlite-wasm/browser-worker`: worker entrypoint using the shared mojidata-api worker protocol + +Example worker client: + +```ts +import { createMojidataApiWorkerClient } from "@mandel59/mojidata-api-runtime" + +const worker = new Worker( + new URL("@mandel59/mojidata-api-sqlite-wasm/browser-worker", import.meta.url), + { type: "module" }, +) + +const db = createMojidataApiWorkerClient(worker, { + sqlWasmUrl: "/assets/sqlite3.wasm", + mojidataDbUrl: "/assets/moji.db", + idsfindDbUrl: "/assets/idsfind.db", + sqliteWasm: { + mojidataDbVersion: "moji-db-v1", + idsfindDbVersion: "idsfind-db-v1", + mojidataDbByteLength: 123456, + idsfindDbByteLength: 654321, + }, +}) + +await db.ready +``` + +`sqlWasmUrl` is passed to SQLite wasm as the `sqlite3.wasm` URL. Applications +can also pass `sqliteWasm.wasmUrl` to override it, or `sqliteWasm.wasmBinary` +when the app already fetched the wasm bytes through its own asset pipeline. +Custom workers can call `getSqliteWasm({ wasmUrl, wasmBinary, locateFile })` +directly when they need lower-level control. + +The browser worker initializes SQLite and the OPFS pool during `ready`, but DB +assets are imported lazily. `moji.db` is downloaded/imported on the first +mojidata query, and `idsfind.db` is downloaded/imported on the first IDS query. + +`opfs-sahpool` is only available from Worker contexts with OPFS APIs. In +unsupported contexts, use `tryEnsureOpfsSAHPoolDatabase()` or catch +`SqliteWasmOpfsError` and fall back to the existing SQL.js backend. + +The package does not copy or serve `moji.db`, `idsfind.db`, or `sqlite3.wasm` +assets. Applications should keep asset URL/versioning, Cache Storage prefetch, +and framework middleware in the application layer. diff --git a/packages/mojidata-api-sqlite-wasm/browser-worker.ts b/packages/mojidata-api-sqlite-wasm/browser-worker.ts new file mode 100644 index 0000000..6ea98cd --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/browser-worker.ts @@ -0,0 +1 @@ +import "./lib/browser-worker.js" diff --git a/packages/mojidata-api-sqlite-wasm/index.ts b/packages/mojidata-api-sqlite-wasm/index.ts new file mode 100644 index 0000000..8dd3f60 --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/index.ts @@ -0,0 +1,33 @@ +export { + createSqliteWasmDb, + createSqliteWasmDbFromOpfsSAHPool, + createSqliteWasmExecutorProvider, + createSqliteWasmMojidataDbProvider, + openOpfsSAHPoolDatabase, + type CreateSqliteWasmDbFromOpfsSAHPoolOptions, +} from "./lib/sqlite-wasm-runtime.js" +export { + createSqliteWasmExecutor, + type SqliteWasmDatabaseLike, +} from "./lib/sqlite-wasm-executor.js" +export { installMojidataSqliteWasmFunctions } from "./lib/sqlite-wasm-functions.js" +export { + DEFAULT_OPFS_MANIFEST_DIRECTORY, + SqliteWasmOpfsError, + ensureOpfsSAHPoolDatabase, + getSqliteWasm, + installOpfsSAHPool, + isOpfsSAHPoolSupported, + readOpfsSAHPoolManifest, + tryInstallOpfsSAHPool, + tryEnsureOpfsSAHPoolDatabase, + writeOpfsSAHPoolManifest, + type EnsureOpfsSAHPoolDatabaseResult, + type OpfsSAHPoolInstallOptions, + type OpfsSAHPoolManifest, + type OpfsSAHPoolMaterializeOptions, + type SqliteWasmOpfsFailure, + type SqliteWasmOpfsFailureReason, + type SqliteWasmInitOptions, + type SqliteWasmSAHPoolUtil, +} from "./lib/opfs-sahpool.js" diff --git a/packages/mojidata-api-sqlite-wasm/lib/browser-worker.ts b/packages/mojidata-api-sqlite-wasm/lib/browser-worker.ts new file mode 100644 index 0000000..58befed --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/lib/browser-worker.ts @@ -0,0 +1,76 @@ +import type { MojidataApiDb } from "@mandel59/mojidata-api-core" +import type { + WorkerRequest, + WorkerResponse, +} from "@mandel59/mojidata-api-runtime/lib/worker-protocol" +import type { WorkerInit } from "@mandel59/mojidata-api-runtime/lib/worker-protocol" + +import { + createSqliteWasmDbFromOpfsSAHPool, + getSqliteWasm, + installOpfsSAHPool, +} from "../index.js" + +let api: MojidataApiDb | undefined + +function serializeError(error: unknown) { + if (error instanceof Error) { + return { message: error.message, stack: error.stack } + } + return { message: String(error) } +} + +async function initWorker(init: WorkerInit) { + const wasmUrl = init.sqliteWasm?.wasmUrl ?? init.sqlWasmUrl + const sqlite3 = await getSqliteWasm({ + wasmUrl: wasmUrl || undefined, + wasmBinary: init.sqliteWasm?.wasmBinary, + }) + const poolUtil = await installOpfsSAHPool(sqlite3, { + name: init.sqliteWasm?.opfsName, + directory: init.sqliteWasm?.opfsDirectory, + initialCapacity: init.sqliteWasm?.initialCapacity ?? 6, + clearOnInit: init.sqliteWasm?.clearOnInit, + }) + const manifestDirectory = init.sqliteWasm?.manifestDirectory + api = await createSqliteWasmDbFromOpfsSAHPool({ + poolUtil, + mojidata: { + name: init.sqliteWasm?.mojidataDbName ?? "/mojidata/moji.db", + assetUrl: init.mojidataDbUrl, + assetVersion: init.sqliteWasm?.mojidataDbVersion ?? init.mojidataDbUrl, + byteLength: init.sqliteWasm?.mojidataDbByteLength, + manifestDirectory, + }, + idsfind: { + name: init.sqliteWasm?.idsfindDbName ?? "/mojidata/idsfind.db", + assetUrl: init.idsfindDbUrl, + assetVersion: init.sqliteWasm?.idsfindDbVersion ?? init.idsfindDbUrl, + byteLength: init.sqliteWasm?.idsfindDbByteLength, + manifestDirectory, + }, + }) +} + +globalThis.addEventListener("message", async (ev: MessageEvent) => { + const req = ev.data as WorkerRequest + const post = (res: WorkerResponse) => (globalThis as unknown as Worker).postMessage(res) + + try { + if (req.type === "init") { + await initWorker(req.init) + post({ id: req.id, ok: true, result: null }) + return + } + + if (!api) { + throw new Error("Worker is not initialized; call init first.") + } + + const { method, args } = req.call + const result = await (api as any)[method](...args) + post({ id: req.id, ok: true, result }) + } catch (error) { + post({ id: req.id, ok: false, error: serializeError(error) }) + } +}) diff --git a/packages/mojidata-api-sqlite-wasm/lib/opfs-sahpool.ts b/packages/mojidata-api-sqlite-wasm/lib/opfs-sahpool.ts new file mode 100644 index 0000000..9e3c8e0 --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/lib/opfs-sahpool.ts @@ -0,0 +1,355 @@ +import sqlite3InitModule, { + type Database, + type SAHPoolUtil, + type Sqlite3Static, +} from "@sqlite.org/sqlite-wasm" + +export const DEFAULT_OPFS_MANIFEST_DIRECTORY = "mojidata-api-sqlite-wasm" + +export type SqliteWasmSAHPoolUtil = Omit & { + OpfsSAHPoolDb: new (filename: string, flags?: string) => Database +} + +export type OpfsSAHPoolInstallOptions = Parameters< + Sqlite3Static["installOpfsSAHPoolVfs"] +>[0] & { + forceReinitIfPreviouslyFailed?: boolean +} + +export type OpfsSAHPoolMaterializeOptions = { + name: string + assetUrl: string + assetVersion: string + byteLength?: number + manifestKey?: string + manifestDirectory?: string + fetch?: (input: string) => Promise +} + +export type OpfsSAHPoolManifest = { + name: string + assetUrl: string + assetVersion: string + byteLength: number + importedAt: string +} + +export type EnsureOpfsSAHPoolDatabaseResult = { + name: string + manifest: OpfsSAHPoolManifest + status: "imported" | "reused" +} + +export type SqliteWasmOpfsFailureReason = + | "unsupported" + | "pool-init-failed" + | "manifest-read-failed" + | "fetch-failed" + | "byte-length-mismatch" + | "import-failed" + | "manifest-write-failed" + +export type SqliteWasmOpfsFailure = { + ok: false + reason: SqliteWasmOpfsFailureReason + error: Error +} + +export class SqliteWasmOpfsError extends Error { + constructor( + readonly reason: SqliteWasmOpfsFailureReason, + message: string, + readonly cause?: unknown, + ) { + super(message) + this.name = "SqliteWasmOpfsError" + } +} + +let sqlite3Promise: Promise | undefined + +export type SqliteWasmInitOptions = { + wasmUrl?: string | URL + wasmBinary?: ArrayBuffer | Uint8Array + locateFile?: (path: string, scriptDirectory: string) => string + instantiateWasm?: ( + imports: WebAssembly.Imports, + successCallback: ( + instance: WebAssembly.Instance, + module: WebAssembly.Module, + ) => void, + ) => WebAssembly.Exports | Promise | void +} + +type EmscriptenSqliteWasmInitOptions = Omit + +type SqliteWasmInitModule = ( + options?: EmscriptenSqliteWasmInitOptions, +) => Promise + +function toEmscriptenSqliteWasmInitOptions( + options: SqliteWasmInitOptions = {}, +): EmscriptenSqliteWasmInitOptions { + const { wasmUrl, locateFile, ...rest } = options + if (wasmUrl === undefined) { + return locateFile === undefined ? rest : { ...rest, locateFile } + } + const sqliteWasmUrl = String(wasmUrl) + return { + ...rest, + locateFile: (path, scriptDirectory) => { + if (path === "sqlite3.wasm") { + return sqliteWasmUrl + } + return locateFile?.(path, scriptDirectory) ?? `${scriptDirectory}${path}` + }, + } +} + +export function getSqliteWasm(options?: SqliteWasmInitOptions): Promise { + sqlite3Promise ??= (sqlite3InitModule as SqliteWasmInitModule)( + options === undefined ? undefined : toEmscriptenSqliteWasmInitOptions(options), + ) + return sqlite3Promise +} + +function getNavigatorStorage(): { getDirectory?: () => Promise } | undefined { + return (globalThis.navigator as { storage?: { getDirectory?: () => Promise } } | undefined) + ?.storage +} + +function isWorkerLikeScope() { + const workerCtor = (globalThis as { WorkerGlobalScope?: unknown }).WorkerGlobalScope + return typeof workerCtor === "function" && globalThis instanceof workerCtor +} + +export function isOpfsSAHPoolSupported() { + return isWorkerLikeScope() && typeof getNavigatorStorage()?.getDirectory === "function" +} + +export async function installOpfsSAHPool( + sqlite3: Sqlite3Static, + options: OpfsSAHPoolInstallOptions = {}, +): Promise { + if (!isOpfsSAHPoolSupported()) { + throw new SqliteWasmOpfsError( + "unsupported", + "SQLite wasm opfs-sahpool requires OPFS APIs in a Worker context.", + ) + } + try { + return (await sqlite3.installOpfsSAHPoolVfs(options)) as SqliteWasmSAHPoolUtil + } catch (error) { + throw new SqliteWasmOpfsError( + "pool-init-failed", + "Failed to initialize SQLite wasm opfs-sahpool.", + error, + ) + } +} + +export async function tryInstallOpfsSAHPool( + sqlite3: Sqlite3Static, + options: OpfsSAHPoolInstallOptions = {}, +): Promise<{ ok: true; poolUtil: SqliteWasmSAHPoolUtil } | SqliteWasmOpfsFailure> { + try { + return { ok: true, poolUtil: await installOpfsSAHPool(sqlite3, options) } + } catch (error) { + return toFailure(error) + } +} + +function normalizeOpfsName(name: string) { + if (!name.startsWith("/")) { + throw new Error(`opfs-sahpool database names must be absolute paths: ${name}`) + } + return name +} + +function getManifestSegments(options: OpfsSAHPoolMaterializeOptions) { + const key = options.manifestKey ?? options.name + const normalized = key.replace(/^\/+/u, "").replace(/\/+$/u, "") + const segments = normalized.split("/").filter(Boolean) + if (segments.length === 0) { + return ["root.json"] + } + const leaf = segments[segments.length - 1] + return [...segments.slice(0, -1), `${leaf}.json`] +} + +async function getManifestFileHandle( + options: OpfsSAHPoolMaterializeOptions, + create: boolean, +) { + const storage = getNavigatorStorage() + if (typeof storage?.getDirectory !== "function") { + throw new Error("navigator.storage.getDirectory() is not available.") + } + const root = await storage.getDirectory() + const baseDirectory = options.manifestDirectory ?? DEFAULT_OPFS_MANIFEST_DIRECTORY + const path = [baseDirectory, ...getManifestSegments(options)].filter(Boolean) + const fileName = path[path.length - 1] + let dir = root + for (const segment of path.slice(0, -1)) { + dir = await dir.getDirectoryHandle(segment, { create }) + } + return await dir.getFileHandle(fileName, { create }) +} + +export async function readOpfsSAHPoolManifest( + options: OpfsSAHPoolMaterializeOptions, +): Promise { + try { + const handle = await getManifestFileHandle(options, false) + const file = await handle.getFile() + return JSON.parse(await file.text()) as OpfsSAHPoolManifest + } catch (error) { + if (error instanceof DOMException && error.name === "NotFoundError") { + return null + } + throw error + } +} + +export async function writeOpfsSAHPoolManifest( + options: OpfsSAHPoolMaterializeOptions, + manifest: OpfsSAHPoolManifest, +) { + const handle = await getManifestFileHandle(options, true) + const writable = await handle.createWritable() + try { + await writable.write(JSON.stringify(manifest, null, 2)) + } finally { + await writable.close() + } +} + +function isManifestCurrent( + poolUtil: SqliteWasmSAHPoolUtil, + options: OpfsSAHPoolMaterializeOptions, + manifest: OpfsSAHPoolManifest | null, +) { + return Boolean( + manifest && + manifest.name === options.name && + manifest.assetUrl === options.assetUrl && + manifest.assetVersion === options.assetVersion && + (options.byteLength === undefined || manifest.byteLength === options.byteLength) && + poolUtil.getFileNames().includes(options.name), + ) +} + +async function fetchDatabaseBytes(options: OpfsSAHPoolMaterializeOptions) { + const fetcher = options.fetch ?? globalThis.fetch + if (typeof fetcher !== "function") { + throw new Error("fetch() is not available.") + } + const response = await fetcher(options.assetUrl) + if (!response.ok) { + throw new Error( + `Failed to fetch SQLite database asset: ${options.assetUrl} (${response.status} ${response.statusText})`, + ) + } + const bytes = new Uint8Array(await response.arrayBuffer()) + if (options.byteLength !== undefined && bytes.byteLength !== options.byteLength) { + throw new SqliteWasmOpfsError( + "byte-length-mismatch", + `SQLite database asset size mismatch for ${options.assetUrl}: expected ${options.byteLength}, got ${bytes.byteLength}.`, + ) + } + return bytes +} + +export async function ensureOpfsSAHPoolDatabase( + poolUtil: SqliteWasmSAHPoolUtil, + options: OpfsSAHPoolMaterializeOptions, +): Promise { + normalizeOpfsName(options.name) + + let existingManifest: OpfsSAHPoolManifest | null + try { + existingManifest = await readOpfsSAHPoolManifest(options) + } catch (error) { + throw new SqliteWasmOpfsError( + "manifest-read-failed", + `Failed to read OPFS manifest for ${options.name}.`, + error, + ) + } + + if (isManifestCurrent(poolUtil, options, existingManifest)) { + return { name: options.name, manifest: existingManifest!, status: "reused" } + } + + let bytes: Uint8Array + try { + bytes = await fetchDatabaseBytes(options) + } catch (error) { + if (error instanceof SqliteWasmOpfsError) { + throw error + } + throw new SqliteWasmOpfsError( + "fetch-failed", + `Failed to fetch SQLite database asset for ${options.name}.`, + error, + ) + } + + try { + await poolUtil.importDb(options.name, bytes) + } catch (error) { + throw new SqliteWasmOpfsError( + "import-failed", + `Failed to import SQLite database asset into OPFS: ${options.name}.`, + error, + ) + } + + const manifest: OpfsSAHPoolManifest = { + name: options.name, + assetUrl: options.assetUrl, + assetVersion: options.assetVersion, + byteLength: bytes.byteLength, + importedAt: new Date().toISOString(), + } + + try { + await writeOpfsSAHPoolManifest(options, manifest) + } catch (error) { + throw new SqliteWasmOpfsError( + "manifest-write-failed", + `Failed to write OPFS manifest for ${options.name}.`, + error, + ) + } + + return { name: options.name, manifest, status: "imported" } +} + +function toFailure(error: unknown): SqliteWasmOpfsFailure { + if (error instanceof SqliteWasmOpfsError) { + return { ok: false, reason: error.reason, error } + } + const wrapped = new SqliteWasmOpfsError("unsupported", String(error), error) + return { ok: false, reason: wrapped.reason, error: wrapped } +} + +export async function tryEnsureOpfsSAHPoolDatabase( + poolUtil: SqliteWasmSAHPoolUtil, + options: OpfsSAHPoolMaterializeOptions, +): Promise<{ ok: true; result: EnsureOpfsSAHPoolDatabaseResult } | SqliteWasmOpfsFailure> { + try { + if (!isOpfsSAHPoolSupported()) { + throw new SqliteWasmOpfsError( + "unsupported", + "SQLite wasm opfs-sahpool requires OPFS APIs in a Worker context.", + ) + } + return { + ok: true, + result: await ensureOpfsSAHPoolDatabase(poolUtil, options), + } + } catch (error) { + return toFailure(error) + } +} diff --git a/packages/mojidata-api-sqlite-wasm/lib/sqlite-wasm-executor.ts b/packages/mojidata-api-sqlite-wasm/lib/sqlite-wasm-executor.ts new file mode 100644 index 0000000..a701c48 --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/lib/sqlite-wasm-executor.ts @@ -0,0 +1,30 @@ +import type { BindingSpec, Database } from "@sqlite.org/sqlite-wasm" + +import type { SqlExecutor, SqlParams, SqlRow } from "@mandel59/mojidata-api-core" + +export type SqliteWasmDatabaseLike = Pick + +function bindParams(params?: SqlParams): BindingSpec | undefined { + return params as BindingSpec | undefined +} + +function normalizeRow(row: T): T { + if (row && typeof row === "object" && Object.getPrototypeOf(row) === null) { + return { ...row } as T + } + return row +} + +export function createSqliteWasmExecutor(db: SqliteWasmDatabaseLike): SqlExecutor { + return { + async query(sql: string, params?: SqlParams): Promise { + return (db.selectObjects(sql, bindParams(params)) as T[]).map((row) => + normalizeRow(row), + ) + }, + async queryOne(sql: string, params?: SqlParams): Promise { + const row = (db.selectObject(sql, bindParams(params)) as T | undefined) ?? null + return row === null ? null : normalizeRow(row) + }, + } +} diff --git a/packages/mojidata-api-sqlite-wasm/lib/sqlite-wasm-functions.ts b/packages/mojidata-api-sqlite-wasm/lib/sqlite-wasm-functions.ts new file mode 100644 index 0000000..a837ff5 --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/lib/sqlite-wasm-functions.ts @@ -0,0 +1,17 @@ +import type { Database, SqlValue } from "@sqlite.org/sqlite-wasm" + +import { installMojidataSqlFunctions } from "@mandel59/mojidata-api-core" + +type SqliteWasmFunctionDb = Pick + +export function installMojidataSqliteWasmFunctions(db: SqliteWasmFunctionDb) { + installMojidataSqlFunctions((name, fn) => { + db.createFunction({ + name, + arity: fn.length, + deterministic: true, + innocuous: true, + xFunc: (_ctxPtr, ...args) => fn(...(args as never[])) as SqlValue, + }) + }) +} diff --git a/packages/mojidata-api-sqlite-wasm/lib/sqlite-wasm-runtime.ts b/packages/mojidata-api-sqlite-wasm/lib/sqlite-wasm-runtime.ts new file mode 100644 index 0000000..c866bf7 --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/lib/sqlite-wasm-runtime.ts @@ -0,0 +1,74 @@ +import { createSqlApiDb, type MojidataApiDb, type SqlExecutor } from "@mandel59/mojidata-api-core" +import type { Database } from "@sqlite.org/sqlite-wasm" + +import { + ensureOpfsSAHPoolDatabase, + type OpfsSAHPoolMaterializeOptions, + type SqliteWasmSAHPoolUtil, +} from "./opfs-sahpool.js" +import { createSqliteWasmExecutor } from "./sqlite-wasm-executor.js" +import { installMojidataSqliteWasmFunctions } from "./sqlite-wasm-functions.js" + +export type DatabaseOpener = () => Promise | Database + +export function createSqliteWasmMojidataDbProvider(openDatabase: DatabaseOpener) { + let dbPromise: Promise | undefined + return function getMojidataDb(): Promise { + dbPromise ??= Promise.resolve(openDatabase()).then((db) => { + installMojidataSqliteWasmFunctions(db) + return createSqliteWasmExecutor(db) + }) + return dbPromise + } +} + +export function createSqliteWasmExecutorProvider(openDatabase: DatabaseOpener) { + let dbPromise: Promise | undefined + return function getDb(): Promise { + dbPromise ??= Promise.resolve(openDatabase()).then((db) => createSqliteWasmExecutor(db)) + return dbPromise + } +} + +export function createSqliteWasmDb({ + getMojidataDb, + getIdsfindDb, +}: { + getMojidataDb: () => Promise + getIdsfindDb: () => Promise +}): MojidataApiDb { + return createSqlApiDb({ getMojidataDb, getIdsfindDb }) +} + +export function openOpfsSAHPoolDatabase( + poolUtil: SqliteWasmSAHPoolUtil, + name: string, + flags = "r", +): Database { + const db = new poolUtil.OpfsSAHPoolDb(name, flags) + db.exec("PRAGMA temp_store=memory") + return db +} + +export type CreateSqliteWasmDbFromOpfsSAHPoolOptions = { + poolUtil: SqliteWasmSAHPoolUtil + mojidata: OpfsSAHPoolMaterializeOptions + idsfind: OpfsSAHPoolMaterializeOptions +} + +export async function createSqliteWasmDbFromOpfsSAHPool({ + poolUtil, + mojidata, + idsfind, +}: CreateSqliteWasmDbFromOpfsSAHPoolOptions): Promise { + return createSqliteWasmDb({ + getMojidataDb: createSqliteWasmMojidataDbProvider(async () => { + await ensureOpfsSAHPoolDatabase(poolUtil, mojidata) + return openOpfsSAHPoolDatabase(poolUtil, mojidata.name) + }), + getIdsfindDb: createSqliteWasmExecutorProvider(async () => { + await ensureOpfsSAHPoolDatabase(poolUtil, idsfind) + return openOpfsSAHPoolDatabase(poolUtil, idsfind.name) + }), + }) +} diff --git a/packages/mojidata-api-sqlite-wasm/package.json b/packages/mojidata-api-sqlite-wasm/package.json new file mode 100644 index 0000000..9e63686 --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/package.json @@ -0,0 +1,55 @@ +{ + "name": "@mandel59/mojidata-api-sqlite-wasm", + "version": "1.8.0", + "description": "SQLite wasm OPFS integration package for mojidata-api", + "type": "module", + "repository": { + "type": "git", + "url": "https://github.com/mandel59/mojidata.git", + "directory": "packages/mojidata-api-sqlite-wasm" + }, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + }, + "./browser-worker": { + "types": "./dist/browser-worker.d.ts", + "import": "./dist/browser-worker.js", + "default": "./dist/browser-worker.js" + }, + "./lib/*": { + "types": "./dist/lib/*.d.ts", + "import": "./dist/lib/*.js", + "default": "./dist/lib/*.js" + } + }, + "files": [ + "dist", + "CHANGELOG.md", + "README.md", + "LICENSE.md" + ], + "scripts": { + "prepare": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && ../../.yarn/sdks/typescript/bin/tsc", + "prepare:test-deps": "(cd ../mojidata-api-core && ../../.yarn/sdks/typescript/bin/tsc) && (cd ../mojidata-api-runtime && node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && ../../.yarn/sdks/typescript/bin/tsc)", + "test": "yarn run prepare:test-deps && yarn prepare && node --import tsx --test ./tests/*.test.ts", + "test:ci": "yarn run prepare:test-deps && yarn prepare && node --import tsx --test ./tests/*.test.ts", + "publish": "yarn prepare && yarn npm publish --access public" + }, + "author": "Ryusei Yamaguchi", + "license": "MIT", + "dependencies": { + "@mandel59/mojidata-api-core": "^1.8.0", + "@mandel59/mojidata-api-runtime": "^1.8.0", + "@sqlite.org/sqlite-wasm": "^3.53.0-build1" + }, + "devDependencies": { + "@types/node": "^24.10.4", + "tsx": "^4.21.0", + "typescript": "^5.9.3" + } +} diff --git a/packages/mojidata-api-sqlite-wasm/tests/sqlite-wasm-executor.test.ts b/packages/mojidata-api-sqlite-wasm/tests/sqlite-wasm-executor.test.ts new file mode 100644 index 0000000..0818b18 --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/tests/sqlite-wasm-executor.test.ts @@ -0,0 +1,146 @@ +import assert from "node:assert/strict" +import { describe, test } from "node:test" + +import sqlite3InitModule from "@sqlite.org/sqlite-wasm" + +import { + createSqliteWasmExecutor, + createSqliteWasmDbFromOpfsSAHPool, + installMojidataSqliteWasmFunctions, + isOpfsSAHPoolSupported, + openOpfsSAHPoolDatabase, + tryEnsureOpfsSAHPoolDatabase, + tryInstallOpfsSAHPool, + type SqliteWasmSAHPoolUtil, +} from "../index.js" + +describe("createSqliteWasmExecutor", () => { + test("supports positional and named parameters", async () => { + const sqlite3 = await sqlite3InitModule() + const db = new sqlite3.oo1.DB(":memory:") + try { + db.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + db.exec("INSERT INTO items (value) VALUES ('alpha'), ('beta')") + + const executor = createSqliteWasmExecutor(db) + const rows = await executor.query<{ value?: string }>( + "SELECT value FROM items WHERE id >= ? ORDER BY id", + [1], + ) + const row = await executor.queryOne<{ value?: string }>( + "SELECT value FROM items WHERE id = $id", + { $id: 2 }, + ) + + assert.deepEqual(rows, [{ value: "alpha" }, { value: "beta" }]) + assert.deepEqual(row, { value: "beta" }) + } finally { + db.close() + } + }) + + test("returns null when queryOne has no rows", async () => { + const sqlite3 = await sqlite3InitModule() + const db = new sqlite3.oo1.DB(":memory:") + try { + const executor = createSqliteWasmExecutor(db) + assert.equal(await executor.queryOne("SELECT 1 WHERE 0"), null) + } finally { + db.close() + } + }) +}) + +describe("installMojidataSqliteWasmFunctions", () => { + test("registers mojidata SQL functions", async () => { + const sqlite3 = await sqlite3InitModule() + const db = new sqlite3.oo1.DB(":memory:") + try { + installMojidataSqliteWasmFunctions(db) + const executor = createSqliteWasmExecutor(db) + const row = await executor.queryOne<{ + regexp?: number + parse_int?: number + regexp_all?: string + }>( + "SELECT regexp('[0-9]+', 'abc123') AS regexp, parse_int('10', 16) AS parse_int, regexp_all('a1b2', '[0-9]') AS regexp_all", + ) + + assert.equal(row?.regexp, 1) + assert.equal(row?.parse_int, 16) + assert.match(row?.regexp_all ?? "", /"substr":"1"/) + assert.match(row?.regexp_all ?? "", /"substr":"2"/) + } finally { + db.close() + } + }) +}) + +describe("OPFS fallback helpers", () => { + test("does not materialize OPFS databases during DB initialization", async () => { + const poolUtil = { + getFileNames() { + throw new Error("database should not be materialized during initialization") + }, + OpfsSAHPoolDb: class {}, + } as unknown as SqliteWasmSAHPoolUtil + + await assert.doesNotReject(() => + createSqliteWasmDbFromOpfsSAHPool({ + poolUtil, + mojidata: { + name: "/mojidata/moji.db", + assetUrl: "https://example.test/moji.db", + assetVersion: "moji-db-v1", + }, + idsfind: { + name: "/mojidata/idsfind.db", + assetUrl: "https://example.test/idsfind.db", + assetVersion: "idsfind-db-v1", + }, + }), + ) + }) + + test("configures OPFS databases to keep temporary storage in memory", () => { + const execCalls: string[] = [] + const poolUtil = { + OpfsSAHPoolDb: class { + constructor( + readonly filename: string, + readonly flags?: string, + ) {} + + exec(sql: string) { + execCalls.push(sql) + } + }, + } as unknown as SqliteWasmSAHPoolUtil + + openOpfsSAHPoolDatabase(poolUtil, "/mojidata/moji.db") + + assert.deepEqual(execCalls, ["PRAGMA temp_store=memory"]) + }) + + test("reports unsupported OPFS contexts without throwing", async () => { + assert.equal(isOpfsSAHPoolSupported(), false) + const sqlite3 = await sqlite3InitModule() + + const installResult = await tryInstallOpfsSAHPool(sqlite3) + assert.equal(installResult.ok, false) + if (!installResult.ok) { + assert.equal(installResult.reason, "unsupported") + } + + const result = await tryEnsureOpfsSAHPoolDatabase({} as SqliteWasmSAHPoolUtil, { + name: "/mojidata/moji.db", + assetUrl: "https://example.test/moji.db", + assetVersion: "test", + }) + + assert.equal(result.ok, false) + if (!result.ok) { + assert.equal(result.reason, "unsupported") + } + }) +}) diff --git a/packages/mojidata-api-sqlite-wasm/tsconfig.json b/packages/mojidata-api-sqlite-wasm/tsconfig.json new file mode 100644 index 0000000..dfc7dd2 --- /dev/null +++ b/packages/mojidata-api-sqlite-wasm/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "strict": true, + "esModuleInterop": true, + "declaration": true, + "outDir": "dist", + "rootDir": ".", + "skipLibCheck": true, + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022", "DOM"] + }, + "exclude": ["dist", "tests"] +} diff --git a/packages/mojidata-api/README.md b/packages/mojidata-api/README.md index bc997c9..4104859 100644 --- a/packages/mojidata-api/README.md +++ b/packages/mojidata-api/README.md @@ -13,6 +13,7 @@ It also has split packages for composing the API yourself: - `@mandel59/mojidata-api-hono`: Hono app wiring - `@mandel59/mojidata-api-runtime`: worker client helpers and shared runtime utilities - `@mandel59/mojidata-api-sqljs`: explicit `sql.js` backend package +- `@mandel59/mojidata-api-sqlite-wasm`: browser SQLite wasm OPFS backend package The original `@mandel59/mojidata-api/*` subpath entrypoints remain available as compatibility facades and forward to the split packages. @@ -25,6 +26,8 @@ const db = createNodeDb() ``` For new code that selects the backend explicitly, prefer `@mandel59/mojidata-api-sqljs`. +For browser SPAs that need persistent OPFS storage, use +`@mandel59/mojidata-api-sqlite-wasm` explicitly. Native Node backends are published separately so installing `@mandel59/mojidata-api` does not pull native SQLite concerns into the default path: diff --git a/yarn.lock b/yarn.lock index 9eea0d8..20a3fbc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -945,6 +945,19 @@ __metadata: languageName: unknown linkType: soft +"@mandel59/mojidata-api-sqlite-wasm@workspace:packages/mojidata-api-sqlite-wasm": + version: 0.0.0-use.local + resolution: "@mandel59/mojidata-api-sqlite-wasm@workspace:packages/mojidata-api-sqlite-wasm" + dependencies: + "@mandel59/mojidata-api-core": "npm:^1.8.0" + "@mandel59/mojidata-api-runtime": "npm:^1.8.0" + "@sqlite.org/sqlite-wasm": "npm:^3.53.0-build1" + "@types/node": "npm:^24.10.4" + tsx: "npm:^4.21.0" + typescript: "npm:^5.9.3" + languageName: unknown + linkType: soft + "@mandel59/mojidata-api-sqljs@npm:^1.7.0, @mandel59/mojidata-api-sqljs@npm:^1.8.0, @mandel59/mojidata-api-sqljs@workspace:packages/mojidata-api-sqljs": version: 0.0.0-use.local resolution: "@mandel59/mojidata-api-sqljs@workspace:packages/mojidata-api-sqljs" @@ -1302,6 +1315,13 @@ __metadata: languageName: node linkType: hard +"@sqlite.org/sqlite-wasm@npm:^3.53.0-build1": + version: 3.53.0-build1 + resolution: "@sqlite.org/sqlite-wasm@npm:3.53.0-build1" + checksum: 10/7e8aedffb6dbb5790c681728b62febbd918eba9094c4eb5b5e5d4ac965a31b5cc371a1e0fe9eab48e93b23c09ae0c23b198667de271858a695e9b4aca145105a + languageName: node + linkType: hard + "@szmarczak/http-timer@npm:^1.1.2": version: 1.1.2 resolution: "@szmarczak/http-timer@npm:1.1.2"