fix(sqlite): report lock contention as retryable

docs/api.md

@@ -242,6 +242,7 @@ parsing the human-readable `message` field.
 | `PAYLOAD_TOO_LARGE` | `413` | No | An `async_mode` request body exceeds `memorize.async_queue.max_payload_bytes`; the batch was not accepted |
 | `QUEUE_SATURATED` | `429` | Yes | The memorize queue is at its per-session or global cap; the batch was not accepted — retry later or keep it client-side |
 | `EXTERNAL_SERVICE_UNAVAILABLE` | `503` | **Yes** | An external service (LLM, embedding, rerank) returned an error or timed out |
+| `DATABASE_BUSY` | `503` | **Yes** | SQLite's write lock was held past `[sqlite].busy_timeout_ms`; the transaction rolled back and the response carries `Retry-After` |
 | `CAPABILITY_UNAVAILABLE` | `503` | No | A required server-side capability is missing (e.g. `everos[multimodal]` extra not installed, LibreOffice absent) — requires admin action, not retry |
 | `CONFIGURATION_ERROR` | `500` | No | A required configuration is missing or invalid (e.g. embedding model not set) |
 | `INTERNAL_ERROR` | `500` | No | Unhandled exception (internal details are logged, never leaked) |

src/everos/core/errors.py

@@ -35,6 +35,7 @@ class ErrorCode(StrEnum):
     BAD_REQUEST = "BAD_REQUEST"
     UNSUPPORTED_FORMAT = "UNSUPPORTED_FORMAT"
     EXTERNAL_SERVICE_UNAVAILABLE = "EXTERNAL_SERVICE_UNAVAILABLE"
+    DATABASE_BUSY = "DATABASE_BUSY"
     CAPABILITY_UNAVAILABLE = "CAPABILITY_UNAVAILABLE"
     CONFIGURATION_ERROR = "CONFIGURATION_ERROR"
     INTERNAL_ERROR = "INTERNAL_ERROR"
@@ -171,6 +172,15 @@ class StorageError(InfrastructureError):
     """A markdown or SQLite persistence operation failed."""
 
 
+class DatabaseBusyError(StorageError):
+    """SQLite's write lock was held past ``[sqlite].busy_timeout_ms``.
+
+    The transaction was rolled back and nothing was committed. This is
+    transient: the caller can retry the same request after the backoff in
+    the API's ``Retry-After`` header.
+    """
+
+
 class VectorStoreError(InfrastructureError):
     """A LanceDB vector-store operation failed."""
 

src/everos/entrypoints/api/exception_handlers.py

@@ -37,7 +37,11 @@ class -> status code, ``ErrorCode``, optional fixed message and optional log
 
 
 def _error_response(
-    request: Request, status_code: int, code: ErrorCode, message: str
+    request: Request,
+    status_code: int,
+    code: ErrorCode,
+    message: str,
+    headers: Mapping[str, str] | None = None,
 ) -> JSONResponse:
     """Build a JSONResponse with the canonical error envelope."""
     body = ErrorResponse(
@@ -49,7 +53,9 @@ def _error_response(
             path=str(request.url.path),
         ),
     )
-    return JSONResponse(status_code=status_code, content=body.model_dump())
+    return JSONResponse(
+        status_code=status_code, content=body.model_dump(), headers=headers
+    )
 
 
 def _exception_fields(exc: Exception) -> Mapping[str, object]:
@@ -72,6 +78,8 @@ class _Rule:
     log: tuple[Literal["warning", "error"], str] | None = None
     """``(level, event)`` of the log line, or ``None`` for no log line."""
     log_fields: Callable[[Exception], Mapping[str, object]] = _exception_fields
+    headers: Mapping[str, str] | None = None
+    """Extra response headers (e.g. ``Retry-After``)."""
 
 
 _ERROR_RULES: Mapping[type[Exception], _Rule] = {
@@ -88,6 +96,15 @@ class _Rule:
     errors.ConflictError: _Rule(409, ErrorCode.CONFLICT),
     errors.ExtractionEmptyError: _Rule(422, ErrorCode.EXTRACTION_EMPTY),
     errors.InvalidInputError: _Rule(422, ErrorCode.INVALID_INPUT),
+    # Transient, retryable: SQLite lock contention has a concrete backoff,
+    # unlike the generic infrastructure bucket.
+    errors.DatabaseBusyError: _Rule(
+        503,
+        ErrorCode.DATABASE_BUSY,
+        message="SQLite is busy; retry the request",
+        log=("warning", "database_busy"),
+        headers={"Retry-After": "1"},
+    ),
     # Transient, retryable.
     errors.InfrastructureError: _Rule(
         503,
@@ -125,7 +142,13 @@ async def handle(request: Request, exc: Exception) -> JSONResponse:
             fields = rule.log_fields(exc)
             getattr(logger, level)(event, path=str(request.url.path), **fields)
         message = str(exc) if rule.message is None else rule.message
-        return _error_response(request, rule.status_code, rule.code, message)
+        return _error_response(
+            request,
+            rule.status_code,
+            rule.code,
+            message,
+            headers=rule.headers,
+        )
 
     return handle
 

src/everos/infra/persistence/sqlite/_base/session.py

@@ -2,11 +2,15 @@
 
 from __future__ import annotations
 
+import sqlite3
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
 
+from sqlalchemy.exc import OperationalError
 from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
 
+from everos.core.errors import DatabaseBusyError
+
 
 def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
     """Build an :class:`async_sessionmaker` bound to ``engine``.
@@ -40,6 +44,24 @@ async def session_scope(
     async with session_factory() as session:
         try:
             yield session
+        except OperationalError as exc:
+            await session.rollback()
+            if _is_locked_error(exc):
+                raise DatabaseBusyError("SQLite is busy; retry the request") from exc
+            raise
         except Exception:
             await session.rollback()
             raise
+
+
+def _is_locked_error(exc: OperationalError) -> bool:
+    """Whether SQLAlchemy's ``OperationalError`` is SQLite lock contention."""
+    orig = getattr(exc, "orig", None)
+    code = getattr(orig, "sqlite_errorcode", None)
+    if isinstance(code, int) and (code & 0xFF) in (
+        sqlite3.SQLITE_BUSY,
+        sqlite3.SQLITE_LOCKED,
+    ):
+        return True
+    message = str(exc).lower()
+    return "database is locked" in message or "database table is locked" in message

tests/integration/test_api/test_sqlite_busy_retry.py

@@ -0,0 +1,83 @@
+"""SQLite lock contention -> 503 DATABASE_BUSY -> retry succeeds.
+
+Drives the production exception handler and the real ``session_scope``
+against a real SQLite file. A raw ``BEGIN IMMEDIATE`` holds the write lock;
+the route's commit fails with ``database is locked`` and must surface as a
+retryable 503 with ``Retry-After``, not as a 500. Releasing the lock makes
+the same request succeed.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+
+from fastapi import FastAPI
+from httpx import ASGITransport, AsyncClient
+from sqlmodel import Field, SQLModel
+
+from everos.config import SqliteSettings
+from everos.entrypoints.api.exception_handlers import register_handlers
+from everos.infra.persistence import MemoryRoot
+from everos.infra.persistence.sqlite import (
+    create_session_factory,
+    create_system_engine,
+    session_scope,
+)
+
+
+class _BusyRow(SQLModel, table=True):
+    __tablename__ = "_busy_retry_row"  # type: ignore[assignment]
+    id: int | None = Field(default=None, primary_key=True)
+    note: str
+
+
+def _make_app(factory: object) -> FastAPI:
+    app = FastAPI()
+    register_handlers(app)
+
+    @app.get("/busy")
+    async def _busy() -> dict[str, bool]:
+        async with session_scope(factory) as session:  # type: ignore[arg-type]
+            session.add(_BusyRow(note="row"))
+            await session.commit()
+        return {"ok": True}
+
+    return app
+
+
+async def test_locked_sqlite_returns_503_then_succeeds(
+    tmp_path: Path,
+) -> None:
+    root = MemoryRoot(tmp_path)
+    root.ensure()
+    engine = create_system_engine(root.system_db, SqliteSettings(busy_timeout_ms=0))
+    factory = create_session_factory(engine)
+    try:
+        async with engine.begin() as conn:
+            await conn.run_sync(SQLModel.metadata.create_all)
+
+        blocker = sqlite3.connect(root.system_db, timeout=0)
+        blocker.execute("BEGIN IMMEDIATE")
+        try:
+            async with AsyncClient(
+                transport=ASGITransport(app=_make_app(factory)),
+                base_url="http://test",
+            ) as client:
+                busy = await client.get("/busy")
+                assert busy.status_code == 503
+                assert busy.json()["error"]["code"] == "DATABASE_BUSY"
+                assert busy.headers["Retry-After"] == "1"
+        finally:
+            blocker.rollback()
+            blocker.close()
+
+        async with AsyncClient(
+            transport=ASGITransport(app=_make_app(factory)),
+            base_url="http://test",
+        ) as client:
+            ok = await client.get("/busy")
+            assert ok.status_code == 200
+            assert ok.json() == {"ok": True}
+    finally:
+        await engine.dispose()

tests/unit/test_entrypoints/test_api/test_exception_handlers.py

@@ -13,6 +13,7 @@
 from httpx import ASGITransport, AsyncClient
 
 from everos.core.errors import (
+    DatabaseBusyError,
     DocumentNotFoundError,
     DuplicateDocumentError,
     EmbeddingServiceError,
@@ -74,6 +75,10 @@ async def _multimodal_not_enabled() -> None:
     async def _storage() -> None:
         raise StorageError("disk full")
 
+    @app.get("/raise/database-busy")
+    async def _database_busy() -> None:
+        raise DatabaseBusyError("SQLite is busy; retry the request")
+
     @app.get("/raise/embedding")
     async def _embedding() -> None:
         raise EmbeddingServiceError("embedding provider timeout")
@@ -231,9 +236,22 @@ async def test_storage_error_returns_503(client: AsyncClient) -> None:
     assert resp.status_code == 503
     data = resp.json()
     assert data["error"]["code"] == "EXTERNAL_SERVICE_UNAVAILABLE"
+    assert "Retry-After" not in resp.headers
     _assert_envelope(data, path="/raise/storage")
 
 
+async def test_database_busy_returns_503_with_retry_after(
+    client: AsyncClient,
+) -> None:
+    """DatabaseBusyError → 503, code=DATABASE_BUSY, Retry-After present."""
+    resp = await client.get("/raise/database-busy")
+    assert resp.status_code == 503
+    assert resp.headers["Retry-After"] == "1"
+    data = resp.json()
+    assert data["error"]["code"] == "DATABASE_BUSY"
+    _assert_envelope(data, path="/raise/database-busy")
+
+
 # ---------------------------------------------------------------------------
 # MRO dispatch tests
 # ---------------------------------------------------------------------------

tests/unit/test_infra/test_sqlite/test_base/test_session.py

@@ -2,13 +2,16 @@
 
 from __future__ import annotations
 
+import sqlite3
 from pathlib import Path
 
 import pytest
 from sqlalchemy import text
+from sqlalchemy.exc import OperationalError
 from sqlmodel import Field, SQLModel
 
 from everos.config import SqliteSettings
+from everos.core.errors import DatabaseBusyError
 from everos.infra.persistence import MemoryRoot
 from everos.infra.persistence.sqlite import (
     create_session_factory,
@@ -76,3 +79,45 @@ async def test_session_scope_rolls_back_on_exception(
             assert count[0] == 0
     finally:
         await engine.dispose()
+
+
+async def test_session_scope_maps_lock_contention_to_database_busy(
+    memory_root: MemoryRoot,
+) -> None:
+    """A real ``BEGIN IMMEDIATE`` lock becomes DatabaseBusyError, not a 500."""
+    engine = create_system_engine(
+        memory_root.system_db, SqliteSettings(busy_timeout_ms=0)
+    )
+    factory = create_session_factory(engine)
+    try:
+        async with engine.begin() as conn:
+            await conn.run_sync(SQLModel.metadata.create_all)
+
+        blocker = sqlite3.connect(memory_root.system_db, timeout=0)
+        blocker.execute("BEGIN IMMEDIATE")
+        try:
+            with pytest.raises(DatabaseBusyError):
+                async with session_scope(factory) as s:
+                    s.add(_Sample(note="blocked"))
+                    await s.commit()
+        finally:
+            blocker.rollback()
+            blocker.close()
+    finally:
+        await engine.dispose()
+
+
+async def test_session_scope_leaves_other_operational_errors_unchanged(
+    memory_root: MemoryRoot,
+) -> None:
+    engine = create_system_engine(memory_root.system_db, SqliteSettings())
+    factory = create_session_factory(engine)
+    try:
+        async with engine.begin() as conn:
+            await conn.run_sync(SQLModel.metadata.create_all)
+
+        with pytest.raises(OperationalError):
+            async with session_scope(factory) as s:
+                await s.execute(text("SELECT * FROM _missing_session_scope_table"))
+    finally:
+        await engine.dispose()