Resolve Mai-Fura backlog: PII decisions persistence (#274), per-route body size (#267), AppModule decomposition (#256)

app/ai-service/api/decorators.py

@@ -0,0 +1,45 @@
+"""Route decorators for the AI service.
+
+This module currently exposes ``with_body_size`` so individual routes
+can declare a larger body-size cap than the service-wide default
+(``MAX_REQUEST_BODY_BYTES``/10 MiB).  See the ``MaxRequestBodySizeMiddleware``
+implementation in ``main.py`` for the matching lookup.
+"""
+
+from __future__ import annotations
+
+from typing import Callable, TypeVar
+
+F = TypeVar("F", bound=Callable[..., object])
+
+
+def with_body_size(max_bytes: int) -> Callable[[F], F]:
+    """Decorator: mark a route handler with a custom body-size limit.
+
+    Usage::
+
+        from api.decorators import with_body_size
+
+        @router.post("/ai/upload-large")
+        @with_body_size(64 * 1024 * 1024)  # 64 MiB
+        async def upload_large(...): ...
+
+    The annotation is stored on ``func._max_body_size`` and read by
+    ``main.py::lifespan`` while walking ``app.routes``.  Decorators
+    return the original function untouched so FastAPI's introspection
+    continues to see ``endpoint`` as a regular async callable.
+    """
+    if max_bytes is None or max_bytes <= 0:
+        raise ValueError(
+            "with_body_size requires a positive integer byte limit; "
+            "use 0 only to disable globally, not for a single route."
+        )
+
+    def decorator(func: F) -> F:
+        # FastAPI keeps a reference to the original callable via
+        # ``route.endpoint``.  Setting the attribute on the wrapped
+        # function (which is what the route stores) is enough.
+        func._max_body_size = int(max_bytes)  # type: ignore[attr-defined]
+        return func
+
+    return decorator

app/ai-service/api/v1/anonymize.py

@@ -1,20 +1,75 @@
 """
 v1 anonymization endpoint.
+
+Records aggregate audit metadata (counts only, never raw or anonymized
+text) for every successful anonymization when ``pii_decisions_enabled``
+is true.  See ``persistence/pii_decisions.py`` for the storage layer.
 """
 
+import hashlib
 import logging
+import time
 
-from fastapi import APIRouter, HTTPException
+from fastapi import APIRouter, BackgroundTasks, HTTPException
 
 from schemas.anonymization import AnonymizeRequest, AnonymizeResponse
+from persistence.pii_decisions import PIIDecisionRecord, new_record_id
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter(tags=["anonymization"])
 
 
+def _text_fingerprint(text: str) -> str:
+    """SHA-256 hex digest so duplicate redact audits can be detected
+    without ever rehydrating the source text."""
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _record_pii_decision(
+    text: str,
+    result: dict,
+    model_version: str | None,
+) -> None:
+    """BackgroundTasks callback: persist aggregate audit metadata.
+
+    Reads ``settings`` lazily inside the worker to avoid import cycles
+    and to apply the latest config at the moment of write.
+    """
+    try:
+        from config import settings
+        if not settings.pii_decisions_enabled:
+            return
+        # Late import keeps the persistence module out of the request
+        # path so import failures don't break the anonymize endpoint.
+        from persistence.pii_decisions import PIIDecisionStore
+
+        record = PIIDecisionRecord(
+            id=new_record_id(),
+            created_at=time.time(),
+            original_length=result.get("original_length", len(text)),
+            pii_summary=result.get("pii_summary", {}),
+            token_counts=result.get("token_counts", {}),
+            text_fingerprint=_text_fingerprint(text),
+            model_version=model_version,
+        )
+        store = PIIDecisionStore(settings.pii_decisions_db_path)
+        store.save_decision(record, settings.pii_decisions_retention_days)
+        logger.info(
+            "stored pii_decision id=%s total=%d",
+            record.id,
+            record.pii_summary.get("total", 0),
+        )
+    except Exception as exc:  # pragma: no cover - defensive
+        # Never break the request because of audit-log failure.
+        logger.error("pii_decision persistence failed: %s", exc)
+
+
 @router.post("/ai/anonymize", response_model=AnonymizeResponse)
-async def anonymize_text(request: AnonymizeRequest):
+async def anonymize_text(
+    request: AnonymizeRequest,
+    background_tasks: BackgroundTasks,
+):
     """Anonymize names, locations, and dates before text is sent to external LLMs."""
     import main as _main
 
@@ -25,8 +80,22 @@ async def anonymize_text(request: AnonymizeRequest):
         from config import settings
         active_p = settings.get_active_provider()
         model_version = settings.groq_model if active_p == "groq" else settings.openai_model
+
+        # Aggregate-only audit record; never touches raw or
+        # anonymized text.  The fingerprint is enough to detect
+        # duplicate scrub jobs without a privacy leak.  Skip the
+        # BackgroundTasks dispatch entirely when persistence is
+        # disabled so the default configuration incurs no per-request
+        # overhead.
+        if settings.pii_decisions_enabled:
+            background_tasks.add_task(
+                _record_pii_decision,
+                request.text,
+                result,
+                model_version,
+            )
+
         return AnonymizeResponse(success=True, model_version=model_version, **result)
     except Exception as e:
         logger.error(f"Anonymization failed: {str(e)}", exc_info=True)
         raise HTTPException(status_code=500, detail="Failed to anonymize text")
-

app/ai-service/api/v1/pii_decisions.py

@@ -0,0 +1,90 @@
+"""
+Auditor search endpoint for PII decision records.
+
+Returns only aggregate metadata — never raw or anonymized text — so
+operators can answer "we redacted N PII entities at time T in a
+document of length L" without rehydrating sensitive content.
+
+Pair with the periodic retention sweep started in
+``main.py::lifespan`` to keep the underlying SQLite table bounded.
+"""
+
+import logging
+from datetime import datetime
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException, Query
+
+from schemas.anonymization import (
+    PIIDecisionSummary,
+    PIIDecisionRecord,
+    PIIDecisionsResponse,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["pii-decisions"])
+
+# Cap page size so a misconfigured caller cannot exhaust the DB in a
+# single request.  500 rows of metadata is still a tiny working set.
+MAX_PAGE_SIZE = 500
+DEFAULT_PAGE_SIZE = 100
+
+
+@router.get("/ai/pii-decisions", response_model=PIIDecisionsResponse)
+async def list_pii_decisions(
+    limit: int = Query(
+        DEFAULT_PAGE_SIZE,
+        ge=1,
+        le=MAX_PAGE_SIZE,
+        description="Maximum number of decision records to return (newest first).",
+    ),
+) -> PIIDecisionsResponse:
+    """Return recent PII decision records for auditor review."""
+    from config import settings
+    from persistence.pii_decisions import PIIDecisionStore
+
+    if not settings.pii_decisions_enabled:
+        raise HTTPException(
+            status_code=404,
+            detail={
+                "code": "pii_decisions_disabled",
+                "message": (
+                    "PII decisions persistence is disabled; set "
+                    "PII_DECISIONS_ENABLED=true to enable auditing."
+                ),
+            },
+        )
+
+    try:
+        store = PIIDecisionStore(settings.pii_decisions_db_path)
+        rows = store.get_recent_decisions(limit=limit)
+    except Exception as exc:
+        logger.error("pii_decisions lookup failed: %s", exc)
+        raise HTTPException(
+            status_code=500,
+            detail={
+                "code": "pii_decisions_lookup_failed",
+                "message": "Failed to query PII decisions store",
+            },
+        )
+
+    decisions = []
+    for r in rows:
+        decisions.append(
+            PIIDecisionRecord(
+                id=r["id"],
+                created_at=datetime.fromtimestamp(r["created_at"]),
+                original_length=r["original_length"],
+                pii_summary=PIIDecisionSummary(**r["pii_summary"]),
+                token_counts=r.get("token_counts", {}),
+                text_fingerprint=r["text_fingerprint"],
+                model_version=r.get("model_version"),
+            )
+        )
+
+    return PIIDecisionsResponse(
+        success=True,
+        count=len(decisions),
+        decisions=decisions,
+    )

app/ai-service/api/v1/router.py

@@ -16,6 +16,8 @@
     humanitarian,
     fraud,
     artifacts,
+    pii_decisions,
+    upload_large,
 )
 
 v1_router = APIRouter(prefix="/v1")
@@ -27,3 +29,7 @@
 v1_router.include_router(humanitarian.router)
 v1_router.include_router(fraud.router)
 v1_router.include_router(artifacts.router)
+# Auditor endpoint for PII decision records (Issue #274).
+v1_router.include_router(pii_decisions.router)
+# Large-payload ingest with per-route 64 MiB cap (Issue #267).
+v1_router.include_router(upload_large.router)

app/ai-service/api/v1/upload_large.py

@@ -0,0 +1,63 @@
+"""
+v1 upload-large endpoint — accepts up to 64 MiB requests.
+
+Issue #267: ``MaxRequestBodySizeMiddleware`` defaults to the
+service-wide ``MAX_REQUEST_BODY_BYTES`` (10 MiB) but per-route overrides
+are required for routes like this one that handle genomic evidence or
+large OCR pre-flight payloads.  The ``@with_body_size(...)`` marker is
+read by ``main.py::lifespan`` and converted into a per-route regex
+lookup so an oversized body on this route is rejected with 413 only
+once the actual cap is exceeded; the default 10 MiB cap stays in force
+for every other POST.
+"""
+
+import logging
+from typing import Annotated
+
+from fastapi import APIRouter, File, HTTPException, Request, UploadFile
+
+from api.decorators import with_body_size
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["upload"])
+
+# 64 MiB per the acceptance criteria of issue #267.  Stored as a
+# module-level constant so the rate-limit middleware, observability,
+# and tests all read the same value.
+MAX_UPLOAD_BYTES = 64 * 1024 * 1024
+
+
+@router.post("/ai/upload-large")
+@with_body_size(MAX_UPLOAD_BYTES)
+async def upload_large(
+    request: Request,
+    file: Annotated[UploadFile, File(description="Large evidence or document blob")],
+):
+    """Accept an upload of up to 64 MiB.  Larger uploads are rejected
+    with HTTP 413 by ``MaxRequestBodySizeMiddleware``."""
+    contents = await file.read()
+    size = len(contents)
+
+    if size == 0:
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "code": "empty_payload",
+                "message": "Uploaded payload is empty",
+            },
+        )
+
+    logger.info(
+        "upload-large accepted filename=%s size=%d limit=%d",
+        file.filename or "<unnamed>",
+        size,
+        MAX_UPLOAD_BYTES,
+    )
+
+    return {
+        "success": True,
+        "filename": file.filename,
+        "size": size,
+        "limit_bytes": MAX_UPLOAD_BYTES,
+    }

app/ai-service/config.py

@@ -43,6 +43,17 @@ class Settings(BaseSettings):
             '/hooks/') match any path with that prefix. The built-in
             infrastructure defaults (/health, /, /ai/metrics, /docs,
             /redoc, /openapi.json) are always merged in.
+        PII_DECISIONS_ENABLED: When truthy, every successful anonymize()
+            call writes an aggregate audit record to the PII decisions
+            store. Default: false (opt-in so routine tests don't have to
+            clean up rows).
+        PII_DECISIONS_DB_PATH: Filesystem path to the SQLite file backing
+            the PII decisions store. Parent directories are created on
+            first write. Default: ./data/pii_decisions.db.
+        PII_DECISIONS_RETENTION_DAYS: Number of days a decision record
+            survives before the periodic sweeper removes it. Default: 30.
+        PII_DECISIONS_SWEEP_INTERVAL_SECONDS: How often the in-process
+            retention sweep runs. Default: 3600 (1 hour).
     """
 
     # API Keys
@@ -87,6 +98,14 @@ class Settings(BaseSettings):
     # always appended so operators cannot accidentally expose themselves.
     request_body_bypass_paths: str = ""
 
+    # PII decisions persistence (Issue #274). Disabled by default so that
+    # the store is opt-in for tests; the runtime behaviour of
+    # /v1/ai/anonymize is unchanged when disabled.
+    pii_decisions_enabled: bool = False
+    pii_decisions_db_path: str = "./data/pii_decisions.db"
+    pii_decisions_retention_days: int = 30
+    pii_decisions_sweep_interval_seconds: int = 3600
+
     # Legacy route deprecation/retirement date (Sunset header)
     legacy_retirement_date: str = "Wed, 01 Oct 2026 00:00:00 GMT"