feat: deployable Helm chart + analytics-agent bootstrap CLI

.github/workflows/helm.yml

@@ -0,0 +1,124 @@
+name: Helm chart
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "helm/**"
+      - "docker/Dockerfile"
+  pull_request:
+    branches: [main]
+    paths:
+      - "helm/**"
+      - "docker/Dockerfile"
+
+jobs:
+  helm-chart:
+    name: Lint & validate
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: v3.18.0
+
+      - name: Lint chart
+        run: helm lint helm/analytics-agent/
+
+      - name: Render templates
+        run: |
+          helm template test-release helm/analytics-agent/ \
+            --set image.repository=analytics-agent \
+            --set image.tag=test \
+            > /tmp/rendered.yaml
+
+      - name: Verify port consistency (Dockerfile EXPOSE vs Helm targetPort)
+        run: |
+          DOCKERFILE_PORT=$(grep '^EXPOSE' docker/Dockerfile | awk '{print $2}')
+          HELM_PORT=$(grep 'targetPort:' helm/analytics-agent/values.yaml | awk '{print $2}')
+          echo "Dockerfile EXPOSE: $DOCKERFILE_PORT"
+          echo "Helm targetPort:   $HELM_PORT"
+          [ "$DOCKERFILE_PORT" = "$HELM_PORT" ] || \
+            { echo "ERROR: port mismatch — update helm/analytics-agent/values.yaml"; exit 1; }
+
+      - name: Verify hook ordering (SA and Secret weight < bootstrap Job weight)
+        run: |
+          pip install -q pyyaml
+          python3 - <<'PYEOF'
+          import sys, yaml
+
+          docs = list(yaml.safe_load_all(open("/tmp/rendered.yaml")))
+          hooks = {}
+          for doc in docs:
+              if not doc:
+                  continue
+              ann = (doc.get("metadata") or {}).get("annotations") or {}
+              if "pre-install" in ann.get("helm.sh/hook", ""):
+                  hooks[doc["kind"]] = int(ann.get("helm.sh/hook-weight", 0))
+
+          print("Hook weights:", hooks)
+          sa, secret, job = hooks.get("ServiceAccount"), hooks.get("Secret"), hooks.get("Job")
+          assert sa is not None, "ServiceAccount must be a hook"
+          assert secret is not None, "Secret must be a hook"
+          assert job is not None, "bootstrap Job must be a hook"
+          assert sa < job,     f"ServiceAccount weight ({sa}) must be < Job ({job})"
+          assert secret < job, f"Secret weight ({secret}) must be < Job ({job})"
+          print("Hook ordering OK")
+          PYEOF
+
+  helm-install:
+    name: Integration test — helm install
+    runs-on: ubuntu-latest
+    needs: helm-chart
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Create kind cluster
+        uses: helm/kind-action@v1
+        with:
+          cluster_name: kind
+          wait: 60s
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: v3.18.0
+
+      - name: Build image
+        run: docker build -f docker/Dockerfile -t analytics-agent:test .
+
+      - name: Load image into kind
+        run: kind load docker-image analytics-agent:test
+
+      - name: Start Postgres
+        run: |
+          kubectl run postgres \
+            --image=postgres:15 \
+            --env=POSTGRES_PASSWORD=secret \
+            --env=POSTGRES_DB=analytics \
+            --env=POSTGRES_USER=analytics
+          kubectl expose pod postgres --port=5432
+          kubectl wait --for=condition=Ready pod/postgres --timeout=60s
+
+      - name: Helm install
+        run: |
+          helm install aa-test helm/analytics-agent/ \
+            --set image.repository=analytics-agent \
+            --set image.tag=test \
+            --set image.pullPolicy=Never \
+            --set "config.env.DATABASE_URL=postgresql+asyncpg://analytics:secret@postgres:5432/analytics" \
+            --set "config.env.ENCRYPTION_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa=" \
+            --timeout=120s
+
+      - name: Wait for deployment rollout
+        run: kubectl rollout status deployment/aa-test-analytics-agent --timeout=120s
+
+      - name: Verify /health endpoint
+        run: |
+          kubectl port-forward svc/aa-test-analytics-agent 8080:80 &
+          sleep 3
+          curl -sf http://localhost:8080/health
+          echo ""
+          curl -sf http://localhost:8080/api/engines

AGENTS.md

@@ -52,15 +52,17 @@ cd frontend && pnpm dev
 
 **DataHub credentials**: run `datahub init --sso --host https://your-instance.acryl.io/gms` once. The app reads `~/.datahubenv` automatically; or set `DATAHUB_GMS_URL` + `DATAHUB_GMS_TOKEN` in `config.yaml` / `.env`.
 
-**Database**: SQLite at `./data/dev.db` by default. Alembic runs automatically on startup. For Postgres set `DATABASE_URL=postgresql+asyncpg://...`.
+**Database**: SQLite at `./data/dev.db` by default. For Postgres set `DATABASE_URL=postgresql+asyncpg://...`.
+
+**Bootstrap (migrations + seed)**: The FastAPI lifespan no longer runs migrations or seeds — it does read-only initialization (loading engines from the DB, propagating env vars, validating the encryption key). All DB-mutating bootstrap work lives in `analytics_agent.bootstrap` and is invoked via the CLI: `analytics-agent bootstrap` runs Alembic migrate → seed-integrations → seed-context-platforms → seed-defaults, idempotently. Run it before the first `uvicorn` start and after any release that adds migrations. The Helm chart runs it automatically as a `pre-install`/`pre-upgrade` hook.
 
 ---
 
 ## Key file map
 
 | Path | What it does |
 |------|-------------|
-| `backend/src/analytics_agent/main.py` | FastAPI app factory + lifespan (runs Alembic, seeds integrations, mounts SPA) |
+| `backend/src/analytics_agent/main.py` | FastAPI app factory + lifespan (read-only init: loads engines, validates encryption key, mounts SPA — no migrations/seeds) |
 | `backend/src/analytics_agent/agent/graph.py` | LangGraph `StateGraph`: ReAct agent → conditional chart node |
 | `backend/src/analytics_agent/agent/streaming.py` | `astream_events` → SSE event dicts; handles `on_tool_error` |
 | `backend/src/analytics_agent/agent/history.py` | Reconstructs LangChain message history from DB rows; pads orphaned tool calls |

CLAUDE.md

@@ -34,11 +34,13 @@ cd frontend && pnpm tsc --noEmit   # type-check without building
 
 ### Restarting the backend
 
-When `.env`, `config.yaml`, or Python source changes, the backend needs a restart:
+When `.env`, `config.yaml`, or Python source changes, the backend needs a restart. Run `analytics-agent bootstrap` whenever `config.yaml` changes or after pulling a release that adds migrations.
 ```bash
 pkill -f "analytics_agent.main"; sleep 2
 set -a && source .env && set +a
-nohup uv run uvicorn analytics_agent.main:app --port 8100 > /tmp/analytics_agent.log 2>&1 &
+uv run analytics-agent bootstrap   # apply migrations + seed config.yaml
+nohup uv run uvicorn analytics_agent.main:app --port 8100 \
+  > /tmp/analytics_agent.log 2>&1 &
 sleep 5 && curl -s http://localhost:8100/api/engines
 ```
 

README.md

@@ -85,6 +85,18 @@ Open **http://localhost:8100** — a setup wizard handles the LLM key and connec
 
 > **Without `just`:** `uv sync && cd frontend && pnpm install && pnpm build && cd .. && uv run uvicorn analytics_agent.main:app --port 8100`
 
+### First-time setup
+
+Before the first `uvicorn` start (or after pulling a release that adds migrations), run:
+
+```bash
+uv run analytics-agent bootstrap
+```
+
+This applies Alembic migrations, seeds engines and context platforms from `config.yaml`, and writes first-run setting defaults. The command is idempotent — re-running it on an up-to-date database is a no-op.
+
+For Kubernetes deployments, the Helm chart runs `analytics-agent bootstrap` automatically as a `pre-install`/`pre-upgrade` hook (see `helm/analytics-agent/README.md`).
+
 ### Optional: pre-configure via `.env`
 
 ```bash

backend/src/analytics_agent/bootstrap.py

@@ -0,0 +1,218 @@
+"""DB-mutating bootstrap functions.
+
+Pure async helpers, no FastAPI coupling. Each is independently callable, idempotent,
+and intended to be invoked from the analytics-agent CLI (typically via a Helm
+pre-install/pre-upgrade hook). All write logic that used to live inside the
+FastAPI lifespan now lives here.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+
+from analytics_agent.config import settings
+
+
+def run_migrations() -> None:
+    """Run Alembic migrations synchronously (Alembic is a sync tool)."""
+    from alembic import command
+    from alembic.config import Config
+
+    if "sqlite" in settings.database_url:
+        db_path = settings.database_url.replace("sqlite+aiosqlite:///", "")
+        Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+    elif "mysql" in settings.database_url:
+        _ensure_mysql_schema()
+
+    alembic_cfg = Config("alembic.ini")
+    command.upgrade(alembic_cfg, "head")
+
+
+def _ensure_mysql_schema() -> None:
+    """Create the analytics_agent MySQL schema if it doesn't exist."""
+    import pymysql
+
+    match = re.search(r"/([^/?]+)(\?|$)", settings.database_url)
+    if not match:
+        return
+    schema = match.group(1)
+
+    url_no_schema = re.sub(r"mysql\+aiomysql://", "", settings.database_url)
+    creds_match = re.match(r"([^:]+):([^@]+)@([^:/]+):?(\d+)?", url_no_schema)
+    if not creds_match:
+        return
+    user, password, host, port = creds_match.groups()
+
+    try:
+        conn = pymysql.connect(
+            host=host,
+            port=int(port or 3306),
+            user=user,
+            password=password,
+            connect_timeout=5,
+        )
+        with conn.cursor() as cur:
+            cur.execute(
+                f"CREATE SCHEMA IF NOT EXISTS `{schema}` "
+                "DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
+            )
+        conn.commit()
+        conn.close()
+    except Exception as exc:
+        logging.getLogger(__name__).warning("Could not ensure MySQL schema '%s': %s", schema, exc)
+
+
+async def seed_integrations_from_yaml() -> None:
+    """Upsert config.yaml engines into the integrations table.
+
+    Skips rows whose ``source == "ui"`` (the user has edited them via Settings UI).
+    Removes yaml-source rows no longer in config. Migrates any legacy
+    ``snowflake_oauth:*`` entries from the settings table to credentials.
+    Does NOT register engines in the in-memory factory — that is done at pod
+    boot by ``main.register_engines_from_db()``.
+    """
+    import uuid
+
+    import orjson
+
+    from analytics_agent.db.base import _get_session_factory
+    from analytics_agent.db.repository import (
+        CredentialRepo,
+        IntegrationRepo,
+        SettingsRepo,
+    )
+
+    logger = logging.getLogger(__name__)
+    factory = _get_session_factory()
+    async with factory() as session:
+        integration_repo = IntegrationRepo(session)
+        cred_repo = CredentialRepo(session)
+        settings_repo = SettingsRepo(session)
+
+        config_engine_names = {cfg.effective_name for cfg in settings.load_engines_config()}
+        for cfg in settings.load_engines_config():
+            engine_type = cfg.type
+            engine_name = cfg.effective_name
+            connection = cfg.connection
+            label = f"{engine_type.capitalize()} ({engine_name})"
+            existing = await integration_repo.get(engine_name)
+            if existing is not None and existing.source == "ui":
+                logger.info(
+                    "Skipping yaml seed for '%s' — user-managed via Settings UI",
+                    engine_name,
+                )
+                continue
+            await integration_repo.upsert(
+                id=str(uuid.uuid5(uuid.NAMESPACE_DNS, f"yaml:{engine_name}")),
+                name=engine_name,
+                type=engine_type,
+                label=label,
+                config=orjson.dumps(connection).decode(),
+                source="yaml",
+            )
+
+        for intg in await integration_repo.list_all():
+            if intg.source == "yaml" and intg.name not in config_engine_names:
+                logger.info(
+                    "Removing stale yaml integration '%s' (no longer in config.yaml)",
+                    intg.name,
+                )
+                await integration_repo.delete(intg.name)
+
+        # Legacy oauth_token migration
+        all_integrations = await integration_repo.list_all()
+        for intg in all_integrations:
+            old_key = f"snowflake_oauth:{intg.name}"
+            raw = await settings_repo.get(old_key)
+            if raw and intg.credential is None:
+                try:
+                    data = orjson.loads(raw)
+                    method = data.get("method", "")
+                    user = data.get("username") or data.get("user", "")
+                    if method == "externalbrowser" and user:
+                        await cred_repo.upsert(
+                            id=str(uuid.uuid4()),
+                            integration_name=intg.name,
+                            auth_type="sso_externalbrowser",
+                            username=user,
+                        )
+                        await settings_repo.delete(old_key)
+                except Exception:
+                    pass
+
+
+async def seed_context_platforms_from_yaml() -> None:
+    """Upsert config.yaml context_platforms into the DB.
+
+    Preserves user-edited credentials and ``_discovered_tools`` metadata for
+    yaml-source rows. Removes yaml-source rows no longer in config. Does NOT
+    propagate DataHub env vars — that is done at pod boot by
+    ``main.propagate_datahub_env()``.
+    """
+    import contextlib
+    import uuid
+
+    import orjson
+
+    from analytics_agent.db.base import _get_session_factory
+    from analytics_agent.db.repository import ContextPlatformRepo
+
+    logger = logging.getLogger(__name__)
+    factory = _get_session_factory()
+    async with factory() as session:
+        repo = ContextPlatformRepo(session)
+
+        config_platform_names = {cfg.name for cfg in settings.load_context_platforms_config()}
+        for cfg in settings.load_context_platforms_config():
+            existing = await repo.get(cfg.name)
+            if existing:
+                changed = False
+                new_label = cfg.label or cfg.type.capitalize()
+                if existing.label != new_label:
+                    existing.label = new_label
+                    changed = True
+                if existing.source == "yaml":
+                    stored: dict = {}
+                    with contextlib.suppress(Exception):
+                        stored = orjson.loads(existing.config)
+                    yaml_cfg_dict = cfg.model_dump()
+                    new_env = yaml_cfg_dict.get("env", {})
+                    if new_env and stored.get("env") != new_env:
+                        stored["env"] = new_env
+                        existing.config = orjson.dumps(stored).decode()
+                        changed = True
+                if changed:
+                    await session.commit()
+            else:
+                await repo.upsert(
+                    id=str(uuid.uuid5(uuid.NAMESPACE_DNS, f"yaml:{cfg.name}")),
+                    type=cfg.type,
+                    name=cfg.name,
+                    label=cfg.label or cfg.type.capitalize(),
+                    config=orjson.dumps(cfg.model_dump()).decode(),
+                    source="yaml",
+                )
+
+        for plat in await repo.list_all():
+            if plat.source == "yaml" and plat.name not in config_platform_names:
+                logger.info("Removing stale yaml context platform '%s'", plat.name)
+                await repo.delete(plat.name)
+
+
+async def seed_default_settings() -> None:
+    """Write first-run defaults to the settings table (no-op if already set)."""
+    import orjson
+
+    from analytics_agent.db.base import _get_session_factory
+    from analytics_agent.db.repository import SettingsRepo
+
+    factory = _get_session_factory()
+    async with factory() as session:
+        repo = SettingsRepo(session)
+        if await repo.get("enabled_mutation_tools") is None:
+            await repo.set(
+                "enabled_mutation_tools",
+                orjson.dumps(["publish_analysis", "save_correction"]).decode(),
+            )