From 3553b02375d8a5a0b301b852804f626283b0ec5f Mon Sep 17 00:00:00 2001 From: craig Date: Mon, 27 Apr 2026 17:48:22 -0700 Subject: [PATCH 01/22] chore: add baseline helm chart and /health endpoint Adds the initial helm/analytics-agent chart (Chart.yaml, deployment, service, ingress, secret-env, serviceaccount) and a /health endpoint in main.py used by the chart's liveness/readiness probes. --- backend/src/analytics_agent/main.py | 4 + helm/analytics-agent/Chart.yaml | 6 ++ helm/analytics-agent/templates/_helpers.tpl | 60 ++++++++++++++ .../analytics-agent/templates/deployment.yaml | 75 +++++++++++++++++ helm/analytics-agent/templates/ingress.yaml | 41 ++++++++++ .../analytics-agent/templates/secret-env.yaml | 11 +++ helm/analytics-agent/templates/service.yaml | 16 ++++ .../templates/serviceaccount.yaml | 13 +++ helm/analytics-agent/values.yaml | 80 +++++++++++++++++++ 9 files changed, 306 insertions(+) create mode 100644 helm/analytics-agent/Chart.yaml create mode 100644 helm/analytics-agent/templates/_helpers.tpl create mode 100644 helm/analytics-agent/templates/deployment.yaml create mode 100644 helm/analytics-agent/templates/ingress.yaml create mode 100644 helm/analytics-agent/templates/secret-env.yaml create mode 100644 helm/analytics-agent/templates/service.yaml create mode 100644 helm/analytics-agent/templates/serviceaccount.yaml create mode 100644 helm/analytics-agent/values.yaml diff --git a/backend/src/analytics_agent/main.py b/backend/src/analytics_agent/main.py index 88f30e7..df15fdc 100644 --- a/backend/src/analytics_agent/main.py +++ b/backend/src/analytics_agent/main.py @@ -517,6 +517,10 @@ def create_app() -> FastAPI: app.include_router(api_router) + @app.get("/health", include_in_schema=False) + async def _health() -> dict[str, str]: + return {"status": "ok"} + setup_tracing(app) # ── Serve built React SPA ────────────────────────────────────────────── diff --git a/helm/analytics-agent/Chart.yaml b/helm/analytics-agent/Chart.yaml new file mode 100644 index 0000000..45c3d68 --- /dev/null +++ b/helm/analytics-agent/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: analytics-agent +description: A Helm chart for the Analytics Agent (FastAPI + React SPA) +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/helm/analytics-agent/templates/_helpers.tpl b/helm/analytics-agent/templates/_helpers.tpl new file mode 100644 index 0000000..03c6dfd --- /dev/null +++ b/helm/analytics-agent/templates/_helpers.tpl @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "analytics-agent.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "analytics-agent.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "analytics-agent.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "analytics-agent.labels" -}} +helm.sh/chart: {{ include "analytics-agent.chart" . }} +{{ include "analytics-agent.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "analytics-agent.selectorLabels" -}} +app.kubernetes.io/name: {{ include "analytics-agent.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "analytics-agent.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "analytics-agent.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm/analytics-agent/templates/deployment.yaml b/helm/analytics-agent/templates/deployment.yaml new file mode 100644 index 0000000..015e677 --- /dev/null +++ b/helm/analytics-agent/templates/deployment.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "analytics-agent.fullname" . }} + labels: + {{- include "analytics-agent.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "analytics-agent.selectorLabels" . | nindent 6 }} + component-group: app + template: + metadata: + annotations: + checksum/secret-env: {{ include (print $.Template.BasePath "/secret-env.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "analytics-agent.labels" . | nindent 8 }} + component-group: app + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "analytics-agent.serviceAccountName" . }} + terminationGracePeriodSeconds: 30 + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: app + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: app-http + containerPort: {{ .Values.service.http.targetPort }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + envFrom: + - secretRef: + name: {{ include "analytics-agent.fullname" . }}-env + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/analytics-agent/templates/ingress.yaml b/helm/analytics-agent/templates/ingress.yaml new file mode 100644 index 0000000..1c5f804 --- /dev/null +++ b/helm/analytics-agent/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "analytics-agent.fullname" . }} + labels: + {{- include "analytics-agent.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "analytics-agent.fullname" $ }} + port: + name: {{ $.Values.ingress.servicePortName }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/analytics-agent/templates/secret-env.yaml b/helm/analytics-agent/templates/secret-env.yaml new file mode 100644 index 0000000..023d6c5 --- /dev/null +++ b/helm/analytics-agent/templates/secret-env.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "analytics-agent.fullname" . }}-env + labels: + {{- include "analytics-agent.labels" . | nindent 4 }} +type: Opaque +stringData: + {{- range $key, $value := .Values.config.env }} + {{ $key }}: {{ tpl (toString $value) $ | quote }} + {{- end }} diff --git a/helm/analytics-agent/templates/service.yaml b/helm/analytics-agent/templates/service.yaml new file mode 100644 index 0000000..3488c5e --- /dev/null +++ b/helm/analytics-agent/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "analytics-agent.fullname" . }} + labels: + {{- include "analytics-agent.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.http.port }} + targetPort: {{ .Values.service.http.targetPort }} + protocol: TCP + name: http + selector: + {{- include "analytics-agent.selectorLabels" . | nindent 4 }} + component-group: app diff --git a/helm/analytics-agent/templates/serviceaccount.yaml b/helm/analytics-agent/templates/serviceaccount.yaml new file mode 100644 index 0000000..476961d --- /dev/null +++ b/helm/analytics-agent/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "analytics-agent.serviceAccountName" . }} + labels: + {{- include "analytics-agent.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/helm/analytics-agent/values.yaml b/helm/analytics-agent/values.yaml new file mode 100644 index 0000000..980528f --- /dev/null +++ b/helm/analytics-agent/values.yaml @@ -0,0 +1,80 @@ +# Default values for analytics-agent. +replicaCount: 1 + +# Environment variables rendered into a Secret and mounted via envFrom. +# Override per-deployment in your environment values file. +config: + env: + LOG_LEVEL: INFO + +image: + # Override per-deployment with your registry/repository. + repository: analytics-agent + pullPolicy: IfNotPresent + # Defaults to .Chart.AppVersion when unset. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + automount: true + annotations: {} + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: {} + +securityContext: {} + +service: + type: ClusterIP + http: + port: 80 + targetPort: 8000 + +ingress: + enabled: false + className: "" + servicePortName: http + annotations: {} + hosts: + - host: chart-example.local + paths: + - path: / + pathType: Prefix + tls: [] + +resources: {} + +livenessProbe: + httpGet: + path: /health + port: app-http + initialDelaySeconds: 30 + periodSeconds: 10 +readinessProbe: + httpGet: + path: /health + port: app-http + initialDelaySeconds: 5 + periodSeconds: 5 + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 10 + targetCPUUtilizationPercentage: 80 + +volumes: [] +volumeMounts: [] + +nodeSelector: {} + +tolerations: [] + +affinity: {} From 48af4fd79b59779044257c1d02ab75bab80e1947 Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 10:25:35 -0700 Subject: [PATCH 02/22] chore: add click dep and analytics-agent CLI entrypoint --- pyproject.toml | 4 ++++ uv.lock | 2 ++ 2 files changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 737c688..2ff7bb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ "opentelemetry-instrumentation-fastapi>=0.46b0", "opentelemetry-instrumentation-langchain>=0.32.1", # Utilities + "click>=8.1.0", "orjson>=3.10.0", "jsonschema>=4.23.0", "tiktoken>=0.7.0", @@ -66,6 +67,9 @@ Homepage = "https://github.com/datahub-project/analytics-agent" Repository = "https://github.com/datahub-project/analytics-agent" "Bug Tracker" = "https://github.com/datahub-project/analytics-agent/issues" +[project.scripts] +analytics-agent = "analytics_agent.cli:cli" + [project.optional-dependencies] dev = [ "pytest>=8.0.0", diff --git a/uv.lock b/uv.lock index d784363..b155def 100644 --- a/uv.lock +++ b/uv.lock @@ -720,6 +720,7 @@ dependencies = [ { name = "aiosqlite" }, { name = "alembic" }, { name = "asyncpg" }, + { name = "click" }, { name = "cryptography" }, { name = "datahub-agent-context", extra = ["langchain"] }, { name = "fastapi" }, @@ -774,6 +775,7 @@ requires-dist = [ { name = "aiosqlite", specifier = ">=0.20.0" }, { name = "alembic", specifier = ">=1.13.0" }, { name = "asyncpg", specifier = ">=0.29.0" }, + { name = "click", specifier = ">=8.1.0" }, { name = "cryptography", specifier = ">=42.0.0" }, { name = "datahub-agent-context", extras = ["langchain"], specifier = ">=1.5.0" }, { name = "fastapi", specifier = ">=0.115.0" }, From dbc270a95a67c54259abcd647b7805bae06cd03d Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 10:26:59 -0700 Subject: [PATCH 03/22] feat: add bootstrap module and cli skeleton --- backend/src/analytics_agent/bootstrap.py | 9 +++++++++ backend/src/analytics_agent/cli.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 backend/src/analytics_agent/bootstrap.py create mode 100644 backend/src/analytics_agent/cli.py diff --git a/backend/src/analytics_agent/bootstrap.py b/backend/src/analytics_agent/bootstrap.py new file mode 100644 index 0000000..89ebe02 --- /dev/null +++ b/backend/src/analytics_agent/bootstrap.py @@ -0,0 +1,9 @@ +"""DB-mutating bootstrap functions. + +Pure async helpers, no FastAPI coupling. Each is independently callable, idempotent, +and intended to be invoked from the analytics-agent CLI (typically via a Helm +pre-install/pre-upgrade hook). All write logic that used to live inside the +FastAPI lifespan now lives here. +""" + +from __future__ import annotations diff --git a/backend/src/analytics_agent/cli.py b/backend/src/analytics_agent/cli.py new file mode 100644 index 0000000..ebeb07f --- /dev/null +++ b/backend/src/analytics_agent/cli.py @@ -0,0 +1,17 @@ +"""Click CLI for analytics-agent — bootstrap operations.""" + +from __future__ import annotations + +import logging + +import click + + +@click.group() +@click.version_option(package_name="datahub-analytics-agent") +def cli() -> None: + """Analytics-agent admin CLI.""" + logging.basicConfig( + level=logging.INFO, + format="%(levelname)s [%(name)s] %(message)s", + ) From 9844f1543d36c4ef57d05421219cfdb32b499f2e Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 13:20:04 -0700 Subject: [PATCH 04/22] feat: extract run_migrations into bootstrap module --- backend/src/analytics_agent/bootstrap.py | 57 ++++++++++++++++++++++++ tests/unit/test_bootstrap.py | 42 +++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 tests/unit/test_bootstrap.py diff --git a/backend/src/analytics_agent/bootstrap.py b/backend/src/analytics_agent/bootstrap.py index 89ebe02..5c21181 100644 --- a/backend/src/analytics_agent/bootstrap.py +++ b/backend/src/analytics_agent/bootstrap.py @@ -7,3 +7,60 @@ """ from __future__ import annotations + +import logging +import re +from pathlib import Path + +from analytics_agent.config import settings + + +def run_migrations() -> None: + """Run Alembic migrations synchronously (Alembic is a sync tool).""" + from alembic import command + from alembic.config import Config + + if "sqlite" in settings.database_url: + db_path = settings.database_url.replace("sqlite+aiosqlite:///", "") + Path(db_path).parent.mkdir(parents=True, exist_ok=True) + elif "mysql" in settings.database_url: + _ensure_mysql_schema() + + alembic_cfg = Config("alembic.ini") + command.upgrade(alembic_cfg, "head") + + +def _ensure_mysql_schema() -> None: + """Create the analytics_agent MySQL schema if it doesn't exist.""" + import pymysql + + match = re.search(r"/([^/?]+)(\?|$)", settings.database_url) + if not match: + return + schema = match.group(1) + + url_no_schema = re.sub(r"mysql\+aiomysql://", "", settings.database_url) + creds_match = re.match(r"([^:]+):([^@]+)@([^:/]+):?(\d+)?", url_no_schema) + if not creds_match: + return + user, password, host, port = creds_match.groups() + + try: + conn = pymysql.connect( + host=host, + port=int(port or 3306), + user=user, + password=password, + connect_timeout=5, + ) + with conn.cursor() as cur: + cur.execute( + f"CREATE SCHEMA IF NOT EXISTS `{schema}` " + "DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" + ) + conn.commit() + conn.close() + except Exception as exc: + logging.getLogger(__name__).warning( + "Could not ensure MySQL schema '%s': %s", schema, exc + ) diff --git a/tests/unit/test_bootstrap.py b/tests/unit/test_bootstrap.py new file mode 100644 index 0000000..821894e --- /dev/null +++ b/tests/unit/test_bootstrap.py @@ -0,0 +1,42 @@ +from pathlib import Path + +import pytest +from analytics_agent import bootstrap +from sqlalchemy import create_engine, inspect + + +@pytest.fixture +def sqlite_db(tmp_path, monkeypatch): + """Point settings at a fresh sqlite file for the duration of the test.""" + db_path = tmp_path / "test.db" + url = f"sqlite+aiosqlite:///{db_path}" + monkeypatch.setenv("DATABASE_URL", url) + # Force the settings singleton to pick up the new URL. + from analytics_agent import config as _config + + monkeypatch.setattr(_config.settings, "database_url", url) + # Reset the cached engine and session factory globals so they pick up the + # new URL on next call. The names match db/base.py exactly. + from analytics_agent.db import base as _base + + monkeypatch.setattr(_base, "_engine", None, raising=False) + monkeypatch.setattr(_base, "_AsyncSessionFactory", None, raising=False) + return db_path + + +def test_run_migrations_creates_tables(sqlite_db, monkeypatch): + # Alembic must be invoked from the repo root where alembic.ini lives. + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.chdir(repo_root) + + bootstrap.run_migrations() + + sync_url = f"sqlite:///{sqlite_db}" + engine = create_engine(sync_url) + tables = set(inspect(engine).get_table_names()) + engine.dispose() + + assert "alembic_version" in tables + assert "integrations" in tables + assert "context_platforms" in tables + assert "settings" in tables From 592224cc08d23df76f88edee8cb31e43837d99a4 Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 13:23:22 -0700 Subject: [PATCH 05/22] feat: extract seed_integrations_from_yaml into bootstrap module --- backend/src/analytics_agent/bootstrap.py | 81 +++++++++++++++++++ tests/unit/test_bootstrap.py | 99 ++++++++++++++++++++++++ 2 files changed, 180 insertions(+) diff --git a/backend/src/analytics_agent/bootstrap.py b/backend/src/analytics_agent/bootstrap.py index 5c21181..0710eb9 100644 --- a/backend/src/analytics_agent/bootstrap.py +++ b/backend/src/analytics_agent/bootstrap.py @@ -64,3 +64,84 @@ def _ensure_mysql_schema() -> None: logging.getLogger(__name__).warning( "Could not ensure MySQL schema '%s': %s", schema, exc ) + + +async def seed_integrations_from_yaml() -> None: + """Upsert config.yaml engines into the integrations table. + + Skips rows whose ``source == "ui"`` (the user has edited them via Settings UI). + Removes yaml-source rows no longer in config. Migrates any legacy + ``snowflake_oauth:*`` entries from the settings table to credentials. + Does NOT register engines in the in-memory factory — that is done at pod + boot by ``main.register_engines_from_db()``. + """ + import uuid + + import orjson + + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import ( + CredentialRepo, + IntegrationRepo, + SettingsRepo, + ) + + logger = logging.getLogger(__name__) + factory = _get_session_factory() + async with factory() as session: + integration_repo = IntegrationRepo(session) + cred_repo = CredentialRepo(session) + settings_repo = SettingsRepo(session) + + config_engine_names = { + cfg.effective_name for cfg in settings.load_engines_config() + } + for cfg in settings.load_engines_config(): + engine_type = cfg.type + engine_name = cfg.effective_name + connection = cfg.connection + label = f"{engine_type.capitalize()} ({engine_name})" + existing = await integration_repo.get(engine_name) + if existing is not None and existing.source == "ui": + logger.info( + "Skipping yaml seed for '%s' — user-managed via Settings UI", + engine_name, + ) + continue + await integration_repo.upsert( + id=str(uuid.uuid5(uuid.NAMESPACE_DNS, f"yaml:{engine_name}")), + name=engine_name, + type=engine_type, + label=label, + config=orjson.dumps(connection).decode(), + source="yaml", + ) + + for intg in await integration_repo.list_all(): + if intg.source == "yaml" and intg.name not in config_engine_names: + logger.info( + "Removing stale yaml integration '%s' (no longer in config.yaml)", + intg.name, + ) + await integration_repo.delete(intg.name) + + # Legacy oauth_token migration + all_integrations = await integration_repo.list_all() + for intg in all_integrations: + old_key = f"snowflake_oauth:{intg.name}" + raw = await settings_repo.get(old_key) + if raw and intg.credential is None: + try: + data = orjson.loads(raw) + method = data.get("method", "") + user = data.get("username") or data.get("user", "") + if method == "externalbrowser" and user: + await cred_repo.upsert( + id=str(uuid.uuid4()), + integration_name=intg.name, + auth_type="sso_externalbrowser", + username=user, + ) + await settings_repo.delete(old_key) + except Exception: + pass diff --git a/tests/unit/test_bootstrap.py b/tests/unit/test_bootstrap.py index 821894e..7635430 100644 --- a/tests/unit/test_bootstrap.py +++ b/tests/unit/test_bootstrap.py @@ -1,5 +1,6 @@ from pathlib import Path +import orjson import pytest from analytics_agent import bootstrap from sqlalchemy import create_engine, inspect @@ -40,3 +41,101 @@ def test_run_migrations_creates_tables(sqlite_db, monkeypatch): assert "integrations" in tables assert "context_platforms" in tables assert "settings" in tables + + +@pytest.mark.asyncio +async def test_seed_integrations_idempotent(sqlite_db, monkeypatch): + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.chdir(repo_root) + bootstrap.run_migrations() + + # Stub out load_engines_config to return a single yaml engine. + from analytics_agent import config as _config + + fake_cfg = _config.EngineConfig( + type="snowflake", + name="test_sf", + connection={"account": "x", "user": "y"}, + ) + monkeypatch.setattr( + _config.Settings, + "load_engines_config", + lambda self: [fake_cfg], + ) + + await bootstrap.seed_integrations_from_yaml() + await bootstrap.seed_integrations_from_yaml() # second run must be a no-op + + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import IntegrationRepo + + factory = _get_session_factory() + async with factory() as session: + rows = await IntegrationRepo(session).list_all() + + assert len(rows) == 1 + assert rows[0].name == "test_sf" + assert rows[0].source == "yaml" + assert orjson.loads(rows[0].config) == {"account": "x", "user": "y"} + + +@pytest.mark.asyncio +async def test_seed_integrations_removes_stale_yaml(sqlite_db, monkeypatch): + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.chdir(repo_root) + bootstrap.run_migrations() + + from analytics_agent import config as _config + + cfg_a = _config.EngineConfig(type="snowflake", name="a", connection={}) + cfg_b = _config.EngineConfig(type="snowflake", name="b", connection={}) + + monkeypatch.setattr(_config.Settings, "load_engines_config", lambda self: [cfg_a, cfg_b]) + await bootstrap.seed_integrations_from_yaml() + + monkeypatch.setattr(_config.Settings, "load_engines_config", lambda self: [cfg_a]) + await bootstrap.seed_integrations_from_yaml() + + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import IntegrationRepo + + factory = _get_session_factory() + async with factory() as session: + rows = await IntegrationRepo(session).list_all() + + names = {r.name for r in rows} + assert names == {"a"} + + +@pytest.mark.asyncio +async def test_seed_integrations_skips_ui_managed(sqlite_db, monkeypatch): + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.chdir(repo_root) + bootstrap.run_migrations() + + from analytics_agent import config as _config + + cfg = _config.EngineConfig( + type="snowflake", name="x", connection={"v": "yaml"} + ) + monkeypatch.setattr(_config.Settings, "load_engines_config", lambda self: [cfg]) + + await bootstrap.seed_integrations_from_yaml() + + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import IntegrationRepo + + factory = _get_session_factory() + async with factory() as session: + repo = IntegrationRepo(session) + row = await repo.get("x") + row.source = "ui" + row.config = orjson.dumps({"v": "ui-edited"}).decode() + await session.commit() + + await bootstrap.seed_integrations_from_yaml() # must not overwrite ui edits + + async with factory() as session: + row = await IntegrationRepo(session).get("x") + assert row.source == "ui" + assert orjson.loads(row.config) == {"v": "ui-edited"} From f601511d1d1ed07ba42898c4e864651569df2030 Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 13:25:39 -0700 Subject: [PATCH 06/22] feat: extract seed_context_platforms_from_yaml into bootstrap module --- backend/src/analytics_agent/bootstrap.py | 62 ++++++++++++++++++++ tests/unit/test_bootstrap.py | 72 ++++++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/backend/src/analytics_agent/bootstrap.py b/backend/src/analytics_agent/bootstrap.py index 0710eb9..51d2071 100644 --- a/backend/src/analytics_agent/bootstrap.py +++ b/backend/src/analytics_agent/bootstrap.py @@ -145,3 +145,65 @@ async def seed_integrations_from_yaml() -> None: await settings_repo.delete(old_key) except Exception: pass + + +async def seed_context_platforms_from_yaml() -> None: + """Upsert config.yaml context_platforms into the DB. + + Preserves user-edited credentials and ``_discovered_tools`` metadata for + yaml-source rows. Removes yaml-source rows no longer in config. Does NOT + propagate DataHub env vars — that is done at pod boot by + ``main.propagate_datahub_env()``. + """ + import contextlib + import uuid + + import orjson + + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import ContextPlatformRepo + + logger = logging.getLogger(__name__) + factory = _get_session_factory() + async with factory() as session: + repo = ContextPlatformRepo(session) + + config_platform_names = { + cfg.name for cfg in settings.load_context_platforms_config() + } + for cfg in settings.load_context_platforms_config(): + existing = await repo.get(cfg.name) + if existing: + changed = False + new_label = cfg.label or cfg.type.capitalize() + if existing.label != new_label: + existing.label = new_label + changed = True + if existing.source == "yaml": + stored: dict = {} + with contextlib.suppress(Exception): + stored = orjson.loads(existing.config) + yaml_cfg_dict = cfg.model_dump() + new_env = yaml_cfg_dict.get("env", {}) + if new_env and stored.get("env") != new_env: + stored["env"] = new_env + existing.config = orjson.dumps(stored).decode() + changed = True + if changed: + await session.commit() + else: + await repo.upsert( + id=str(uuid.uuid5(uuid.NAMESPACE_DNS, f"yaml:{cfg.name}")), + type=cfg.type, + name=cfg.name, + label=cfg.label or cfg.type.capitalize(), + config=orjson.dumps(cfg.model_dump()).decode(), + source="yaml", + ) + + for plat in await repo.list_all(): + if plat.source == "yaml" and plat.name not in config_platform_names: + logger.info( + "Removing stale yaml context platform '%s'", plat.name + ) + await repo.delete(plat.name) diff --git a/tests/unit/test_bootstrap.py b/tests/unit/test_bootstrap.py index 7635430..f66e4f4 100644 --- a/tests/unit/test_bootstrap.py +++ b/tests/unit/test_bootstrap.py @@ -139,3 +139,75 @@ async def test_seed_integrations_skips_ui_managed(sqlite_db, monkeypatch): row = await IntegrationRepo(session).get("x") assert row.source == "ui" assert orjson.loads(row.config) == {"v": "ui-edited"} + + +@pytest.mark.asyncio +async def test_seed_context_platforms_idempotent(sqlite_db, monkeypatch): + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.chdir(repo_root) + bootstrap.run_migrations() + + from analytics_agent import config as _config + + fake = _config.DataHubPlatformConfig( + type="datahub", + name="default", + label="DataHub", + url="http://gms", + token="t", + ) + # Pydantic BaseSettings rejects instance attribute assignment, so patch + # on the class. (The Task 4 implementer hit the same issue.) + monkeypatch.setattr( + _config.Settings, + "load_context_platforms_config", + lambda self: [fake], + ) + + await bootstrap.seed_context_platforms_from_yaml() + await bootstrap.seed_context_platforms_from_yaml() + + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import ContextPlatformRepo + + factory = _get_session_factory() + async with factory() as session: + rows = await ContextPlatformRepo(session).list_all() + + assert len(rows) == 1 + assert rows[0].name == "default" + assert rows[0].source == "yaml" + + +@pytest.mark.asyncio +async def test_seed_context_platforms_removes_stale(sqlite_db, monkeypatch): + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.chdir(repo_root) + bootstrap.run_migrations() + + from analytics_agent import config as _config + + a = _config.DataHubPlatformConfig(type="datahub", name="a", url="http://a", token="t") + b = _config.DataHubPlatformConfig(type="datahub", name="b", url="http://b", token="t") + + monkeypatch.setattr( + _config.Settings, + "load_context_platforms_config", + lambda self: [a, b], + ) + await bootstrap.seed_context_platforms_from_yaml() + + monkeypatch.setattr( + _config.Settings, + "load_context_platforms_config", + lambda self: [a], + ) + await bootstrap.seed_context_platforms_from_yaml() + + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import ContextPlatformRepo + + factory = _get_session_factory() + async with factory() as session: + rows = await ContextPlatformRepo(session).list_all() + assert {r.name for r in rows} == {"a"} From 960753ba2364a6b149e5f03e46bb7968d45b0e48 Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 13:27:01 -0700 Subject: [PATCH 07/22] feat: extract seed_default_settings into bootstrap module --- backend/src/analytics_agent/bootstrap.py | 17 ++++++++++ tests/unit/test_bootstrap.py | 42 ++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/backend/src/analytics_agent/bootstrap.py b/backend/src/analytics_agent/bootstrap.py index 51d2071..9a4782e 100644 --- a/backend/src/analytics_agent/bootstrap.py +++ b/backend/src/analytics_agent/bootstrap.py @@ -207,3 +207,20 @@ async def seed_context_platforms_from_yaml() -> None: "Removing stale yaml context platform '%s'", plat.name ) await repo.delete(plat.name) + + +async def seed_default_settings() -> None: + """Write first-run defaults to the settings table (no-op if already set).""" + import orjson + + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import SettingsRepo + + factory = _get_session_factory() + async with factory() as session: + repo = SettingsRepo(session) + if await repo.get("enabled_mutation_tools") is None: + await repo.set( + "enabled_mutation_tools", + orjson.dumps(["publish_analysis", "save_correction"]).decode(), + ) diff --git a/tests/unit/test_bootstrap.py b/tests/unit/test_bootstrap.py index f66e4f4..1135984 100644 --- a/tests/unit/test_bootstrap.py +++ b/tests/unit/test_bootstrap.py @@ -211,3 +211,45 @@ async def test_seed_context_platforms_removes_stale(sqlite_db, monkeypatch): async with factory() as session: rows = await ContextPlatformRepo(session).list_all() assert {r.name for r in rows} == {"a"} + + +@pytest.mark.asyncio +async def test_seed_default_settings_writes_first_run_defaults( + sqlite_db, monkeypatch +): + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.chdir(repo_root) + bootstrap.run_migrations() + + await bootstrap.seed_default_settings() + + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import SettingsRepo + + factory = _get_session_factory() + async with factory() as session: + raw = await SettingsRepo(session).get("enabled_mutation_tools") + + assert orjson.loads(raw) == ["publish_analysis", "save_correction"] + + +@pytest.mark.asyncio +async def test_seed_default_settings_does_not_overwrite(sqlite_db, monkeypatch): + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.chdir(repo_root) + bootstrap.run_migrations() + + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import SettingsRepo + + factory = _get_session_factory() + async with factory() as session: + await SettingsRepo(session).set( + "enabled_mutation_tools", orjson.dumps(["custom"]).decode() + ) + + await bootstrap.seed_default_settings() + + async with factory() as session: + raw = await SettingsRepo(session).get("enabled_mutation_tools") + assert orjson.loads(raw) == ["custom"] From f91da60931f5a035e5c2d8c67c343204851557a9 Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 13:28:57 -0700 Subject: [PATCH 08/22] feat: wire click subcommands for bootstrap operations --- backend/src/analytics_agent/cli.py | 49 ++++++++++++++++++ tests/unit/test_cli.py | 83 ++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 tests/unit/test_cli.py diff --git a/backend/src/analytics_agent/cli.py b/backend/src/analytics_agent/cli.py index ebeb07f..e4b1039 100644 --- a/backend/src/analytics_agent/cli.py +++ b/backend/src/analytics_agent/cli.py @@ -2,10 +2,13 @@ from __future__ import annotations +import asyncio import logging import click +from analytics_agent import bootstrap + @click.group() @click.version_option(package_name="datahub-analytics-agent") @@ -15,3 +18,49 @@ def cli() -> None: level=logging.INFO, format="%(levelname)s [%(name)s] %(message)s", ) + + +@cli.command("migrate") +def migrate() -> None: + """Apply Alembic migrations to the configured database.""" + click.echo("→ Running database migrations…") + bootstrap.run_migrations() + click.echo("✓ Migrations complete.") + + +@cli.command("seed-integrations") +def seed_integrations() -> None: + """Upsert config.yaml engines into the integrations table.""" + click.echo("→ Seeding integrations from config.yaml…") + asyncio.run(bootstrap.seed_integrations_from_yaml()) + click.echo("✓ Integrations seeded.") + + +@cli.command("seed-context-platforms") +def seed_context_platforms() -> None: + """Upsert config.yaml context platforms into the DB.""" + click.echo("→ Seeding context platforms from config.yaml…") + asyncio.run(bootstrap.seed_context_platforms_from_yaml()) + click.echo("✓ Context platforms seeded.") + + +@cli.command("seed-defaults") +def seed_defaults() -> None: + """Write first-run defaults to the settings table.""" + click.echo("→ Writing first-run default settings…") + asyncio.run(bootstrap.seed_default_settings()) + click.echo("✓ Defaults written.") + + +@cli.command("bootstrap") +def bootstrap_cmd() -> None: + """Run migrations + all seeds (idempotent). Intended for Helm hooks.""" + click.echo("→ Running migrations…") + bootstrap.run_migrations() + click.echo("→ Seeding integrations…") + asyncio.run(bootstrap.seed_integrations_from_yaml()) + click.echo("→ Seeding context platforms…") + asyncio.run(bootstrap.seed_context_platforms_from_yaml()) + click.echo("→ Writing first-run defaults…") + asyncio.run(bootstrap.seed_default_settings()) + click.echo("✓ Bootstrap complete.") diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 0000000..0bd1c2c --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,83 @@ +from pathlib import Path + +import orjson +import pytest +from analytics_agent.cli import cli +from click.testing import CliRunner +from sqlalchemy import create_engine, inspect + + +@pytest.fixture +def sqlite_db(tmp_path, monkeypatch): + db_path = tmp_path / "test.db" + url = f"sqlite+aiosqlite:///{db_path}" + monkeypatch.setenv("DATABASE_URL", url) + from analytics_agent import config as _config + + monkeypatch.setattr(_config.settings, "database_url", url) + from analytics_agent.db import base as _base + + monkeypatch.setattr(_base, "_engine", None, raising=False) + monkeypatch.setattr(_base, "_AsyncSessionFactory", None, raising=False) + return db_path + + +def _chdir_repo_root(monkeypatch): + monkeypatch.chdir(Path(__file__).resolve().parents[2]) + + +def test_cli_migrate_creates_schema(sqlite_db, monkeypatch): + _chdir_repo_root(monkeypatch) + result = CliRunner().invoke(cli, ["migrate"]) + assert result.exit_code == 0, result.output + + engine = create_engine(f"sqlite:///{sqlite_db}") + tables = set(inspect(engine).get_table_names()) + engine.dispose() + assert {"alembic_version", "integrations", "context_platforms", "settings"} <= tables + + +def test_cli_bootstrap_runs_all_steps_idempotent(sqlite_db, monkeypatch): + _chdir_repo_root(monkeypatch) + + from analytics_agent import config as _config + + # Pydantic BaseSettings — patch on the class. + monkeypatch.setattr(_config.Settings, "load_engines_config", lambda self: []) + monkeypatch.setattr( + _config.Settings, "load_context_platforms_config", lambda self: [] + ) + + runner = CliRunner() + result1 = runner.invoke(cli, ["bootstrap"]) + assert result1.exit_code == 0, result1.output + result2 = runner.invoke(cli, ["bootstrap"]) + assert result2.exit_code == 0, result2.output + + # Default settings should have been written + import asyncio + + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import SettingsRepo + + async def _read(): + factory = _get_session_factory() + async with factory() as session: + return await SettingsRepo(session).get("enabled_mutation_tools") + + raw = asyncio.run(_read()) + assert orjson.loads(raw) == ["publish_analysis", "save_correction"] + + +def test_cli_bootstrap_fails_fast_on_migration_error(monkeypatch, tmp_path): + """If migrate fails, the umbrella command must exit non-zero.""" + _chdir_repo_root(monkeypatch) + # Point at an unwritable directory to break sqlite mkdir + bad_url = "sqlite+aiosqlite:////this/path/does/not/exist/test.db" + monkeypatch.setenv("DATABASE_URL", bad_url) + from analytics_agent import config as _config + + monkeypatch.setattr(_config.settings, "database_url", bad_url) + + result = CliRunner().invoke(cli, ["bootstrap"], catch_exceptions=True) + assert result.exit_code != 0 From fda9257c61e8feef99d5f9602060d8607673c5e1 Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 14:33:40 -0700 Subject: [PATCH 09/22] refactor: slim lifespan to read-only init; bootstrap moved to CLI --- backend/src/analytics_agent/main.py | 299 ++++++---------------------- 1 file changed, 57 insertions(+), 242 deletions(-) diff --git a/backend/src/analytics_agent/main.py b/backend/src/analytics_agent/main.py index df15fdc..e83fd92 100644 --- a/backend/src/analytics_agent/main.py +++ b/backend/src/analytics_agent/main.py @@ -1,6 +1,5 @@ from __future__ import annotations -import contextlib import logging import os from contextlib import asynccontextmanager @@ -20,194 +19,81 @@ from analytics_agent.config import settings -@asynccontextmanager -async def lifespan(app: FastAPI): - # Run Alembic migrations at startup (sync — Alembic is not async) - _run_migrations() +async def register_engines_from_db() -> None: + """Populate the in-memory engine factory from rows in the integrations table. - # Seed integrations from config.yaml and load all into engine registry - await _seed_integrations() + This is a per-pod read-only operation. Yaml→DB seeding lives in + ``analytics_agent.bootstrap.seed_integrations_from_yaml`` and runs as a + Helm pre-install/pre-upgrade hook. + """ + import orjson - # Seed context platforms (DataHub, etc.) from config.yaml into DB - await _seed_context_platforms() + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import IntegrationRepo + from analytics_agent.engines.factory import register_engine - # Fail fast if rows are encrypted but the master key is absent - await _check_encryption_key_consistency() + logger = logging.getLogger(__name__) + factory = _get_session_factory() + async with factory() as session: + all_integrations = await IntegrationRepo(session).list_all() - # Apply first-run defaults (skipped if already configured) - await _seed_default_settings() + for intg in all_integrations: + try: + conn_cfg = orjson.loads(intg.config) + register_engine(intg.name, intg.type, conn_cfg) + except Exception as e: + logger.warning("Failed to register engine %s: %s", intg.name, e) - # Load LLM credentials stored via the onboarding wizard into the singleton. - # This is what makes the app work with zero env vars. - await _load_llm_config_from_db() - # Kick off tool discovery for any MCP connections that haven't been discovered yet. - # Runs in the background so it doesn't delay startup. - import asyncio as _asyncio +async def propagate_datahub_env() -> None: + """Copy the first DataHub context platform's URL/token into ``os.environ``. - _asyncio.create_task(_discover_mcp_tools_on_boot()) + Sync callers (agent tools, ``datahub.py``) read these env vars. This is a + per-pod read-only operation; yaml→DB seeding for context platforms lives in + ``analytics_agent.bootstrap.seed_context_platforms_from_yaml``. + """ + import orjson - yield + from analytics_agent.db.base import _get_session_factory + from analytics_agent.db.repository import ContextPlatformRepo - # Cleanup engine connections - from analytics_agent.engines.factory import close_all + factory = _get_session_factory() + async with factory() as session: + all_platforms = await ContextPlatformRepo(session).list_all() - await close_all() + for plat in all_platforms: + if plat.type == "datahub": + parsed = orjson.loads(plat.config) + if parsed.get("url"): + os.environ["DATAHUB_GMS_URL"] = parsed["url"] + if parsed.get("token"): + os.environ["DATAHUB_GMS_TOKEN"] = parsed["token"] + break -async def _seed_integrations() -> None: - """ - Upsert config.yaml engines into the integrations table, then load all - integrations (yaml + ui-created) into the engine factory registry. - Also migrates any legacy oauth_token:* entries from settings table. - """ - import uuid +@asynccontextmanager +async def lifespan(app: FastAPI): + # Fail fast if rows are encrypted but the master key is absent. + # Must run BEFORE any DB read that would deserialize EncryptedJSON columns. + await _check_encryption_key_consistency() - import orjson + # Per-pod read-only init. All DB-mutating bootstrap work (migrations, + # yaml→DB seeds, first-run defaults) is now done by the analytics-agent + # CLI, run as a Helm pre-install/pre-upgrade hook. + await register_engines_from_db() + await propagate_datahub_env() + await _load_llm_config_from_db() - from analytics_agent.db.base import _get_session_factory - from analytics_agent.db.repository import CredentialRepo, IntegrationRepo, SettingsRepo - from analytics_agent.engines.factory import register_engine + # Background MCP tool discovery — non-blocking, per-platform retry. + import asyncio as _asyncio - factory = _get_session_factory() - async with factory() as session: - integration_repo = IntegrationRepo(session) - cred_repo = CredentialRepo(session) - settings_repo = SettingsRepo(session) - - # 1. Upsert config.yaml engines and remove orphans (yaml-source entries no longer in config). - # If the user has edited a yaml-seeded row via the Settings UI the row's source is flipped - # to "ui" (see api/settings.py::update_connection); we skip those so the user's edits - # survive restarts. - config_engine_names = {cfg.effective_name for cfg in settings.load_engines_config()} - for cfg in settings.load_engines_config(): - engine_type = cfg.type - engine_name = cfg.effective_name - connection = cfg.connection - label = f"{engine_type.capitalize()} ({engine_name})" - existing = await integration_repo.get(engine_name) - if existing is not None and existing.source == "ui": - logging.getLogger(__name__).info( - "Skipping yaml seed for '%s' — user-managed via Settings UI", engine_name - ) - continue - await integration_repo.upsert( - id=str(uuid.uuid5(uuid.NAMESPACE_DNS, f"yaml:{engine_name}")), - name=engine_name, - type=engine_type, - label=label, - config=orjson.dumps(connection).decode(), - source="yaml", - ) - # Delete yaml-source integrations that are no longer in config.yaml - for intg in await integration_repo.list_all(): - if intg.source == "yaml" and intg.name not in config_engine_names: - logging.getLogger(__name__).info( - "Removing stale yaml integration '%s' (no longer in config.yaml)", intg.name - ) - await integration_repo.delete(intg.name) - - # 2. Migrate legacy oauth_token:* from settings table - all_integrations = await integration_repo.list_all() - for intg in all_integrations: - old_key = f"snowflake_oauth:{intg.name}" - raw = await settings_repo.get(old_key) - if raw and intg.credential is None: - try: - data = orjson.loads(raw) - method = data.get("method", "") - user = data.get("username") or data.get("user", "") - if method == "externalbrowser" and user: - await cred_repo.upsert( - id=str(uuid.uuid4()), - integration_name=intg.name, - auth_type="sso_externalbrowser", - username=user, - ) - await settings_repo.delete(old_key) - except Exception: - pass - - # 3. Register all integrations in engine factory - all_integrations = await integration_repo.list_all() - for intg in all_integrations: - try: - conn_cfg = orjson.loads(intg.config) - register_engine(intg.name, intg.type, conn_cfg) - except Exception as e: - logging.getLogger(__name__).warning( - "Failed to register engine %s: %s", intg.name, e - ) - - -async def _seed_context_platforms() -> None: - """Upsert config.yaml context_platforms into the DB, then propagate DataHub config to env.""" - import uuid + _asyncio.create_task(_discover_mcp_tools_on_boot()) - import orjson + yield - from analytics_agent.db.base import _get_session_factory - from analytics_agent.db.repository import ContextPlatformRepo + from analytics_agent.engines.factory import close_all - factory = _get_session_factory() - async with factory() as session: - repo = ContextPlatformRepo(session) - - config_platform_names = {cfg.name for cfg in settings.load_context_platforms_config()} - for cfg in settings.load_context_platforms_config(): - existing = await repo.get(cfg.name) - if existing: - # For yaml-sourced platforms, sync label and env vars (but preserve - # user-edited credentials and _discovered_tools metadata). - changed = False - new_label = cfg.label or cfg.type.capitalize() - if existing.label != new_label: - existing.label = new_label - changed = True - if existing.source == "yaml": - # Re-apply env dict from config.yaml so new vars (e.g. TOOLS_IS_MUTATION_ENABLED) - # take effect without requiring a manual DB delete. - stored: dict = {} - with contextlib.suppress(Exception): - stored = orjson.loads(existing.config) - yaml_cfg_dict = cfg.model_dump() - new_env = yaml_cfg_dict.get("env", {}) - if new_env and stored.get("env") != new_env: - stored["env"] = new_env - # Preserve cached tools - existing.config = orjson.dumps(stored).decode() - changed = True - if changed: - await session.commit() - else: - # First-time creation: seed from config.yaml - await repo.upsert( - id=str(uuid.uuid5(uuid.NAMESPACE_DNS, f"yaml:{cfg.name}")), - type=cfg.type, - name=cfg.name, - label=cfg.label or cfg.type.capitalize(), - config=orjson.dumps(cfg.model_dump()).decode(), - source="yaml", - ) - # Remove yaml-source platforms no longer in config.yaml - for plat in await repo.list_all(): - if plat.source == "yaml" and plat.name not in config_platform_names: - logging.getLogger(__name__).info( - "Removing stale yaml context platform '%s'", plat.name - ) - await repo.delete(plat.name) - - # Propagate the first DataHub platform to os.environ so sync callers - # (agent tools, datahub.py) continue working without DB access. - all_platforms = await repo.list_all() - for plat in all_platforms: - if plat.type == "datahub": - parsed = orjson.loads(plat.config) - if parsed.get("url"): - os.environ["DATAHUB_GMS_URL"] = parsed["url"] - if parsed.get("token"): - os.environ["DATAHUB_GMS_TOKEN"] = parsed["token"] - break + await close_all() async def _check_encryption_key_consistency() -> None: @@ -241,23 +127,6 @@ async def _check_encryption_key_consistency() -> None: raise RuntimeError(msg) -async def _seed_default_settings() -> None: - """Write first-run defaults to the settings table (no-op if already set).""" - import orjson - - from analytics_agent.db.base import _get_session_factory - from analytics_agent.db.repository import SettingsRepo - - factory = _get_session_factory() - async with factory() as session: - repo = SettingsRepo(session) - if await repo.get("enabled_mutation_tools") is None: - await repo.set( - "enabled_mutation_tools", - orjson.dumps(["publish_analysis", "save_correction"]).decode(), - ) - - async def _discover_mcp_tools_on_boot() -> None: """Background task: discover tools for any MCP context platforms that don't have them yet. @@ -422,60 +291,6 @@ async def _load_llm_config_from_db() -> None: os.environ["ENABLE_PROMPT_CACHE"] = "true" if flag_on else "false" -def _run_migrations() -> None: - """Run Alembic migrations synchronously (Alembic is a sync tool).""" - from alembic import command - from alembic.config import Config - - if "sqlite" in settings.database_url: - db_path = settings.database_url.replace("sqlite+aiosqlite:///", "") - Path(db_path).parent.mkdir(parents=True, exist_ok=True) - elif "mysql" in settings.database_url: - _ensure_mysql_schema() - - alembic_cfg = Config("alembic.ini") - command.upgrade(alembic_cfg, "head") - - -def _ensure_mysql_schema() -> None: - """Create the analytics_agent MySQL schema if it doesn't exist.""" - import re - - import pymysql - - # Extract schema name from URL: mysql+aiomysql://user:pass@host:port/schema - match = re.search(r"/([^/?]+)(\?|$)", settings.database_url) - if not match: - return - schema = match.group(1) - - # Build connection params without the schema (connect to server root) - url_no_schema = re.sub(r"/[^/?]+(\?|$)", "/\1", settings.database_url) - url_no_schema = re.sub(r"mysql\+aiomysql://", "", settings.database_url) - # Parse user:pass@host:port - creds_match = re.match(r"([^:]+):([^@]+)@([^:/]+):?(\d+)?", url_no_schema) - if not creds_match: - return - user, password, host, port = creds_match.groups() - - try: - conn = pymysql.connect( - host=host, - port=int(port or 3306), - user=user, - password=password, - connect_timeout=5, - ) - with conn.cursor() as cur: - cur.execute( - f"CREATE SCHEMA IF NOT EXISTS `{schema}` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" - ) - conn.commit() - conn.close() - except Exception as exc: - logging.getLogger(__name__).warning("Could not ensure MySQL schema '%s': %s", schema, exc) - - def create_app() -> FastAPI: import os From 956685af3f79f7612617b5af6ac45ce0d8587133 Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 16:44:34 -0700 Subject: [PATCH 10/22] feat(helm): add bootstrap Job as pre-install/pre-upgrade hook --- .../templates/bootstrap-job.yaml | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 helm/analytics-agent/templates/bootstrap-job.yaml diff --git a/helm/analytics-agent/templates/bootstrap-job.yaml b/helm/analytics-agent/templates/bootstrap-job.yaml new file mode 100644 index 0000000..bbd9d28 --- /dev/null +++ b/helm/analytics-agent/templates/bootstrap-job.yaml @@ -0,0 +1,65 @@ +{{- if .Values.bootstrap.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "analytics-agent.fullname" . }}-bootstrap + labels: + {{- include "analytics-agent.labels" . | nindent 4 }} + component-group: bootstrap + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + backoffLimit: 0 + template: + metadata: + {{- with .Values.bootstrap.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "analytics-agent.labels" . | nindent 8 }} + component-group: bootstrap + spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "analytics-agent.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: bootstrap + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["analytics-agent", "bootstrap"] + envFrom: + - secretRef: + name: {{ include "analytics-agent.fullname" . }}-env + resources: + {{- toYaml .Values.bootstrap.resources | nindent 12 }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} From 5d7cbcdb9cd2cadcade1bbaafb44056799e6cee0 Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 16:45:44 -0700 Subject: [PATCH 11/22] feat(helm): add bootstrap.enabled values default true --- helm/analytics-agent/values.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/helm/analytics-agent/values.yaml b/helm/analytics-agent/values.yaml index 980528f..48022c4 100644 --- a/helm/analytics-agent/values.yaml +++ b/helm/analytics-agent/values.yaml @@ -78,3 +78,12 @@ nodeSelector: {} tolerations: [] affinity: {} + +# One-shot bootstrap Job (pre-install/pre-upgrade hook). +# Runs `analytics-agent bootstrap` to apply Alembic migrations and seed +# config.yaml entries into the database. Disable when migrations are managed +# externally. +bootstrap: + enabled: true + resources: {} + podAnnotations: {} From 6fdd666048066ca77837d1bbb35ebdab665f4b88 Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 16:47:38 -0700 Subject: [PATCH 12/22] docs: document analytics-agent bootstrap CLI and Helm hook --- AGENTS.md | 6 ++++-- CLAUDE.md | 6 ++++-- README.md | 12 ++++++++++++ helm/analytics-agent/README.md | 21 +++++++++++++++++++++ 4 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 helm/analytics-agent/README.md diff --git a/AGENTS.md b/AGENTS.md index 417610b..f85f781 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,7 +52,9 @@ cd frontend && pnpm dev **DataHub credentials**: run `datahub init --sso --host https://your-instance.acryl.io/gms` once. The app reads `~/.datahubenv` automatically; or set `DATAHUB_GMS_URL` + `DATAHUB_GMS_TOKEN` in `config.yaml` / `.env`. -**Database**: SQLite at `./data/dev.db` by default. Alembic runs automatically on startup. For Postgres set `DATABASE_URL=postgresql+asyncpg://...`. +**Database**: SQLite at `./data/dev.db` by default. For Postgres set `DATABASE_URL=postgresql+asyncpg://...`. + +**Bootstrap (migrations + seed)**: The FastAPI lifespan no longer runs migrations or seeds — it does read-only initialization (loading engines from the DB, propagating env vars, validating the encryption key). All DB-mutating bootstrap work lives in `analytics_agent.bootstrap` and is invoked via the CLI: `analytics-agent bootstrap` runs Alembic migrate → seed-integrations → seed-context-platforms → seed-defaults, idempotently. Run it before the first `uvicorn` start and after any release that adds migrations. The Helm chart runs it automatically as a `pre-install`/`pre-upgrade` hook. --- @@ -60,7 +62,7 @@ cd frontend && pnpm dev | Path | What it does | |------|-------------| -| `backend/src/analytics_agent/main.py` | FastAPI app factory + lifespan (runs Alembic, seeds integrations, mounts SPA) | +| `backend/src/analytics_agent/main.py` | FastAPI app factory + lifespan (read-only init: loads engines, validates encryption key, mounts SPA — no migrations/seeds) | | `backend/src/analytics_agent/agent/graph.py` | LangGraph `StateGraph`: ReAct agent → conditional chart node | | `backend/src/analytics_agent/agent/streaming.py` | `astream_events` → SSE event dicts; handles `on_tool_error` | | `backend/src/analytics_agent/agent/history.py` | Reconstructs LangChain message history from DB rows; pads orphaned tool calls | diff --git a/CLAUDE.md b/CLAUDE.md index ca78083..130554d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,11 +34,13 @@ cd frontend && pnpm tsc --noEmit # type-check without building ### Restarting the backend -When `.env`, `config.yaml`, or Python source changes, the backend needs a restart: +When `.env`, `config.yaml`, or Python source changes, the backend needs a restart. Run `analytics-agent bootstrap` whenever `config.yaml` changes or after pulling a release that adds migrations. ```bash pkill -f "analytics_agent.main"; sleep 2 set -a && source .env && set +a -nohup uv run uvicorn analytics_agent.main:app --port 8100 > /tmp/analytics_agent.log 2>&1 & +uv run analytics-agent bootstrap # apply migrations + seed config.yaml +nohup uv run uvicorn analytics_agent.main:app --port 8100 \ + > /tmp/analytics_agent.log 2>&1 & sleep 5 && curl -s http://localhost:8100/api/engines ``` diff --git a/README.md b/README.md index aca795e..9d9496b 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,18 @@ Open **http://localhost:8100** — a setup wizard handles the LLM key and connec > **Without `just`:** `uv sync && cd frontend && pnpm install && pnpm build && cd .. && uv run uvicorn analytics_agent.main:app --port 8100` +### First-time setup + +Before the first `uvicorn` start (or after pulling a release that adds migrations), run: + +```bash +uv run analytics-agent bootstrap +``` + +This applies Alembic migrations, seeds engines and context platforms from `config.yaml`, and writes first-run setting defaults. The command is idempotent — re-running it on an up-to-date database is a no-op. + +For Kubernetes deployments, the Helm chart runs `analytics-agent bootstrap` automatically as a `pre-install`/`pre-upgrade` hook (see `helm/analytics-agent/README.md`). + ### Optional: pre-configure via `.env` ```bash diff --git a/helm/analytics-agent/README.md b/helm/analytics-agent/README.md new file mode 100644 index 0000000..2bf7f4b --- /dev/null +++ b/helm/analytics-agent/README.md @@ -0,0 +1,21 @@ +# analytics-agent Helm chart + +## Bootstrap hook + +A `Job` runs `analytics-agent bootstrap` as a `helm.sh/hook: pre-install,pre-upgrade` +hook. It applies Alembic migrations and seeds `config.yaml` entries into the +database before any pod starts. + +| Value | Default | Purpose | +|-------|---------|---------| +| `bootstrap.enabled` | `true` | Set to `false` to skip the hook (e.g. when migrations are managed externally). | +| `bootstrap.resources` | `{}` | Resource requests/limits for the bootstrap container. | +| `bootstrap.podAnnotations` | `{}` | Extra annotations on the Job pod. | + +The Job uses the same image, secret (`{fullname}-env`), `volumes`, `volumeMounts`, +`nodeSelector`, `tolerations`, and `affinity` as the deployment. It has +`backoffLimit: 0` — failures are immediate and visible via `kubectl logs`. + +If a pod starts before the bootstrap hook has run (e.g. you disabled +`bootstrap.enabled` and forgot to migrate externally), the lifespan crashes with +a SQLAlchemy "no such table" error. From 0d47ea9a1ee27e92405fd1c2172b164fb45b0cee Mon Sep 17 00:00:00 2001 From: craig Date: Tue, 28 Apr 2026 17:11:23 -0700 Subject: [PATCH 13/22] fix(ci): apply ruff format and bootstrap before e2e uvicorn --- backend/src/analytics_agent/bootstrap.py | 16 ++++------------ tests/e2e/playwright.config.ts | 2 +- tests/unit/test_bootstrap.py | 12 +++--------- tests/unit/test_cli.py | 4 +--- 4 files changed, 9 insertions(+), 25 deletions(-) diff --git a/backend/src/analytics_agent/bootstrap.py b/backend/src/analytics_agent/bootstrap.py index 9a4782e..53a330d 100644 --- a/backend/src/analytics_agent/bootstrap.py +++ b/backend/src/analytics_agent/bootstrap.py @@ -61,9 +61,7 @@ def _ensure_mysql_schema() -> None: conn.commit() conn.close() except Exception as exc: - logging.getLogger(__name__).warning( - "Could not ensure MySQL schema '%s': %s", schema, exc - ) + logging.getLogger(__name__).warning("Could not ensure MySQL schema '%s': %s", schema, exc) async def seed_integrations_from_yaml() -> None: @@ -93,9 +91,7 @@ async def seed_integrations_from_yaml() -> None: cred_repo = CredentialRepo(session) settings_repo = SettingsRepo(session) - config_engine_names = { - cfg.effective_name for cfg in settings.load_engines_config() - } + config_engine_names = {cfg.effective_name for cfg in settings.load_engines_config()} for cfg in settings.load_engines_config(): engine_type = cfg.type engine_name = cfg.effective_name @@ -168,9 +164,7 @@ async def seed_context_platforms_from_yaml() -> None: async with factory() as session: repo = ContextPlatformRepo(session) - config_platform_names = { - cfg.name for cfg in settings.load_context_platforms_config() - } + config_platform_names = {cfg.name for cfg in settings.load_context_platforms_config()} for cfg in settings.load_context_platforms_config(): existing = await repo.get(cfg.name) if existing: @@ -203,9 +197,7 @@ async def seed_context_platforms_from_yaml() -> None: for plat in await repo.list_all(): if plat.source == "yaml" and plat.name not in config_platform_names: - logger.info( - "Removing stale yaml context platform '%s'", plat.name - ) + logger.info("Removing stale yaml context platform '%s'", plat.name) await repo.delete(plat.name) diff --git a/tests/e2e/playwright.config.ts b/tests/e2e/playwright.config.ts index 90992a1..6fa32ac 100644 --- a/tests/e2e/playwright.config.ts +++ b/tests/e2e/playwright.config.ts @@ -25,7 +25,7 @@ export default defineConfig({ projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], webServer: { - command: `uv run uvicorn analytics_agent.main:app --port ${TEST_PORT}`, + command: `uv run analytics-agent bootstrap && uv run uvicorn analytics_agent.main:app --port ${TEST_PORT}`, url: `http://localhost:${TEST_PORT}/api/engines`, cwd: join(__dirname, "../.."), // repo root so Alembic finds its config reuseExistingServer: false, diff --git a/tests/unit/test_bootstrap.py b/tests/unit/test_bootstrap.py index 1135984..aac6d89 100644 --- a/tests/unit/test_bootstrap.py +++ b/tests/unit/test_bootstrap.py @@ -115,9 +115,7 @@ async def test_seed_integrations_skips_ui_managed(sqlite_db, monkeypatch): from analytics_agent import config as _config - cfg = _config.EngineConfig( - type="snowflake", name="x", connection={"v": "yaml"} - ) + cfg = _config.EngineConfig(type="snowflake", name="x", connection={"v": "yaml"}) monkeypatch.setattr(_config.Settings, "load_engines_config", lambda self: [cfg]) await bootstrap.seed_integrations_from_yaml() @@ -214,9 +212,7 @@ async def test_seed_context_platforms_removes_stale(sqlite_db, monkeypatch): @pytest.mark.asyncio -async def test_seed_default_settings_writes_first_run_defaults( - sqlite_db, monkeypatch -): +async def test_seed_default_settings_writes_first_run_defaults(sqlite_db, monkeypatch): repo_root = Path(__file__).resolve().parents[2] monkeypatch.chdir(repo_root) bootstrap.run_migrations() @@ -244,9 +240,7 @@ async def test_seed_default_settings_does_not_overwrite(sqlite_db, monkeypatch): factory = _get_session_factory() async with factory() as session: - await SettingsRepo(session).set( - "enabled_mutation_tools", orjson.dumps(["custom"]).decode() - ) + await SettingsRepo(session).set("enabled_mutation_tools", orjson.dumps(["custom"]).decode()) await bootstrap.seed_default_settings() diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 0bd1c2c..0519313 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -44,9 +44,7 @@ def test_cli_bootstrap_runs_all_steps_idempotent(sqlite_db, monkeypatch): # Pydantic BaseSettings — patch on the class. monkeypatch.setattr(_config.Settings, "load_engines_config", lambda self: []) - monkeypatch.setattr( - _config.Settings, "load_context_platforms_config", lambda self: [] - ) + monkeypatch.setattr(_config.Settings, "load_context_platforms_config", lambda self: []) runner = CliRunner() result1 = runner.invoke(cli, ["bootstrap"]) From 14723560cbbcacfc8ee67edf126084813bfe34aa Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sat, 2 May 2026 15:16:54 -0700 Subject: [PATCH 14/22] fix: run analytics-agent bootstrap before uvicorn in all justfile start recipes Ensures migrations and yaml seeds are applied on first run (and are a no-op on subsequent runs) for serve, dev, start, and dev-full targets. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- justfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/justfile b/justfile index b1d4b3d..777cf83 100644 --- a/justfile +++ b/justfile @@ -34,10 +34,12 @@ typecheck: # Start backend (blocks; use 'dev' for background) serve: + uv run analytics-agent bootstrap uv run uvicorn analytics_agent.main:app --reload --port {{dev_port}} # Build frontend if stale, then start backend in background (with auto-reload on Python changes) dev: build-if-stale + uv run analytics-agent bootstrap pkill -f "analytics_agent.main" || true nohup uv run uvicorn analytics_agent.main:app --reload --port {{dev_port}} > {{log}} 2>&1 & sleep 3 && curl -s http://localhost:{{dev_port}}/api/engines | head -c 120 @@ -45,6 +47,7 @@ dev: build-if-stale # Start backend, rebuilding frontend if stale start: build-if-stale + uv run analytics-agent bootstrap pkill -f "analytics_agent.main" || true nohup uv run uvicorn analytics_agent.main:app --port {{port}} > {{log}} 2>&1 & sleep 3 && curl -s http://localhost:{{port}}/api/engines | head -c 120 @@ -57,6 +60,7 @@ frontend: # Start backend (reload mode) + Vite dev server in parallel dev-full: + uv run analytics-agent bootstrap pkill -f "analytics_agent.main" || true nohup uv run uvicorn analytics_agent.main:app --reload --port {{dev_port}} > {{log}} 2>&1 & @echo "Backend → http://localhost:{{dev_port}}" From b753196fd2a4da6770775468d17620a4f39cb091 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sat, 2 May 2026 15:18:17 -0700 Subject: [PATCH 15/22] fix(helm): correct containerPort and service targetPort to 8100 Dockerfile binds uvicorn to 8100; values.yaml had the legacy default of 8000 which would have caused liveness/readiness probe failures. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- helm/analytics-agent/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/analytics-agent/values.yaml b/helm/analytics-agent/values.yaml index 48022c4..d240fec 100644 --- a/helm/analytics-agent/values.yaml +++ b/helm/analytics-agent/values.yaml @@ -35,7 +35,7 @@ service: type: ClusterIP http: port: 80 - targetPort: 8000 + targetPort: 8100 ingress: enabled: false From 5c32b5d1ca8e221b10c1a8830f0c7019ae61b67f Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sat, 2 May 2026 15:36:24 -0700 Subject: [PATCH 16/22] fix: resolve four issues found during local Helm deploy test - docker/Dockerfile: add .venv/bin to PATH so analytics-agent CLI is reachable by name in the container (Helm job command was exit 127) - pyproject.toml: add psycopg2-binary so Alembic's sync engine can connect to PostgreSQL (asyncpg is async-only; Alembic needs psycopg2) - helm/templates: make ServiceAccount and Secret pre-install/pre-upgrade hooks at weight -1 so they exist before the bootstrap Job (weight 0) fires; without this the Job pod fails with SA/Secret not found - cli.py: consolidate bootstrap seeds into one asyncio.run() call; multiple sequential asyncio.run() calls in the same process cause "Future attached to a different loop" with asyncpg because the cached SQLAlchemy engine is bound to the first (now-closed) event loop Co-Authored-By: Claude Sonnet 4.6 (1M context) --- backend/src/analytics_agent/cli.py | 14 ++--- docker/Dockerfile | 1 + .../analytics-agent/templates/secret-env.yaml | 4 ++ .../templates/serviceaccount.yaml | 7 ++- pyproject.toml | 1 + uv.lock | 54 +++++++++++++++++++ 6 files changed, 73 insertions(+), 8 deletions(-) diff --git a/backend/src/analytics_agent/cli.py b/backend/src/analytics_agent/cli.py index e4b1039..1ac0603 100644 --- a/backend/src/analytics_agent/cli.py +++ b/backend/src/analytics_agent/cli.py @@ -55,12 +55,14 @@ def seed_defaults() -> None: @cli.command("bootstrap") def bootstrap_cmd() -> None: """Run migrations + all seeds (idempotent). Intended for Helm hooks.""" + + async def _run_all_seeds() -> None: + await bootstrap.seed_integrations_from_yaml() + await bootstrap.seed_context_platforms_from_yaml() + await bootstrap.seed_default_settings() + click.echo("→ Running migrations…") bootstrap.run_migrations() - click.echo("→ Seeding integrations…") - asyncio.run(bootstrap.seed_integrations_from_yaml()) - click.echo("→ Seeding context platforms…") - asyncio.run(bootstrap.seed_context_platforms_from_yaml()) - click.echo("→ Writing first-run defaults…") - asyncio.run(bootstrap.seed_default_settings()) + click.echo("→ Seeding integrations, context platforms, and defaults…") + asyncio.run(_run_all_seeds()) click.echo("✓ Bootstrap complete.") diff --git a/docker/Dockerfile b/docker/Dockerfile index 7792253..4db6350 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,6 +18,7 @@ COPY pyproject.toml uv.lock README.md ./ COPY backend/ backend/ RUN uv sync --no-dev +ENV PATH="/app/.venv/bin:$PATH" COPY alembic.ini config.yaml.example ./ # Use example as the default config; operators override via volume mount or ENGINES_CONFIG env var diff --git a/helm/analytics-agent/templates/secret-env.yaml b/helm/analytics-agent/templates/secret-env.yaml index 023d6c5..04b1beb 100644 --- a/helm/analytics-agent/templates/secret-env.yaml +++ b/helm/analytics-agent/templates/secret-env.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "analytics-agent.fullname" . }}-env labels: {{- include "analytics-agent.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-1" + "helm.sh/hook-delete-policy": before-hook-creation type: Opaque stringData: {{- range $key, $value := .Values.config.env }} diff --git a/helm/analytics-agent/templates/serviceaccount.yaml b/helm/analytics-agent/templates/serviceaccount.yaml index 476961d..f138349 100644 --- a/helm/analytics-agent/templates/serviceaccount.yaml +++ b/helm/analytics-agent/templates/serviceaccount.yaml @@ -5,9 +5,12 @@ metadata: name: {{ include "analytics-agent.serviceAccountName" . }} labels: {{- include "analytics-agent.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-1" + "helm.sh/hook-delete-policy": before-hook-creation + {{- with .Values.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} automountServiceAccountToken: {{ .Values.serviceAccount.automount }} {{- end }} diff --git a/pyproject.toml b/pyproject.toml index 2ff7bb4..a6426a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "sqlalchemy[asyncio]>=2.0.0", "alembic>=1.13.0", "asyncpg>=0.29.0", + "psycopg2-binary>=2.9.0", "aiosqlite>=0.20.0", "aiomysql>=0.2.0", # Config diff --git a/uv.lock b/uv.lock index b155def..13539df 100644 --- a/uv.lock +++ b/uv.lock @@ -738,6 +738,7 @@ dependencies = [ { name = "opentelemetry-instrumentation-langchain" }, { name = "opentelemetry-sdk" }, { name = "orjson" }, + { name = "psycopg2-binary" }, { name = "pydantic-settings" }, { name = "pymysql" }, { name = "python-dotenv" }, @@ -795,6 +796,7 @@ requires-dist = [ { name = "opentelemetry-instrumentation-langchain", specifier = ">=0.32.1" }, { name = "opentelemetry-sdk", specifier = ">=1.25.0" }, { name = "orjson", specifier = ">=3.10.0" }, + { name = "psycopg2-binary", specifier = ">=2.9.0" }, { name = "pydantic-settings", specifier = ">=2.5.0" }, { name = "pymysql", specifier = ">=1.1.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, @@ -2815,6 +2817,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, ] +[[package]] +name = "psycopg2-binary" +version = "2.9.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2a/60/a3624f79acea344c16fbef3a94d28b89a8042ddfb8f3e4ca83f538671409/psycopg2_binary-2.9.12.tar.gz", hash = "sha256:5ac9444edc768c02a6b6a591f070b8aae28ff3a99be57560ac996001580f294c", size = 379686, upload-time = "2026-04-21T09:40:34.304Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/19/d4ce60954f3bb9d8e3bc5e5c4d1f2487de2d3851bf2391d54954c9df12a6/psycopg2_binary-2.9.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5c8ce6c61bd1b1f6b9c24ee32211599f6166af2c55abb19456090a21fd16554b", size = 3712338, upload-time = "2026-04-20T23:34:03.961Z" }, + { url = "https://files.pythonhosted.org/packages/53/71/c85409ee0d78890f0660eff262e815e7dd2bb741a17611d82e9e8cd9dc5e/psycopg2_binary-2.9.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b4a9eaa6e7f4ff91bec10aa3fb296878e75187bced5cc4bafe17dc40915e1326", size = 3822407, upload-time = "2026-04-20T23:34:05.977Z" }, + { url = "https://files.pythonhosted.org/packages/3c/ed/60486c2c7f0d4d1ede2bfb1ed27e2498477ce646bc7f6b2759906303117e/psycopg2_binary-2.9.12-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c6528cefc8e50fcc6f4a107e27a672058b36cc5736d665476aeb413ba88dbb06", size = 4578425, upload-time = "2026-04-20T23:34:08.246Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b9/656cb03fad9f4f49f2145c334b1126ee75189929ca4e6187d485a2d59951/psycopg2_binary-2.9.12-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4e184b1fb6072bf05388aa41c697e1b2d01b3473f107e7ec44f186a32cfd0b8", size = 4273709, upload-time = "2026-04-20T23:34:10.974Z" }, + { url = "https://files.pythonhosted.org/packages/99/66/08cf0da0e25cc6fb142c89be45fc8418792858f0c4cbff5e24530ff02cd6/psycopg2_binary-2.9.12-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4766ab678563054d3f1d064a4db19cc4b5f9e3a8d9018592a8285cf200c248f3", size = 5893779, upload-time = "2026-04-20T23:34:13.905Z" }, + { url = "https://files.pythonhosted.org/packages/17/d7/eecd9ce8e146d3721115d82d3836efdbb712187e4590325df549989d18f4/psycopg2_binary-2.9.12-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5a0253224780c978746cb9be55a946bcdaf40fe3519c0f622924cdabdafe2c39", size = 4109308, upload-time = "2026-04-20T23:34:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2e/b1dc289b362cc8d45697b57eefbd673186f49a4ea0906928988e3affcc98/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0dc9228d47c46bda253d2ecd6bb93b56a9f2d7ad33b684a1fa3622bf74ffe30c", size = 3654405, upload-time = "2026-04-20T23:34:19.303Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e4/4c4aea6473214dbdbd0fbba11aa4691e76dc01722c55724c5951719865ff/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f921f3cd87035ef7df233383011d7a53ea1d346224752c1385f1edfd790ceb6a", size = 3299187, upload-time = "2026-04-20T23:34:21.206Z" }, + { url = "https://files.pythonhosted.org/packages/ba/5d/b03b99986446a4f57b170ed9a2579fb7ff9783ca0fa5226b19db99737fee/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3d999bd982a723113c1a45b55a7a6a90d64d0ed2278020ed625c490ff7bef96c", size = 3047716, upload-time = "2026-04-20T23:34:23.077Z" }, + { url = "https://files.pythonhosted.org/packages/14/86/382ee4afbd1d97500c9d2862b20c2fdeddf4b7335e984df3fb4309f64108/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29d4d134bd0ab46ffb04e94aa3c5fa3ef582e9026609165e2f758ff76fc3a3be", size = 3349237, upload-time = "2026-04-20T23:34:25.211Z" }, + { url = "https://files.pythonhosted.org/packages/a8/16/9a57c75ba1eda7165c017342f526810d5f5a12647dde749c99ae9a7141d7/psycopg2_binary-2.9.12-cp311-cp311-win_amd64.whl", hash = "sha256:cb4a1dacdd48077150dc762a9e5ddbf32c256d66cb46f80839391aa458774936", size = 2757036, upload-time = "2026-04-20T23:34:27.77Z" }, + { url = "https://files.pythonhosted.org/packages/e2/9f/ef4ef3c8e15083df90ca35265cfd1a081a2f0cc07bb229c6314c6af817f4/psycopg2_binary-2.9.12-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5cdc05117180c5fa9c40eea8ea559ce64d73824c39d928b7da9fb5f6a9392433", size = 3712459, upload-time = "2026-04-20T23:34:30.549Z" }, + { url = "https://files.pythonhosted.org/packages/b5/01/3dd14e46ba48c1e1a6ec58ee599fa1b5efa00c246d5046cd903d0eeb1af1/psycopg2_binary-2.9.12-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3227a3bc228c10d21011a99245edca923e4e8bf461857e869a507d9a41fe9f6", size = 3822936, upload-time = "2026-04-20T23:34:32.77Z" }, + { url = "https://files.pythonhosted.org/packages/a6/f7/0640e4901119d8a9f7a1784b927f494e2198e213ceb593753d1f2c8b1b30/psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:995ce929eede89db6254b50827e2b7fd61e50d11f0b116b29fffe4a2e53c4580", size = 4578676, upload-time = "2026-04-20T23:34:35.18Z" }, + { url = "https://files.pythonhosted.org/packages/b0/55/44df3965b5f297c50cc0b1b594a31c67d6127a9d133045b8a66611b14dfb/psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9fe06d93e72f1c048e731a2e3e7854a5bfaa58fc736068df90b352cefe66f03f", size = 4274917, upload-time = "2026-04-20T23:34:37.982Z" }, + { url = "https://files.pythonhosted.org/packages/b0/4b/74535248b1eac0c9336862e8617c765ac94dac76f9e25d7c4a79588c8907/psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40e7b28b63aaf737cb3a1edc3a9bbc9a9f4ad3dcb7152e8c1130e4050eddcb7d", size = 5894843, upload-time = "2026-04-20T23:34:40.856Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ba/f1bf8d2ae71868ad800b661099086ee52bc0f8d9f05be1acd8ebb06757cc/psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:89d19a9f7899e8eb0656a2b3a08e0da04c720a06db6e0033eab5928aabe60fa9", size = 4110556, upload-time = "2026-04-20T23:34:44.016Z" }, + { url = "https://files.pythonhosted.org/packages/45/46/c15706c338403b7c420bcc0c2905aad116cc064545686d8bf85f1999ea00/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:612b965daee295ae2da8f8218ce1d274645dc76ef3f1abf6a0a94fd57eff876d", size = 3655714, upload-time = "2026-04-20T23:34:46.233Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7c/a2d5dc09b64a4564db242a0fe418fde7d33f6f8259dd2c5b9d7def00fb5a/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b9a339b79d37c1b45f3235265f07cdeb0cb5ad7acd2ac7720a5920989c17c24e", size = 3301154, upload-time = "2026-04-20T23:34:49.528Z" }, + { url = "https://files.pythonhosted.org/packages/c0/e8/cc8c9a4ce71461f9ec548d38cadc41dc184b34c73e6455450775a9334ccd/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:3471336e1acfd9c7fe507b8bad5af9317b6a89294f9eb37bd9a030bb7bebcdc6", size = 3048882, upload-time = "2026-04-20T23:34:51.86Z" }, + { url = "https://files.pythonhosted.org/packages/19/6a/31e2296bc0787c5ab75d3d118e40b239db8151b5192b90b77c72bc9256e9/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7af18183109e23502c8b2ae7f6926c0882766f35b5175a4cd737ad825e4d7a1b", size = 3351298, upload-time = "2026-04-20T23:34:54.124Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a8/75f4e3e11203b590150abed2cf7794b9c9c9f7eceddae955191138b44dde/psycopg2_binary-2.9.12-cp312-cp312-win_amd64.whl", hash = "sha256:398fcd4db988c7d7d3713e2b8e18939776fd3fb447052daae4f24fa39daede4c", size = 2757230, upload-time = "2026-04-20T23:34:56.242Z" }, + { url = "https://files.pythonhosted.org/packages/91/bb/4608c96f970f6e0c56572e87027ef4404f709382a3503e9934526d7ba051/psycopg2_binary-2.9.12-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7c729a73c7b1b84de3582f73cdd27d905121dc2c531f3d9a3c32a3011033b965", size = 3712419, upload-time = "2026-04-20T23:34:58.754Z" }, + { url = "https://files.pythonhosted.org/packages/5e/af/48f76af9d50d61cf390f8cd657b503168b089e2e9298e48465d029fcc713/psycopg2_binary-2.9.12-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4413d0caef93c5cf50b96863df4c2efe8c269bf2267df353225595e7e15e8df7", size = 3822990, upload-time = "2026-04-20T23:35:00.821Z" }, + { url = "https://files.pythonhosted.org/packages/7a/df/aba0f99397cd811d32e06fc0cc781f1f3ce98bc0e729cb423925085d781a/psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4dfcf8e45ebb0c663be34a3442f65e17311f3367089cd4e5e3a3e8e62c978777", size = 4578696, upload-time = "2026-04-20T23:35:03.409Z" }, + { url = "https://files.pythonhosted.org/packages/95/9c/eaa74021ac4e4d5c2f83d82fc6615a63f4fe6c94dc4e94c3990427053f67/psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c41321a14dd74aceb6a9a643b9253a334521babfa763fa873e33d89cfa122fb5", size = 4274982, upload-time = "2026-04-20T23:35:05.583Z" }, + { url = "https://files.pythonhosted.org/packages/35/ed/c25deff98bd26187ba48b3b250a3ffc3037c46c5b89362534a15d200e0db/psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83946ba43979ebfdc99a3cd0ee775c89f221df026984ba19d46133d8d75d3cd9", size = 5894867, upload-time = "2026-04-20T23:35:07.902Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/8d0e21ca77373c6c9589e5c4528f6e8f0c08c62cafc76fb0bddb7a2cee22/psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:411e85815652d13560fbe731878daa5d92378c4995a22302071890ec3397d019", size = 4110578, upload-time = "2026-04-20T23:35:10.149Z" }, + { url = "https://files.pythonhosted.org/packages/00/fc/f481e2435bd8f742d0123309174aae4165160ad3ef17c1b99c3622c241d2/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c8ad4c08e00f7679559eaed7aff1edfffc60c086b976f93972f686384a95e2c", size = 3655816, upload-time = "2026-04-20T23:35:12.56Z" }, + { url = "https://files.pythonhosted.org/packages/53/79/b9f46466bdbe9f239c96cde8be33c1aace4842f06013b47b730dc9759187/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:00814e40fa23c2b37ef0a1e3c749d89982c73a9cb5046137f0752a22d432e82f", size = 3301307, upload-time = "2026-04-20T23:35:15.029Z" }, + { url = "https://files.pythonhosted.org/packages/3f/19/7dc003b32fe35024df89b658104f7c8538a8b2dcbde7a4e746ce929742e7/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:98062447aebc20ed20add1f547a364fd0ef8933640d5372ff1873f8deb9b61be", size = 3048968, upload-time = "2026-04-20T23:35:16.757Z" }, + { url = "https://files.pythonhosted.org/packages/91/58/2dbd7db5c604d45f4950d988506aae672a14126ec22998ced5021cbb76bb/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:66a7685d7e548f10fb4ce32fb01a7b7f4aa702134de92a292c7bd9e0d3dbd290", size = 3351369, upload-time = "2026-04-20T23:35:18.933Z" }, + { url = "https://files.pythonhosted.org/packages/42/ee/dee8dcaad07f735824de3d6563bc67119fa6c28257b17977a8d624f02fab/psycopg2_binary-2.9.12-cp313-cp313-win_amd64.whl", hash = "sha256:b6937f5fe4e180aeee87de907a2fa982ded6f7f15d7218f78a083e4e1d68f2a0", size = 2757347, upload-time = "2026-04-20T23:35:21.283Z" }, + { url = "https://files.pythonhosted.org/packages/13/1b/708c0dca874acfad6d65314271859899a79007686f3a1f74e82a2ed4b645/psycopg2_binary-2.9.12-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6f3b3de8a74ef8db215f22edffb19e32dc6fa41340456de7ec99efdc8a7b3ec2", size = 3712428, upload-time = "2026-04-20T23:35:23.453Z" }, + { url = "https://files.pythonhosted.org/packages/d6/39/ddbea9d4b4de6aca9431b6ed253f530f8a02d3b8f9bcfd0dbfe2b3de6fe4/psycopg2_binary-2.9.12-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1006fb62f0f0bc5ce256a832356c6262e91be43f5e4eb15b5eaf38079464caf2", size = 3823184, upload-time = "2026-04-20T23:35:25.92Z" }, + { url = "https://files.pythonhosted.org/packages/bf/a0/bc2fef74b106fa345567122a0659e6d94512ed7dc0131ec44c9e5aba3725/psycopg2_binary-2.9.12-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:840066105706cd2eb29b9a1c2329620056582a4bf3e8169dec5c447042d0869f", size = 4579157, upload-time = "2026-04-20T23:35:28.542Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/d4e3b2005d3de607ca4fbb0e8742e248056e52184a6b94ebda3c1c2c329b/psycopg2_binary-2.9.12-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:863f5d12241ebe1c76a72a04c2113b6dc905f90b9cef0e9be0efd994affd9354", size = 4274970, upload-time = "2026-04-20T23:35:30.418Z" }, + { url = "https://files.pythonhosted.org/packages/2e/42/c9853f8db3967fe08bcde11f53d53b85d351750cae726ce001cb68afa9c1/psycopg2_binary-2.9.12-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a99eaab34a9010f1a086b126de467466620a750634d114d20455f3a824aae033", size = 5895175, upload-time = "2026-04-20T23:35:33.584Z" }, + { url = "https://files.pythonhosted.org/packages/eb/fd/b82b5601a97630308bef079f545ffec481bbbc795c2ba5ec416a01d03f60/psycopg2_binary-2.9.12-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ffdd7dc5463ccd61845ac37b7012d0f35a1548df9febe14f8dd549be4a0bc81e", size = 4110658, upload-time = "2026-04-20T23:35:35.638Z" }, + { url = "https://files.pythonhosted.org/packages/62/8c/32ca69b0389ef25dd22937bf9e8fbe2ce27aea20b05ded48c4ce4cb42475/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:54a0dfecab1b48731f934e06139dfe11e24219fb6d0ceb32177cf0375f14c7b5", size = 3656251, upload-time = "2026-04-20T23:35:37.854Z" }, + { url = "https://files.pythonhosted.org/packages/c4/29/96992a2b59e3b9d730fcf9612d0a387305025dc867a9fc490a9e496e074e/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:96937c9c5d891f772430f418a7a8b4691a90c3e6b93cf72b5bd7cad8cbca32a5", size = 3301810, upload-time = "2026-04-20T23:35:39.927Z" }, + { url = "https://files.pythonhosted.org/packages/56/ad/44b06659949b243ae10112cd3b20a197f9bf3e81d5651379b9eb889bfaad/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:77b348775efd4cdab410ec6609d81ccecd1139c90265fa583a7255c8064bc03d", size = 3048977, upload-time = "2026-04-20T23:35:41.806Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/10a1bcebadb6aa55e280e1f58975c36a7b560ea525184c7aa4064c466633/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:527e6342b3e44c2f0544f6b8e927d60de7f163f5723b8f1dfa7d2a84298738cd", size = 3351466, upload-time = "2026-04-20T23:35:43.993Z" }, + { url = "https://files.pythonhosted.org/packages/20/be/b732c8418ffa5bcfda002890f5dc4c869fc17db66ff11f53b17cfe44afc0/psycopg2_binary-2.9.12-cp314-cp314-win_amd64.whl", hash = "sha256:f12ae41fcafadb39b2785e64a40f9db05d6de2ac114077457e0e7c597f3af980", size = 2848762, upload-time = "2026-04-20T23:35:46.421Z" }, +] + [[package]] name = "pyarrow" version = "23.0.1" From e19a074297a50e290252e9582af93c3a7fcaf4d9 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sat, 2 May 2026 15:39:23 -0700 Subject: [PATCH 17/22] ci: add Helm chart lint, port consistency, and hook ordering checks Three checks that would have caught the bugs found during local deploy: - helm lint: template syntax validation - port check: asserts Dockerfile EXPOSE == values.yaml targetPort - hook ordering: asserts SA and Secret weight < bootstrap Job weight Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .github/workflows/ci.yml | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88493b5..e24f59d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,62 @@ jobs: - name: Run e2e tests run: npx --prefix frontend playwright test --config tests/e2e/playwright.config.ts + # ── Helm chart ────────────────────────────────────────────────────────────── + helm-chart: + name: Helm — lint & validate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Helm + uses: azure/setup-helm@v4 + with: + version: v3.18.0 + + - name: Lint chart + run: helm lint helm/analytics-agent/ + + - name: Render templates + run: | + helm template test-release helm/analytics-agent/ \ + --set image.repository=analytics-agent \ + --set image.tag=test \ + > /tmp/rendered.yaml + + - name: Verify port consistency (Dockerfile EXPOSE vs Helm targetPort) + run: | + DOCKERFILE_PORT=$(grep '^EXPOSE' docker/Dockerfile | awk '{print $2}') + HELM_PORT=$(grep 'targetPort:' helm/analytics-agent/values.yaml | awk '{print $2}') + echo "Dockerfile EXPOSE: $DOCKERFILE_PORT" + echo "Helm targetPort: $HELM_PORT" + [ "$DOCKERFILE_PORT" = "$HELM_PORT" ] || \ + { echo "ERROR: port mismatch — update helm/analytics-agent/values.yaml"; exit 1; } + + - name: Verify hook ordering (SA and Secret weight < bootstrap Job weight) + run: | + pip install -q pyyaml + python3 - <<'PYEOF' + import sys, yaml + + docs = list(yaml.safe_load_all(open("/tmp/rendered.yaml"))) + hooks = {} + for doc in docs: + if not doc: + continue + ann = (doc.get("metadata") or {}).get("annotations") or {} + if "pre-install" in ann.get("helm.sh/hook", ""): + hooks[doc["kind"]] = int(ann.get("helm.sh/hook-weight", 0)) + + print("Hook weights:", hooks) + sa, secret, job = hooks.get("ServiceAccount"), hooks.get("Secret"), hooks.get("Job") + assert sa is not None, "ServiceAccount must be a hook" + assert secret is not None, "Secret must be a hook" + assert job is not None, "bootstrap Job must be a hook" + assert sa < job, f"ServiceAccount weight ({sa}) must be < Job ({job})" + assert secret < job, f"Secret weight ({secret}) must be < Job ({job})" + print("Hook ordering OK") + PYEOF + # ── Frontend ──────────────────────────────────────────────────────────────── frontend-lint: name: Frontend — typecheck & lint From a028c896ce529a8d37e7179155e84261e361c7a7 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sat, 2 May 2026 15:47:04 -0700 Subject: [PATCH 18/22] ci: move Helm checks to dedicated helm.yml workflow with path filters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Triggers only on changes to helm/** or docker/Dockerfile — not on every Python or frontend change. Removes the helm-chart job from ci.yml. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .github/workflows/ci.yml | 56 ------------------------------- .github/workflows/helm.yml | 69 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/helm.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e24f59d..88493b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,62 +87,6 @@ jobs: - name: Run e2e tests run: npx --prefix frontend playwright test --config tests/e2e/playwright.config.ts - # ── Helm chart ────────────────────────────────────────────────────────────── - helm-chart: - name: Helm — lint & validate - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Helm - uses: azure/setup-helm@v4 - with: - version: v3.18.0 - - - name: Lint chart - run: helm lint helm/analytics-agent/ - - - name: Render templates - run: | - helm template test-release helm/analytics-agent/ \ - --set image.repository=analytics-agent \ - --set image.tag=test \ - > /tmp/rendered.yaml - - - name: Verify port consistency (Dockerfile EXPOSE vs Helm targetPort) - run: | - DOCKERFILE_PORT=$(grep '^EXPOSE' docker/Dockerfile | awk '{print $2}') - HELM_PORT=$(grep 'targetPort:' helm/analytics-agent/values.yaml | awk '{print $2}') - echo "Dockerfile EXPOSE: $DOCKERFILE_PORT" - echo "Helm targetPort: $HELM_PORT" - [ "$DOCKERFILE_PORT" = "$HELM_PORT" ] || \ - { echo "ERROR: port mismatch — update helm/analytics-agent/values.yaml"; exit 1; } - - - name: Verify hook ordering (SA and Secret weight < bootstrap Job weight) - run: | - pip install -q pyyaml - python3 - <<'PYEOF' - import sys, yaml - - docs = list(yaml.safe_load_all(open("/tmp/rendered.yaml"))) - hooks = {} - for doc in docs: - if not doc: - continue - ann = (doc.get("metadata") or {}).get("annotations") or {} - if "pre-install" in ann.get("helm.sh/hook", ""): - hooks[doc["kind"]] = int(ann.get("helm.sh/hook-weight", 0)) - - print("Hook weights:", hooks) - sa, secret, job = hooks.get("ServiceAccount"), hooks.get("Secret"), hooks.get("Job") - assert sa is not None, "ServiceAccount must be a hook" - assert secret is not None, "Secret must be a hook" - assert job is not None, "bootstrap Job must be a hook" - assert sa < job, f"ServiceAccount weight ({sa}) must be < Job ({job})" - assert secret < job, f"Secret weight ({secret}) must be < Job ({job})" - print("Hook ordering OK") - PYEOF - # ── Frontend ──────────────────────────────────────────────────────────────── frontend-lint: name: Frontend — typecheck & lint diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml new file mode 100644 index 0000000..5801295 --- /dev/null +++ b/.github/workflows/helm.yml @@ -0,0 +1,69 @@ +name: Helm chart + +on: + push: + branches: [main] + paths: + - "helm/**" + - "docker/Dockerfile" + pull_request: + branches: [main] + paths: + - "helm/**" + - "docker/Dockerfile" + +jobs: + helm-chart: + name: Lint & validate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Helm + uses: azure/setup-helm@v4 + with: + version: v3.18.0 + + - name: Lint chart + run: helm lint helm/analytics-agent/ + + - name: Render templates + run: | + helm template test-release helm/analytics-agent/ \ + --set image.repository=analytics-agent \ + --set image.tag=test \ + > /tmp/rendered.yaml + + - name: Verify port consistency (Dockerfile EXPOSE vs Helm targetPort) + run: | + DOCKERFILE_PORT=$(grep '^EXPOSE' docker/Dockerfile | awk '{print $2}') + HELM_PORT=$(grep 'targetPort:' helm/analytics-agent/values.yaml | awk '{print $2}') + echo "Dockerfile EXPOSE: $DOCKERFILE_PORT" + echo "Helm targetPort: $HELM_PORT" + [ "$DOCKERFILE_PORT" = "$HELM_PORT" ] || \ + { echo "ERROR: port mismatch — update helm/analytics-agent/values.yaml"; exit 1; } + + - name: Verify hook ordering (SA and Secret weight < bootstrap Job weight) + run: | + pip install -q pyyaml + python3 - <<'PYEOF' + import sys, yaml + + docs = list(yaml.safe_load_all(open("/tmp/rendered.yaml"))) + hooks = {} + for doc in docs: + if not doc: + continue + ann = (doc.get("metadata") or {}).get("annotations") or {} + if "pre-install" in ann.get("helm.sh/hook", ""): + hooks[doc["kind"]] = int(ann.get("helm.sh/hook-weight", 0)) + + print("Hook weights:", hooks) + sa, secret, job = hooks.get("ServiceAccount"), hooks.get("Secret"), hooks.get("Job") + assert sa is not None, "ServiceAccount must be a hook" + assert secret is not None, "Secret must be a hook" + assert job is not None, "bootstrap Job must be a hook" + assert sa < job, f"ServiceAccount weight ({sa}) must be < Job ({job})" + assert secret < job, f"Secret weight ({secret}) must be < Job ({job})" + print("Hook ordering OK") + PYEOF From a73fd603c0879ae9746c01747cfefda6998e829d Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sat, 2 May 2026 15:49:39 -0700 Subject: [PATCH 19/22] ci(helm): add kind-based helm install integration test Spins up a kind cluster, builds the image, starts Postgres, runs helm install, and verifies the /health endpoint responds 200. Runs after the static lint job passes (needs: helm-chart). Docker layer caching keeps repeat runs fast. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .github/workflows/helm.yml | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 5801295..0b80978 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -67,3 +67,73 @@ jobs: assert secret < job, f"Secret weight ({secret}) must be < Job ({job})" print("Hook ordering OK") PYEOF + + helm-install: + name: Integration test — helm install + runs-on: ubuntu-latest + needs: helm-chart + steps: + - uses: actions/checkout@v4 + + - name: Install kind + run: | + curl -Lo /usr/local/bin/kind \ + https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64 + chmod +x /usr/local/bin/kind + + - name: Install Helm + uses: azure/setup-helm@v4 + with: + version: v3.18.0 + + - name: Create kind cluster + run: kind create cluster --wait 60s + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker + + - name: Build image + uses: docker/build-push-action@v6 + with: + context: . + file: docker/Dockerfile + tags: analytics-agent:test + push: false + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Load image into kind + run: kind load docker-image analytics-agent:test + + - name: Start Postgres + run: | + kubectl run postgres \ + --image=postgres:15 \ + --env=POSTGRES_PASSWORD=secret \ + --env=POSTGRES_DB=analytics \ + --env=POSTGRES_USER=analytics + kubectl expose pod postgres --port=5432 + kubectl wait --for=condition=Ready pod/postgres --timeout=60s + + - name: Helm install + run: | + helm install aa-test helm/analytics-agent/ \ + --set image.repository=analytics-agent \ + --set image.tag=test \ + --set image.pullPolicy=Never \ + --set "config.env.DATABASE_URL=postgresql+asyncpg://analytics:secret@postgres:5432/analytics" \ + --set "config.env.ENCRYPTION_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa=" \ + --timeout=120s + + - name: Wait for deployment rollout + run: kubectl rollout status deployment/aa-test-analytics-agent --timeout=120s + + - name: Verify /health endpoint + run: | + kubectl port-forward svc/aa-test-analytics-agent 8080:80 & + sleep 3 + curl -sf http://localhost:8080/health + echo "" + curl -sf http://localhost:8080/api/engines From fad3c809b4289057674d2dc0691bfe3544c978dd Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sat, 2 May 2026 15:57:11 -0700 Subject: [PATCH 20/22] fix(ci): use helm/kind-action to install kind in CI runner Direct curl+chmod to /usr/local/bin fails on GitHub-hosted runners (permission denied). helm/kind-action handles installation correctly. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .github/workflows/helm.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 0b80978..0359146 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -75,20 +75,16 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install kind - run: | - curl -Lo /usr/local/bin/kind \ - https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64 - chmod +x /usr/local/bin/kind + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + wait: 60s - name: Install Helm uses: azure/setup-helm@v4 with: version: v3.18.0 - - name: Create kind cluster - run: kind create cluster --wait 60s - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: From be81c2930232d7410f58994649b8e5bdaadade67 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sat, 2 May 2026 15:59:42 -0700 Subject: [PATCH 21/22] fix(ci): use plain docker build instead of build-push-action GHA cache export is unsupported with the docker buildx driver, which is required for kind load docker-image to work. Plain docker build is simpler and sufficient for a local kind-based integration test. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .github/workflows/helm.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 0359146..97980c8 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -85,20 +85,8 @@ jobs: with: version: v3.18.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver: docker - - name: Build image - uses: docker/build-push-action@v6 - with: - context: . - file: docker/Dockerfile - tags: analytics-agent:test - push: false - cache-from: type=gha - cache-to: type=gha,mode=max + run: docker build -f docker/Dockerfile -t analytics-agent:test . - name: Load image into kind run: kind load docker-image analytics-agent:test From e35b0443927cac3ba5810474c5d5b651dfd288be Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sat, 2 May 2026 16:01:42 -0700 Subject: [PATCH 22/22] fix(ci): set cluster_name=kind for helm/kind-action The action defaults to cluster name "chart-testing"; kind load docker-image expects "kind". Explicitly set the name to match. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .github/workflows/helm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 97980c8..055cff9 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -78,6 +78,7 @@ jobs: - name: Create kind cluster uses: helm/kind-action@v1 with: + cluster_name: kind wait: 60s - name: Install Helm