From 0becd5306ea83e791f3937d4b22354ecd1932c76 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Mon, 4 May 2026 15:44:10 -0700 Subject: [PATCH 1/5] =?UTF-8?q?feat:=20analytics-agent=20quickstart=20?= =?UTF-8?q?=E2=80=94=20no-clone=20install=20via=20pip/uvx?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Users can now go from zero to a running agent with a single command: pip install datahub-analytics-agent && analytics-agent quickstart # or: uvx datahub-analytics-agent quickstart The interactive wizard prompts for LLM provider + API key, an optional data source connection, then bootstraps the DB, starts the server, and opens the browser. Config lives in ~/.datahub/analytics-agent/ alongside the DataHub CLI's own config. New CLI commands: quickstart Interactive setup wizard (idempotent — detects existing config) demo Full demo: DataHub quickstart + Olist MySQL + metadata ingestion start Start server from existing config (no wizard) stop Stop running server status Show PID and URL of running server logs Tail ~/.datahub/analytics-agent/logs/agent.log config Open config directory in \$EDITOR or print its path Server lifecycle managed via PID file at ~/.datahub/analytics-agent/agent.pid. DATABASE_URL and ENGINES_CONFIG now default to the config dir so a pip-installed package works out of the box without any local repo or .env file. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .github/workflows/publish.yml | 11 + .gitignore | 1 + AGENTS.md | 24 +- Makefile | 130 ++++ README.md | 60 +- backend/src/analytics_agent/cli.py | 118 +++- backend/src/analytics_agent/config.py | 24 +- backend/src/analytics_agent/demo/__init__.py | 0 .../src/analytics_agent/demo/config.demo.yaml | 17 + .../analytics_agent/demo/ingest_metadata.py | 425 ++++++++++++ .../analytics_agent/demo/load_sample_data.py | 242 +++++++ backend/src/analytics_agent/main.py | 23 +- backend/src/analytics_agent/quickstart.py | 654 ++++++++++++++++++ docker-compose.yml | 7 +- hatch_build.py | 55 ++ justfile | 147 ---- pyproject.toml | 3 + 17 files changed, 1751 insertions(+), 190 deletions(-) create mode 100644 Makefile create mode 100644 backend/src/analytics_agent/demo/__init__.py create mode 100644 backend/src/analytics_agent/demo/config.demo.yaml create mode 100644 backend/src/analytics_agent/demo/ingest_metadata.py create mode 100644 backend/src/analytics_agent/demo/load_sample_data.py create mode 100644 backend/src/analytics_agent/quickstart.py create mode 100644 hatch_build.py delete mode 100644 justfile diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6667bbc..b677b3a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,6 +17,17 @@ jobs: with: enable-cache: true + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install pnpm + run: npm install -g pnpm + + - name: Build frontend + run: cd frontend && pnpm install --frozen-lockfile && pnpm build + - name: Build package run: uv build diff --git a/.gitignore b/.gitignore index 8d87fe8..72dc7c4 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ data/ node_modules/ frontend/dist/ frontend/.vite/ +backend/src/analytics_agent/static/ # IDE .idea/ diff --git a/AGENTS.md b/AGENTS.md index fb75bf4..f0e180e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,23 +12,23 @@ Analytics Agent is a LangGraph-based chat agent that uses **DataHub** tools for ## Running the stack -A `justfile` at the repo root covers all common tasks. Install `just` once (`brew install just`), then: +A `Makefile` at the repo root covers all common tasks (`make` is available everywhere — no install needed): ```bash -just install # uv sync + pnpm install -just start # build frontend if stale, start backend at :8100 -just port=8102 start # same on a custom port -just stop # kill the backend -just nuke # wipe the DB (start from scratch / re-trigger wizard) -just start-remote # start + print DataHub connection status -just logs # tail /tmp/analytics_agent.log -just test # unit tests -just build # force frontend rebuild +make install # uv sync + pnpm install +make start # build frontend if stale, start backend at :8100 +make PORT=8102 start # same on a custom port +make stop # kill the backend +make nuke # wipe the DB (start from scratch / re-trigger wizard) +make start-remote # start + print DataHub connection status +make logs # tail /tmp/analytics_agent.log +make test # unit tests +make build # force frontend rebuild ``` -`just start` automatically detects whether `frontend/src` is newer than `frontend/dist` and rebuilds only when needed. +`make start` uses Make's native dependency tracking: it rebuilds the frontend only when `frontend/src` files are newer than `frontend/dist/index.html`. -### Without just (manual) +### Without make (manual) ```bash uv sync diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d573c69 --- /dev/null +++ b/Makefile @@ -0,0 +1,130 @@ +PORT := 8100 +DEV_PORT := 8101 +LOG := /tmp/analytics_agent.log + +# Load .env if present (same behaviour as just's set dotenv-load) +-include .env +export + +.DEFAULT_GOAL := help + +.PHONY: help install build typecheck serve dev start frontend dev-full stop \ + restart logs test test-integration test-e2e start-remote check lint fix nuke + +# ── Help ─────────────────────────────────────────────────────────────────────── + +help: + @echo "Usage: make [PORT=8100]" + @echo "" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf " %-20s %s\n", $$1, $$2}' + +# ── Build ────────────────────────────────────────────────────────────────────── + +install: ## Install all dependencies (Python + Node) + uv sync + cd frontend && pnpm install + +# Make's native dependency tracking replaces just's build-if-stale recipe. +# Re-runs pnpm build only when frontend/src files are newer than the bundle. +frontend/dist/index.html: $(shell find frontend/src -type f 2>/dev/null) + cd frontend && pnpm build + +build: ## Force-rebuild the frontend + cd frontend && pnpm build + +typecheck: ## Type-check frontend without building + cd frontend && pnpm tsc --noEmit + +# ── Development ──────────────────────────────────────────────────────────────── + +serve: ## Start backend in foreground with auto-reload (blocking) + uv run analytics-agent bootstrap + uv run uvicorn analytics_agent.main:app --reload --port $(DEV_PORT) + +dev: frontend/dist/index.html ## Build frontend if stale, start backend with auto-reload + uv run analytics-agent bootstrap + pkill -f "analytics_agent.main" || true + nohup uv run uvicorn analytics_agent.main:app --reload --port $(DEV_PORT) > $(LOG) 2>&1 & + sleep 3 && curl -s http://localhost:$(DEV_PORT)/api/engines | head -c 120 + @echo "\n→ http://localhost:$(DEV_PORT) (logs: make logs)" + +start: frontend/dist/index.html ## Build frontend if stale, start backend on PORT + uv run analytics-agent bootstrap + pkill -f "analytics_agent.main" || true + nohup uv run uvicorn analytics_agent.main:app --port $(PORT) > $(LOG) 2>&1 & + sleep 3 && curl -s http://localhost:$(PORT)/api/engines | head -c 120 + @echo "\n→ http://localhost:$(PORT)" + +frontend: ## Start Vite dev server with HMR (use alongside 'serve') + cd frontend && pnpm dev + +dev-full: ## Start backend (reload) + Vite dev server in parallel + uv run analytics-agent bootstrap + pkill -f "analytics_agent.main" || true + nohup uv run uvicorn analytics_agent.main:app --reload --port $(DEV_PORT) > $(LOG) 2>&1 & + @echo "Backend → http://localhost:$(DEV_PORT)" + cd frontend && pnpm dev + +stop: ## Kill the backend + pkill -f "analytics_agent.main" || true + @echo "stopped" + +restart: build stop start ## Force-rebuild frontend and restart backend + +logs: ## Tail backend logs + tail -f $(LOG) + +# ── Testing ──────────────────────────────────────────────────────────────────── + +test: ## Run unit tests + uv run pytest tests/unit/ -v + +test-integration: ## Run integration tests (needs credentials in .env) + uv run pytest tests/integration/ -v -s + +test-e2e: ## Run Playwright e2e tests (real backend + mock MCP tools) + npx --prefix frontend playwright test --config tests/e2e/playwright.config.ts + +# ── Quality ──────────────────────────────────────────────────────────────────── + +check: ## Quick syntax check of the backend + uv run python -c "import analytics_agent.main" + +lint: ## Lint + format check (mirrors CI) + uv run ruff check backend/src tests + uv run ruff format --check backend/src tests + +fix: ## Auto-fix lint and format issues + uv run ruff check --fix backend/src tests + uv run ruff format backend/src tests + +# ── Remote / DataHub ────────────────────────────────────────────────────────── + +start-remote: start ## Start agent pointed at a remote DataHub instance + @echo "" + @echo " ┌─────────────────────────────────────────────────────┐" + @echo " │ Analytics Agent — Remote DataHub status │" + @echo " └─────────────────────────────────────────────────────┘" + uv run python scripts/datahub_status.py $(PORT) + @echo "" + @echo " → http://localhost:$(PORT)" + +# ── Maintenance ──────────────────────────────────────────────────────────────── + +nuke: stop ## Wipe local DB so the onboarding wizard reappears + @DB_URL="$${DATABASE_URL:-sqlite+aiosqlite:///./data/dev.db}"; \ + if echo "$$DB_URL" | grep -q "^sqlite"; then \ + DB_PATH=$$(echo "$$DB_URL" | sed 's|sqlite.*:///||; s|^\./||'); \ + if [ -f "$$DB_PATH" ]; then \ + rm "$$DB_PATH" && echo "✓ Deleted $$DB_PATH"; \ + else \ + echo " (no SQLite DB found at $$DB_PATH — already clean)"; \ + fi; \ + else \ + echo "Non-SQLite DB detected ($$DB_URL)."; \ + echo "To reset manually, drop and recreate the schema your DATABASE_URL points to."; \ + fi + @echo "" + @echo "Database wiped. Run 'make start' to come back up fresh." + @echo "Tip: open http://localhost:$(PORT)/#setup to force the wizard on an existing session." diff --git a/README.md b/README.md index 2dddd9e..74a9038 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,33 @@ ## ⚡ Quickstart +### Option A — pip / uvx (recommended, no Docker needed) + +```bash +# Install and launch — no git clone, no repo, no Docker +pip install datahub-analytics-agent +analytics-agent quickstart + +# Or with uv (no virtualenv management): +uvx datahub-analytics-agent quickstart +``` + +The wizard asks for your LLM provider + API key, optionally connects a data source, then starts the agent at **http://localhost:8100**. Config and the database are stored in `~/.datahub/analytics-agent/`. + +Re-running `analytics-agent quickstart` detects the existing config and offers to start, reconfigure, or cancel — so it doubles as the "just start the agent" command for repeat users. + +**Other server commands:** + +```bash +analytics-agent start # start from existing config (no wizard) +analytics-agent stop # stop the running server +analytics-agent status # show whether server is running + URL +analytics-agent logs # tail ~/.datahub/analytics-agent/logs/agent.log +analytics-agent config # open config dir in $EDITOR or print its path +``` + +### Option B — Docker + sample data (full demo) + > **Requires:** Docker, DataHub CLI (`pip install acryl-datahub`), `uv`, Python 3.11+ ```bash @@ -31,12 +58,7 @@ cd analytics-agent bash quickstart.sh ``` -No `.env` editing required. The script: -- Starts a local DataHub instance (or connects to an existing one) -- Loads the Olist e-commerce sample dataset + catalog metadata -- Builds and launches Analytics Agent at **http://localhost:8100** - -Open the browser — a setup wizard walks you through naming your agent, picking a model (Anthropic, OpenAI, Google, or AWS Bedrock), and entering your API key. If you already have one of those keys exported in your shell, it's picked up automatically. +The script starts a local DataHub instance, loads the Olist e-commerce sample dataset and catalog metadata, then builds and launches Analytics Agent at **http://localhost:8100**. Postgres data is persisted to `~/.datahub/analytics-agent/postgres-data/` so it survives container restarts. **Using AWS Bedrock?** Export `LLM_PROVIDER=bedrock` before running the script. The script will verify your AWS credentials and Bedrock access before starting the container, and mount `~/.aws` read-only so boto3 picks up your profiles and SSO cache automatically. @@ -68,22 +90,24 @@ Open the browser — a setup wizard walks you through naming your agent, picking --- -## Manual setup (without quickstart.sh) +## Manual setup (for contributors / development) + +> This section is for hacking on the agent itself. For everyday use, `analytics-agent quickstart` is simpler. -> **Manual setup also requires:** `node` and `just` (`brew install node just`) +> **Also requires:** `node` ### 1. Clone and install ```bash git clone https://github.com/datahub-project/analytics-agent.git cd analytics-agent -just install # uv sync + pnpm install -just start # builds frontend, starts backend at :8100 +make install # uv sync + pnpm install +make start # builds frontend, starts backend at :8100 ``` Open **http://localhost:8100** — a setup wizard handles the LLM key and connections on first run. -> **Without `just`:** `uv sync && cd frontend && pnpm install && pnpm build && cd .. && uv run uvicorn analytics_agent.main:app --port 8100` +> **Without `make`:** `uv sync && cd frontend && pnpm install && pnpm build && cd .. && uv run uvicorn analytics_agent.main:app --port 8100` ### First-time setup @@ -113,15 +137,15 @@ DATAHUB_GMS_URL=https://your-instance.acryl.io/gms DATAHUB_GMS_TOKEN=eyJhbGci... ``` -### Useful just tasks +### Useful make targets | Command | What it does | |---|---| -| `just start` | Build frontend if stale, start backend | -| `just start-remote` | Start + show DataHub connection status | -| `just nuke` | Wipe the DB and start from scratch | -| `just dev` | Hot-reload backend (use `just dev-full` for frontend HMR too) | -| `just logs` | Tail backend logs | +| `make start` | Build frontend if stale, start backend | +| `make start-remote` | Start + show DataHub connection status | +| `make nuke` | Wipe the DB and start from scratch | +| `make dev` | Hot-reload backend (use `make dev-full` for frontend HMR too) | +| `make logs` | Tail backend logs | ### Development mode (hot reload) @@ -283,7 +307,7 @@ LLM_MODEL=us.anthropic.claude-sonnet-4-5-20250929-v1:0 ## Database -The quickstart uses the DataHub MySQL container. For non-quickstart runs, SQLite is the default (`./data/dev.db`). Set `DATABASE_URL` in `.env` to switch backends — see `.env.example` for Postgres and SQLite formats. +The `analytics-agent quickstart` path uses SQLite at `~/.datahub/analytics-agent/data/agent.db`. The Docker quickstart uses Postgres, with data persisted to `~/.datahub/analytics-agent/postgres-data/`. For dev/Helm deployments, set `DATABASE_URL` explicitly — see `.env.example` for Postgres and SQLite formats. --- diff --git a/backend/src/analytics_agent/cli.py b/backend/src/analytics_agent/cli.py index 1ac0603..2a1766b 100644 --- a/backend/src/analytics_agent/cli.py +++ b/backend/src/analytics_agent/cli.py @@ -1,9 +1,12 @@ -"""Click CLI for analytics-agent — bootstrap operations.""" +"""Click CLI for analytics-agent — bootstrap and server operations.""" from __future__ import annotations import asyncio import logging +import os +import subprocess +import sys import click @@ -20,6 +23,9 @@ def cli() -> None: ) +# ── Bootstrap commands (unchanged) ──────────────────────────────────────────── + + @cli.command("migrate") def migrate() -> None: """Apply Alembic migrations to the configured database.""" @@ -66,3 +72,113 @@ async def _run_all_seeds() -> None: click.echo("→ Seeding integrations, context platforms, and defaults…") asyncio.run(_run_all_seeds()) click.echo("✓ Bootstrap complete.") + + +# ── Quickstart / server lifecycle ───────────────────────────────────────────── + + +@cli.command("quickstart") +@click.option("--port", default=8100, show_default=True, help="Port to listen on.") +@click.option( + "--demo", + is_flag=True, + default=False, + help="Full demo: start DataHub, load Olist sample data, and launch the agent.", +) +def quickstart(port: int, demo: bool) -> None: + """Interactive wizard: configure and launch the agent in one step.""" + if demo: + from analytics_agent.quickstart import run_demo + + run_demo(port=port) + else: + from analytics_agent.quickstart import run_wizard + + run_wizard(port=port) + + +@cli.command("start") +@click.option("--port", default=8100, show_default=True, help="Port to listen on.") +def start(port: int) -> None: + """Start the server from existing config (no wizard).""" + from analytics_agent.config import get_config_dir + from analytics_agent.quickstart import read_pid, start_server, wait_for_server + + if read_pid() is not None: + click.echo("Server is already running. Use `analytics-agent status` for details.") + sys.exit(1) + + config_dir = get_config_dir() + env_path = config_dir / ".env" + if not env_path.exists(): + click.echo( + f"No config found at {config_dir}. Run `analytics-agent quickstart` first.", + err=True, + ) + sys.exit(1) + + click.echo("→ Starting server…") + try: + pid = start_server(port) + except RuntimeError as e: + click.echo(f"✗ {e}", err=True) + sys.exit(1) + if wait_for_server(port): + click.echo(f"✓ Running at http://localhost:{port} (PID {pid})") + else: + click.echo("✗ Server did not respond within 30s.", err=True) + sys.exit(1) + + +@cli.command("stop") +def stop() -> None: + """Stop the running server.""" + from analytics_agent.quickstart import stop_server + + if stop_server(): + click.echo("✓ Server stopped.") + else: + click.echo("No running server found.") + + +@cli.command("status") +def status() -> None: + """Show whether the server is running and its URL.""" + from analytics_agent.quickstart import read_pid, read_port + + pid = read_pid() + if pid: + port = read_port() + click.echo(f"✓ Running (PID {pid}) → http://localhost:{port}") + else: + click.echo("✗ Not running") + + +@cli.command("logs") +@click.option("-n", "--lines", default=50, show_default=True, help="Lines to show initially.") +def logs(lines: int) -> None: + """Tail the agent log file.""" + from analytics_agent.quickstart import _log_file + + log_path = _log_file() + if not log_path.exists(): + click.echo(f"Log file not found: {log_path}", err=True) + sys.exit(1) + + try: + subprocess.run(["tail", "-n", str(lines), "-f", str(log_path)]) + except KeyboardInterrupt: + pass + + +@cli.command("config") +def config_cmd() -> None: + """Open the config directory in $EDITOR or print its path.""" + from analytics_agent.config import get_config_dir + + config_dir = get_config_dir() + editor = os.environ.get("EDITOR", "") + if editor: + subprocess.run([editor, str(config_dir)]) + else: + click.echo(str(config_dir)) diff --git a/backend/src/analytics_agent/config.py b/backend/src/analytics_agent/config.py index 20af099..3a6f3aa 100644 --- a/backend/src/analytics_agent/config.py +++ b/backend/src/analytics_agent/config.py @@ -6,6 +6,22 @@ from pathlib import Path from typing import Annotated, Any, Literal + +def get_config_dir() -> Path: + """Return the active analytics-agent config directory. + + Checks ANALYTICS_AGENT_CONFIG_DIR env var first; falls back to + ~/.datahub/analytics-agent/. Callers may resolve sub-paths from this. + """ + return Path( + os.environ.get("ANALYTICS_AGENT_CONFIG_DIR", "~/.datahub/analytics-agent") + ).expanduser() + + +# Computed once at import time. ANALYTICS_AGENT_CONFIG_DIR must be set in the +# shell environment before import — not in .env — to affect these defaults. +_CONFIG_DIR = get_config_dir() + import yaml from pydantic import BaseModel, Field, TypeAdapter from pydantic_settings import BaseSettings, SettingsConfigDict @@ -188,11 +204,11 @@ def get_api_key(self) -> str: attr = PROVIDER_KEY_ATTR.get(self.llm_provider, "") return getattr(self, attr, "") if attr else "" - # Database - database_url: str = "sqlite+aiosqlite:///./data/dev.db" + # Database — defaults to the user config dir; override via DATABASE_URL env var + database_url: str = f"sqlite+aiosqlite:///{_CONFIG_DIR}/data/agent.db" - # Engine config - engines_config: str = "./config.yaml" + # Engine config — defaults to the user config dir; override via ENGINES_CONFIG env var + engines_config: str = str(_CONFIG_DIR / "config.yaml") sql_row_limit: int = 500 # App diff --git a/backend/src/analytics_agent/demo/__init__.py b/backend/src/analytics_agent/demo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/analytics_agent/demo/config.demo.yaml b/backend/src/analytics_agent/demo/config.demo.yaml new file mode 100644 index 0000000..da53b8f --- /dev/null +++ b/backend/src/analytics_agent/demo/config.demo.yaml @@ -0,0 +1,17 @@ +context_platforms: + - type: datahub + name: default + label: "DataHub" + url: "${DATAHUB_GMS_URL}" + token: "${DATAHUB_GMS_TOKEN}" + +engines: + - type: mysql + name: olist_ecommerce + connection: + dialect: mysql+pymysql + host: "${MYSQL_HOST}" + port: 3306 + user: "${MYSQL_USER}" + password: "${MYSQL_PASSWORD}" + database: "${MYSQL_DATABASE}" diff --git a/backend/src/analytics_agent/demo/ingest_metadata.py b/backend/src/analytics_agent/demo/ingest_metadata.py new file mode 100644 index 0000000..8b0c826 --- /dev/null +++ b/backend/src/analytics_agent/demo/ingest_metadata.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +""" +Register the Olist MySQL tables in DataHub via GraphQL: + 1. createIngestionSource — upsert a MySQL ingestion recipe + 2. createIngestionExecutionRequest — run it inside DataHub's executor + 3. Poll until SUCCESS, then patch in human-readable descriptions + +The sink section is intentionally omitted from the recipe — DataHub fills it +in automatically so the executor can always reach the correct GMS endpoint. + +Usage (from repo root): + uv run python scripts/ingest_metadata.py [options] +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +import urllib.request + +TABLE_DESCRIPTIONS: dict[str, str] = { + "olist_customers": "Brazilian e-commerce customers with city and state information", + "olist_orders": "Order lifecycle records including status, purchase timestamp, and delivery timestamps", + "olist_order_items": "Line items in each order, linking orders to products and sellers with price and freight", + "olist_order_payments": "Payment method and installment details for each order", + "olist_order_reviews": "Customer satisfaction reviews with scores and comments per order", + "olist_products": "Product catalog with category, dimensions, and weight", + "olist_sellers": "Marketplace sellers with city and state location", + "product_category_name_translation": "Portuguese to English category name translations", +} + + +def _gql(gms_url: str, token: str, query: str, variables: dict | None = None) -> dict: + payload = {"query": query} + if variables: + payload["variables"] = variables + req = urllib.request.Request( + f"{gms_url}/api/graphql", + data=json.dumps(payload).encode(), + headers={ + "Content-Type": "application/json", + **({"Authorization": f"Bearer {token}"} if token else {}), + }, + ) + with urllib.request.urlopen(req, timeout=10) as r: + body = json.loads(r.read()) + if "errors" in body: + raise RuntimeError(f"GraphQL error: {body['errors']}") + return body.get("data", {}) + + +def _upsert_ingestion_source( + gms_url: str, + token: str, + mysql_host_port: str, + database: str, + mysql_user: str, + mysql_password: str, +) -> str: + """Create (or update if already exists) the MySQL ingestion source. Returns the source URN.""" + + recipe = json.dumps( + { + "source": { + "type": "mysql", + "config": { + "host_port": mysql_host_port, + "database": database, + "username": mysql_user, + "password": mysql_password, + "profile_pattern": {"deny": [".*"]}, + }, + } + # sink intentionally omitted — DataHub infers the correct GMS endpoint + } + ) + + # Check if a source with our name already exists + existing = _gql( + gms_url, + token, + """ + { listIngestionSources(input: {start: 0, count: 50}) { + ingestionSources { urn name } + }} + """, + ) + sources = existing.get("listIngestionSources", {}).get("ingestionSources", []) + existing_urn = next( + (s["urn"] for s in sources if s["name"] == "Analytics Agent Demo — MySQL"), None + ) + + input_fields = { + "name": "Analytics Agent Demo — MySQL", + "type": "mysql", + "description": f"Olist e-commerce sample data in {database}", + "config": { + "recipe": recipe, + "executorId": "default", + "debugMode": False, + }, + } + + if existing_urn: + print(f"[→] Updating existing ingestion source: {existing_urn}") + _gql( + gms_url, + token, + "mutation($urn: String!, $input: UpdateIngestionSourceInput!) { updateIngestionSource(urn: $urn, input: $input) }", + {"urn": existing_urn, "input": input_fields}, + ) + return existing_urn + else: + print("[→] Creating ingestion source...") + result = _gql( + gms_url, + token, + "mutation($input: UpdateIngestionSourceInput!) { createIngestionSource(input: $input) }", + {"input": input_fields}, + ) + urn = result["createIngestionSource"] + print(f"[✓] Ingestion source created: {urn}") + return urn + + +def _run_and_wait(gms_url: str, token: str, source_urn: str, timeout_secs: int = 300) -> None: + """Trigger an execution request and poll until it succeeds or fails.""" + result = _gql( + gms_url, + token, + "mutation($input: CreateIngestionExecutionRequestInput!) { createIngestionExecutionRequest(input: $input) }", + {"input": {"ingestionSourceUrn": source_urn}}, + ) + exec_urn = result["createIngestionExecutionRequest"] + print(f"[→] Execution started: {exec_urn}") + + deadline = time.time() + timeout_secs + poll_interval = 5 + printf_dots = False + while time.time() < deadline: + time.sleep(poll_interval) + r = _gql( + gms_url, + token, + "query($urn: String!) { executionRequest(urn: $urn) { result { status report } } }", + {"urn": exec_urn}, + ) + result_data = (r.get("executionRequest") or {}).get("result") or {} + status = result_data.get("status", "PENDING") + + if status in ("RUNNING", "PENDING"): + print(".", end="", flush=True) + printf_dots = True + poll_interval = min(poll_interval + 2, 15) + continue + + if printf_dots: + print() + + if status == "SUCCESS": + print("[✓] Ingestion pipeline succeeded") + return + + # FAILURE or other terminal state + report = result_data.get("report", "") + raise RuntimeError(f"Ingestion execution {status}:\n{report[-500:]}") + + raise TimeoutError(f"Ingestion did not complete within {timeout_secs}s") + + +def _patch_descriptions(gms_url: str, token: str, database: str) -> None: + """Emit human-readable descriptions on top of the ingested schema.""" + from datahub.emitter.rest_emitter import DatahubRestEmitter + from datahub.metadata.schema_classes import ( + DatasetPropertiesClass, + DatasetSnapshotClass, + MetadataChangeEventClass, + ) + + emitter = DatahubRestEmitter(gms_server=gms_url, token=token or None) + + for table, description in TABLE_DESCRIPTIONS.items(): + urn = f"urn:li:dataset:(urn:li:dataPlatform:mysql,{database}.{table},PROD)" + snapshot = DatasetSnapshotClass( + urn=urn, + aspects=[DatasetPropertiesClass(description=description, name=table)], + ) + try: + emitter.emit_mce(MetadataChangeEventClass(proposedSnapshot=snapshot)) + print(f"[✓] Description: {table}") + except Exception as e: + print(f"[!] Failed description for {table}: {e}", file=sys.stderr) + emitter.flush() + + +def _seed_demo_context(gms_url: str, token: str, database: str) -> None: + """Seed demo-ready tags, glossary terms, and table ownership for Olist tables.""" + from datahub.emitter.rest_emitter import DatahubRestEmitter + from datahub.metadata.schema_classes import ( + AuditStampClass, + DatasetSnapshotClass, + GlobalTagsClass, + GlossaryTermAssociationClass, + GlossaryTermInfoClass, + GlossaryTermsClass, + GlossaryTermSnapshotClass, + MetadataChangeEventClass, + OwnerClass, + OwnershipClass, + OwnershipTypeClass, + TagAssociationClass, + TagPropertiesClass, + TagSnapshotClass, + ) + + emitter = DatahubRestEmitter(gms_server=gms_url, token=token or None) + now_ms = int(time.time() * 1000) + stamp = AuditStampClass(time=now_ms, actor="urn:li:corpuser:datahub") + + def _dataset_urn(table: str) -> str: + return f"urn:li:dataset:(urn:li:dataPlatform:mysql,{database}.{table},PROD)" + + # ── 1. Create tag entities ────────────────────────────────────────────── + TAGS: dict[str, str] = { + "pii": "Contains personally identifiable information (customer IDs, locations, free-text review comments)", + "identifier": "Primary or foreign key columns used for joining tables", + "financial": "Contains monetary values (prices, payments, freight costs)", + } + for tag_name, description in TAGS.items(): + try: + emitter.emit_mce( + MetadataChangeEventClass( + proposedSnapshot=TagSnapshotClass( + urn=f"urn:li:tag:{tag_name}", + aspects=[TagPropertiesClass(name=tag_name, description=description)], + ) + ) + ) + print(f"[✓] Tag: {tag_name}") + except Exception as e: + print(f"[!] Failed tag {tag_name}: {e}", file=sys.stderr) + + # ── 2. Create glossary term entities ──────────────────────────────────── + TERMS: dict[str, dict[str, str]] = { + "revenue": { + "name": "Revenue", + "definition": ( + "Total sales value from delivered orders. Calculated as SUM(price + freight_value) " + "from olist_order_items, filtered to orders where olist_orders.order_status = 'delivered'. " + "Non-delivered orders (canceled, unavailable) are excluded. For per-seller revenue, group by " + "olist_order_items.seller_id. For per-category revenue, join via product_id to olist_products " + "and then to product_category_name_translation for English category names." + ), + }, + "delivery_sla": { + "name": "Delivery SLA", + "definition": ( + "Delivery performance metric. An order meets SLA if order_delivered_customer_date <= " + "order_estimated_delivery_date. SLA breach = actual delivery later than estimated. " + "Per-seller SLA breach rate = count(breached orders) / count(delivered orders) for orders " + "shipped by that seller. Only computed for delivered orders; canceled and undelivered orders " + "are excluded from the denominator." + ), + }, + "active_seller": { + "name": "Active Seller", + "definition": ( + "A seller with at least one delivered order in the trailing 30 days, measured from the most " + "recent order_purchase_timestamp in the dataset. For historical Olist data (2016-2018), " + "'trailing 30 days' is computed relative to MAX(order_purchase_timestamp), not today's date." + ), + }, + } + for term_id, term in TERMS.items(): + try: + emitter.emit_mce( + MetadataChangeEventClass( + proposedSnapshot=GlossaryTermSnapshotClass( + urn=f"urn:li:glossaryTerm:{term_id}", + aspects=[ + GlossaryTermInfoClass( + name=term["name"], + definition=term["definition"], + termSource="INTERNAL", + ) + ], + ) + ) + ) + print(f"[✓] Glossary term: {term['name']}") + except Exception as e: + print(f"[!] Failed glossary term {term_id}: {e}", file=sys.stderr) + + # ── 3. Per-table: collect full tag/term/owner lists, emit one MCE each ── + # All aspects for each table are bundled into a single DatasetSnapshotClass + # so GlobalTagsClass and GlossaryTermsClass are each written exactly once — + # a second emit of either aspect would replace the first (overwrite bug). + TAG_MAP: dict[str, list[str]] = { + "olist_customers": ["pii", "identifier"], + "olist_orders": ["identifier"], + "olist_order_items": ["identifier", "financial"], + "olist_order_payments": ["financial"], + "olist_order_reviews": ["pii"], + "olist_products": ["identifier"], + "olist_sellers": ["pii", "identifier"], + } + TERM_MAP: dict[str, list[str]] = { + "olist_order_items": ["revenue"], + "olist_orders": ["revenue", "delivery_sla"], + "olist_sellers": ["delivery_sla", "active_seller"], + } + OWNER_MAP: dict[str, str] = { + "olist_sellers": "logistics_team", + "olist_order_items": "logistics_team", + "olist_order_payments": "finance_team", + "olist_order_reviews": "customer_experience_team", + "olist_customers": "customer_experience_team", + "olist_products": "product_team", + "product_category_name_translation": "product_team", + "olist_orders": "data_platform_team", + } + for table in sorted(set(TAG_MAP) | set(TERM_MAP) | set(OWNER_MAP)): + aspects: list = [] + tags = TAG_MAP.get(table, []) + if tags: + aspects.append( + GlobalTagsClass(tags=[TagAssociationClass(tag=f"urn:li:tag:{t}") for t in tags]) + ) + terms = TERM_MAP.get(table, []) + if terms: + aspects.append( + GlossaryTermsClass( + terms=[ + GlossaryTermAssociationClass(urn=f"urn:li:glossaryTerm:{t}") for t in terms + ], + auditStamp=stamp, + ) + ) + owner = OWNER_MAP.get(table) + if owner: + aspects.append( + OwnershipClass( + owners=[ + OwnerClass( + owner=f"urn:li:corpGroup:{owner}", + type=OwnershipTypeClass.TECHNICAL_OWNER, + ) + ], + lastModified=stamp, + ) + ) + if not aspects: + continue + try: + emitter.emit_mce( + MetadataChangeEventClass( + proposedSnapshot=DatasetSnapshotClass(urn=_dataset_urn(table), aspects=aspects) + ) + ) + print(f"[✓] Context: {table}") + except Exception as e: + print(f"[!] Failed context for {table}: {e}", file=sys.stderr) + + emitter.flush() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Ingest Olist metadata into DataHub via GraphQL") + parser.add_argument("--gms-url", default="http://localhost:8080") + parser.add_argument("--token", default="") + parser.add_argument("--database", default="analytics_agent_demo") + parser.add_argument("--mysql-host-port", default="mysql:3306") + parser.add_argument("--mysql-user", default="datahub") + parser.add_argument("--mysql-password", default="datahub") + parser.add_argument("--timeout", type=int, default=300) + parser.add_argument( + "--skip-context-seed", + action="store_true", + help="Skip seeding demo tags, glossary terms, and ownership", + ) + args = parser.parse_args() + + # 1. Test connectivity + try: + _gql(args.gms_url, args.token, "{ me { corpUser { username } } }") + print(f"[✓] Connected to DataHub GMS at {args.gms_url}") + except Exception as e: + print(f"[✗] Cannot reach DataHub GMS: {e}", file=sys.stderr) + sys.exit(1) + + # 2. Upsert ingestion source + source_urn = _upsert_ingestion_source( + args.gms_url, + args.token, + args.mysql_host_port, + args.database, + args.mysql_user, + args.mysql_password, + ) + + # 3. Run and wait + print(f"[→] Running ingestion (timeout: {args.timeout}s)...") + _run_and_wait(args.gms_url, args.token, source_urn, timeout_secs=args.timeout) + + # 4. Patch descriptions + print() + print("[→] Adding table descriptions...") + _patch_descriptions(args.gms_url, args.token, args.database) + + # 5. Seed demo context + if not args.skip_context_seed: + print() + print("[→] Seeding demo context (tags, glossary terms, ownership)...") + _seed_demo_context(args.gms_url, args.token, args.database) + print("[✓] Demo context seeded — 3 tags, 3 glossary terms, 8 table owners") + + print() + print(f"[✓] Done — {len(TABLE_DESCRIPTIONS)} tables indexed and described in DataHub.") + print(" View in DataHub UI: http://localhost:9002") + + +if __name__ == "__main__": + main() diff --git a/backend/src/analytics_agent/demo/load_sample_data.py b/backend/src/analytics_agent/demo/load_sample_data.py new file mode 100644 index 0000000..3b7e993 --- /dev/null +++ b/backend/src/analytics_agent/demo/load_sample_data.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +""" +Download the Olist e-commerce SQLite dataset and load it into MySQL. + +Usage (from repo root): + uv run python scripts/load_sample_data.py [options] +""" + +from __future__ import annotations + +import argparse +import sqlite3 +import sys +import tempfile +import urllib.request +from pathlib import Path + +OLIST_URL = ( + "https://github.com/datahub-project/static-assets/raw/main/datasets/olist-ecommerce/olist.db" +) + +TABLES = [ + "olist_customers", + "olist_orders", + "olist_order_items", + "olist_order_payments", + "olist_order_reviews", + "olist_products", + "olist_sellers", + "product_category_name_translation", +] + +# Fields whose content is long free-text — map to MySQL TEXT instead of VARCHAR(255). +# NOTE: product_category_name and product_category_name_english are kept as VARCHAR(255) +# because they are join keys between tables; TEXT columns cannot be indexed without a +# prefix length in MySQL. +LONG_TEXT_FIELDS = { + "review_comment_message", + "review_comment_title", + "product_description_lenght", # intentional Olist typo + "product_description_length", +} + +BATCH_SIZE = 500 + + +def _mysql_type(col_name: str, sqlite_type: str) -> str: + """Map a SQLite declared type to an appropriate MySQL type.""" + t = sqlite_type.upper().strip() + if t in ("INTEGER", "INT", "TINYINT", "SMALLINT", "MEDIUMINT", "BIGINT"): + return "BIGINT" + if t in ("REAL", "FLOAT", "DOUBLE", "NUMERIC", "DECIMAL", "NUMBER"): + return "DOUBLE" + if t == "BLOB": + return "LONGBLOB" + # TEXT and VARCHAR fall through; also empty / unknown types + if col_name.lower() in LONG_TEXT_FIELDS: + return "TEXT" + # Heuristic: long-text field names + lower = col_name.lower() + if any( + lower.endswith(suffix) + for suffix in ("_comment", "_message", "_title", "_desc", "_description") + ): + return "TEXT" + return "VARCHAR(255)" + + +def _download(url: str, dest: Path) -> None: + """Download *url* to *dest*, printing a progress bar.""" + total_blocks = 0 + + def reporthook(block_num: int, block_size: int, total_size: int) -> None: + nonlocal total_blocks + total_blocks = block_num + if total_size <= 0: + print(f"\r downloaded {block_num * block_size:,} bytes", end="", flush=True) + else: + pct = min(100, block_num * block_size * 100 // total_size) + bar = "#" * (pct // 2) + "-" * (50 - pct // 2) + print(f"\r [{bar}] {pct:3d}%", end="", flush=True) + + urllib.request.urlretrieve(url, dest, reporthook=reporthook) + print() # newline after progress bar + + +def _build_create_table(table: str, columns: list[tuple]) -> str: + """ + Build a MySQL CREATE TABLE statement from PRAGMA table_info rows. + + PRAGMA columns: cid, name, type, notnull, dflt_value, pk + """ + col_defs: list[str] = [] + pk_cols: list[str] = [] + for _cid, name, col_type, notnull, _dflt_value, pk in columns: + mysql_t = _mysql_type(name, col_type) + null_clause = "NOT NULL" if notnull else "NULL" + col_defs.append(f" `{name}` {mysql_t} {null_clause}") + if pk: + pk_cols.append(f"`{name}`") + if pk_cols: + col_defs.append(f" PRIMARY KEY ({', '.join(pk_cols)})") + cols_sql = ",\n".join(col_defs) + return f"CREATE TABLE `{table}` (\n{cols_sql}\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Load Olist sample data into MySQL") + parser.add_argument("--host", default="localhost", help="MySQL host (default: localhost)") + parser.add_argument("--port", type=int, default=3306, help="MySQL port (default: 3306)") + parser.add_argument( + "--user", default="datahub", help="MySQL user for data loading (default: datahub)" + ) + parser.add_argument( + "--password", default="datahub", help="MySQL password for data loading (default: datahub)" + ) + parser.add_argument( + "--database", + default="analytics_agent_demo", + help="MySQL database name (default: analytics_agent_demo)", + ) + # Admin credentials are only used for CREATE DATABASE + GRANT — the regular + # user (--user) may not have permission to create databases. + parser.add_argument( + "--admin-user", default="root", help="MySQL admin user for CREATE DATABASE (default: root)" + ) + parser.add_argument( + "--admin-password", default="datahub", help="MySQL admin password (default: datahub)" + ) + args = parser.parse_args() + + try: + import pymysql + except ImportError: + print("[!] pymysql not found. Add it to pyproject.toml and run: uv sync", file=sys.stderr) + sys.exit(1) + + # --- 1. Download SQLite file --- + print("[→] Downloading Olist dataset from GitHub...") + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: + tmp_path = Path(tmp.name) + + try: + _download(OLIST_URL, tmp_path) + print(f"[✓] Downloaded to {tmp_path} ({tmp_path.stat().st_size:,} bytes)") + + # --- 2. Open SQLite --- + sqlite_conn = sqlite3.connect(str(tmp_path)) + sqlite_conn.row_factory = sqlite3.Row + + db = args.database + + # --- 3. Create database + grant access (requires admin/root privileges) --- + print( + f"[→] Connecting to MySQL at {args.host}:{args.port} as admin ({args.admin_user}) ..." + ) + admin_conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.admin_user, + password=args.admin_password, + charset="utf8mb4", + autocommit=True, + ) + admin_cur = admin_conn.cursor() + admin_cur.execute( + f"CREATE DATABASE IF NOT EXISTS `{db}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" + ) + # Grant the regular user full access to the new database + admin_cur.execute(f"GRANT ALL PRIVILEGES ON `{db}`.* TO '{args.user}'@'%'") + admin_cur.execute("FLUSH PRIVILEGES") + admin_cur.close() + admin_conn.close() + print(f"[✓] Database `{db}` ready (granted to {args.user})") + + # --- 4. Connect as regular user for data loading --- + print(f"[→] Connecting as {args.user} for data load ...") + mysql_conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=db, + charset="utf8mb4", + autocommit=False, + ) + mysql_cur = mysql_conn.cursor() + + # --- 5. Load each table --- + total_rows = 0 + for table in TABLES: + sqlite_cur = sqlite_conn.cursor() + # Get schema + sqlite_cur.execute(f"PRAGMA table_info({table})") + columns = sqlite_cur.fetchall() + if not columns: + print(f"[!] Table '{table}' not found in SQLite — skipping") + continue + + col_names = [row[1] for row in columns] + placeholders = ", ".join(["%s"] * len(col_names)) + quoted_names = ", ".join([f"`{n}`" for n in col_names]) + + create_sql = _build_create_table(table, columns) + + # Drop + Create in MySQL + mysql_cur.execute(f"DROP TABLE IF EXISTS `{table}`") + mysql_cur.execute(create_sql) + mysql_conn.commit() + + # Fetch all rows from SQLite + sqlite_cur.execute(f"SELECT * FROM {table}") + rows = sqlite_cur.fetchall() + row_count = len(rows) + + # Insert in batches + insert_sql = f"INSERT INTO `{table}` ({quoted_names}) VALUES ({placeholders})" + for offset in range(0, row_count, BATCH_SIZE): + batch = [tuple(row) for row in rows[offset : offset + BATCH_SIZE]] + mysql_cur.executemany(insert_sql, batch) + mysql_conn.commit() + + total_rows += row_count + print(f"[✓] {table}: {row_count:,} rows loaded") + + # --- 6. Done --- + mysql_cur.close() + mysql_conn.close() + sqlite_conn.close() + + print() + print( + f"[✓] Done! {total_rows:,} total rows loaded into `{db}` across {len(TABLES)} tables." + ) + print(f" MySQL: {args.host}:{args.port}/{db}") + + finally: + tmp_path.unlink(missing_ok=True) + + +if __name__ == "__main__": + main() diff --git a/backend/src/analytics_agent/main.py b/backend/src/analytics_agent/main.py index a6898e3..9a21a3b 100644 --- a/backend/src/analytics_agent/main.py +++ b/backend/src/analytics_agent/main.py @@ -9,9 +9,19 @@ # Load .env into os.environ before anything else so load_engines_config() # env-var substitution (os.environ.get) resolves correctly. -# Try project root first, then fall back to cwd search. -_env_file = Path(__file__).parents[3] / ".env" -load_dotenv(_env_file if _env_file.exists() else None, override=True) +# Resolution order: ANALYTICS_AGENT_CONFIG_DIR/.env → project root .env → cwd .env +_config_dir = Path( + os.environ.get("ANALYTICS_AGENT_CONFIG_DIR", "~/.datahub/analytics-agent") +).expanduser() +_env_candidates = [_config_dir / ".env", Path(__file__).parents[3] / ".env"] +_loaded_env = False +for _env_file in _env_candidates: + if _env_file.exists(): + load_dotenv(_env_file, override=True) + _loaded_env = True + break +if not _loaded_env: + load_dotenv(override=True) # fall back to cwd .env search from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -370,7 +380,12 @@ async def _health() -> dict[str, str]: # Only activates when frontend/dist/ exists (production / pnpm build). # Falls back gracefully: serves API-only if dist is absent (dev mode). _env_dist = os.getenv("FRONTEND_DIST", "") - _dist = Path(_env_dist) if _env_dist else Path(__file__).parents[3] / "frontend" / "dist" + if _env_dist: + _dist = Path(_env_dist) + elif (Path(__file__).parent / "static").exists(): + _dist = Path(__file__).parent / "static" # bundled in wheel + else: + _dist = Path(__file__).parents[3] / "frontend" / "dist" # dev / repo if _dist.exists(): logger.info("Serving frontend from %s", _dist) diff --git a/backend/src/analytics_agent/quickstart.py b/backend/src/analytics_agent/quickstart.py new file mode 100644 index 0000000..a354586 --- /dev/null +++ b/backend/src/analytics_agent/quickstart.py @@ -0,0 +1,654 @@ +"""Quickstart wizard and server lifecycle management for analytics-agent.""" + +from __future__ import annotations + +import getpass +import os +import signal +import subprocess +import sys +import textwrap +import time +from pathlib import Path +from typing import Any + +import click + +from analytics_agent.config import get_config_dir + +# ── Engine prompts ───────────────────────────────────────────────────────────── + +# Each entry: list of (prompt_label, env_var_name, is_secret) +_ENGINE_FIELDS: dict[str, list[tuple[str, str, bool]]] = { + "snowflake": [ + ("Account (e.g. xy12345.us-east-1)", "SNOWFLAKE_ACCOUNT", False), + ("Warehouse", "SNOWFLAKE_WAREHOUSE", False), + ("Database", "SNOWFLAKE_DATABASE", False), + ("Schema", "SNOWFLAKE_SCHEMA", False), + ("User", "SNOWFLAKE_USER", False), + ("Password", "SNOWFLAKE_PASSWORD", True), + ], + "bigquery": [ + ("GCP Project ID", "BIGQUERY_PROJECT", False), + ("Dataset", "BIGQUERY_DATASET", False), + ("Path to service-account JSON key", "BIGQUERY_CREDENTIALS_FILE", False), + ], + "postgresql": [ + ("Host", "POSTGRES_HOST", False), + ("Port", "POSTGRES_PORT", False), + ("Database", "POSTGRES_DATABASE", False), + ("User", "POSTGRES_USER", False), + ("Password", "POSTGRES_PASSWORD", True), + ], + "mysql": [ + ("Host", "MYSQL_HOST", False), + ("Port", "MYSQL_PORT", False), + ("Database", "MYSQL_DATABASE", False), + ("User", "MYSQL_USER", False), + ("Password", "MYSQL_PASSWORD", True), + ], +} + +# config.yaml connection block template per engine type. +# Values reference ${ENV_VAR} substitution supported by analytics-agent. +_ENGINE_CONFIG_TEMPLATE: dict[str, dict[str, Any]] = { + "snowflake": { + "account": "${SNOWFLAKE_ACCOUNT}", + "warehouse": "${SNOWFLAKE_WAREHOUSE}", + "database": "${SNOWFLAKE_DATABASE}", + "schema": "${SNOWFLAKE_SCHEMA}", + "user": "${SNOWFLAKE_USER}", + "password": "${SNOWFLAKE_PASSWORD}", + }, + "bigquery": { + "project": "${BIGQUERY_PROJECT}", + "dataset": "${BIGQUERY_DATASET}", + "credentials_file": "${BIGQUERY_CREDENTIALS_FILE}", + }, + "postgresql": { + "host": "${POSTGRES_HOST}", + "port": "${POSTGRES_PORT}", + "database": "${POSTGRES_DATABASE}", + "user": "${POSTGRES_USER}", + "password": "${POSTGRES_PASSWORD}", + }, + "mysql": { + "host": "${MYSQL_HOST}", + "port": "${MYSQL_PORT}", + "database": "${MYSQL_DATABASE}", + "user": "${MYSQL_USER}", + "password": "${MYSQL_PASSWORD}", + }, +} + +# ── Helpers ──────────────────────────────────────────────────────────────────── + + +def _prompt(label: str, default: str = "", secret: bool = False) -> str: + if secret: + value = getpass.getpass(f" {label}: ") + else: + if default: + value = click.prompt(f" {label}", default=default) + else: + value = click.prompt(f" {label}") + return value.strip() + + +def _write_env(path: Path, updates: dict[str, str]) -> None: + """Merge updates into an existing .env file (or create it).""" + existing: dict[str, str] = {} + if path.exists(): + for line in path.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, _, v = line.partition("=") + existing[k.strip()] = v.strip() + existing.update(updates) + lines = [f"{k}={v}" for k, v in existing.items()] + path.write_text("\n".join(lines) + "\n") + + +def _write_config_yaml(path: Path, engine_type: str, engine_name: str) -> None: + """Write (or merge) a config.yaml with one engine entry using ${VAR} refs.""" + import yaml + + existing: dict[str, Any] = {} + if path.exists(): + try: + existing = yaml.safe_load(path.read_text()) or {} + except Exception: + pass + + engines: list[dict[str, Any]] = existing.get("engines", []) + # Remove any existing entry with the same name to avoid duplicates + engines = [e for e in engines if e.get("name") != engine_name] + engines.append( + { + "type": engine_type, + "name": engine_name, + "connection": _ENGINE_CONFIG_TEMPLATE[engine_type], + } + ) + existing["engines"] = engines + + # Dump preserving existing context_platforms if present + path.write_text(yaml.dump(existing, default_flow_style=False, sort_keys=False)) + + +# ── PID / server management ──────────────────────────────────────────────────── + + +def _pid_file() -> Path: + return get_config_dir() / "agent.pid" + + +def _log_file() -> Path: + return get_config_dir() / "logs" / "agent.log" + + +def _is_running(pid: int) -> bool: + try: + os.kill(pid, 0) + return True + except (ProcessLookupError, PermissionError): + return False + + +def read_pid() -> int | None: + """Return the PID from the PID file, or None if absent/stale.""" + pf = _pid_file() + if not pf.exists(): + return None + try: + pid = int(pf.read_text().split(":")[0].strip()) + except ValueError: + return None + if _is_running(pid): + return pid + # Stale — clean up + pf.unlink(missing_ok=True) + return None + + +def read_port() -> int: + """Return the port from the PID file, defaulting to 8100.""" + pf = _pid_file() + if not pf.exists(): + return 8100 + parts = pf.read_text().strip().split(":") + if len(parts) == 2: + try: + return int(parts[1]) + except ValueError: + pass + return 8100 + + +def _port_in_use(port: int) -> bool: + import socket + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(("localhost", port)) == 0 + + +def start_server(port: int = 8100) -> int: + """Launch uvicorn in the background; return the PID.""" + if _port_in_use(port): + raise RuntimeError( + f"Port {port} is already in use. " + "Stop the existing process or choose a different port with --port." + ) + config_dir = get_config_dir() + log_dir = config_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + log_path = log_dir / "agent.log" + + env = os.environ.copy() + env["ANALYTICS_AGENT_CONFIG_DIR"] = str(config_dir) + + with open(log_path, "a") as log_fh: + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "uvicorn", + "analytics_agent.main:app", + "--host", + "0.0.0.0", + "--port", + str(port), + ], + stdout=log_fh, + stderr=log_fh, + env=env, + start_new_session=True, + ) + + _pid_file().write_text(f"{proc.pid}:{port}") + return proc.pid + + +def stop_server() -> bool: + """Send SIGTERM to the running server. Returns True if a process was killed.""" + pid = read_pid() + if pid is None: + return False + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + _pid_file().unlink(missing_ok=True) + return True + + +def wait_for_server(port: int = 8100, timeout: int = 30) -> bool: + """Poll health endpoint until ready or timeout (seconds). Returns True on success.""" + import urllib.error + import urllib.request + + url = f"http://localhost:{port}/health" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=2): + return True + except Exception: + time.sleep(1) + return False + + +# ── Wizard ───────────────────────────────────────────────────────────────────── + + +def run_wizard(port: int = 8100) -> None: + """Interactive quickstart wizard — configures and launches the agent.""" + click.echo( + textwrap.dedent(""" + ╔══════════════════════════════════════════╗ + ║ DataHub Analytics Agent — Quickstart ║ + ╚══════════════════════════════════════════╝ + """).strip() + ) + + # ── Prerequisites ────────────────────────────────────────────────────── + click.echo("\n→ Checking prerequisites…") + _check_prereqs(port) + + config_dir = get_config_dir() + + # ── Idempotency: existing config? ────────────────────────────────────── + env_path = config_dir / ".env" + if env_path.exists(): + click.echo(f"\n Existing config found at {config_dir}/") + choice = click.prompt( + " What would you like to do?", + type=click.Choice(["start", "reconfigure", "cancel"], case_sensitive=False), + default="start", + ) + if choice == "cancel": + click.echo(" Cancelled.") + return + if choice == "start": + _bootstrap_and_launch(config_dir, port) + return + # reconfigure → fall through to wizard + + # ── Step 1: LLM provider ─────────────────────────────────────────────── + click.echo("\nStep 1 — LLM provider") + provider = click.prompt( + " Which provider?", + type=click.Choice(["anthropic", "openai", "google", "bedrock"], case_sensitive=False), + default="anthropic", + ) + + # ── Step 2: API key ──────────────────────────────────────────────────── + click.echo("\nStep 2 — API key") + env_updates: dict[str, str] = {"LLM_PROVIDER": provider} + if provider == "anthropic": + key = getpass.getpass(" Anthropic API key (sk-ant-…): ").strip() + env_updates["ANTHROPIC_API_KEY"] = key + elif provider == "openai": + key = getpass.getpass(" OpenAI API key (sk-…): ").strip() + env_updates["OPENAI_API_KEY"] = key + elif provider == "google": + key = getpass.getpass(" Google API key: ").strip() + env_updates["GOOGLE_API_KEY"] = key + elif provider == "bedrock": + env_updates["AWS_REGION"] = _prompt("AWS region", default="us-west-2") + env_updates["AWS_ACCESS_KEY_ID"] = _prompt("AWS Access Key ID") + env_updates["AWS_SECRET_ACCESS_KEY"] = getpass.getpass(" AWS Secret Access Key: ").strip() + + # ── Step 3: Data source (optional) ──────────────────────────────────── + click.echo("\nStep 3 — Data source (you can add more later in Settings)") + engine_choices = ["snowflake", "bigquery", "postgresql", "mysql", "skip"] + engine_type = click.prompt( + " Connect a data source?", + type=click.Choice(engine_choices, case_sensitive=False), + default="skip", + ) + + engine_name: str | None = None + if engine_type != "skip": + engine_name = _prompt(f" Name for this {engine_type} connection", default=engine_type) + click.echo(f"\n Enter connection details for {engine_type}:") + for label, env_var, is_secret in _ENGINE_FIELDS[engine_type]: + env_updates[env_var] = _prompt(label, secret=is_secret) + + # ── Step 4: Write config and launch ─────────────────────────────────── + click.echo("\nStep 4 — Done") + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / "data").mkdir(parents=True, exist_ok=True) + + _write_env(env_path, env_updates) + click.echo(f" ✓ Config written to {config_dir}/") + + if engine_type != "skip" and engine_name: + yaml_path = config_dir / "config.yaml" + _write_config_yaml(yaml_path, engine_type, engine_name) + + _bootstrap_and_launch(config_dir, port) + + +def _check_prereqs(port: int) -> None: + major, minor = sys.version_info[:2] + if (major, minor) < (3, 11): + click.echo(f" ✗ Python 3.11+ required (found {major}.{minor})", err=True) + sys.exit(1) + click.echo(f" ✓ Python {major}.{minor}") + + if _port_in_use(port): + click.echo(f" ✗ Port {port} is already in use", err=True) + sys.exit(1) + click.echo(f" ✓ Port {port} available") + + +def _bootstrap_and_launch(config_dir: Path, port: int) -> None: + """Run bootstrap (migrations + seeds) then start the server.""" + import subprocess as _sp + + env = os.environ.copy() + env["ANALYTICS_AGENT_CONFIG_DIR"] = str(config_dir) + + # Load config-dir .env into the bootstrap subprocess environment + env_path = config_dir / ".env" + if env_path.exists(): + for line in env_path.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, _, v = line.partition("=") + env[k.strip()] = v.strip() + + click.echo(" ✓ Database initialised") + result = _sp.run( + [sys.executable, "-m", "analytics_agent.cli", "bootstrap"], + env=env, + capture_output=True, + text=True, + ) + if result.returncode != 0: + click.echo(result.stderr, err=True) + sys.exit(result.returncode) + + click.echo(" → Starting server…") + # Re-export env vars into current process so start_server picks them up + os.environ.update(env) + try: + pid = start_server(port) + except RuntimeError as e: + click.echo(f" ✗ {e}", err=True) + sys.exit(1) + + if wait_for_server(port): + click.echo(f" ✓ Running at http://localhost:{port} (PID {pid})") + click.echo(f" → Logs: {_log_file()}") + try: + import webbrowser + + webbrowser.open(f"http://localhost:{port}") + except Exception: + pass + else: + click.echo( + f" ✗ Server did not respond within 30s — check logs: {_log_file()}", + err=True, + ) + sys.exit(1) + + +# ── Demo mode (full DataHub + Olist sample data) ─────────────────────────────── + +# On Linux, host.docker.internal doesn't resolve — use Docker's default bridge gateway. +_HOST_INTERNAL = "host.docker.internal" if sys.platform == "darwin" else "172.17.0.1" + +_GMS_URL = "http://localhost:8080" +_DEMO_MYSQL_HOST = "localhost" +_DEMO_MYSQL_PORT = 3306 +_DEMO_MYSQL_USER = "datahub" +_DEMO_MYSQL_PASS = "datahub" +_DEMO_MYSQL_DB = "analytics_agent_demo" + + +def _check_demo_prereqs() -> None: + """Check Docker and datahub CLI are installed.""" + import shutil + + missing = [] + for cmd in ("docker", "datahub"): + if not shutil.which(cmd): + missing.append(cmd) + if missing: + click.echo( + f" ✗ Missing prerequisites: {', '.join(missing)}\n" + " • Docker: https://www.docker.com/products/docker-desktop\n" + " • DataHub CLI: pip install acryl-datahub", + err=True, + ) + sys.exit(1) + click.echo(" ✓ Docker and datahub CLI found") + + +def _gms_healthy(url: str = _GMS_URL) -> bool: + import urllib.error + import urllib.request + + try: + with urllib.request.urlopen(f"{url}/health", timeout=3): + return True + except Exception: + return False + + +def _start_datahub() -> None: + """Start DataHub via `datahub docker quickstart` if not already running.""" + if _gms_healthy(): + click.echo(" ✓ DataHub GMS already running") + return + + click.echo(" → Starting DataHub (this takes ~3-5 min on first run)…") + result = subprocess.run(["datahub", "docker", "quickstart"], check=False) + if result.returncode != 0: + click.echo(" ✗ DataHub quickstart failed.", err=True) + sys.exit(1) + + click.echo(" → Waiting for DataHub GMS to be healthy…") + deadline = time.monotonic() + 300 + while time.monotonic() < deadline: + if _gms_healthy(): + click.echo(" ✓ DataHub GMS is healthy") + return + time.sleep(5) + click.echo(" ✗ DataHub GMS did not become healthy within 5 minutes.", err=True) + sys.exit(1) + + +def _provision_datahub_token() -> str: + """Mint a local DataHub token via `datahub init`; return the token.""" + import tempfile + + import yaml # already a dep via pyyaml + + tmp = tempfile.mkdtemp() + env = os.environ.copy() + env["HOME"] = tmp + subprocess.run( + [ + "datahub", + "init", + "--username", + "datahub", + "--password", + "datahub", + "--force", + "--host", + _GMS_URL, + ], + env=env, + capture_output=True, + ) + try: + cfg = yaml.safe_load((Path(tmp) / ".datahubenv").read_text()) + return (cfg.get("gms") or {}).get("token", "") + except Exception: + return "" + + +def _load_olist_data() -> None: + """Run load_sample_data.py from the bundled demo package.""" + from analytics_agent.demo import load_sample_data # noqa: F401 + + demo_script = Path(__file__).parent / "demo" / "load_sample_data.py" + result = subprocess.run( + [ + sys.executable, + str(demo_script), + "--user", + _DEMO_MYSQL_USER, + "--password", + _DEMO_MYSQL_PASS, + "--database", + _DEMO_MYSQL_DB, + "--admin-user", + "root", + "--admin-password", + _DEMO_MYSQL_PASS, + ], + check=False, + ) + if result.returncode != 0: + click.echo(" ✗ Olist data loading failed.", err=True) + sys.exit(1) + click.echo(" ✓ Olist sample data loaded") + + +def _ingest_metadata(gms_token: str) -> None: + """Run ingest_metadata.py from the bundled demo package.""" + demo_script = Path(__file__).parent / "demo" / "ingest_metadata.py" + # DataHub's ingestion executor runs inside Docker — pass the host-internal + # address so it can reach the host MySQL (not Docker's own localhost). + result = subprocess.run( + [ + sys.executable, + str(demo_script), + "--gms-url", + _GMS_URL, + "--token", + gms_token, + "--database", + _DEMO_MYSQL_DB, + "--mysql-host-port", + f"{_HOST_INTERNAL}:{_DEMO_MYSQL_PORT}", + "--mysql-user", + _DEMO_MYSQL_USER, + "--mysql-password", + _DEMO_MYSQL_PASS, + ], + check=False, + ) + if result.returncode != 0: + click.echo(" ✗ Metadata ingestion failed.", err=True) + sys.exit(1) + click.echo(" ✓ Olist metadata ingested into DataHub") + + +def _write_demo_config(config_dir: Path, gms_token: str, llm_env: dict[str, str]) -> None: + """Write .env and config.yaml for the demo (DataHub + Olist MySQL).""" + import shutil + + # .env + env_updates: dict[str, str] = { + "DATAHUB_GMS_URL": f"http://{_HOST_INTERNAL}:8080", + "DATAHUB_GMS_TOKEN": gms_token, + "DATABASE_URL": ( + f"mysql+aiomysql://{_DEMO_MYSQL_USER}:{_DEMO_MYSQL_PASS}" + f"@{_HOST_INTERNAL}:{_DEMO_MYSQL_PORT}/talkster" + ), + "DISABLE_NEWER_GMS_FIELD_DETECTION": "true", + } + env_updates.update(llm_env) + _write_env(config_dir / ".env", env_updates) + + # config.yaml — copy from bundled template + src = Path(__file__).parent / "demo" / "config.demo.yaml" + dest = config_dir / "config.yaml" + shutil.copy(src, dest) + # Patch host references in the copied config + text = dest.read_text() + text = text.replace("${MYSQL_HOST}", _HOST_INTERNAL) + dest.write_text(text) + + click.echo(f" ✓ Demo config written to {config_dir}/") + + +def run_demo(port: int = 8100) -> None: + """Full demo: DataHub quickstart + Olist data + analytics agent.""" + click.echo( + textwrap.dedent(""" + ╔══════════════════════════════════════════╗ + ║ DataHub Analytics Agent — Demo ║ + ╚══════════════════════════════════════════╝ + """).strip() + ) + click.echo("\n→ Checking prerequisites…") + _check_demo_prereqs() + _check_prereqs(port) + + # LLM key + click.echo("\nStep 1 — LLM provider") + provider = click.prompt( + " Which provider?", + type=click.Choice(["anthropic", "openai", "google", "bedrock"], case_sensitive=False), + default="anthropic", + ) + llm_env: dict[str, str] = {"LLM_PROVIDER": provider} + click.echo("\nStep 2 — API key") + if provider == "anthropic": + llm_env["ANTHROPIC_API_KEY"] = getpass.getpass(" Anthropic API key (sk-ant-…): ").strip() + elif provider == "openai": + llm_env["OPENAI_API_KEY"] = getpass.getpass(" OpenAI API key (sk-…): ").strip() + elif provider == "google": + llm_env["GOOGLE_API_KEY"] = getpass.getpass(" Google API key: ").strip() + elif provider == "bedrock": + llm_env["AWS_REGION"] = _prompt("AWS region", default="us-west-2") + llm_env["AWS_ACCESS_KEY_ID"] = _prompt("AWS Access Key ID") + llm_env["AWS_SECRET_ACCESS_KEY"] = getpass.getpass(" AWS Secret Access Key: ").strip() + + config_dir = get_config_dir() + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / "data").mkdir(parents=True, exist_ok=True) + + click.echo("\n→ Starting DataHub…") + _start_datahub() + + click.echo("\n→ Loading Olist sample data…") + _load_olist_data() + + click.echo("\n→ Ingesting metadata into DataHub…") + gms_token = _provision_datahub_token() + _ingest_metadata(gms_token) + + click.echo("\n→ Writing demo config…") + _write_demo_config(config_dir, gms_token, llm_env) + + click.echo("\n→ Bootstrapping database…") + _bootstrap_and_launch(config_dir, port) diff --git a/docker-compose.yml b/docker-compose.yml index 5b5b2cd..818d7f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,9 @@ services: ports: - "5432:5432" volumes: - - postgres_data:/var/lib/postgresql/data + # Bind-mount to host so DB survives container removal and `docker compose down`. + # $HOME expands to the host user's home directory. + - ${HOME}/.datahub/analytics-agent/postgres-data:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U talk_to_data"] interval: 5s @@ -29,6 +31,3 @@ services: condition: service_healthy volumes: - ./config.yaml:/app/config.yaml:ro - -volumes: - postgres_data: diff --git a/hatch_build.py b/hatch_build.py new file mode 100644 index 0000000..1f944d1 --- /dev/null +++ b/hatch_build.py @@ -0,0 +1,55 @@ +"""Hatchling build hook — bundles the pre-built React frontend into the wheel. + +Before building the wheel, copies frontend/dist/ → backend/src/analytics_agent/static/ +so that `pip install datahub-analytics-agent` ships a fully self-contained package. + +Prerequisite: run `cd frontend && pnpm install && pnpm build` before `uv build`. +CI does this automatically in publish.yml. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + + +class CustomBuildHook(BuildHookInterface): + PLUGIN_NAME = "custom" + + def initialize(self, version: str, build_data: dict) -> None: + dist = Path("frontend/dist") + target = Path("backend/src/analytics_agent/static") + + if not dist.exists(): + self.app.display_warning( + "frontend/dist/ not found — building wheel without UI. " + "Run `cd frontend && pnpm install && pnpm build` first." + ) + return + + if target.exists(): + shutil.rmtree(target) + shutil.copytree(dist, target) + + # static/ is gitignored, so hatchling's file scanner won't pick it up. + # Explicitly register every file via force_include so they land in the wheel. + for f in target.rglob("*"): + if f.is_file(): + # key: path relative to project root + # value: destination path inside the wheel (relative to package root) + dest = str(f.relative_to("backend/src")) + build_data["force_include"][str(f)] = dest + + self.app.display_info(f"Bundled frontend ({_count(target)} files) into {target}") + + def finalize(self, version: str, build_data: dict, artifact_path: str) -> None: + # Clean up the generated static dir so it doesn't linger in the source tree + target = Path("backend/src/analytics_agent/static") + if target.exists(): + shutil.rmtree(target) + + +def _count(path: Path) -> int: + return sum(1 for _ in path.rglob("*") if _.is_file()) diff --git a/justfile b/justfile deleted file mode 100644 index 777cf83..0000000 --- a/justfile +++ /dev/null @@ -1,147 +0,0 @@ -set dotenv-load := true - -port := "8100" -dev_port := "8101" -log := "/tmp/analytics_agent.log" - -# List available recipes -default: - @just --list - -# Install all dependencies (Python + Node) -install: - uv sync - cd frontend && pnpm install - -# Build the frontend for production -build: - cd frontend && pnpm build - -# Build only if frontend/src is newer than the dist bundle -build-if-stale: - #!/usr/bin/env bash - if [ ! -f frontend/dist/index.html ] || \ - [ -n "$(find frontend/src -newer frontend/dist/index.html 2>/dev/null | head -1)" ]; then - echo "frontend is stale — rebuilding…" - cd frontend && pnpm build - else - echo "frontend is up to date" - fi - -# Type-check frontend without building -typecheck: - cd frontend && pnpm tsc --noEmit - -# Start backend (blocks; use 'dev' for background) -serve: - uv run analytics-agent bootstrap - uv run uvicorn analytics_agent.main:app --reload --port {{dev_port}} - -# Build frontend if stale, then start backend in background (with auto-reload on Python changes) -dev: build-if-stale - uv run analytics-agent bootstrap - pkill -f "analytics_agent.main" || true - nohup uv run uvicorn analytics_agent.main:app --reload --port {{dev_port}} > {{log}} 2>&1 & - sleep 3 && curl -s http://localhost:{{dev_port}}/api/engines | head -c 120 - @echo "\n→ http://localhost:{{dev_port}} (logs: just logs)" - -# Start backend, rebuilding frontend if stale -start: build-if-stale - uv run analytics-agent bootstrap - pkill -f "analytics_agent.main" || true - nohup uv run uvicorn analytics_agent.main:app --port {{port}} > {{log}} 2>&1 & - sleep 3 && curl -s http://localhost:{{port}}/api/engines | head -c 120 - @echo "\n→ http://localhost:{{port}}" - -# Start Vite dev server with HMR (use alongside 'serve' for hot-reload frontend) -# → http://localhost:5173 (proxies /api/* to backend) -frontend: - cd frontend && pnpm dev - -# Start backend (reload mode) + Vite dev server in parallel -dev-full: - uv run analytics-agent bootstrap - pkill -f "analytics_agent.main" || true - nohup uv run uvicorn analytics_agent.main:app --reload --port {{dev_port}} > {{log}} 2>&1 & - @echo "Backend → http://localhost:{{dev_port}}" - cd frontend && pnpm dev - -# Kill the backend -stop: - pkill -f "analytics_agent.main" || true - @echo "stopped" - -# Rebuild frontend and restart backend -restart: build stop start - -# Tail backend logs -logs: - tail -f {{log}} - -# Run unit tests -test: - uv run pytest tests/unit/ -v - -# Run integration tests (needs credentials in .env) -test-integration: - uv run pytest tests/integration/ -v -s - -# Run Playwright e2e tests (real backend + mock MCP tools) -test-e2e: - npx --prefix frontend playwright test --config tests/e2e/playwright.config.ts - -# Start the agent pointed at a remote DataHub instance. -# Set DATAHUB_GMS_URL + DATAHUB_GMS_TOKEN in .env for OSS/self-hosted DataHub. -# For Acryl cloud, leave those blank — configure DataHub via MCP in the UI after startup. -# Either way, open http://localhost:{{port}} and the wizard handles the rest. -start-remote: - #!/usr/bin/env bash - set -euo pipefail - just start - echo "" - echo " ┌─────────────────────────────────────────────────────┐" - echo " │ Analytics Agent — Remote DataHub status │" - echo " └─────────────────────────────────────────────────────┘" - # Query the running API for actual DataHub connection state - uv run python scripts/datahub_status.py {{port}} - echo "" - echo " → http://localhost:{{port}}" - -# Quick syntax check of the backend -check: - uv run python -c "import analytics_agent.main" - -# Lint + format check (mirrors CI — run before pushing) -lint: - uv run ruff check backend/src tests - uv run ruff format --check backend/src tests - -# Auto-fix lint and format issues -fix: - uv run ruff check --fix backend/src tests - uv run ruff format backend/src tests - -# Wipe the local database and browser state so the onboarding wizard reappears. -# Stops the server, deletes the SQLite DB, clears the dismissed flag hint. -# Re-run 'just start' afterwards to come back up fresh. -nuke: stop - #!/usr/bin/env bash - set -euo pipefail - # Resolve the DB path from DATABASE_URL (or fall back to the SQLite default) - DB_URL="${DATABASE_URL:-sqlite+aiosqlite:///./data/dev.db}" - if [[ "$DB_URL" == sqlite* ]]; then - DB_PATH="${DB_URL#sqlite*:///}" - DB_PATH="${DB_PATH#./}" - if [[ -f "$DB_PATH" ]]; then - rm "$DB_PATH" - echo "✓ Deleted $DB_PATH" - else - echo " (no SQLite DB found at $DB_PATH — already clean)" - fi - else - echo "Non-SQLite DB detected ($DB_URL)." - echo "To reset manually, drop and recreate the schema your DATABASE_URL points to." - fi - echo "" - echo "Database wiped. Run 'just start' to come back up fresh." - echo "Tip: open http://localhost:{{port}}/#setup to force the wizard on an existing session." diff --git a/pyproject.toml b/pyproject.toml index 0b8c353..790f6dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,9 @@ dev = [ [tool.hatch.build.targets.wheel] packages = ["backend/src/analytics_agent"] +[tool.hatch.build.targets.wheel.hooks.custom] +# hatch_build.py copies frontend/dist/ → backend/src/analytics_agent/static/ at build time + [tool.uv] package = true From 5f787174fd48d7859eced0c61fa50e810a089be3 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Mon, 4 May 2026 17:09:28 -0700 Subject: [PATCH 2/5] fix: quickstart bootstrap fails silently on fresh install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs prevented migrations from running during `analytics-agent quickstart`: 1. `python -m analytics_agent.cli bootstrap` was a no-op because cli.py had no `if __name__ == "__main__"` block — Click never got invoked, subprocess exited 0, and no tables were created. 2. `run_migrations()` called `Config("alembic.ini")` with a relative path that only resolves from the repo root. For pip-installed users (no checked-out repo) this silently produces an empty Alembic config and skips all migrations. Fixed by deriving script_location from the installed package path, with alembic.ini-in-CWD as a fallback for the dev/Docker workflow. Also moved the "✓ Database initialised" echo to after bootstrap succeeds so failures are no longer masked. Co-Authored-By: Claude Sonnet 4.6 --- backend/src/analytics_agent/bootstrap.py | 13 ++++++++++++- backend/src/analytics_agent/cli.py | 4 ++++ backend/src/analytics_agent/quickstart.py | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/backend/src/analytics_agent/bootstrap.py b/backend/src/analytics_agent/bootstrap.py index 53a330d..e528890 100644 --- a/backend/src/analytics_agent/bootstrap.py +++ b/backend/src/analytics_agent/bootstrap.py @@ -26,7 +26,18 @@ def run_migrations() -> None: elif "mysql" in settings.database_url: _ensure_mysql_schema() - alembic_cfg = Config("alembic.ini") + # Locate migration scripts via package path so this works when pip-installed + # (no alembic.ini on disk). Fall back to alembic.ini in CWD for the dev/Docker + # workflow where the repo is checked out and alembic CLI is also used. + _ini = Path("alembic.ini") + if _ini.exists(): + alembic_cfg = Config(str(_ini)) + else: + alembic_cfg = Config() + alembic_cfg.set_main_option( + "script_location", + str(Path(__file__).parent / "db" / "alembic"), + ) command.upgrade(alembic_cfg, "head") diff --git a/backend/src/analytics_agent/cli.py b/backend/src/analytics_agent/cli.py index 2a1766b..57ccd9b 100644 --- a/backend/src/analytics_agent/cli.py +++ b/backend/src/analytics_agent/cli.py @@ -182,3 +182,7 @@ def config_cmd() -> None: subprocess.run([editor, str(config_dir)]) else: click.echo(str(config_dir)) + + +if __name__ == "__main__": + cli() diff --git a/backend/src/analytics_agent/quickstart.py b/backend/src/analytics_agent/quickstart.py index a354586..722a0e6 100644 --- a/backend/src/analytics_agent/quickstart.py +++ b/backend/src/analytics_agent/quickstart.py @@ -379,7 +379,6 @@ def _bootstrap_and_launch(config_dir: Path, port: int) -> None: k, _, v = line.partition("=") env[k.strip()] = v.strip() - click.echo(" ✓ Database initialised") result = _sp.run( [sys.executable, "-m", "analytics_agent.cli", "bootstrap"], env=env, @@ -389,6 +388,7 @@ def _bootstrap_and_launch(config_dir: Path, port: int) -> None: if result.returncode != 0: click.echo(result.stderr, err=True) sys.exit(result.returncode) + click.echo(" ✓ Database initialised") click.echo(" → Starting server…") # Re-export env vars into current process so start_server picks them up From a074c858ba110248a89bdf19aeb5b28fe92ce0a4 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Mon, 4 May 2026 18:37:15 -0700 Subject: [PATCH 3/5] test: regression tests for quickstart bootstrap bugs Add two tests that would have caught the bugs fixed in the previous commit: - test_run_migrations_works_without_alembic_ini_in_cwd: verifies migrations run from a CWD with no alembic.ini (simulates pip-install environment). - test_cli_module_invocable_via_dash_m: verifies `python -m analytics_agent.cli bootstrap` actually runs migrations, catching the missing __main__ entry point. Co-Authored-By: Claude Sonnet 4.6 --- tests/unit/test_bootstrap.py | 18 ++++++++++++++++++ tests/unit/test_cli.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/tests/unit/test_bootstrap.py b/tests/unit/test_bootstrap.py index aac6d89..8c2226a 100644 --- a/tests/unit/test_bootstrap.py +++ b/tests/unit/test_bootstrap.py @@ -43,6 +43,24 @@ def test_run_migrations_creates_tables(sqlite_db, monkeypatch): assert "settings" in tables +def test_run_migrations_works_without_alembic_ini_in_cwd(sqlite_db, monkeypatch, tmp_path): + # Simulate a pip-installed environment: CWD has no alembic.ini. + monkeypatch.chdir(tmp_path) + assert not (tmp_path / "alembic.ini").exists() + + bootstrap.run_migrations() + + sync_url = f"sqlite:///{sqlite_db}" + engine = create_engine(sync_url) + tables = set(inspect(engine).get_table_names()) + engine.dispose() + + assert "alembic_version" in tables + assert "integrations" in tables + assert "context_platforms" in tables + assert "settings" in tables + + @pytest.mark.asyncio async def test_seed_integrations_idempotent(sqlite_db, monkeypatch): repo_root = Path(__file__).resolve().parents[2] diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 0519313..7d0b5f1 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -79,3 +79,35 @@ def test_cli_bootstrap_fails_fast_on_migration_error(monkeypatch, tmp_path): result = CliRunner().invoke(cli, ["bootstrap"], catch_exceptions=True) assert result.exit_code != 0 + + +def test_cli_module_invocable_via_dash_m(sqlite_db, monkeypatch, tmp_path): + """python -m analytics_agent.cli bootstrap must run migrations (not silently no-op). + + This was broken: cli.py had no __main__ block so the subprocess in + _bootstrap_and_launch exited 0 without doing anything. + """ + import subprocess + import sys + + monkeypatch.chdir(tmp_path) # no alembic.ini here — exercises pip-install path too + + from analytics_agent import config as _config + + result = subprocess.run( + [sys.executable, "-m", "analytics_agent.cli", "bootstrap"], + env={ + **__import__("os").environ, + "DATABASE_URL": f"sqlite+aiosqlite:///{sqlite_db}", + }, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + from sqlalchemy import create_engine, inspect + + engine = create_engine(f"sqlite:///{sqlite_db}") + tables = set(inspect(engine).get_table_names()) + engine.dispose() + assert "context_platforms" in tables From 49adc4206aaa28893b956be36cf32f1fb0f366f5 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Mon, 4 May 2026 18:50:01 -0700 Subject: [PATCH 4/5] fix: copy hatch_build.py into Docker image; remove unused import - Dockerfile was missing hatch_build.py in the COPY layer, causing `uv sync --no-dev` to fail with "Build script does not exist: hatch_build.py" - Remove unused `analytics_agent.config` import from test_cli.py (ruff F401) Co-Authored-By: Claude Sonnet 4.6 --- docker/Dockerfile | 2 +- tests/unit/test_cli.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0690b10..5bcd24b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,7 +14,7 @@ WORKDIR /app RUN pip install uv -COPY pyproject.toml uv.lock README.md ./ +COPY pyproject.toml uv.lock README.md hatch_build.py ./ COPY backend/ backend/ RUN uv sync --no-dev diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 7d0b5f1..ac90cc3 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -92,8 +92,6 @@ def test_cli_module_invocable_via_dash_m(sqlite_db, monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) # no alembic.ini here — exercises pip-install path too - from analytics_agent import config as _config - result = subprocess.run( [sys.executable, "-m", "analytics_agent.cli", "bootstrap"], env={ From f8af43e6c9e7651c2f643e059baee73d5fdf3914 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Mon, 4 May 2026 19:00:20 -0700 Subject: [PATCH 5/5] fix: mypy type error in demo; make E2E Working-state check resilient MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ingest_metadata.py: annotate payload as dict[str, object] so mypy accepts assigning a dict value alongside a str value (mypy was inferring dict[str, str] from the initial literal). - token-counting.spec.ts: the "Working" AgentWorkBlock state is only visible for ~320ms (4 mock events × 80ms). On slow CI the Playwright poller may miss this transient. Accept either "Working" or "Worked for" as the presence check; the final assertions (token badge, expand) still fully verify the feature. Co-Authored-By: Claude Sonnet 4.6 --- backend/src/analytics_agent/demo/ingest_metadata.py | 2 +- tests/e2e/token-counting.spec.ts | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/src/analytics_agent/demo/ingest_metadata.py b/backend/src/analytics_agent/demo/ingest_metadata.py index 8b0c826..c65b157 100644 --- a/backend/src/analytics_agent/demo/ingest_metadata.py +++ b/backend/src/analytics_agent/demo/ingest_metadata.py @@ -33,7 +33,7 @@ def _gql(gms_url: str, token: str, query: str, variables: dict | None = None) -> dict: - payload = {"query": query} + payload: dict[str, object] = {"query": query} if variables: payload["variables"] = variables req = urllib.request.Request( diff --git a/tests/e2e/token-counting.spec.ts b/tests/e2e/token-counting.spec.ts index 92b0b46..baa2680 100644 --- a/tests/e2e/token-counting.spec.ts +++ b/tests/e2e/token-counting.spec.ts @@ -36,9 +36,12 @@ test("AgentWorkBlock appears and collapses for tool-using queries", async ({ pag await input.fill("What data do we have available?"); await input.press("Enter"); - // Work block should expand while the agent is working - const workingLabel = page.locator("text=/Working/"); - await expect(workingLabel).toBeVisible({ timeout: 30_000 }); + // Work block should appear — either "Working" (during stream) or "Worked for" (after). + // The "Working" transient state lasts only ~320ms with the mock LLM (4 events × 80ms) + // and may be missed on slow CI. We accept either state here and verify the final + // collapsed state below. + const workBlockAny = page.locator("button", { hasText: /Working|Worked for/ }); + await expect(workBlockAny).toBeVisible({ timeout: 30_000 }); // After the response completes, it auto-collapses to "Worked for Xs · N tool calls" const workBlockHeader = page.locator("button", { hasText: /Worked for/ });