diff --git a/.github/workflows/deploy-dashboard.yml b/.github/workflows/deploy-dashboard.yml new file mode 100644 index 0000000..cc4ad16 --- /dev/null +++ b/.github/workflows/deploy-dashboard.yml @@ -0,0 +1,104 @@ +name: Deploy Dashboard to Cloudflare Pages + +on: + push: + branches: + - main + - develop + paths: + - 'dashboard/**' + - '.github/workflows/deploy-dashboard.yml' + pull_request: + types: [opened, synchronize, reopened] + paths: + - 'dashboard/**' + - '.github/workflows/deploy-dashboard.yml' + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + contents: read + deployments: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get branch name + id: branch + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + BRANCH_NAME="${{ github.head_ref }}" + else + BRANCH_NAME="${{ github.ref_name }}" + fi + # Sanitize branch name for URL (replace / with -, lowercase) + SANITIZED=$(echo "$BRANCH_NAME" | tr '/' '-' | tr '[:upper:]' '[:lower:]') + # Cloudflare Pages truncates branch names to ~28 chars for preview URLs + TRUNCATED=$(echo "$SANITIZED" | cut -c1-28) + echo "name=$TRUNCATED" >> $GITHUB_OUTPUT + echo "original=$BRANCH_NAME" >> $GITHUB_OUTPUT + + - name: Deploy to Cloudflare Pages + id: deploy + uses: cloudflare/wrangler-action@v3 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + # Deploy with branch name so Cloudflare Pages routes correctly: + # - main branch -> osa-dash.pages.dev (production) + # - develop branch -> develop.osa-dash.pages.dev (preview) + # - PR branches -> {branch}.osa-dash.pages.dev (preview) + command: pages deploy dashboard --project-name=osa-dash --branch=${{ steps.branch.outputs.name }} + + - name: Comment on PR with preview URL + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const branch = '${{ steps.branch.outputs.name }}'; + const previewUrl = `https://${branch}.osa-dash.pages.dev`; + + // Find existing comment from this workflow + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('') + ); + + const sha = '${{ github.sha }}'.substring(0, 7); + const body = [ + '', + '## Dashboard Preview', + '', + '| Name | Link |', + '|------|------|', + `| **Preview URL** | ${previewUrl} |`, + '| **Branch** | \`${{ steps.branch.outputs.original }}\` |', + `| **Commit** | \`${sha}\` |`, + '', + 'This preview will be updated automatically when you push new commits.' + ].join('\n'); + + if (botComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + } diff --git a/dashboard/_redirects b/dashboard/_redirects new file mode 100644 index 0000000..bbb3e7a --- /dev/null +++ b/dashboard/_redirects @@ -0,0 +1 @@ +/* /index.html 200 diff --git a/dashboard/index.html b/dashboard/index.html new file mode 100644 index 0000000..7147083 --- /dev/null +++ b/dashboard/index.html @@ -0,0 +1,762 @@ + + + + + + Open Science Assistant - Dashboard + + + + +
+ +
+
Loading...
+ + + +
+ + + + diff --git a/src/api/main.py b/src/api/main.py index 2dcc139..9d6566a 100644 --- a/src/api/main.py +++ b/src/api/main.py @@ -14,7 +14,12 @@ from pydantic import BaseModel from src.api.config import get_settings -from src.api.routers import create_community_router, metrics_router, sync_router +from src.api.routers import ( + create_community_router, + metrics_public_router, + metrics_router, + sync_router, +) from src.api.routers.health import router as health_router from src.api.routers.widget_test import router as widget_test_router from src.api.scheduler import start_scheduler, stop_scheduler @@ -194,8 +199,9 @@ def register_routes(app: FastAPI) -> None: # Sync router (not community-specific) app.include_router(sync_router) - # Metrics router (global metrics endpoints) + # Metrics routers (admin + public) app.include_router(metrics_router) + app.include_router(metrics_public_router) # Health check router app.include_router(health_router) @@ -239,6 +245,8 @@ async def root() -> dict[str, Any]: endpoints[f"GET /{community_id}/sessions"] = f"List active {name} sessions" endpoints[f"GET /{community_id}/sessions/{{session_id}}"] = "Get session info" endpoints[f"DELETE /{community_id}/sessions/{{session_id}}"] = "Delete a session" + endpoints[f"GET /{community_id}/metrics/public"] = f"Public {name} metrics" + endpoints[f"GET /{community_id}/metrics/public/usage"] = f"Public {name} usage stats" # Add non-community endpoints endpoints["GET /sync/status"] = "Knowledge sync status" @@ -246,6 +254,7 @@ async def root() -> dict[str, Any]: endpoints["POST /sync/trigger"] = "Trigger sync (requires API key)" endpoints["GET /metrics/overview"] = "Metrics overview (requires admin key)" endpoints["GET /metrics/tokens"] = "Token breakdown (requires admin key)" + endpoints["GET /metrics/public/overview"] = "Public metrics overview" endpoints["GET /health"] = "Health check" return { diff --git a/src/api/routers/__init__.py b/src/api/routers/__init__.py index c2a7692..39f85ab 100644 --- a/src/api/routers/__init__.py +++ b/src/api/routers/__init__.py @@ -2,6 +2,12 @@ from src.api.routers.community import create_community_router from src.api.routers.metrics import router as metrics_router +from src.api.routers.metrics_public import router as metrics_public_router from src.api.routers.sync import router as sync_router -__all__ = ["create_community_router", "metrics_router", "sync_router"] +__all__ = [ + "create_community_router", + "metrics_public_router", + "metrics_router", + "sync_router", +] diff --git a/src/api/routers/community.py b/src/api/routers/community.py index 4062ffd..fadf738 100644 --- a/src/api/routers/community.py +++ b/src/api/routers/community.py @@ -7,6 +7,7 @@ import hashlib import json import logging +import sqlite3 import time import uuid from collections.abc import AsyncGenerator @@ -30,11 +31,16 @@ RequestLogEntry, extract_token_usage, extract_tool_names, - get_metrics_connection, log_request, + metrics_connection, now_iso, ) -from src.metrics.queries import get_community_summary, get_usage_stats +from src.metrics.queries import ( + get_community_summary, + get_public_community_summary, + get_public_usage_stats, + get_usage_stats, +) logger = logging.getLogger(__name__) @@ -1054,19 +1060,15 @@ async def get_community_config() -> CommunityConfigResponse: @router.get("/metrics") async def community_metrics(_auth: RequireAdminAuth) -> dict[str, Any]: """Get metrics summary for this community. Requires admin auth.""" - import sqlite3 - - conn = get_metrics_connection() try: - return get_community_summary(community_id, conn) + with metrics_connection() as conn: + return get_community_summary(community_id, conn) except sqlite3.Error: logger.exception("Failed to query metrics for community %s", community_id) raise HTTPException( status_code=503, detail="Metrics database is temporarily unavailable.", ) - finally: - conn.close() @router.get("/metrics/usage") async def community_usage( @@ -1078,11 +1080,9 @@ async def community_usage( ), ) -> dict[str, Any]: """Get time-bucketed usage stats for this community. Requires admin auth.""" - import sqlite3 - - conn = get_metrics_connection() try: - return get_usage_stats(community_id, period, conn) + with metrics_connection() as conn: + return get_usage_stats(community_id, period, conn) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) from e except sqlite3.Error: @@ -1091,8 +1091,52 @@ async def community_usage( status_code=503, detail="Metrics database is temporarily unavailable.", ) - finally: - conn.close() + + # ----------------------------------------------------------------------- + # Per-community Public Metrics Endpoints (no auth required) + # ----------------------------------------------------------------------- + + @router.get("/metrics/public") + async def community_metrics_public() -> dict[str, Any]: + """Get public metrics summary for this community. + + Returns request counts, error rate, and top tools. + No tokens, costs, or model information exposed. + """ + try: + with metrics_connection() as conn: + return get_public_community_summary(community_id, conn) + except sqlite3.Error: + logger.exception("Failed to query public metrics for community %s", community_id) + raise HTTPException( + status_code=503, + detail="Metrics database is temporarily unavailable.", + ) + + @router.get("/metrics/public/usage") + async def community_usage_public( + period: str = Query( + default="daily", + description="Time bucket period", + pattern="^(daily|weekly|monthly)$", + ), + ) -> dict[str, Any]: + """Get public time-bucketed usage stats for this community. + + Returns request counts and errors per time bucket. + No tokens or costs exposed. + """ + try: + with metrics_connection() as conn: + return get_public_usage_stats(community_id, period, conn) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + except sqlite3.Error: + logger.exception("Failed to query public usage stats for community %s", community_id) + raise HTTPException( + status_code=503, + detail="Metrics database is temporarily unavailable.", + ) return router diff --git a/src/api/routers/metrics_public.py b/src/api/routers/metrics_public.py new file mode 100644 index 0000000..2aaf1f9 --- /dev/null +++ b/src/api/routers/metrics_public.py @@ -0,0 +1,39 @@ +"""Public metrics API endpoints. + +Exposes non-sensitive aggregate metrics (request counts, error rates) +without authentication. No tokens, costs, or model information. + +Per-community public metrics are served from the community router +at /{community_id}/metrics/public. +""" + +import logging +import sqlite3 +from typing import Any + +from fastapi import APIRouter, HTTPException + +from src.metrics.db import metrics_connection +from src.metrics.queries import get_public_overview + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/metrics/public", tags=["Public Metrics"]) + + +@router.get("/overview") +async def public_overview() -> dict[str, Any]: + """Get public metrics overview across all communities. + + Returns total requests, error rate, active community count, + and per-community request counts. No tokens, costs, or model info. + """ + try: + with metrics_connection() as conn: + return get_public_overview(conn) + except sqlite3.Error: + logger.exception("Failed to query metrics database for public overview") + raise HTTPException( + status_code=503, + detail="Metrics database is temporarily unavailable.", + ) diff --git a/src/metrics/db.py b/src/metrics/db.py index f716046..c72842f 100644 --- a/src/metrics/db.py +++ b/src/metrics/db.py @@ -7,6 +7,8 @@ import json import logging import sqlite3 +from collections.abc import Generator +from contextlib import contextmanager from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path @@ -93,6 +95,22 @@ def get_metrics_connection(db_path: Path | None = None) -> sqlite3.Connection: return conn +@contextmanager +def metrics_connection(db_path: Path | None = None) -> Generator[sqlite3.Connection, None, None]: + """Context manager for a metrics database connection. + + Ensures the connection is closed after use, even if an exception occurs. + + Args: + db_path: Optional path override (for testing). + """ + conn = get_metrics_connection(db_path) + try: + yield conn + finally: + conn.close() + + def init_metrics_db(db_path: Path | None = None) -> None: """Initialize the metrics database schema. Idempotent. diff --git a/src/metrics/queries.py b/src/metrics/queries.py index ec67e0a..6281639 100644 --- a/src/metrics/queries.py +++ b/src/metrics/queries.py @@ -11,6 +11,54 @@ logger = logging.getLogger(__name__) +# SQLite strftime patterns for time-bucketed queries +_PERIOD_FORMAT_MAP = { + "daily": "%Y-%m-%d", + "weekly": "%Y-W%W", + "monthly": "%Y-%m", +} + + +def _validate_period(period: str) -> str: + """Validate and return the strftime format for a period. + + Raises ValueError if period is not one of: daily, weekly, monthly. + """ + if period not in _PERIOD_FORMAT_MAP: + raise ValueError(f"Invalid period: {period}. Must be one of: daily, weekly, monthly") + return _PERIOD_FORMAT_MAP[period] + + +def _count_tools( + community_id: str, conn: sqlite3.Connection, limit: int = 5 +) -> list[dict[str, Any]]: + """Count tool usage from JSON arrays in the tools_called column. + + Returns top tools sorted by count descending. + """ + tool_rows = conn.execute( + """ + SELECT tools_called + FROM request_log + WHERE community_id = ? AND tools_called IS NOT NULL + """, + (community_id,), + ).fetchall() + tool_counts: dict[str, int] = {} + for tr in tool_rows: + try: + tools = json.loads(tr["tools_called"]) + for tool in tools: + tool_counts[tool] = tool_counts.get(tool, 0) + 1 + except (json.JSONDecodeError, TypeError): + logger.warning( + "Malformed tools_called data in request_log for community %s: %r", + community_id, + tr["tools_called"], + ) + top = sorted(tool_counts.items(), key=lambda x: x[1], reverse=True)[:limit] + return [{"tool": name, "count": count} for name, count in top] + def get_community_summary(community_id: str, conn: sqlite3.Connection) -> dict[str, Any]: """Get summary statistics for a single community. @@ -20,8 +68,9 @@ def get_community_summary(community_id: str, conn: sqlite3.Connection) -> dict[s conn: SQLite connection (with row_factory=sqlite3.Row). Returns: - Dict with total_requests, total_tokens, avg_duration_ms, - error_rate, top_models, top_tools. + Dict with total_requests, total_input_tokens, total_output_tokens, + total_tokens, avg_duration_ms, total_estimated_cost, error_rate, + top_models, top_tools. """ row = conn.execute( """ @@ -55,29 +104,6 @@ def get_community_summary(community_id: str, conn: sqlite3.Connection) -> dict[s (community_id,), ).fetchall() - # Top tools (from JSON array in tools_called column) - tool_rows = conn.execute( - """ - SELECT tools_called - FROM request_log - WHERE community_id = ? AND tools_called IS NOT NULL - """, - (community_id,), - ).fetchall() - tool_counts: dict[str, int] = {} - for tr in tool_rows: - try: - tools = json.loads(tr["tools_called"]) - for tool in tools: - tool_counts[tool] = tool_counts.get(tool, 0) + 1 - except (json.JSONDecodeError, TypeError): - logger.warning( - "Malformed tools_called data in request_log for community %s: %r", - community_id, - tr["tools_called"], - ) - top_tools = sorted(tool_counts.items(), key=lambda x: x[1], reverse=True)[:5] - return { "community_id": community_id, "total_requests": total, @@ -88,7 +114,7 @@ def get_community_summary(community_id: str, conn: sqlite3.Connection) -> dict[s "total_estimated_cost": round(row["total_estimated_cost"], 4), "error_rate": round(error_rate, 4), "top_models": [{"model": r["model"], "count": r["count"]} for r in model_rows], - "top_tools": [{"tool": t[0], "count": t[1]} for t in top_tools], + "top_tools": _count_tools(community_id, conn), } @@ -107,18 +133,9 @@ def get_usage_stats( Returns: Dict with period, community_id, and buckets list. """ - # SQLite strftime patterns for bucketing - format_map = { - "daily": "%Y-%m-%d", - "weekly": "%Y-W%W", - "monthly": "%Y-%m", - } - if period not in format_map: - raise ValueError(f"Invalid period: {period}. Must be one of: daily, weekly, monthly") - - fmt = format_map[period] + fmt = _validate_period(period) - # Safe to use f-string: fmt is from a hardcoded whitelist, not user input + # Safe to use f-string: fmt is from _PERIOD_FORMAT_MAP whitelist, not user input rows = conn.execute( f""" SELECT @@ -288,3 +305,141 @@ def get_token_breakdown( for r in by_key_source ], } + + +# --------------------------------------------------------------------------- +# Public query functions (no tokens, costs, or model info) +# --------------------------------------------------------------------------- + + +def get_public_overview(conn: sqlite3.Connection) -> dict[str, Any]: + """Get public metrics overview with only non-sensitive data. + + Returns request counts and error rates; no tokens, costs, or model info. + + Args: + conn: SQLite connection. + + Returns: + Dict with total_requests, error_rate, communities_active, + and per-community request counts. + """ + totals = conn.execute( + """ + SELECT + COUNT(*) as total_requests, + COUNT(CASE WHEN status_code >= 400 THEN 1 END) as total_errors + FROM request_log + """ + ).fetchone() + + total_req = totals["total_requests"] + + community_rows = conn.execute( + """ + SELECT + community_id, + COUNT(*) as requests, + COUNT(CASE WHEN status_code >= 400 THEN 1 END) as errors + FROM request_log + WHERE community_id IS NOT NULL + GROUP BY community_id + ORDER BY requests DESC + """ + ).fetchall() + + return { + "total_requests": total_req, + "error_rate": round(totals["total_errors"] / total_req, 4) if total_req > 0 else 0.0, + "communities_active": len(community_rows), + "communities": [ + { + "community_id": r["community_id"], + "requests": r["requests"], + "error_rate": round(r["errors"] / r["requests"], 4) if r["requests"] > 0 else 0.0, + } + for r in community_rows + ], + } + + +def get_public_community_summary(community_id: str, conn: sqlite3.Connection) -> dict[str, Any]: + """Get public summary for a single community. + + Returns request counts and top tools; no tokens, costs, or model info. + + Args: + community_id: The community identifier. + conn: SQLite connection. + + Returns: + Dict with community_id, total_requests, error_rate, top_tools. + """ + row = conn.execute( + """ + SELECT + COUNT(*) as total_requests, + COUNT(CASE WHEN status_code >= 400 THEN 1 END) as error_count + FROM request_log + WHERE community_id = ? + """, + (community_id,), + ).fetchone() + + total = row["total_requests"] + error_rate = row["error_count"] / total if total > 0 else 0.0 + + return { + "community_id": community_id, + "total_requests": total, + "error_rate": round(error_rate, 4), + "top_tools": _count_tools(community_id, conn), + } + + +def get_public_usage_stats( + community_id: str, + period: str, + conn: sqlite3.Connection, +) -> dict[str, Any]: + """Get public time-bucketed usage statistics. + + Returns request counts and errors per bucket; no tokens or costs. + + Args: + community_id: The community identifier. + period: One of "daily", "weekly", "monthly". + conn: SQLite connection. + + Returns: + Dict with period, community_id, and buckets list. + """ + fmt = _validate_period(period) + + # Safe to use f-string: fmt is from _PERIOD_FORMAT_MAP whitelist, not user input + rows = conn.execute( + f""" + SELECT + strftime('{fmt}', timestamp) as bucket, + COUNT(*) as requests, + COUNT(CASE WHEN status_code >= 400 THEN 1 END) as errors + FROM request_log + WHERE community_id = ? + GROUP BY bucket + ORDER BY bucket + """, + (community_id,), + ).fetchall() + + return { + "community_id": community_id, + "period": period, + "buckets": [ + { + "bucket": r["bucket"], + "requests": r["requests"], + "errors": r["errors"], + } + for r in rows + ], + } diff --git a/tests/test_api/test_dashboard.py b/tests/test_api/test_dashboard.py new file mode 100644 index 0000000..d15b00d --- /dev/null +++ b/tests/test_api/test_dashboard.py @@ -0,0 +1,87 @@ +"""Tests for the dashboard static HTML page. + +The dashboard is a standalone static site in dashboard/index.html, +deployed separately to Cloudflare Pages. These tests verify the HTML +contains the expected structure and API references. +""" + +from pathlib import Path + +DASHBOARD_HTML_PATH = Path(__file__).parent.parent.parent / "dashboard" / "index.html" + + +class TestDashboardHTML: + """Tests for dashboard/index.html static file.""" + + def test_file_exists(self) -> None: + assert DASHBOARD_HTML_PATH.exists(), "dashboard/index.html must exist" + + def test_is_valid_html(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + assert "" in content + assert "" in content + + def test_contains_page_title(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + assert "Open Science Assistant" in content + + def test_contains_chart_js_cdn(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + assert "chart.js" in content + + def test_references_public_overview_api(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + assert "/metrics/public/overview" in content + + def test_references_community_public_metrics_api(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + # Should use /{community}/metrics/public pattern + assert "/metrics/public" in content + assert "/metrics/public/usage" in content + + def test_has_client_side_routing(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + assert "getRoute" in content + assert "window.location.pathname" in content + + def test_has_aggregate_view(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + assert "renderAggregateView" in content + assert "Questions Answered" in content + + def test_has_community_view(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + assert "loadCommunityView" in content + + def test_has_tab_bar(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + assert "tabBar" in content + assert "tab-link" in content + assert "renderTabs" in content + + def test_has_admin_key_input(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + assert "adminKeyInput" in content + assert "Admin Access" in content + + def test_admin_section_hidden_by_default(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + assert "admin-section" in content + assert "display: none" in content or "display:none" in content + + def test_has_period_toggle(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + assert "changePeriod" in content + assert "daily" in content + assert "weekly" in content + assert "monthly" in content + + def test_api_base_configurable(self) -> None: + content = DASHBOARD_HTML_PATH.read_text() + # Should support ?api= query param or window.OSA_API_BASE override + assert "OSA_API_BASE" in content + + def test_cloudflare_redirects_file_exists(self) -> None: + redirects_path = DASHBOARD_HTML_PATH.parent / "_redirects" + assert redirects_path.exists(), "_redirects needed for Cloudflare Pages SPA routing" diff --git a/tests/test_api/test_metrics_public.py b/tests/test_api/test_metrics_public.py new file mode 100644 index 0000000..8f8f427 --- /dev/null +++ b/tests/test_api/test_metrics_public.py @@ -0,0 +1,395 @@ +"""Tests for public metrics API endpoints. + +Global overview: GET /metrics/public/overview (no auth) +Per-community: GET /{community_id}/metrics/public (no auth) +Per-community usage: GET /{community_id}/metrics/public/usage (no auth) +""" + +import os +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from src.api.main import app +from src.metrics.db import ( + RequestLogEntry, + init_metrics_db, + log_request, +) + + +@pytest.fixture +def metrics_db(tmp_path): + """Create isolated metrics database with sample data.""" + db_path = tmp_path / "metrics.db" + init_metrics_db(db_path) + + entries = [ + RequestLogEntry( + request_id="r1", + timestamp="2025-01-15T10:00:00+00:00", + endpoint="/hed/ask", + method="POST", + community_id="hed", + duration_ms=200.0, + status_code=200, + model="qwen/qwen3-235b", + input_tokens=100, + output_tokens=50, + total_tokens=150, + estimated_cost=0.001, + tools_called=["search_docs", "validate_hed"], + key_source="platform", + ), + RequestLogEntry( + request_id="r2", + timestamp="2025-01-15T11:00:00+00:00", + endpoint="/hed/chat", + method="POST", + community_id="hed", + duration_ms=300.0, + status_code=200, + model="qwen/qwen3-235b", + input_tokens=200, + output_tokens=100, + total_tokens=300, + tools_called=["search_docs"], + key_source="byok", + ), + RequestLogEntry( + request_id="r3", + timestamp="2025-01-16T09:00:00+00:00", + endpoint="/hed/ask", + method="POST", + community_id="hed", + duration_ms=100.0, + status_code=500, + ), + RequestLogEntry( + request_id="r4", + timestamp="2025-01-15T12:00:00+00:00", + endpoint="/bids/ask", + method="POST", + community_id="bids", + duration_ms=250.0, + status_code=200, + model="anthropic/claude-sonnet", + input_tokens=150, + output_tokens=75, + total_tokens=225, + key_source="platform", + ), + ] + for e in entries: + log_request(e, db_path=db_path) + + return db_path + + +@pytest.fixture +def isolated_metrics(metrics_db): + """Patch metrics DB path for all metrics code.""" + with patch("src.metrics.db.get_metrics_db_path", return_value=metrics_db): + yield metrics_db + + +@pytest.fixture +def noauth_env(): + """Disable auth requirement.""" + from src.api.config import get_settings + + os.environ["REQUIRE_API_AUTH"] = "false" + get_settings.cache_clear() + yield + del os.environ["REQUIRE_API_AUTH"] + get_settings.cache_clear() + + +@pytest.fixture +def client(): + """Create test client.""" + return TestClient(app) + + +class TestPublicOverview: + """Tests for GET /metrics/public/overview.""" + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_returns_200_without_auth(self, client): + response = client.get("/metrics/public/overview") + assert response.status_code == 200 + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_response_structure(self, client): + response = client.get("/metrics/public/overview") + data = response.json() + assert "total_requests" in data + assert "error_rate" in data + assert "communities_active" in data + assert "communities" in data + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_no_sensitive_fields(self, client): + response = client.get("/metrics/public/overview") + data = response.json() + assert "total_tokens" not in data + assert "total_estimated_cost" not in data + assert "avg_duration_ms" not in data + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_request_counts_correct(self, client): + response = client.get("/metrics/public/overview") + data = response.json() + assert data["total_requests"] == 4 + assert data["communities_active"] == 2 + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_error_rate_includes_errors(self, client): + response = client.get("/metrics/public/overview") + data = response.json() + # 1 error out of 4 requests = 0.25 + assert data["error_rate"] == 0.25 + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_community_breakdown_no_sensitive_fields(self, client): + response = client.get("/metrics/public/overview") + data = response.json() + for community in data["communities"]: + assert "community_id" in community + assert "requests" in community + assert "error_rate" in community + assert "tokens" not in community + assert "estimated_cost" not in community + + +class TestCommunityPublicMetrics: + """Tests for GET /{community_id}/metrics/public.""" + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_returns_200_without_auth(self, client): + response = client.get("/hed/metrics/public") + assert response.status_code == 200 + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_response_structure(self, client): + response = client.get("/hed/metrics/public") + data = response.json() + assert data["community_id"] == "hed" + assert "total_requests" in data + assert "error_rate" in data + assert "top_tools" in data + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_no_sensitive_fields(self, client): + response = client.get("/hed/metrics/public") + data = response.json() + assert "total_tokens" not in data + assert "total_estimated_cost" not in data + assert "top_models" not in data + assert "avg_duration_ms" not in data + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_top_tools_populated(self, client): + response = client.get("/hed/metrics/public") + data = response.json() + tools = {t["tool"]: t["count"] for t in data["top_tools"]} + assert "search_docs" in tools + assert tools["search_docs"] == 2 + + +class TestCommunityPublicUsage: + """Tests for GET /{community_id}/metrics/public/usage.""" + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_daily_usage_returns_200(self, client): + response = client.get("/hed/metrics/public/usage", params={"period": "daily"}) + assert response.status_code == 200 + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_daily_usage_structure(self, client): + response = client.get("/hed/metrics/public/usage", params={"period": "daily"}) + data = response.json() + assert data["community_id"] == "hed" + assert data["period"] == "daily" + assert "buckets" in data + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_buckets_no_sensitive_fields(self, client): + response = client.get("/hed/metrics/public/usage", params={"period": "daily"}) + data = response.json() + for bucket in data["buckets"]: + assert "bucket" in bucket + assert "requests" in bucket + assert "errors" in bucket + assert "tokens" not in bucket + assert "estimated_cost" not in bucket + assert "avg_duration_ms" not in bucket + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_monthly_usage(self, client): + response = client.get("/hed/metrics/public/usage", params={"period": "monthly"}) + assert response.status_code == 200 + data = response.json() + assert data["period"] == "monthly" + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_weekly_usage(self, client): + response = client.get("/hed/metrics/public/usage", params={"period": "weekly"}) + assert response.status_code == 200 + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_invalid_period_returns_422(self, client): + response = client.get("/hed/metrics/public/usage", params={"period": "hourly"}) + assert response.status_code == 422 + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_default_period_is_daily(self, client): + response = client.get("/hed/metrics/public/usage") + assert response.status_code == 200 + data = response.json() + assert data["period"] == "daily" + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_daily_buckets_count_and_errors(self, client): + """Verify bucket count and error values match fixture data.""" + response = client.get("/hed/metrics/public/usage", params={"period": "daily"}) + data = response.json() + buckets = data["buckets"] + # Fixture has HED requests on 2025-01-15 and 2025-01-16 + assert len(buckets) == 2 + bucket_map = {b["bucket"]: b for b in buckets} + # 2025-01-16 has one request with status_code=500 + assert bucket_map["2025-01-16"]["errors"] == 1 + + +class TestPublicAdminBoundary: + """Verify public endpoints work without auth while admin endpoints require it.""" + + @pytest.fixture + def auth_env(self): + """Enable auth with a test API key so admin endpoints reject anonymous requests.""" + from src.api.config import get_settings + + os.environ["REQUIRE_API_AUTH"] = "true" + os.environ["API_KEYS"] = "test-secret-key" + get_settings.cache_clear() + yield + del os.environ["REQUIRE_API_AUTH"] + del os.environ["API_KEYS"] + get_settings.cache_clear() + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_public_overview_no_auth_200(self, client): + response = client.get("/metrics/public/overview") + assert response.status_code == 200 + + @pytest.mark.usefixtures("isolated_metrics", "auth_env") + def test_admin_overview_no_auth_rejected(self, client): + """Admin endpoint must reject unauthenticated requests.""" + response = client.get("/metrics/overview") + assert response.status_code in (401, 403) + + @pytest.mark.usefixtures("isolated_metrics", "auth_env") + def test_admin_tokens_no_auth_rejected(self, client): + """Admin token endpoint must reject unauthenticated requests.""" + response = client.get("/metrics/tokens") + assert response.status_code in (401, 403) + + @pytest.mark.usefixtures("isolated_metrics", "auth_env") + def test_public_overview_accessible_with_auth_enabled(self, client): + """Public overview must return 200 even when auth is required.""" + response = client.get("/metrics/public/overview") + assert response.status_code == 200 + + @pytest.mark.usefixtures("isolated_metrics", "auth_env") + def test_community_public_metrics_accessible_with_auth_enabled(self, client): + """Per-community public metrics must return 200 even when auth is required.""" + response = client.get("/hed/metrics/public") + assert response.status_code == 200 + + @pytest.mark.usefixtures("isolated_metrics", "auth_env") + def test_community_public_usage_accessible_with_auth_enabled(self, client): + """Per-community public usage must return 200 even when auth is required.""" + response = client.get("/hed/metrics/public/usage") + assert response.status_code == 200 + + +class TestEmptyDatabase: + """Verify public endpoints handle empty databases gracefully.""" + + @pytest.fixture + def empty_metrics_db(self, tmp_path): + db_path = tmp_path / "empty_metrics.db" + init_metrics_db(db_path) + return db_path + + @pytest.fixture + def isolated_empty_metrics(self, empty_metrics_db): + with patch("src.metrics.db.get_metrics_db_path", return_value=empty_metrics_db): + yield + + @pytest.mark.usefixtures("isolated_empty_metrics", "noauth_env") + def test_overview_empty_db(self, client): + response = client.get("/metrics/public/overview") + assert response.status_code == 200 + data = response.json() + assert data["total_requests"] == 0 + assert data["error_rate"] == 0.0 + assert data["communities_active"] == 0 + assert data["communities"] == [] + + @pytest.mark.usefixtures("isolated_empty_metrics", "noauth_env") + def test_community_metrics_empty_db(self, client): + response = client.get("/hed/metrics/public") + assert response.status_code == 200 + data = response.json() + assert data["total_requests"] == 0 + assert data["error_rate"] == 0.0 + assert data["top_tools"] == [] + + @pytest.mark.usefixtures("isolated_empty_metrics", "noauth_env") + def test_community_usage_empty_db(self, client): + response = client.get("/hed/metrics/public/usage") + assert response.status_code == 200 + data = response.json() + assert data["buckets"] == [] + + +class TestCommunityMetricsValues: + """Verify computed values per community match fixture data dynamically.""" + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_community_values_from_overview(self, client): + """Check each community's request count and error rate from overview.""" + response = client.get("/metrics/public/overview") + data = response.json() + checked = 0 + for community in data["communities"]: + cid = community["community_id"] + resp = client.get(f"/{cid}/metrics/public") + if resp.status_code != 200: + continue # community route not registered in test app + detail = resp.json() + assert detail["total_requests"] == community["requests"] + assert detail["total_requests"] > 0 + assert detail["error_rate"] == community["error_rate"] + checked += 1 + assert checked > 0, "Expected at least one community with a registered route" + + @pytest.mark.usefixtures("isolated_metrics", "noauth_env") + def test_per_community_tool_counts_consistent(self, client): + """Each tool count should be a positive integer.""" + response = client.get("/metrics/public/overview") + checked = 0 + for community in response.json()["communities"]: + cid = community["community_id"] + resp = client.get(f"/{cid}/metrics/public") + if resp.status_code != 200: + continue # community route not registered in test app + detail = resp.json() + for tool_entry in detail["top_tools"]: + assert isinstance(tool_entry["tool"], str) + assert tool_entry["count"] > 0 + checked += 1 + assert checked > 0, "Expected at least one community with a registered route"