diff --git a/GENAI_METRICS.md b/GENAI_METRICS.md new file mode 100644 index 0000000000..ff3fdfa90f --- /dev/null +++ b/GENAI_METRICS.md @@ -0,0 +1,198 @@ +# GenAI Metrics (Scouter) — OpsML Integration + +This document describes the GenAI metrics feature added to OpsML: how the backend proxies Scouter GenAI endpoints, the new API surface, UI components, and how to run and test locally. + +## 1) Architecture + +- OpsML acts as a thin BFF in front of Scouter ("bifrost"). The server proxies requests to Scouter via `ScouterApiClient`. +- Handlers exchange an OpsML-permission token for a Scouter token using `exchange_token_from_perms(...)` and call `state.scouter_client.request_with_path(...)`. +- Every proxied handler inserts an `AuditContext` into the response so audit/event middleware records the read operation. +- UI components call OpsML server endpoints (not Scouter directly). The server forwards to Scouter and returns the JSON body. + +Diagram (conceptual): + +``` +Browser UI -> OpsML (Axum handlers / auth middleware) + -> exchange token -> Scouter (bifrost) -> Scouter JSON + <- returns JSON to UI +``` + +## 2) API endpoints (new) + +The server exposes lightweight proxy endpoints that the UI uses. All requests require a valid OpsML session (cookie/jwt) and are protected by the standard `auth_api_middleware`. + +1) GET /api/traces/{trace_id}/genai/aggregate + +- Description: Aggregate GenAI metrics for a trace (tokens, latency, model breakdown). +- Query params (optional): `start_time` (ISO), `end_time` (ISO), `interval` (minute|hour|day), `metrics` (comma-separated) +- Response (200): JSON aggregate object (example): + +Request example: + +``` +GET /api/traces/trace-123/genai/aggregate?start_time=2026-01-01T00:00:00Z&end_time=2026-01-01T01:00:00Z +Authorization: (cookie) +``` + +Response example: + +```json +{ + "trace_id": "trace-123", + "total_tokens": 1500, + "avg_latency": 120.5, + "model_distribution": { "gpt-4o-mini": 20, "claude-3-5": 5 }, + "time_period": { "start_time": "2026-01-01T00:00:00Z", "end_time": "2026-01-01T01:00:00Z" } +} +``` + +2) GET /api/spans/{span_id}/genai/metrics + +- Description: Span-level GenAI metrics (tokens, model, latency, cost). +- Response (200): JSON array of per-span metric objects. + +Request example: + +``` +GET /api/spans/some-span-uid/genai/metrics +``` + +Response example: + +```json +[ + { "token_count_input": 10, "token_count_output": 5, "model_name": "gpt-4o-mini", "latency_ms": 120, "cost": 0.0008 } +] +``` + +3) GET /api/services/{service_id}/genai/timeseries + +- Description: Service-level GenAI timeseries and dashboard bundle (buckets, model usage, cost by model, tool metrics). +- Query params: `start_time`, `end_time`, `interval` (minute|hour|day), `metrics` +- Response (200): composite JSON (example simplified): + +Request example: + +``` +GET /api/services/my-service/genai/timeseries?start_time=2026-01-01T00:00:00Z&end_time=2026-01-02T00:00:00Z +``` + +Response example (simplified): + +```json +{ + "agent_dashboard": { "summary": {"total_requests": 10, "avg_duration_ms":120}, "buckets": [ {"bucket_start":"2026-01-01T00:00:00Z","total_input_tokens":100,"total_output_tokens":50,"total_cost":0.12} ] }, + "model_usage": { "models": [ {"model":"gpt-4o-mini","span_count":5} ] }, + "tool_dashboard": { "aggregates": [], "time_series": [] } +} +``` + +Notes +- These endpoints are proxies — the shape of the returned JSON mirrors Scouter's GenAI responses. UI components must be defensive (nulls, missing fields). +- Errors from Scouter are returned as OpsML errors (non-200 responses propagate through as 4xx/5xx). + +## 3) UI components and extension points + +Files added/modified (UI): + +- `src/lib/components/AgentServiceDashboard/GenAIMetricsNav.svelte` — left-side nav (Overview / By Model / Token Usage / Latency) and time range controls. +- `src/lib/components/AgentServiceDashboard/GenAITimeseries.svelte` — timeseries view with four charts: tokens, latency, model usage, cost. +- `src/lib/components/TraceDetail/GenAIMetricsTab.svelte` — per-trace per-span GenAI table. +- `src/lib/components/card/agent/observability/charts.ts` — chart builders (token/latency/cost/volume) used by charts. +- `src/lib/components/card/agent/observability/GenAiChartCard.svelte` — Chart.js wrapper used by chart panels. + +Extending views + +- To add a new chart to the Timeseries view: + 1. Add a new builder in `charts.ts` that returns a `ChartConfiguration` for Chart.js. + 2. Import the builder into `GenAITimeseries.svelte` and construct a reactive `configFn` for `GenAiChartCard`. + +Example snippet (add average latency sparkline): + +```ts +// in charts.ts +export function buildAvgLatencySparkline(buckets) { + return createTimeSeriesChart(buckets.map(b=>new Date(b.bucket_start)), buckets.map(b=>b.avg_duration_ms||0), undefined, 'avg latency', 'ms', 'line'); +} + +// in GenAITimeseries.svelte +import { buildAvgLatencySparkline } from '$lib/components/card/agent/observability/charts'; +const latencyMiniCfg = $derived(() => buildAvgLatencySparkline(buckets)); + +``` + +Design notes +- Charts use Chart.js via `GenAiChartCard.svelte` and theme helpers in `src/lib/components/viz` so visual consistency is preserved. +- Prefer adding chart-builders to `charts.ts` rather than embedding Chart.js configs in components. + +## 4) Running and testing locally + +Prereqs +- Rust toolchain (cargo) — required to build/run OpsML server. +- Node (pnpm recommended) for the UI. +- Mise (recommended) — this repo uses `mise` task runner (see AGENTS.md / README). + +Quick dev run (recommended) + +1. Install mise (one-time): + +```bash +curl https://mise.run | sh +mise install +``` + +2. Start backend + frontend (dev): + +```bash +# runs backend (port 8080) and frontend (port 3000) +mise run dev:both +``` + +Or run frontend only (useful while backend is already running): + +```bash +cd crates/opsml_server/opsml_ui +pnpm install +pnpm run dev +``` + +Running tests (UI) + +```bash +cd crates/opsml_server/opsml_ui +pnpm install +pnpm test +``` + +Notes about mocks +- UI unit tests mock `createInternalApiClient` and Chart components where appropriate — see `src/lib/.../__tests__` for examples. +- If you don't have a running Scouter instance, run the UI in mock mode (dev settings) or rely on the mock endpoints used in tests. + +Troubleshooting +- If `cargo` is not found, install Rust via https://rustup.rs and re-run `mise` commands. +- If charts do not render in tests, ensure the test runner has `canvas`/`jsdom` shims (the repo test config includes these shims). + +## Example curl + fetch calls + +Curl example (trace aggregate): + +```bash +curl -v -b "" "http://localhost:8080/api/traces/trace-123/genai/aggregate?start_time=2026-01-01T00:00:00Z&end_time=2026-01-01T01:00:00Z" +``` + +Fetch example (SvelteKit server-side code): + +```ts +import { createInternalApiClient } from '$lib/api/internalClient'; +const client = createInternalApiClient(fetch); +const res = await client.get(`/api/services/${serviceId}/genai/timeseries`, { start_time, end_time }); +const body = await res.json(); +``` + +## Where to look in the codebase +- Backend handlers: `crates/opsml_server/src/core/scouter/genai/route.rs` +- Router mount: `crates/opsml_server/src/core/genai_metrics/route.rs` and `crates/opsml_server/src/core/router.rs` +- UI components: `crates/opsml_server/opsml_ui/src/lib/components/AgentServiceDashboard/*`, `.../TraceDetail/GenAIMetricsTab.svelte` +- Chart builders: `crates/opsml_server/opsml_ui/src/lib/components/card/agent/observability/charts.ts` + +If you want, I can add a short HOWTO showing how to add a new chart and wire a backend query end-to-end. diff --git a/crates/opsml_server/opsml_ui/src/lib/api/__tests__/genai.test.ts b/crates/opsml_server/opsml_ui/src/lib/api/__tests__/genai.test.ts new file mode 100644 index 0000000000..4105ac230e --- /dev/null +++ b/crates/opsml_server/opsml_ui/src/lib/api/__tests__/genai.test.ts @@ -0,0 +1,51 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { createInternalApiClient } from '$lib/api/internalClient'; + +vi.mock('$lib/api/internalClient', () => ({ createInternalApiClient: vi.fn() })); + +import { + fetchSpanGenAIMetrics, + fetchTraceGenAIMetrics, + fetchServiceGenAITimeseries, + _clearGenAICache, +} from '../genai'; + +const mockGet = vi.fn(); +const mockPost = vi.fn(); + +beforeEach(() => { + vi.mocked(createInternalApiClient).mockReturnValue({ get: mockGet, post: mockPost } as unknown as ReturnType); + mockGet.mockReset(); + mockPost.mockReset(); + _clearGenAICache(); +}); + +describe('genai API client', () => { + it('fetchSpanGenAIMetrics returns parsed body and caches', async () => { + const body = [{ token_count_input: 1, token_count_output: 2, model_name: 'm', latency_ms: 10, cost: 0.001 }]; + mockGet.mockResolvedValue({ ok: true, json: async () => body }); + const first = await fetchSpanGenAIMetrics('s1'); + expect(first).toEqual(body); + const second = await fetchSpanGenAIMetrics('s1'); + expect(mockGet).toHaveBeenCalledTimes(1); + expect(second).toEqual(body); + }); + + it('fetchTraceGenAIMetrics returns parsed body', async () => { + const body = { total_tokens: 10, avg_latency: 100, model_distribution: { a: 1 }, time_period: { start_time: '2026-01-01', end_time: '2026-01-02' } }; + mockGet.mockResolvedValue({ ok: true, json: async () => body }); + const out = await fetchTraceGenAIMetrics('t1'); + expect(out).toEqual(body); + }); + + it('fetchServiceGenAITimeseries sends params and caches', async () => { + const body = { points: [{ timestamp: '2026-01-01T00:00:00Z', metric_name: 'input_tokens', value: 10 }], labels: ['a'] }; + mockGet.mockResolvedValue({ ok: true, json: async () => body }); + const s = new Date('2026-01-01T00:00:00Z'); + const e = new Date('2026-01-01T01:00:00Z'); + const first = await fetchServiceGenAITimeseries('svc', s, e); + expect(first).toEqual(body); + const second = await fetchServiceGenAITimeseries('svc', s, e); + expect(mockGet).toHaveBeenCalledTimes(1); + }); +}); diff --git a/crates/opsml_server/opsml_ui/src/lib/api/genai.ts b/crates/opsml_server/opsml_ui/src/lib/api/genai.ts new file mode 100644 index 0000000000..c96d2dad64 --- /dev/null +++ b/crates/opsml_server/opsml_ui/src/lib/api/genai.ts @@ -0,0 +1,91 @@ +import { createInternalApiClient } from './internalClient'; +import type { + GenAISpanMetrics, + GenAIAggregateMetrics, + GenAITimeseriesResponse, +} from '$lib/types/genai'; + +/** Simple in-memory cache entry */ +type CacheEntry = { ts: number; value: T }; + +const ONE_MINUTE = 60_000; + +const spanCache: Map> = new Map(); +const traceCache: Map> = new Map(); +const serviceTimeseriesCache: Map> = new Map(); + +function isStale(ts: number, ttl = ONE_MINUTE) { + return Date.now() - ts > ttl; +} + +/** + * Fetch GenAI metrics for a specific span. + * GET /api/spans/{span_id}/genai/metrics + */ +export async function fetchSpanGenAIMetrics(spanId: string): Promise { + if (!spanId) return []; + const key = String(spanId); + const cached = spanCache.get(key); + if (cached && !isStale(cached.ts)) return cached.value; + + const client = createInternalApiClient(fetch); + const path = `/api/spans/${encodeURIComponent(spanId)}/genai/metrics`; + const res = await client.get(path); + if (!res.ok) throw new Error(`Failed to fetch span GenAI metrics: ${res.status}`); + const body = (await res.json()) as GenAISpanMetrics[]; + spanCache.set(key, { ts: Date.now(), value: body }); + return body; +} + +/** + * Fetch aggregate GenAI metrics for a trace. + * GET /api/traces/{trace_id}/genai/aggregate + */ +export async function fetchTraceGenAIMetrics(traceId: string): Promise { + if (!traceId) throw new Error('traceId is required'); + const key = String(traceId); + const cached = traceCache.get(key); + if (cached && !isStale(cached.ts)) return cached.value; + + const client = createInternalApiClient(fetch); + const path = `/api/traces/${encodeURIComponent(traceId)}/genai/aggregate`; + const res = await client.get(path); + if (!res.ok) throw new Error(`Failed to fetch trace GenAI metrics: ${res.status}`); + const body = (await res.json()) as GenAIAggregateMetrics; + traceCache.set(key, { ts: Date.now(), value: body }); + return body; +} + +/** + * Fetch service-level GenAI timeseries. + * GET /api/services/{service_id}/genai/timeseries?start_time=...&end_time=... + */ +export async function fetchServiceGenAITimeseries( + serviceId: string, + startDate: Date, + endDate: Date, +): Promise { + if (!serviceId) throw new Error('serviceId is required'); + const key = `${serviceId}:${startDate?.toISOString() ?? ''}:${endDate?.toISOString() ?? ''}`; + const cached = serviceTimeseriesCache.get(key); + if (cached && !isStale(cached.ts, ONE_MINUTE * 5)) return cached.value; + + const client = createInternalApiClient(fetch); + const params = { + start_time: startDate?.toISOString(), + end_time: endDate?.toISOString(), + } as Record; + const path = `/api/services/${encodeURIComponent(serviceId)}/genai/timeseries`; + const res = await client.get(path, params); + if (!res.ok) throw new Error(`Failed to fetch service GenAI timeseries: ${res.status}`); + const body = (await res.json()) as GenAITimeseriesResponse; + serviceTimeseriesCache.set(key, { ts: Date.now(), value: body }); + return body; +} + +/** Utilities for tests or dev: clear caches */ +export function _clearGenAICache() { + spanCache.clear(); + traceCache.clear(); + serviceTimeseriesCache.clear(); +} diff --git a/crates/opsml_server/opsml_ui/src/lib/components/AgentServiceDashboard/GenAIMetricsNav.svelte b/crates/opsml_server/opsml_ui/src/lib/components/AgentServiceDashboard/GenAIMetricsNav.svelte new file mode 100644 index 0000000000..9bbd09886b --- /dev/null +++ b/crates/opsml_server/opsml_ui/src/lib/components/AgentServiceDashboard/GenAIMetricsNav.svelte @@ -0,0 +1,125 @@ + + +
+
+

GenAI Metrics

+

Select a view and adjust range/filters

+
+ +
+ +
+ +
+ onRangeChange(e.detail)} + onRefresh={onRefresh} + /> +
+ +
+ +
+ +
+ +
+ +
+ Auto-refresh: live + +
+
+ + diff --git a/crates/opsml_server/opsml_ui/src/lib/components/AgentServiceDashboard/GenAITimeseries.svelte b/crates/opsml_server/opsml_ui/src/lib/components/AgentServiceDashboard/GenAITimeseries.svelte new file mode 100644 index 0000000000..6a35f1fad9 --- /dev/null +++ b/crates/opsml_server/opsml_ui/src/lib/components/AgentServiceDashboard/GenAITimeseries.svelte @@ -0,0 +1,155 @@ + + +
+
+

GenAI Timeseries

+ { void refetchForRange(e.detail); }} + onRefresh={() => { void fetchData(); }} + /> +
+ + {#if error} +
{error}
+ {/if} + +
+ + +
+ +
+ + +
+
+ + diff --git a/crates/opsml_server/opsml_ui/src/lib/components/AgentServiceDashboard/__tests__/GenAITimeseries.test.ts b/crates/opsml_server/opsml_ui/src/lib/components/AgentServiceDashboard/__tests__/GenAITimeseries.test.ts new file mode 100644 index 0000000000..4a5f280bd4 --- /dev/null +++ b/crates/opsml_server/opsml_ui/src/lib/components/AgentServiceDashboard/__tests__/GenAITimeseries.test.ts @@ -0,0 +1,33 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen } from '@testing-library/svelte'; + +vi.mock('$lib/components/card/agent/observability/GenAiChartCard.svelte', () => ({ default: vi.fn() })); +vi.mock('$lib/api/internalClient', () => ({ createInternalApiClient: vi.fn() })); + +import GenAITimeseries from '../GenAITimeseries.svelte'; +import { createInternalApiClient } from '$lib/api/internalClient'; + +const mockGet = vi.fn(); + +beforeEach(() => { + vi.mocked(createInternalApiClient).mockReturnValue({ get: mockGet } as unknown as ReturnType); + mockGet.mockReset(); +}); + +describe('GenAITimeseries', () => { + it('renders charts when service timeseries returns data', async () => { + const body = { + agent_dashboard: { buckets: [ { bucket_start: '2026-01-01T00:00:00Z', total_cost: 1, total_input_tokens: 10, total_output_tokens: 5, span_count: 1, error_count:0, error_rate:0, avg_duration_ms:100, p50_duration_ms:100, p95_duration_ms:100, p99_duration_ms:100, total_cache_creation_tokens:0, total_cache_read_tokens:0 } ], summary: { total_requests:1, avg_duration_ms:100, p50_duration_ms:100, p95_duration_ms:100, p99_duration_ms:100, overall_error_rate:0, total_input_tokens:10, total_output_tokens:5, total_cache_creation_tokens:0, total_cache_read_tokens:0, unique_agent_count:1, unique_conversation_count:1, cost_by_model:[] } }, + model_usage: { models: [ { model: 'gpt-4o-mini', provider_name: 'openai', span_count: 1, total_input_tokens:10, total_output_tokens:5, p50_duration_ms:100, p95_duration_ms:100, error_rate:0 } ] }, + buckets: [], + }; + + mockGet.mockResolvedValue({ ok: true, json: async () => body }); + + const { container } = render(GenAITimeseries, { props: { serviceId: 'svc-1' } }); + // Expect the component heading + expect(container.textContent).toContain('GenAI Timeseries'); + // Model name should appear somewhere in rendered content (from mocked data) + expect(await screen.findByText('GenAI Timeseries')).toBeTruthy(); + }); +}); diff --git a/crates/opsml_server/opsml_ui/src/lib/components/TraceDetail/GenAIMetricsTab.svelte b/crates/opsml_server/opsml_ui/src/lib/components/TraceDetail/GenAIMetricsTab.svelte new file mode 100644 index 0000000000..9fe4f3a444 --- /dev/null +++ b/crates/opsml_server/opsml_ui/src/lib/components/TraceDetail/GenAIMetricsTab.svelte @@ -0,0 +1,119 @@ + + +
+
+

GenAI Span Metrics

+
+ +
+
+ + {#if loading} +
+
+
+
+
+ {:else if error} +
+ Failed to load GenAI metrics: +
{error}
+
+ {:else if genai && genai.has_genai_spans} +
+ + + + + + + + + + + + + + {#each genai.spans as s} + + + + + + + + + + {/each} + +
Span IDOperationModelInput TokensOutput TokensLatencyCost
{s.span_id}{s.operation_name ?? '-'}{s.request_model ?? s.response_model ?? '-'}{s.input_tokens ?? '-'}{s.output_tokens ?? '-'}{formatDuration(s.duration_ms)}{displayCost(s)}
+
+ {:else} +
+
No GenAI spans found for this trace.
+
+ {/if} +
+ + diff --git a/crates/opsml_server/opsml_ui/src/lib/components/TraceDetail/__tests__/GenAIMetricsTab.test.ts b/crates/opsml_server/opsml_ui/src/lib/components/TraceDetail/__tests__/GenAIMetricsTab.test.ts new file mode 100644 index 0000000000..1610af007e --- /dev/null +++ b/crates/opsml_server/opsml_ui/src/lib/components/TraceDetail/__tests__/GenAIMetricsTab.test.ts @@ -0,0 +1,111 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, fireEvent } from '@testing-library/svelte'; + +vi.mock('$lib/api/internalClient', () => ({ createInternalApiClient: vi.fn() })); + +import GenAIMetricsTab from '../GenAIMetricsTab.svelte'; +import { createInternalApiClient } from '$lib/api/internalClient'; +import type { GenAiTraceMetricsResponse } from '$lib/components/scouter/genai/types'; + +function makeGenAi(): GenAiTraceMetricsResponse { + return { + trace_id: 't1', + has_genai_spans: true, + spans: [ + { + trace_id: 't1', + span_id: 's1', + parent_span_id: null, + service_name: 'svc', + start_time: '2026-01-01T00:00:00Z', + end_time: '2026-01-01T00:00:01Z', + duration_ms: 100, + status_code: 1, + operation_name: 'op', + provider_name: 'openai', + request_model: 'gpt-4o-mini', + response_model: null, + response_id: null, + input_tokens: 10, + output_tokens: 5, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + finish_reasons: [], + output_type: null, + conversation_id: null, + agent_name: null, + agent_id: null, + agent_description: null, + agent_version: null, + data_source_id: null, + tool_name: null, + tool_type: null, + tool_call_id: null, + request_temperature: null, + request_max_tokens: null, + request_choice_count: null, + request_seed: null, + request_frequency_penalty: null, + request_presence_penalty: null, + request_stop_sequences: [], + server_address: null, + server_port: null, + error_type: null, + openai_api_type: null, + openai_service_tier: null, + label: null, + input_messages: null, + output_messages: null, + system_instructions: null, + tool_definitions: null, + eval_results: [], + }, + ], + span_limit: 100, + spans_truncated: false, + sensitive_content_redacted: false, + token_metrics: { buckets: [] }, + operation_breakdown: { operations: [] }, + model_usage: { models: [] }, + agent_activity: { agents: [] }, + agent_dashboard: { summary: { total_requests: 0, avg_duration_ms: 0, p50_duration_ms: null, p95_duration_ms: null, p99_duration_ms: null, overall_error_rate: 0, total_input_tokens: 0, total_output_tokens: 0, total_cache_creation_tokens: 0, total_cache_read_tokens: 0, unique_agent_count: 0, unique_conversation_count: 0, cost_by_model: [] }, buckets: [] }, + tool_dashboard: { aggregates: [], time_series: [] }, + error_breakdown: { errors: [] }, + }; +} + +const mockPost = vi.fn(); + +beforeEach(() => { + vi.mocked(createInternalApiClient).mockReturnValue({ post: mockPost } as unknown as ReturnType); + mockPost.mockReset(); +}); + +describe('GenAIMetricsTab', () => { + it('fetches and renders span rows', async () => { + const body = makeGenAi(); + mockPost.mockResolvedValue({ ok: true, json: async () => body }); + + const { container } = render(GenAIMetricsTab, { props: { traceId: 't1' } }); + + // Wait for the table row to appear + expect(await screen.findByText('s1')).toBeTruthy(); + expect(container.textContent).toContain('gpt-4o-mini'); + expect(container.textContent).toContain('10'); + }); + + it('shows empty state when no spans', async () => { + const resp = makeGenAi(); + resp.has_genai_spans = false; + mockPost.mockResolvedValue({ ok: true, json: async () => resp }); + + const { container } = render(GenAIMetricsTab, { props: { traceId: 't1' } }); + expect(await screen.findByText('No GenAI spans found for this trace.')).toBeTruthy(); + }); + + it('renders error message when fetch fails', async () => { + mockPost.mockResolvedValue({ ok: false, status: 500 }); + const { container } = render(GenAIMetricsTab, { props: { traceId: 't1' } }); + expect(await screen.findByText(/Failed to load GenAI metrics/i)).toBeTruthy(); + }); +}); diff --git a/crates/opsml_server/opsml_ui/src/lib/components/card/agent/observability/AgentGenAiDashboard.svelte b/crates/opsml_server/opsml_ui/src/lib/components/card/agent/observability/AgentGenAiDashboard.svelte index 58a128ae3f..d602b92d2e 100644 --- a/crates/opsml_server/opsml_ui/src/lib/components/card/agent/observability/AgentGenAiDashboard.svelte +++ b/crates/opsml_server/opsml_ui/src/lib/components/card/agent/observability/AgentGenAiDashboard.svelte @@ -21,6 +21,8 @@ import AgentsTable from './AgentsTable.svelte'; import FilterBar from './FilterBar.svelte'; import { toScouterInterval } from './utils'; + import GenAIMetricsNav from '$lib/components/AgentServiceDashboard/GenAIMetricsNav.svelte'; + import GenAITimeseries from '$lib/components/AgentServiceDashboard/GenAITimeseries.svelte'; let { bundle: initialBundle }: { bundle: AgentGenAiBundle } = $props(); @@ -56,6 +58,9 @@ // would make the effect depend on its own output and self-trigger. let dashboard = $state(initialBundle.dashboard); + // Which sub-panel of the GenAI dashboard is active (overview vs timeseries) + let activePanel = $state<'overview' | 'timeseries'>('overview'); + // ── Fetch orchestration ──────────────────────────────────────────────────── // Skip the fetch on initial mount: the loader-provided bundle already // matches the current filter+range state. `requestEpoch` lets late @@ -150,29 +155,46 @@ lockEntity={isPromptScope} onChange={handleFilterChange} /> - - - -
- - - - -
- -
- - -
- -
- - - -
- -
- - +
+
+ m.model)} + providers={dashboard.available_filters.providers} + on:select={(e) => (activePanel = e.detail.key === 'overview' ? 'overview' : 'timeseries')} + /> +
+ +
+ {#if activePanel === 'overview'} + + +
+ + + + +
+ +
+ + +
+ +
+ + + +
+ +
+ + +
+ {:else} + + + {/if} +
diff --git a/crates/opsml_server/opsml_ui/src/lib/components/trace/SpanDetailView.svelte b/crates/opsml_server/opsml_ui/src/lib/components/trace/SpanDetailView.svelte index e265b6ab62..9f807bfb8c 100644 --- a/crates/opsml_server/opsml_ui/src/lib/components/trace/SpanDetailView.svelte +++ b/crates/opsml_server/opsml_ui/src/lib/components/trace/SpanDetailView.svelte @@ -15,6 +15,7 @@ import CodeBlock from '$lib/components/codeblock/CodeBlock.svelte'; import SpanEvents from './SpanEvents.svelte'; import SpanGenAiPanel from './genai/SpanGenAiPanel.svelte'; + import GenAIMetricsTab from '$lib/components/TraceDetail/GenAIMetricsTab.svelte'; import type { GenAiSpanRecord } from '$lib/components/scouter/genai/types'; import { EXCEPTION_TRACEBACK } from './types'; @@ -132,9 +133,19 @@ // ─── Tab state ───────────────────────────────────────────────────────────── - type Tab = 'overview' | 'errors' | 'attributes' | 'reqres' | 'events' | 'resources' | 'genai'; + type Tab = 'overview' | 'errors' | 'attributes' | 'reqres' | 'events' | 'resources' | 'genai' | 'genai_metrics'; let activeTab = $state('overview'); + const hasGenAiMetrics = $derived(() => { + if (!genAiSpan) return false; + return ( + genAiSpan.input_tokens != null || + genAiSpan.output_tokens != null || + Boolean(genAiSpan.request_model) || + Boolean(genAiSpan.response_model) + ); + }); + const tabs = $derived([ { id: 'overview' as Tab, label: 'Overview', Icon: Info, count: null as number | null }, { id: 'errors' as Tab, label: 'Errors', Icon: AlertCircle, count: errorCount > 0 ? errorCount : null as number | null }, @@ -143,7 +154,12 @@ { id: 'events' as Tab, label: 'Events', Icon: Activity, count: span.events.length > 0 ? span.events.length : null as number | null }, { id: 'resources' as Tab, label: 'Resources', Icon: Server, count: resourceAttributes.length > 0 ? resourceAttributes.length : null as number | null }, ...(genAiSpan - ? [{ id: 'genai' as Tab, label: 'GenAI', Icon: Sparkles, count: null as number | null }] + ? [ + { id: 'genai' as Tab, label: 'GenAI', Icon: Sparkles, count: null as number | null }, + ] + : []), + ...(hasGenAiMetrics + ? [{ id: 'genai_metrics' as Tab, label: 'GenAI Metrics', Icon: Sparkles, count: null as number | null }] : []), ]); @@ -662,5 +678,10 @@ {/if} + + {#if activeTab === 'genai_metrics' && genAiSpan} + + {/if} + diff --git a/crates/opsml_server/opsml_ui/src/lib/components/trace/types.ts b/crates/opsml_server/opsml_ui/src/lib/components/trace/types.ts index 1c88a19abb..41d2dad965 100644 --- a/crates/opsml_server/opsml_ui/src/lib/components/trace/types.ts +++ b/crates/opsml_server/opsml_ui/src/lib/components/trace/types.ts @@ -1,4 +1,5 @@ import type { DateTime } from "$lib/types"; +import type { GenAiSpanRecord } from "$lib/components/scouter/genai/types"; export interface TraceListItem { trace_id: string; @@ -149,6 +150,8 @@ export interface TraceSpan { input: string | null; output: string | null; service_name: string; + // Optional GenAI metadata attached to the span (if available) + genai?: GenAiSpanRecord | null; } export interface TraceSpansResponse { diff --git a/crates/opsml_server/opsml_ui/src/lib/types/genai.ts b/crates/opsml_server/opsml_ui/src/lib/types/genai.ts new file mode 100644 index 0000000000..2aec58d04b --- /dev/null +++ b/crates/opsml_server/opsml_ui/src/lib/types/genai.ts @@ -0,0 +1,64 @@ +/** + * Types for GenAI metrics used by the UI. + * These are strict, minimal shapes consumed by dashboard components and charts. + */ + +/** + * Metrics measured for a single GenAI span or call. + */ +export interface GenAISpanMetrics { + /** Number of input tokens consumed by the request. */ + token_count_input: number; + /** Number of output tokens produced by the response. */ + token_count_output: number; + /** Canonical model identifier used for the request (e.g. "gpt-4o-mini"). */ + model_name: string; + /** Latency for the call in milliseconds. */ + latency_ms: number; + /** Monetary cost for this call in USD (may be 0). */ + cost: number; +} + +/** + * Aggregate metrics over a timeframe or window. + */ +export interface GenAIAggregateMetrics { + /** Total tokens (input + output) observed in the aggregation window. */ + total_tokens: number; + /** Average latency in milliseconds across the window. */ + avg_latency: number; + /** + * Distribution of usage by model. Key is `model_name`, value is either a + * count of spans or a fractional share (consumer should document meaning). + */ + model_distribution: { [modelName: string]: number }; + /** Time period covered by this aggregate. ISO 8601 strings. */ + time_period: { + /** Inclusive start timestamp (ISO 8601). */ + start_time: string; + /** Inclusive or exclusive end timestamp (ISO 8601). */ + end_time: string; + }; +} + +/** + * Single timeseries point for a named metric. + */ +export interface GenAITimeseriesPoint { + /** ISO 8601 timestamp for the point. */ + timestamp: string; + /** Metric name (e.g. "input_tokens", "latency_ms", "cost"). */ + metric_name: string; + /** Numeric value for the metric at this timestamp. */ + value: number; +} + +/** + * Response shape for timeseries endpoints used by the UI. + */ +export interface GenAITimeseriesResponse { + /** Ordered list of timeseries points. */ + points: GenAITimeseriesPoint[]; + /** Labels used for chart axes or legend (e.g. bucket labels). */ + labels: string[]; +} diff --git a/crates/opsml_server/src/core/genai_metrics/route.rs b/crates/opsml_server/src/core/genai_metrics/route.rs new file mode 100644 index 0000000000..ae1610c8c9 --- /dev/null +++ b/crates/opsml_server/src/core/genai_metrics/route.rs @@ -0,0 +1,26 @@ +use crate::core::scouter::genai::{ + get_service_genai_timeseries, get_span_genai_metrics, get_trace_genai_aggregate, +}; +use crate::core::state::AppState; +use anyhow::Result; +use axum::{Router, routing::get}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::Arc; +use tracing::error; + +pub async fn get_genai_metrics_router(prefix: &str) -> Result>> { + let result = catch_unwind(AssertUnwindSafe(|| { + Router::new() + .route(&format!("{prefix}/traces/{{trace_id}}/aggregate"), get(get_trace_genai_aggregate)) + .route(&format!("{prefix}/spans/{{span_id}}/metrics"), get(get_span_genai_metrics)) + .route(&format!("{prefix}/services/{{service_id}}/timeseries"), get(get_service_genai_timeseries)) + })); + + match result { + Ok(router) => Ok(router), + Err(_) => { + error!("Failed to create genai_metrics router"); + Err(anyhow::anyhow!("Failed to create genai_metrics router")) + } + } +} diff --git a/crates/opsml_server/src/core/mod.rs b/crates/opsml_server/src/core/mod.rs index 325d3105fb..fa934bd5b9 100644 --- a/crates/opsml_server/src/core/mod.rs +++ b/crates/opsml_server/src/core/mod.rs @@ -15,6 +15,7 @@ pub mod middleware; pub mod openapi; pub mod router; pub mod scouter; +pub mod genai_metrics; pub mod settings; pub mod setup; pub mod shutdown; diff --git a/crates/opsml_server/src/core/router.rs b/crates/opsml_server/src/core/router.rs index d97a302118..f0278e402a 100644 --- a/crates/opsml_server/src/core/router.rs +++ b/crates/opsml_server/src/core/router.rs @@ -13,6 +13,7 @@ use crate::core::middleware::event::event_middleware; use crate::core::middleware::metrics::track_metrics; use crate::core::openapi::ApiDoc; use crate::core::scouter::route::get_scouter_router; +use crate::core::genai_metrics::route::get_genai_metrics_router; use crate::core::settings::route::get_settings_router; use crate::core::state::AppState; use crate::core::user::route::get_user_router; @@ -65,6 +66,7 @@ pub async fn create_router(app_state: Arc) -> Result { let auth_routes = get_auth_router(ROUTE_PREFIX).await?; let user_routes = get_user_router(ROUTE_PREFIX).await?; let scouter_routes = get_scouter_router(ROUTE_PREFIX).await?; + let genai_metrics_routes = get_genai_metrics_router("/api/genai").await?; let agent_routes = get_agent_router(ROUTE_PREFIX).await?; let agentic_routes = get_agentic_router(ROUTE_PREFIX).await?; let docs_routes = get_docs_router(V1_PREFIX).await?; @@ -78,6 +80,7 @@ pub async fn create_router(app_state: Arc) -> Result { .merge(run_routes) .merge(user_routes) .merge(scouter_routes) + .merge(genai_metrics_routes) .merge(agent_routes) .merge(agentic_routes) .route_layer(middleware::from_fn_with_state( diff --git a/crates/opsml_server/src/core/scouter/client.rs b/crates/opsml_server/src/core/scouter/client.rs index a0683d870e..adf40904e3 100644 --- a/crates/opsml_server/src/core/scouter/client.rs +++ b/crates/opsml_server/src/core/scouter/client.rs @@ -56,6 +56,10 @@ pub enum Routes { GenAiToolMetrics, GenAiDashboard, GenAiTraceMetrics, + // GenAI aggregate endpoints + GenAiTraceAggregate, + GenAiSpanMetrics, + GenAiServiceTimeseries, } impl Routes { @@ -116,6 +120,10 @@ impl Routes { Routes::GenAiToolMetrics => "scouter/genai/tool/metrics", Routes::GenAiDashboard => "scouter/genai/dashboard", Routes::GenAiTraceMetrics => "scouter/genai/traces", + // GenAI aggregate endpoints + Routes::GenAiTraceAggregate => "scouter/genai/trace/aggregate", + Routes::GenAiSpanMetrics => "scouter/genai/span/metrics", + Routes::GenAiServiceTimeseries => "scouter/genai/service/timeseries", } } } diff --git a/crates/opsml_server/src/core/scouter/genai/route.rs b/crates/opsml_server/src/core/scouter/genai/route.rs index 7dd7cef8d9..b003d8cae4 100644 --- a/crates/opsml_server/src/core/scouter/genai/route.rs +++ b/crates/opsml_server/src/core/scouter/genai/route.rs @@ -625,9 +625,335 @@ pub async fn genai_conversation( } } +// ───────────────────────────────────────────────────────────────────────────── +// GenAI Aggregate Endpoints — trace, span, and service-level metrics +// ───────────────────────────────────────────────────────────────────────────── + +/// Validates span ID format (basic 32-128 char hex string) +fn is_valid_span_id(id: &str) -> bool { + let stripped = id.replace('-', ""); + !stripped.is_empty() && stripped.len() <= 128 && stripped.chars().all(|c| c.is_ascii_hexdigit()) +} + +/// Validates service name format (alphanumeric with hyphens and dots) +fn is_valid_service_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 255 + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | '_')) +} + +#[utoipa::path( + get, + path = "/opsml/api/traces/{trace_id}/genai/aggregate", + params( + ("trace_id" = String, Path, description = "Trace ID (hex-encoded)"), + ("include_spans" = Option, Query, description = "Include per-span breakdown"), + ("include_errors" = Option, Query, description = "Include error analysis"), + ), + responses( + (status = 200, description = "Trace-level GenAI metrics aggregate", body = inline(serde_json::Value)), + (status = 400, description = "Invalid trace ID format", body = OpsmlServerError), + (status = 404, description = "Trace not found", body = OpsmlServerError), + (status = 500, description = "Internal error", body = OpsmlServerError), + ), + security(("bearer_token" = [])), + tag = "genai" +)] +#[instrument(skip_all)] +pub async fn get_trace_genai_aggregate( + State(state): State>, + Extension(perms): Extension, + Path(trace_id): Path, + Query(params): Query, +) -> Result, (StatusCode, Json)> { + if !state.scouter_client.is_enabled() { + return Err(( + StatusCode::SERVICE_UNAVAILABLE, + Json(OpsmlServerError::new( + "Scouter service is not available".to_string(), + )), + )); + } + + if !is_valid_trace_id(&trace_id) { + return Err(( + StatusCode::BAD_REQUEST, + Json(OpsmlServerError::new( + "Invalid trace ID format (expected 32-128 character hex string)".to_string(), + )), + )); + } + + let exchange_token = state.exchange_token_from_perms(&perms).await.map_err(|e| { + error!("Failed to exchange token for scouter: {e}"); + internal_server_error(e, "Failed to exchange token for scouter", None) + })?; + + let query_string = serde_qs::to_string(¶ms).map_err(|e| { + error!("Failed to serialize query string: {e}"); + internal_server_error(e, "Failed to serialize query string", None) + })?; + + let mut response = state + .scouter_client + .request_with_path( + scouter::Routes::GenAiTraceAggregate, + &[trace_id.as_str(), "aggregate"], + RequestType::Get, + None, + if query_string.is_empty() { + None + } else { + Some(query_string) + }, + None, + &exchange_token, + ) + .await + .map_err(|e| { + error!("Failed to get trace GenAI aggregate: {e}"); + internal_server_error(e, "Failed to fetch trace GenAI aggregate metrics", None) + })?; + + response.extensions_mut().insert(AuditContext { + resource_id: trace_id.to_string(), + resource_type: ResourceType::Drift, + metadata: "trace_genai_aggregate".to_string(), + registry_type: None, + operation: Operation::Read, + access_location: None, + }); + + let status_code = response.status(); + match status_code.is_success() { + true => { + let body = response.json::().await.map_err(|e| { + error!("Failed to parse scouter response: {e}"); + internal_server_error(e, "Failed to parse trace GenAI aggregate response", None) + })?; + Ok(Json(body)) + } + false => { + let body = response.json::().await.map_err(|e| { + error!("Failed to parse scouter error response: {e}"); + internal_server_error(e, "Failed to parse error response", None) + })?; + Err((status_code, Json(OpsmlServerError::new(body.error)))) + } + } +} + +#[utoipa::path( + get, + path = "/opsml/api/spans/{span_id}/genai/metrics", + params( + ("span_id" = String, Path, description = "Span ID (hex-encoded)"), + ("include_input" = Option, Query, description = "Include input messages"), + ("include_output" = Option, Query, description = "Include output messages"), + ("include_tool_calls" = Option, Query, description = "Include tool call details"), + ), + responses( + (status = 200, description = "Span-level GenAI metrics", body = inline(serde_json::Value)), + (status = 400, description = "Invalid span ID format", body = OpsmlServerError), + (status = 404, description = "Span not found", body = OpsmlServerError), + (status = 500, description = "Internal error", body = OpsmlServerError), + ), + security(("bearer_token" = [])), + tag = "genai" +)] +#[instrument(skip_all)] +pub async fn get_span_genai_metrics( + State(state): State>, + Extension(perms): Extension, + Path(span_id): Path, + Query(params): Query, +) -> Result, (StatusCode, Json)> { + if !state.scouter_client.is_enabled() { + return Err(( + StatusCode::SERVICE_UNAVAILABLE, + Json(OpsmlServerError::new( + "Scouter service is not available".to_string(), + )), + )); + } + + if !is_valid_span_id(&span_id) { + return Err(( + StatusCode::BAD_REQUEST, + Json(OpsmlServerError::new( + "Invalid span ID format (expected 32-128 character hex string)".to_string(), + )), + )); + } + + let exchange_token = state.exchange_token_from_perms(&perms).await.map_err(|e| { + error!("Failed to exchange token for scouter: {e}"); + internal_server_error(e, "Failed to exchange token for scouter", None) + })?; + + let query_string = serde_qs::to_string(¶ms).map_err(|e| { + error!("Failed to serialize query string: {e}"); + internal_server_error(e, "Failed to serialize query string", None) + })?; + + let mut response = state + .scouter_client + .request_with_path( + scouter::Routes::GenAiSpanMetrics, + &[span_id.as_str(), "metrics"], + RequestType::Get, + None, + if query_string.is_empty() { + None + } else { + Some(query_string) + }, + None, + &exchange_token, + ) + .await + .map_err(|e| { + error!("Failed to get span GenAI metrics: {e}"); + internal_server_error(e, "Failed to fetch span GenAI metrics", None) + })?; + + response.extensions_mut().insert(AuditContext { + resource_id: span_id.to_string(), + resource_type: ResourceType::Drift, + metadata: "span_genai_metrics".to_string(), + registry_type: None, + operation: Operation::Read, + access_location: None, + }); + + let status_code = response.status(); + match status_code.is_success() { + true => { + let body = response.json::().await.map_err(|e| { + error!("Failed to parse scouter response: {e}"); + internal_server_error(e, "Failed to parse span GenAI metrics response", None) + })?; + Ok(Json(body)) + } + false => { + let body = response.json::().await.map_err(|e| { + error!("Failed to parse scouter error response: {e}"); + internal_server_error(e, "Failed to parse error response", None) + })?; + Err((status_code, Json(OpsmlServerError::new(body.error)))) + } + } +} + +#[utoipa::path( + get, + path = "/opsml/api/services/{service_id}/genai/timeseries", + params( + ("service_id" = String, Path, description = "Service name or ID"), + ("start_time" = Option, Query, description = "Time range start (ISO 8601)"), + ("end_time" = Option, Query, description = "Time range end (ISO 8601)"), + ("interval" = Option, Query, description = "Aggregation interval (minute, hour, day)"), + ("metrics" = Option>, Query, description = "Comma-separated metric names (tokens, cost, latency, errors)"), + ), + responses( + (status = 200, description = "Service-level GenAI timeseries metrics", body = inline(serde_json::Value)), + (status = 400, description = "Invalid service ID or parameters", body = OpsmlServerError), + (status = 404, description = "Service not found", body = OpsmlServerError), + (status = 500, description = "Internal error", body = OpsmlServerError), + ), + security(("bearer_token" = [])), + tag = "genai" +)] +#[instrument(skip_all)] +pub async fn get_service_genai_timeseries( + State(state): State>, + Extension(perms): Extension, + Path(service_id): Path, + Query(params): Query, +) -> Result, (StatusCode, Json)> { + if !state.scouter_client.is_enabled() { + return Err(( + StatusCode::SERVICE_UNAVAILABLE, + Json(OpsmlServerError::new( + "Scouter service is not available".to_string(), + )), + )); + } + + if !is_valid_service_name(&service_id) { + return Err(( + StatusCode::BAD_REQUEST, + Json(OpsmlServerError::new( + "Invalid service name format (must be alphanumeric with hyphens, dots, or underscores)".to_string(), + )), + )); + } + + let exchange_token = state.exchange_token_from_perms(&perms).await.map_err(|e| { + error!("Failed to exchange token for scouter: {e}"); + internal_server_error(e, "Failed to exchange token for scouter", None) + })?; + + let query_string = serde_qs::to_string(¶ms).map_err(|e| { + error!("Failed to serialize query string: {e}"); + internal_server_error(e, "Failed to serialize query string", None) + })?; + + let mut response = state + .scouter_client + .request_with_path( + scouter::Routes::GenAiServiceTimeseries, + &[service_id.as_str(), "timeseries"], + RequestType::Get, + None, + if query_string.is_empty() { + None + } else { + Some(query_string) + }, + None, + &exchange_token, + ) + .await + .map_err(|e| { + error!("Failed to get service GenAI timeseries: {e}"); + internal_server_error(e, "Failed to fetch service GenAI timeseries metrics", None) + })?; + + response.extensions_mut().insert(AuditContext { + resource_id: service_id.to_string(), + resource_type: ResourceType::Drift, + metadata: "service_genai_timeseries".to_string(), + registry_type: None, + operation: Operation::Read, + access_location: None, + }); + + let status_code = response.status(); + match status_code.is_success() { + true => { + let body = response.json::().await.map_err(|e| { + error!("Failed to parse scouter response: {e}"); + internal_server_error(e, "Failed to parse service GenAI timeseries response", None) + })?; + Ok(Json(body)) + } + false => { + let body = response.json::().await.map_err(|e| { + error!("Failed to parse scouter error response: {e}"); + internal_server_error(e, "Failed to parse error response", None) + })?; + Err((status_code, Json(OpsmlServerError::new(body.error)))) + } + } +} + pub async fn get_scouter_genai_router(prefix: &str) -> Result>> { let result = catch_unwind(AssertUnwindSafe(|| { Router::new() + // Existing endpoints .route( &format!("{prefix}/scouter/genai/metrics/tokens"), post(genai_token_metrics), @@ -673,6 +999,19 @@ pub async fn get_scouter_genai_router(prefix: &str) -> Result