diff --git a/ROADMAP.md b/ROADMAP.md index e3dfe76..d7e7090 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -98,9 +98,9 @@ As usage grows, the platform needs stronger derived data pipelines, performance - [x] [P0] Cursor transcript and tool-call extraction mapped onto the normalized session model - [ ] [P1] Cursor native telemetry enrichment for approvals, change shape, and context-usage signals - [ ] [P1] Cursor reliable token and model-usage extraction once source telemetry is trustworthy -- [ ] [P1] Workflow fingerprinting: infer common sequences like search -> read -> edit -> test -> fix +- [x] [P1] Workflow fingerprinting: infer common sequences like search -> read -> edit -> test -> fix - [ ] [P1] Cursor-specific workflow fingerprinting and session archetype mapping -- [ ] [P1] Session archetype detection: debugging, feature delivery, refactor, migration, docs, investigation +- [x] [P1] Session archetype detection: debugging, feature delivery, refactor, migration, docs, investigation - [ ] [P1] Delegation graph capture for multi-agent and subagent workflows - [ ] [P2] Exemplar session library for high-value workflows and onboarding examples - [ ] [P2] Prompt, skill, and template reuse analytics by workflow and outcome @@ -268,7 +268,7 @@ As usage grows, the platform needs stronger derived data pipelines, performance - [x] [P0] Cursor `agent_type` support across capture, sync, ingest, and analytics filters - [ ] [P0] Durable background job system for sync, facet extraction, narratives, and alerts - [ ] [P0] Scalable API key lookup and verification strategy -- [ ] [P1] Source-capability registry so Primer can safely gate analytics by what each agent source actually provides +- [x] [P1] Source-capability registry so Primer can safely gate analytics by what each agent source actually provides - [ ] [P1] OpenTelemetry integration for metrics, traces, and logs - [ ] [P1] Redis-backed caching for analytics query results and high-read metadata - [ ] [P1] Analytics performance work for large orgs and concurrent dashboard usage diff --git a/alembic/versions/44c9b01ccad2_add_session_workflow_profiles.py b/alembic/versions/44c9b01ccad2_add_session_workflow_profiles.py new file mode 100644 index 0000000..a8b90a2 --- /dev/null +++ b/alembic/versions/44c9b01ccad2_add_session_workflow_profiles.py @@ -0,0 +1,62 @@ +"""add session workflow profiles + +Revision ID: 44c9b01ccad2 +Revises: 0f3b0a1e2c91 +Create Date: 2026-03-13 13:02:58.876348 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "44c9b01ccad2" +down_revision: Union[str, None] = "0f3b0a1e2c91" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "session_workflow_profiles", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("session_id", sa.String(length=36), nullable=False), + sa.Column("fingerprint_id", sa.String(length=255), nullable=True), + sa.Column("label", sa.String(length=255), nullable=True), + sa.Column("steps", sa.JSON(), nullable=True), + sa.Column("archetype", sa.String(length=50), nullable=True), + sa.Column("archetype_source", sa.String(length=20), nullable=True), + sa.Column("archetype_reason", sa.Text(), nullable=True), + sa.Column("top_tools", sa.JSON(), nullable=True), + sa.Column("delegation_count", sa.Integer(), nullable=False), + sa.Column("verification_run_count", sa.Integer(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["session_id"], + ["sessions.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_session_workflow_profiles_session_id", + "session_workflow_profiles", + ["session_id"], + unique=True, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("ix_session_workflow_profiles_session_id", table_name="session_workflow_profiles") + op.drop_table("session_workflow_profiles") + # ### end Alembic commands ### diff --git a/frontend/src/components/sessions/__tests__/session-detail-panel.test.tsx b/frontend/src/components/sessions/__tests__/session-detail-panel.test.tsx index f05f09e..74f084f 100644 --- a/frontend/src/components/sessions/__tests__/session-detail-panel.test.tsx +++ b/frontend/src/components/sessions/__tests__/session-detail-panel.test.tsx @@ -90,6 +90,17 @@ const baseSession: SessionDetailResponse = { last_verification_status: "passed", sample_recovery_commands: ["pytest -q"], }, + workflow_profile: { + fingerprint_id: "feature_delivery::read+edit+execute+test+ship", + label: "feature delivery: read -> edit -> execute -> test -> ship", + steps: ["read", "edit", "execute", "test", "ship"], + archetype: "feature_delivery", + archetype_source: "session_type", + archetype_reason: "Mapped from the extracted session type 'code_modification'.", + top_tools: ["Read", "Edit"], + delegation_count: 0, + verification_run_count: 1, + }, } describe("SessionDetailPanel", () => { @@ -165,6 +176,16 @@ describe("SessionDetailPanel", () => { expect(screen.getByText("Sample Recovery Commands")).toBeInTheDocument() }) + it("renders workflow profile when present", () => { + render() + + expect(screen.getByText("Workflow Profile")).toBeInTheDocument() + expect(screen.getByText("Feature Delivery")).toBeInTheDocument() + expect(screen.getByText("Workflow Steps")).toBeInTheDocument() + expect(screen.getByText("Top Tools")).toBeInTheDocument() + expect(screen.getByText("Mapped from the extracted session type 'code_modification'.")).toBeInTheDocument() + }) + it("counts hidden change-shape files from both named overflow and inferred files", () => { render( { expect(screen.queryByText("Facets")).not.toBeInTheDocument() }) + + it("does not render workflow profile card when null", () => { + render( + , + ) + + expect(screen.queryByText("Workflow Profile")).not.toBeInTheDocument() + }) }) diff --git a/frontend/src/components/sessions/session-detail-panel.tsx b/frontend/src/components/sessions/session-detail-panel.tsx index f49627a..2027a89 100644 --- a/frontend/src/components/sessions/session-detail-panel.tsx +++ b/frontend/src/components/sessions/session-detail-panel.tsx @@ -44,10 +44,20 @@ const EXECUTION_STATUS_BADGE: Record< unknown: "secondary", } +const WORKFLOW_ARCHETYPE_LABELS: Record = { + debugging: "Debugging", + feature_delivery: "Feature Delivery", + refactor: "Refactor", + migration: "Migration", + docs: "Docs", + investigation: "Investigation", +} + export function SessionDetailPanel({ session }: SessionDetailPanelProps) { const { facets } = session const changeShape = session.change_shape const recoveryPath = session.recovery_path + const workflowProfile = session.workflow_profile const visibleNamedFiles = changeShape?.named_touched_files?.slice(0, 8) ?? [] const hiddenChangeShapeFiles = changeShape ? changeShape.files_touched_count - visibleNamedFiles.length @@ -235,6 +245,77 @@ export function SessionDetailPanel({ session }: SessionDetailPanelProps) { )} + {workflowProfile && ( + + + Workflow Profile + + +
+
+

Archetype

+

+ {workflowProfile.archetype + ? WORKFLOW_ARCHETYPE_LABELS[workflowProfile.archetype] + : "-"} +

+
+
+

Fingerprint

+

+ {workflowProfile.label ?? "-"} +

+
+
+

Delegations

+

+ {workflowProfile.delegation_count} +

+
+
+

Verification Runs

+

+ {workflowProfile.verification_run_count} +

+
+
+ {workflowProfile.steps && workflowProfile.steps.length > 0 && ( +
+

+ Workflow Steps +

+
+ {workflowProfile.steps.map((step) => ( + + {step.replace(/_/g, " ")} + + ))} +
+
+ )} + {workflowProfile.top_tools && workflowProfile.top_tools.length > 0 && ( +
+

+ Top Tools +

+
+ {workflowProfile.top_tools.map((tool) => ( + + {tool} + + ))} +
+
+ )} + {workflowProfile.archetype_reason && ( +

+ {workflowProfile.archetype_reason} +

+ )} + + + )} + {changeShape && ( diff --git a/frontend/src/pages/__tests__/session-detail.test.tsx b/frontend/src/pages/__tests__/session-detail.test.tsx index 8d4ea45..968da3f 100644 --- a/frontend/src/pages/__tests__/session-detail.test.tsx +++ b/frontend/src/pages/__tests__/session-detail.test.tsx @@ -89,6 +89,7 @@ const mockDetailSession = { execution_evidence: [], change_shape: null, recovery_path: null, + workflow_profile: null, } describe("SessionDetailPage", () => { diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 1e11377..1e32301 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -92,6 +92,24 @@ export interface ModelUsageResponse { export type ExecutionEvidenceType = "test" | "lint" | "build" | "verification" export type ExecutionEvidenceStatus = "passed" | "failed" | "unknown" +export type WorkflowStep = + | "search" + | "read" + | "edit" + | "execute" + | "test" + | "fix" + | "delegate" + | "integrate" + | "ship" +export type SessionArchetype = + | "debugging" + | "feature_delivery" + | "refactor" + | "migration" + | "docs" + | "investigation" +export type ArchetypeSource = "session_type" | "heuristic" export interface SessionExecutionEvidenceResponse { ordinal: number @@ -129,6 +147,18 @@ export interface SessionRecoveryPathResponse { sample_recovery_commands: string[] | null } +export interface SessionWorkflowProfileResponse { + fingerprint_id: string | null + label: string | null + steps: WorkflowStep[] | null + archetype: SessionArchetype | null + archetype_source: ArchetypeSource | null + archetype_reason: string | null + top_tools: string[] | null + delegation_count: number + verification_run_count: number +} + export interface SessionDetailResponse extends SessionResponse { facets: SessionFacetsResponse | null tool_usages: ToolUsageResponse[] @@ -136,6 +166,7 @@ export interface SessionDetailResponse extends SessionResponse { execution_evidence: SessionExecutionEvidenceResponse[] change_shape: SessionChangeShapeResponse | null recovery_path: SessionRecoveryPathResponse | null + workflow_profile: SessionWorkflowProfileResponse | null } export interface OverviewStats { diff --git a/src/primer/common/models.py b/src/primer/common/models.py index c4bd1ed..d08c9bd 100644 --- a/src/primer/common/models.py +++ b/src/primer/common/models.py @@ -148,6 +148,9 @@ class Session(Base): recovery_path: Mapped["SessionRecoveryPath | None"] = relationship( back_populates="session", uselist=False, cascade="all, delete-orphan" ) + workflow_profile: Mapped["SessionWorkflowProfile | None"] = relationship( + back_populates="session", uselist=False, cascade="all, delete-orphan" + ) commits: Mapped[list["SessionCommit"]] = relationship( back_populates="session", cascade="all, delete-orphan" ) @@ -379,6 +382,32 @@ class SessionRecoveryPath(Base): session: Mapped[Session] = relationship(back_populates="recovery_path") +class SessionWorkflowProfile(Base): + __tablename__ = "session_workflow_profiles" + __table_args__ = ( + Index( + "ix_session_workflow_profiles_session_id", + "session_id", + unique=True, + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + session_id: Mapped[str] = mapped_column(ForeignKey("sessions.id"), nullable=False) + fingerprint_id: Mapped[str | None] = mapped_column(String(255), nullable=True) + label: Mapped[str | None] = mapped_column(String(255), nullable=True) + steps: Mapped[list[str] | None] = mapped_column(JSON, nullable=True) + archetype: Mapped[str | None] = mapped_column(String(50), nullable=True) + archetype_source: Mapped[str | None] = mapped_column(String(20), nullable=True) + archetype_reason: Mapped[str | None] = mapped_column(Text, nullable=True) + top_tools: Mapped[list[str] | None] = mapped_column(JSON, nullable=True) + delegation_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + verification_run_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now()) + + session: Mapped[Session] = relationship(back_populates="workflow_profile") + + class Alert(Base): __tablename__ = "alerts" diff --git a/src/primer/common/schemas.py b/src/primer/common/schemas.py index 4ee99e4..6e0b2fd 100644 --- a/src/primer/common/schemas.py +++ b/src/primer/common/schemas.py @@ -23,6 +23,26 @@ class PaginatedResponse[T](BaseModel): InterventionStatus = Literal["planned", "in_progress", "completed", "dismissed"] ExecutionEvidenceType = Literal["test", "lint", "build", "verification"] ExecutionEvidenceStatus = Literal["passed", "failed", "unknown"] +WorkflowStep = Literal[ + "search", + "read", + "edit", + "execute", + "test", + "fix", + "delegate", + "integrate", + "ship", +] +SessionArchetype = Literal[ + "debugging", + "feature_delivery", + "refactor", + "migration", + "docs", + "investigation", +] +ArchetypeSource = Literal["session_type", "heuristic"] RecoveryStrategy = Literal[ "inspect_context", "edit_fix", @@ -233,6 +253,20 @@ class SessionRecoveryPathResponse(BaseModel): model_config = {"from_attributes": True} +class SessionWorkflowProfileResponse(BaseModel): + fingerprint_id: str | None + label: str | None + steps: list[WorkflowStep] | None + archetype: SessionArchetype | None + archetype_source: ArchetypeSource | None + archetype_reason: str | None + top_tools: list[str] | None + delegation_count: int + verification_run_count: int + + model_config = {"from_attributes": True} + + # --- Session Messages --- @@ -368,6 +402,7 @@ class SessionDetailResponse(SessionResponse): execution_evidence: list[SessionExecutionEvidenceResponse] = [] change_shape: SessionChangeShapeResponse | None = None recovery_path: SessionRecoveryPathResponse | None = None + workflow_profile: SessionWorkflowProfileResponse | None = None # --- Analytics --- diff --git a/src/primer/server/services/ingest_service.py b/src/primer/server/services/ingest_service.py index 2e7a8f1..46e4e33 100644 --- a/src/primer/server/services/ingest_service.py +++ b/src/primer/server/services/ingest_service.py @@ -17,6 +17,7 @@ SessionFacets, SessionMessage, SessionRecoveryPath, + SessionWorkflowProfile, ToolUsage, ) from primer.common.models import ( @@ -27,6 +28,7 @@ from primer.server.services.change_shape_service import extract_change_shape from primer.server.services.execution_evidence_service import extract_execution_evidence from primer.server.services.recovery_path_service import extract_recovery_path +from primer.server.services.workflow_profile_service import extract_session_workflow_profile logger = logging.getLogger(__name__) @@ -190,6 +192,8 @@ def upsert_session(db: Session, engineer_id: str, payload: SessionIngestPayload) if payload.messages is not None or payload.facets is not None: _upsert_recovery_path(db, session.id) + _upsert_workflow_profile(db, session.id) + db.flush() return created @@ -317,6 +321,65 @@ def _upsert_recovery_path(db: Session, session_id: str) -> None: db.flush() +def _upsert_workflow_profile(db: Session, session_id: str) -> None: + db.flush() + + session = db.query(SessionModel).filter(SessionModel.id == session_id).first() + if session is None: + return + + tool_usages = db.query(ToolUsage).filter(ToolUsage.session_id == session_id).all() + execution_evidence = ( + db.query(SessionExecutionEvidence) + .filter(SessionExecutionEvidence.session_id == session_id) + .order_by(SessionExecutionEvidence.ordinal) + .all() + ) + change_shape = ( + db.query(SessionChangeShape).filter(SessionChangeShape.session_id == session_id).first() + ) + recovery_path = ( + db.query(SessionRecoveryPath).filter(SessionRecoveryPath.session_id == session_id).first() + ) + facets = db.query(SessionFacets).filter(SessionFacets.session_id == session_id).first() + has_commit = db.query(SessionCommit.id).filter(SessionCommit.session_id == session_id).first() + derived = extract_session_workflow_profile( + session, + tool_usages, + execution_evidence, + change_shape=change_shape, + recovery_path=recovery_path, + facets=facets, + has_commit=has_commit is not None, + ) + + existing = ( + db.query(SessionWorkflowProfile) + .filter(SessionWorkflowProfile.session_id == session_id) + .first() + ) + if derived is None: + if existing is not None: + db.delete(existing) + db.flush() + return + + record = existing or SessionWorkflowProfile(session_id=session_id) + if existing is None: + db.add(record) + + record.fingerprint_id = derived.fingerprint_id + record.label = derived.label + record.steps = derived.steps + record.archetype = derived.archetype + record.archetype_source = derived.archetype_source + record.archetype_reason = derived.archetype_reason + record.top_tools = derived.top_tools + record.delegation_count = derived.delegation_count + record.verification_run_count = derived.verification_run_count + db.flush() + + def log_ingest_event( db: Session, engineer_id: str, diff --git a/src/primer/server/services/project_workspace_service.py b/src/primer/server/services/project_workspace_service.py index d996750..781024d 100644 --- a/src/primer/server/services/project_workspace_service.py +++ b/src/primer/server/services/project_workspace_service.py @@ -11,6 +11,7 @@ SessionCommit, SessionFacets, SessionMessage, + SessionWorkflowProfile, ToolUsage, ) from primer.common.models import Session as SessionModel @@ -1061,6 +1062,19 @@ def _build_workflow_summary( .filter(SessionCommit.session_id.in_(session_ids)) .all() } + profiles_by_session = { + row.session_id: row + for row in ( + db.query( + SessionWorkflowProfile.session_id, + SessionWorkflowProfile.fingerprint_id, + SessionWorkflowProfile.label, + SessionWorkflowProfile.steps, + ) + .filter(SessionWorkflowProfile.session_id.in_(session_ids)) + .all() + ) + } impact_by_type = {item.friction_type: item.impact_score for item in friction_impacts} fingerprinted_sessions = 0 @@ -1072,13 +1086,20 @@ def _build_workflow_summary( facets = facets_by_session.get(session_id, {}) session_type = facets.get("session_type") tool_counts = tool_counts_by_session.get(session_id, Counter()) - steps = infer_workflow_steps(tool_counts, session_id in commit_session_ids) + profile = profiles_by_session.get(session_id) + steps = list(profile.steps or []) if profile is not None else [] + if not steps: + steps = infer_workflow_steps(tool_counts, session_id in commit_session_ids) if not session_type and not steps: continue fingerprinted_sessions += 1 - fingerprint_id = workflow_fingerprint_id(session_type, steps) - label = workflow_fingerprint_label(session_type, steps) + fingerprint_id = profile.fingerprint_id if profile is not None else None + if not fingerprint_id: + fingerprint_id = workflow_fingerprint_id(session_type, steps) + label = profile.label if profile is not None else None + if not label: + label = workflow_fingerprint_label(session_type, steps) label_by_key[fingerprint_id] = label bucket = fingerprints.setdefault( diff --git a/src/primer/server/services/workflow_patterns.py b/src/primer/server/services/workflow_patterns.py index 227faf2..150176d 100644 --- a/src/primer/server/services/workflow_patterns.py +++ b/src/primer/server/services/workflow_patterns.py @@ -2,21 +2,39 @@ from primer.common.tool_classification import classify_tool +_SEARCH_HINTS = ("grep", "glob", "search", "find", "ripgrep") +_READ_HINTS = ("read", "fetch", "open", "cat", "view") +_EDIT_HINTS = ("edit", "write", "patch", "replace", "insert", "delete", "remove", "rename", "move") +_EXECUTE_HINTS = ("bash", "terminal", "exec", "command") +_DELEGATE_HINTS = ("task", "agent", "delegate", "team", "sendmessage", "send_message") +_INTEGRATE_HINTS = ("mcp", "plugin") -def infer_workflow_steps(tool_counts: Counter[str], has_commit: bool) -> list[str]: + +def infer_workflow_steps( + tool_counts: Counter[str], + has_commit: bool, + *, + execution_types: set[str] | None = None, + recovery_strategies: set[str] | None = None, + has_mutations: bool = False, +) -> list[str]: tool_names = set(tool_counts) steps: list[str] = [] - if any(classify_tool(name) == "search" for name in tool_names): + if any(_is_search_tool(name) for name in tool_names): steps.append("search") - if tool_names.intersection({"Read", "WebFetch"}): + if any(_is_read_tool(name) for name in tool_names): steps.append("read") - if tool_names.intersection({"Edit", "Write", "NotebookEdit"}): + if has_mutations or any(_is_edit_tool(name) for name in tool_names): steps.append("edit") - if "Bash" in tool_names: + if execution_types or any(_is_execute_tool(name) for name in tool_names): steps.append("execute") - if any(classify_tool(name) in {"orchestration", "skill"} for name in tool_names): + if execution_types and "test" in execution_types: + steps.append("test") + if recovery_strategies and recovery_strategies.intersection({"edit_fix", "revert_or_reset"}): + steps.append("fix") + if any(_is_delegate_tool(name) for name in tool_names): steps.append("delegate") - if any(classify_tool(name) == "mcp" for name in tool_names): + if any(_is_integrate_tool(name) for name in tool_names): steps.append("integrate") if has_commit: steps.append("ship") @@ -33,3 +51,41 @@ def workflow_fingerprint_label(session_type: str | None, steps: list[str]) -> st type_label = (session_type or "general").replace("_", " ") step_label = " -> ".join(steps) if steps else "reasoning" return f"{type_label}: {step_label}" + + +def _normalized_tool_name(name: str) -> str: + return name.strip().lower() + + +def _is_search_tool(name: str) -> bool: + normalized = _normalized_tool_name(name) + return classify_tool(name) == "search" or any(hint in normalized for hint in _SEARCH_HINTS) + + +def _is_read_tool(name: str) -> bool: + normalized = _normalized_tool_name(name) + return normalized in {"read", "webfetch"} or any(hint in normalized for hint in _READ_HINTS) + + +def _is_edit_tool(name: str) -> bool: + normalized = _normalized_tool_name(name) + return normalized in {"edit", "write", "notebookedit"} or any( + hint in normalized for hint in _EDIT_HINTS + ) + + +def _is_execute_tool(name: str) -> bool: + normalized = _normalized_tool_name(name) + return normalized == "bash" or any(hint in normalized for hint in _EXECUTE_HINTS) + + +def _is_delegate_tool(name: str) -> bool: + normalized = _normalized_tool_name(name) + return classify_tool(name) in {"orchestration", "skill"} or any( + hint in normalized for hint in _DELEGATE_HINTS + ) + + +def _is_integrate_tool(name: str) -> bool: + normalized = _normalized_tool_name(name) + return classify_tool(name) == "mcp" or any(hint in normalized for hint in _INTEGRATE_HINTS) diff --git a/src/primer/server/services/workflow_playbook_service.py b/src/primer/server/services/workflow_playbook_service.py index efa8058..d573fdc 100644 --- a/src/primer/server/services/workflow_playbook_service.py +++ b/src/primer/server/services/workflow_playbook_service.py @@ -4,7 +4,13 @@ from sqlalchemy.orm import Session from primer.common.facet_taxonomy import canonical_outcome, is_success_outcome -from primer.common.models import Engineer, SessionCommit, SessionFacets, ToolUsage +from primer.common.models import ( + Engineer, + SessionCommit, + SessionFacets, + SessionWorkflowProfile, + ToolUsage, +) from primer.common.models import Session as SessionModel from primer.common.schemas import WorkflowPlaybook from primer.server.services.workflow_patterns import ( @@ -135,16 +141,31 @@ def _load_workflow_sessions( .distinct() .all() } + profiles_by_session = { + row.session_id: row + for row in ( + db.query( + SessionWorkflowProfile.session_id, + SessionWorkflowProfile.fingerprint_id, + SessionWorkflowProfile.steps, + ) + .filter(SessionWorkflowProfile.session_id.in_(session_ids)) + .all() + ) + } enriched_rows = [] for row in session_rows: facets = facets_by_session.get(row.id, {}) tool_counts = tool_counts_by_session.get(row.id, Counter()) - steps = infer_workflow_steps(tool_counts, row.id in commit_session_ids) + profile = profiles_by_session.get(row.id) + steps = list(profile.steps or []) if profile is not None else [] + if not steps: + steps = infer_workflow_steps(tool_counts, row.id in commit_session_ids) session_type = facets.get("session_type") - fingerprint_id = ( - workflow_fingerprint_id(session_type, steps) if session_type or steps else None - ) + fingerprint_id = profile.fingerprint_id if profile is not None else None + if not fingerprint_id and (session_type or steps): + fingerprint_id = workflow_fingerprint_id(session_type, steps) enriched_rows.append( { "session_id": row.id, diff --git a/src/primer/server/services/workflow_profile_service.py b/src/primer/server/services/workflow_profile_service.py new file mode 100644 index 0000000..6b1ffaf --- /dev/null +++ b/src/primer/server/services/workflow_profile_service.py @@ -0,0 +1,346 @@ +from __future__ import annotations + +import re +from collections import Counter +from dataclasses import dataclass + +from primer.common.tool_classification import classify_tool +from primer.server.services.workflow_patterns import ( + infer_workflow_steps, + workflow_fingerprint_id, + workflow_fingerprint_label, +) + +_SESSION_TYPE_ARCHETYPES = { + "bug_fix": "debugging", + "code_modification": "feature_delivery", + "debugging": "debugging", + "docs": "docs", + "documentation": "docs", + "exploration": "investigation", + "feature": "feature_delivery", + "feature_delivery": "feature_delivery", + "implementation": "feature_delivery", + "investigation": "investigation", + "migration": "migration", + "refactor": "refactor", + "refactoring": "refactor", + "research": "investigation", +} +_DOC_TEXT_RE = re.compile(r"\b(?:docs?|documentation|readme|changelog|guide)\b") +_MIGRATION_TEXT_RE = re.compile(r"\b(?:migrat\w*|upgrade|moderniz\w*|deprecat\w*|port)\b") + + +@dataclass +class WorkflowProfileRecord: + fingerprint_id: str | None + label: str | None + steps: list[str] + archetype: str | None + archetype_source: str | None + archetype_reason: str | None + top_tools: list[str] + delegation_count: int + verification_run_count: int + + +def extract_session_workflow_profile( + session: object, + tool_usages: list[object], + execution_evidence: list[object], + *, + change_shape: object | None = None, + recovery_path: object | None = None, + facets: object | None = None, + has_commit: bool = False, +) -> WorkflowProfileRecord | None: + tool_counts = _tool_counts(tool_usages) + execution_types = { + evidence_type + for evidence in execution_evidence + if (evidence_type := _string_attr(evidence, "evidence_type")) + } + recovery_strategies = { + strategy + for strategy in (_list_attr(recovery_path, "recovery_strategies") or []) + if isinstance(strategy, str) and strategy + } + session_type = _string_attr(facets, "session_type") + has_mutations = _has_mutations(change_shape) + steps = infer_workflow_steps( + tool_counts, + has_commit, + execution_types=execution_types, + recovery_strategies=recovery_strategies, + has_mutations=has_mutations, + ) + + archetype, archetype_source, archetype_reason = _infer_archetype( + session, + facets, + change_shape, + recovery_path, + session_type=session_type, + steps=steps, + execution_types=execution_types, + has_commit=has_commit, + ) + fingerprint_type = session_type or archetype + fingerprint_id = ( + workflow_fingerprint_id(fingerprint_type, steps) if fingerprint_type or steps else None + ) + label = workflow_fingerprint_label(fingerprint_type, steps) if fingerprint_id else None + top_tools = [tool_name for tool_name, _count in tool_counts.most_common(4)] + delegation_count = sum( + count for tool_name, count in tool_counts.items() if _is_delegation_tool(tool_name) + ) + verification_run_count = sum( + 1 + for evidence in execution_evidence + if _string_attr(evidence, "evidence_type") in {"test", "lint", "build", "verification"} + ) + + if not ( + fingerprint_id + or archetype + or top_tools + or delegation_count > 0 + or verification_run_count > 0 + ): + return None + + return WorkflowProfileRecord( + fingerprint_id=fingerprint_id, + label=label, + steps=steps, + archetype=archetype, + archetype_source=archetype_source, + archetype_reason=archetype_reason, + top_tools=top_tools, + delegation_count=delegation_count, + verification_run_count=verification_run_count, + ) + + +def _infer_archetype( + session: object, + facets: object | None, + change_shape: object | None, + recovery_path: object | None, + *, + session_type: str | None, + steps: list[str], + execution_types: set[str], + has_commit: bool, +) -> tuple[str | None, str | None, str | None]: + normalized_type = session_type.lower() if session_type else None + mapped = _SESSION_TYPE_ARCHETYPES.get(normalized_type or "") + if mapped: + return ( + mapped, + "session_type", + f"Mapped from the extracted session type '{normalized_type}'.", + ) + + text = _hint_text(session, facets) + named_files = _named_files(change_shape) + if _looks_like_docs(text, named_files): + return "docs", "heuristic", "Documentation-heavy prompt or changed files suggest docs work." + + if _looks_like_migration(text, change_shape): + return ( + "migration", + "heuristic", + "Prompt hints and broad file changes suggest a migration or upgrade effort.", + ) + + if _looks_like_debugging(execution_types, recovery_path, steps): + return ( + "debugging", + "heuristic", + "Failed verification or recovery behavior suggests a debugging loop.", + ) + + if _looks_like_refactor(text, change_shape): + return ( + "refactor", + "heuristic", + "Rewrite or rename signals point to a refactor-oriented session.", + ) + + if _looks_like_feature_delivery(change_shape, has_commit, steps): + return ( + "feature_delivery", + "heuristic", + "Mutating changes and shipping signals suggest feature delivery work.", + ) + + if _looks_like_investigation(change_shape, has_commit, steps): + return ( + "investigation", + "heuristic", + "Read-heavy activity without durable changes suggests investigation work.", + ) + + return None, None, None + + +def _tool_counts(tool_usages: list[object]) -> Counter[str]: + counts: Counter[str] = Counter() + for usage in tool_usages: + tool_name = _string_attr(usage, "tool_name") + if not tool_name: + continue + call_count = _int_attr(usage, "call_count") + counts[tool_name] += max(call_count, 0) + return counts + + +def _string_attr(value: object | None, field: str) -> str | None: + if value is None: + return None + candidate = value.get(field) if isinstance(value, dict) else getattr(value, field, None) + if isinstance(candidate, str) and candidate.strip(): + return candidate.strip() + return None + + +def _int_attr(value: object | None, field: str) -> int: + if value is None: + return 0 + candidate = value.get(field) if isinstance(value, dict) else getattr(value, field, None) + try: + return int(candidate or 0) + except (TypeError, ValueError): + return 0 + + +def _list_attr(value: object | None, field: str) -> list[object] | None: + if value is None: + return None + candidate = value.get(field) if isinstance(value, dict) else getattr(value, field, None) + return candidate if isinstance(candidate, list) else None + + +def _hint_text(session: object, facets: object | None) -> str: + parts = [ + _string_attr(session, "first_prompt"), + _string_attr(session, "summary"), + _string_attr(facets, "underlying_goal"), + _string_attr(facets, "brief_summary"), + ] + return " ".join(part.lower() for part in parts if part) + + +def _named_files(change_shape: object | None) -> list[str]: + values = _list_attr(change_shape, "named_touched_files") or [] + return [value for value in values if isinstance(value, str) and value] + + +def _has_mutations(change_shape: object | None) -> bool: + return any( + _int_attr(change_shape, field) > 0 + for field in ( + "files_touched_count", + "diff_size", + "edit_operations", + "create_operations", + "delete_operations", + "rename_operations", + ) + ) + + +def _looks_like_docs(text: str, named_files: list[str]) -> bool: + text_match = bool(_DOC_TEXT_RE.search(text)) + if not named_files: + return False + doc_files = 0 + for path in named_files: + normalized = path.lower() + if normalized.endswith((".md", ".mdx", ".rst", ".txt")) or "/docs/" in normalized: + doc_files += 1 + if text_match and doc_files > 0: + return True + return doc_files > 0 and doc_files >= max(1, len(named_files) // 2) + + +def _looks_like_migration(text: str, change_shape: object | None) -> bool: + if not _MIGRATION_TEXT_RE.search(text): + return False + return ( + _int_attr(change_shape, "files_touched_count") >= 2 + or _int_attr(change_shape, "rename_operations") > 0 + ) + + +def _looks_like_debugging( + execution_types: set[str], + recovery_path: object | None, + steps: list[str], +) -> bool: + if _int_attr(recovery_path, "recovery_step_count") > 0: + return True + if _string_attr(recovery_path, "recovery_result") in {"recovered", "unresolved"}: + return True + if "fix" in steps: + return True + return "test" in execution_types and "edit" in steps + + +def _looks_like_refactor(text: str, change_shape: object | None) -> bool: + if "refactor" in text or "cleanup" in text or "restructure" in text: + return True + return bool( + _int_attr(change_shape, "rename_operations") > 0 + or _int_attr(change_shape, "churn_files_count") > 1 + or _bool_attr(change_shape, "rewrite_indicator") + ) + + +def _looks_like_feature_delivery( + change_shape: object | None, + has_commit: bool, + steps: list[str], +) -> bool: + if has_commit or "ship" in steps: + return True + return bool( + _int_attr(change_shape, "create_operations") > 0 + or _int_attr(change_shape, "diff_size") > 0 + or _int_attr(change_shape, "files_touched_count") > 0 + ) + + +def _looks_like_investigation( + change_shape: object | None, + has_commit: bool, + steps: list[str], +) -> bool: + if has_commit or _has_mutations(change_shape): + return False + return bool(steps) and set(steps).issubset( + {"search", "read", "execute", "delegate", "integrate"} + ) + + +def _is_delegation_tool(tool_name: str) -> bool: + normalized = tool_name.lower() + return classify_tool(tool_name) in {"orchestration", "skill"} or any( + hint in normalized + for hint in ( + "task", + "agent", + "delegate", + "team", + "sendmessage", + "send_message", + ) + ) + + +def _bool_attr(value: object | None, field: str) -> bool: + if value is None: + return False + candidate = value.get(field) if isinstance(value, dict) else getattr(value, field, None) + return bool(candidate) diff --git a/tests/test_analytics.py b/tests/test_analytics.py index 3a6a400..bbd4826 100644 --- a/tests/test_analytics.py +++ b/tests/test_analytics.py @@ -808,6 +808,58 @@ def test_session_detail_includes_recovery_path_when_present( assert data["recovery_path"]["last_verification_status"] == "passed" +def test_session_detail_includes_workflow_profile_when_present( + client, engineer_with_key, admin_headers +): + _eng, api_key = engineer_with_key + sid = _ingest_session( + client, + api_key, + tool_usages=[ + {"tool_name": "Read", "call_count": 2}, + {"tool_name": "Edit", "call_count": 3}, + {"tool_name": "Bash", "call_count": 2}, + ], + messages=[ + { + "ordinal": 0, + "role": "assistant", + "tool_calls": [{"name": "Bash", "input_preview": '{"command":"pytest -q"}'}], + }, + { + "ordinal": 1, + "role": "tool_result", + "tool_results": [{"name": "Bash", "output_preview": "2 passed in 0.12s"}], + }, + ], + facets={"outcome": "success", "session_type": "implementation"}, + commits=[ + { + "sha": "abc123", + "message": "Implement billing export", + "files_changed": 2, + "lines_added": 25, + "lines_deleted": 4, + } + ], + ) + + response = client.get(f"/api/v1/sessions/{sid}", headers=admin_headers) + + assert response.status_code == 200 + data = response.json() + assert data["workflow_profile"]["archetype"] == "feature_delivery" + assert data["workflow_profile"]["archetype_source"] == "session_type" + assert data["workflow_profile"]["steps"] == [ + "read", + "edit", + "execute", + "test", + "ship", + ] + assert data["workflow_profile"]["verification_run_count"] == 1 + + def test_cost_analytics(client, engineer_with_key, admin_headers): _eng, api_key = engineer_with_key _ingest_session( diff --git a/tests/test_workflow_profile_service.py b/tests/test_workflow_profile_service.py new file mode 100644 index 0000000..79cb998 --- /dev/null +++ b/tests/test_workflow_profile_service.py @@ -0,0 +1,172 @@ +from primer.server.services.workflow_profile_service import extract_session_workflow_profile + + +def test_extract_session_workflow_profile_maps_session_type_to_archetype(): + record = extract_session_workflow_profile( + {"first_prompt": "Implement the billing export flow"}, + [ + {"tool_name": "Read", "call_count": 2}, + {"tool_name": "Edit", "call_count": 3}, + {"tool_name": "Bash", "call_count": 2}, + ], + [ + {"evidence_type": "test"}, + {"evidence_type": "build"}, + ], + change_shape={ + "files_touched_count": 3, + "diff_size": 42, + "edit_operations": 2, + }, + facets={"session_type": "implementation"}, + has_commit=True, + ) + + assert record is not None + assert record.archetype == "feature_delivery" + assert record.archetype_source == "session_type" + assert record.steps == ["read", "edit", "execute", "test", "ship"] + assert record.label == "implementation: read -> edit -> execute -> test -> ship" + + +def test_extract_session_workflow_profile_detects_docs_work(): + record = extract_session_workflow_profile( + {"first_prompt": "Update the README and onboarding docs"}, + [ + {"tool_name": "Read", "call_count": 1}, + {"tool_name": "Edit", "call_count": 2}, + ], + [], + change_shape={ + "files_touched_count": 2, + "diff_size": 16, + "edit_operations": 2, + "named_touched_files": ["README.md", "docs/setup.md"], + }, + has_commit=False, + ) + + assert record is not None + assert record.archetype == "docs" + assert record.archetype_source == "heuristic" + assert "docs work" in (record.archetype_reason or "") + + +def test_extract_session_workflow_profile_detects_debugging_fix_loop(): + record = extract_session_workflow_profile( + {"first_prompt": "Figure out why auth tests are failing"}, + [ + {"tool_name": "Edit", "call_count": 1}, + {"tool_name": "Bash", "call_count": 2}, + ], + [{"evidence_type": "test"}], + change_shape={ + "files_touched_count": 1, + "diff_size": 12, + "edit_operations": 1, + }, + recovery_path={ + "recovery_step_count": 2, + "recovery_result": "recovered", + "recovery_strategies": ["edit_fix", "rerun_verification"], + }, + ) + + assert record is not None + assert record.archetype == "debugging" + assert record.steps == ["edit", "execute", "test", "fix"] + assert record.verification_run_count == 1 + + +def test_extract_session_workflow_profile_does_not_treat_docker_as_docs(): + record = extract_session_workflow_profile( + {"first_prompt": "Fix the docker compose build failure"}, + [ + {"tool_name": "Edit", "call_count": 1}, + {"tool_name": "Bash", "call_count": 2}, + ], + [{"evidence_type": "test"}], + change_shape={ + "files_touched_count": 1, + "diff_size": 8, + "edit_operations": 1, + }, + recovery_path={ + "recovery_step_count": 1, + "recovery_result": "recovered", + "recovery_strategies": ["edit_fix"], + }, + ) + + assert record is not None + assert record.archetype == "debugging" + + +def test_extract_session_workflow_profile_does_not_treat_import_as_migration(): + record = extract_session_workflow_profile( + {"first_prompt": "Add import/export helpers for CSV support"}, + [ + {"tool_name": "Read", "call_count": 1}, + {"tool_name": "Edit", "call_count": 2}, + ], + [], + change_shape={ + "files_touched_count": 2, + "diff_size": 20, + "edit_operations": 2, + }, + ) + + assert record is not None + assert record.archetype == "feature_delivery" + + +def test_extract_session_workflow_profile_does_not_treat_guide_text_as_docs_without_doc_files(): + record = extract_session_workflow_profile( + {"first_prompt": "Follow the guide to debug the auth failure"}, + [ + {"tool_name": "Edit", "call_count": 1}, + {"tool_name": "Bash", "call_count": 2}, + ], + [{"evidence_type": "test"}], + change_shape={ + "files_touched_count": 1, + "diff_size": 8, + "edit_operations": 1, + "named_touched_files": ["src/auth.py"], + }, + recovery_path={ + "recovery_step_count": 1, + "recovery_result": "recovered", + "recovery_strategies": ["edit_fix"], + }, + ) + + assert record is not None + assert record.archetype == "debugging" + + +def test_extract_session_workflow_profile_counts_all_verification_runs(): + record = extract_session_workflow_profile( + {"first_prompt": "Stabilize the flaky test suite"}, + [{"tool_name": "Bash", "call_count": 4}], + [ + {"evidence_type": "test"}, + {"evidence_type": "test"}, + {"evidence_type": "lint"}, + {"evidence_type": "verification"}, + ], + change_shape={ + "files_touched_count": 1, + "diff_size": 4, + "edit_operations": 1, + }, + recovery_path={ + "recovery_step_count": 2, + "recovery_result": "recovered", + "recovery_strategies": ["rerun_verification"], + }, + ) + + assert record is not None + assert record.verification_run_count == 4