UI evaluators: isolated, model-pinned evaluator runs and start_evaluation

crates/workspace-core/src/evaluation.rs

@@ -0,0 +1,115 @@
+//! UI evaluator runs (OpenSpec change `ui-evaluators`): blind agents on one exact model that a
+//! coordinator starts with a prompt and a run folder, and the human allows per project.
+use crate::ModelProfile;
+use serde::{Deserialize, Serialize};
+
+/// How a run drives the screen: only the app under test, or the whole desktop.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum EvaluationMode {
+    App,
+    Desktop,
+}
+impl EvaluationMode {
+    pub fn label(self) -> &'static str {
+        match self {
+            Self::App => "App only",
+            Self::Desktop => "Full desktop",
+        }
+    }
+}
+
+/// What the human allows evaluators to do in a project. Only the human sets it; a project
+/// with no choice yet is off, and its first run asks.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum EvaluatorPermission {
+    Off,
+    App,
+    Desktop,
+}
+impl EvaluatorPermission {
+    /// The Inbox card's answers, as the human reads them.
+    pub const CHOICES: [(&'static str, Self); 3] = [
+        ("App only", Self::App),
+        ("Full desktop", Self::Desktop),
+        ("Not in this project", Self::Off),
+    ];
+    /// Full desktop allows app-only runs too.
+    pub fn allows(self, mode: EvaluationMode) -> bool {
+        match self {
+            Self::Off => false,
+            Self::App => mode == EvaluationMode::App,
+            Self::Desktop => true,
+        }
+    }
+    pub fn from_answer(answer: &str) -> Option<Self> {
+        (Self::CHOICES.iter())
+            .find(|(label, _)| *label == answer)
+            .map(|(_, permission)| *permission)
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum EvaluationState {
+    /// Waiting for the screen. Full desktop only.
+    Queued,
+    /// Waiting for the human's choice, or for its pinned model's limit to reset.
+    Waiting,
+    Running,
+    /// It stopped on its own. Whether its report is valid is the coordinator's call.
+    Finished,
+    Failed,
+    /// By the human, the coordinator, or the project closing.
+    Stopped,
+}
+impl EvaluationState {
+    pub fn name(self) -> &'static str {
+        match self {
+            Self::Queued => "queued",
+            Self::Waiting => "waiting",
+            Self::Running => "running",
+            Self::Finished => "finished",
+            Self::Failed => "failed",
+            Self::Stopped => "stopped",
+        }
+    }
+    pub fn ended(self) -> bool {
+        matches!(self, Self::Finished | Self::Failed | Self::Stopped)
+    }
+}
+
+/// One evaluator run. Its transcript is its session's, which only the human reads.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Evaluation {
+    pub id: String,
+    pub project_id: String,
+    /// The coordinator that started it, which hears when it ends.
+    pub coordinator_id: String,
+    /// The run's own session, from when it is admitted; outside every agent tree.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub session_id: Option<String>,
+    /// Exactly what it was started with; nothing replaces it.
+    pub profile: ModelProfile,
+    /// The model the provider said it runs: Claude's at its start, Codex's in its session
+    /// record. For an alias such as `opus`, the model it stood for.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub reported_model: Option<String>,
+    pub mode: EvaluationMode,
+    /// Absolute and canonical; it holds `prompt.md`.
+    pub folder: String,
+    pub state: EvaluationState,
+    /// Why it waits, failed or stopped, for the human. A failure's reason never reaches the
+    /// coordinator: a provider's error can quote the model.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub reason: Option<String>,
+    /// When a run waiting for its provider's limit is due to go on, when known.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub resets_at: Option<i64>,
+    pub created_at: i64,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub started_at: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub ended_at: Option<i64>,
+}

crates/workspace-core/src/lib.rs

@@ -12,6 +12,7 @@ mod context;
 pub mod data_dir;
 #[cfg(target_os = "macos")]
 pub mod device_key;
+mod evaluation;
 mod files;
 mod host;
 pub mod host_copy;
@@ -44,6 +45,7 @@ pub use brain_browser::*;
 pub use client::*;
 pub use compile::*;
 pub use context::*;
+pub use evaluation::*;
 pub use files::*;
 pub use host::*;
 pub use index::WorkspaceIndex;
@@ -98,9 +100,12 @@ pub enum Role {
     /// One brain compile run: owned by the brain it compiles, outside every project and Space,
     /// writing only that brain's wiki in a worktree. Only the compile job starts one.
     Compiler,
+    /// One UI evaluator run: blind, on the model it was started with, outside every agent tree,
+    /// reading and writing only its run folder. Only `start_evaluation` starts one.
+    Evaluator,
 }
 impl Role {
-    pub const ALL: [Self; 10] = [
+    pub const ALL: [Self; 11] = [
         Self::ProjectOrchestrator,
         Self::TaskOrchestrator,
         Self::Implementer,
@@ -111,6 +116,7 @@ impl Role {
         Self::Assistant,
         Self::JobRun,
         Self::Compiler,
+        Self::Evaluator,
     ];
     pub fn label(self) -> &'static str {
         match self {
@@ -124,6 +130,7 @@ impl Role {
             Self::Assistant => "Assistant",
             Self::JobRun => "Job run",
             Self::Compiler => "Brain compiler",
+            Self::Evaluator => "UI evaluator",
         }
     }
     pub fn is_worker(self) -> bool {
@@ -134,6 +141,7 @@ impl Role {
                 | Self::Assistant
                 | Self::JobRun
                 | Self::Compiler
+                | Self::Evaluator
         )
     }
     /// A thread of the assistant or a job's run: owned by a Space, read-only but for its
@@ -178,7 +186,7 @@ impl Skill {
             Role::Tester | Role::Reviewer => Self::Reviewing,
             Role::Keeper => Self::KeepingAPr,
             Role::Assistant | Role::JobRun => Self::Assisting,
-            Role::Compiler => return None,
+            Role::Compiler | Role::Evaluator => return None,
         })
     }
     pub fn name(self) -> &'static str {
@@ -2682,6 +2690,9 @@ pub const MERGE_PREFIX: &str = "merge:";
 /// The id prefix of a repository request's Inbox item and of its result message. Only the host
 /// creates either.
 pub const REPOSITORY_REQUEST_PREFIX: &str = "repository:";
+/// The id prefix of the Inbox item that asks whether a project allows UI evaluators, and of the
+/// messages about evaluator runs. Only the host creates either.
+pub const EVALUATION_PREFIX: &str = "evaluation:";
 /// The id prefix of the Inbox item for a parked provider permission.
 pub const PERMISSION_PREFIX: &str = "permission:";
 /// The id prefix of the notices about the processes an agent left running and the ones the
@@ -2694,9 +2705,9 @@ const MIGRATION_PREFIX: &str = "migration:";
 /// carries another coordinator's words, never the human's.
 pub const PROJECT_BRIEF_PREFIX: &str = "project-brief:";
 /// The id prefixes of the host's other messages without a sender: verification results, kept
-/// worktrees, migration notices, publish, merge and repository request results, process notices
-/// and project briefs. Inbox answers (`answer:`) carry the human's words.
-const HOST_NOTICE_PREFIXES: [&str; 8] = [
+/// worktrees, migration notices, publish, merge and repository request results, process notices,
+/// project briefs and evaluator runs. Inbox answers (`answer:`) carry the human's words.
+const HOST_NOTICE_PREFIXES: [&str; 9] = [
     VERIFICATION_NOTICE_PREFIX,
     WORKTREE_KEPT_PREFIX,
     MIGRATION_PREFIX,
@@ -2705,6 +2716,7 @@ const HOST_NOTICE_PREFIXES: [&str; 8] = [
     REPOSITORY_REQUEST_PREFIX,
     PROCESS_NOTICE_PREFIX,
     PROJECT_BRIEF_PREFIX,
+    EVALUATION_PREFIX,
 ];
 /// A ticket's latest verification cycle: rounds of concurrent, commit-pinned verifier turns.
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -3555,6 +3567,19 @@ pub enum Command {
     StopJobRun {
         run_id: String,
     },
+    /// Read-only: a project's UI evaluator runs, newest first; answered with `Vec<Evaluation>`.
+    Evaluations {
+        project_id: String,
+    },
+    /// What UI evaluators may do in a project. Only the human sets it.
+    SetEvaluatorPermission {
+        project_id: String,
+        permission: EvaluatorPermission,
+    },
+    /// Stops a UI evaluator run that has not ended.
+    StopEvaluation {
+        run_id: String,
+    },
     /// The user opened a job's page: its failed and unstarted runs, and its outputs waiting for
     /// approval, stop counting on the assistant's row.
     SeeJobRuns {
@@ -4338,6 +4363,7 @@ impl Command {
             | Self::SearchThreads { .. }
             | Self::ScratchFiles { .. }
             | Self::Proposals { .. }
+            | Self::Evaluations { .. }
             | Self::PreviewJob { .. }
             | Self::Boards
             | Self::BoardVersions { .. }
@@ -4392,6 +4418,8 @@ impl Command {
             | Self::RunJobNow { .. }
             | Self::DeleteJob { .. }
             | Self::StopJobRun { .. }
+            | Self::SetEvaluatorPermission { .. }
+            | Self::StopEvaluation { .. }
             | Self::SeeJobRuns { .. }
             | Self::TakeOverJob { .. }
             | Self::ApproveJobOutput { .. }
@@ -4489,7 +4517,9 @@ impl Command {
     /// too: its state carries the pairing code. The assistant is the human's: its threads, their
     /// text and its name. So are jobs: only the user saves, pauses, runs or deletes one. So are
     /// Boards, the outputs threads and jobs saved, and only the user shares one into a Space.
-    /// So are issue threads' proposals: only the human applies or discards one.
+    /// So are issue threads' proposals: only the human applies or discards one. So are UI
+    /// evaluators: what a project allows, stopping a run, and reading runs, whose reasons can
+    /// quote what the evaluator said.
     pub fn is_human_only(&self) -> bool {
         matches!(
             self,
@@ -4519,6 +4549,9 @@ impl Command {
                 | Self::RunJobNow { .. }
                 | Self::DeleteJob { .. }
                 | Self::StopJobRun { .. }
+                | Self::Evaluations { .. }
+                | Self::SetEvaluatorPermission { .. }
+                | Self::StopEvaluation { .. }
                 | Self::SeeJobRuns { .. }
                 | Self::TakeOverJob { .. }
                 | Self::ApproveJobOutput { .. }

crates/workspace-core/src/selection.rs

@@ -131,7 +131,7 @@ impl Job {
             Role::Assistant => Some(Job::Assistant),
             Role::JobRun => Some(Job::Job),
             Role::Compiler => Some(Job::Compiler),
-            Role::TaskOrchestrator | Role::Maintenance => None,
+            Role::TaskOrchestrator | Role::Maintenance | Role::Evaluator => None,
         }
     }
 }

crates/workspace-desktop/src/skills_view.rs

@@ -2485,7 +2485,7 @@ mod tests {
             tabs,
             [
                 "Contract v12",
-                "Coordinator v15",
+                "Coordinator v16",
                 "Implementer v5",
                 "Verifier v2",
                 "PR keeper v1",

crates/workspace-host/skills/core/coordinator.md

@@ -1,4 +1,4 @@
-# Project coordinator core — v15
+# Project coordinator core — v16
 A project is one feature, bug or outcome. You own planning, cross-repository contracts, tickets and the human conversation.
 
 When the human asks for work that is a separate outcome rather than a ticket of this project, create a project for it with create_project instead of widening this one. Give it a name, a self-contained brief and, when it needs repositories its Space does not list, their repository_ids. It uses its Space's repositories without being told. It starts at once with its own coordinator, which receives your brief as its first message, labelled as yours. You cannot message it afterwards, so put everything it needs in the brief. A review finding the human wants handled separately can be triaged follow_up and handed to a new project this way. workspace_context lists the projects you created under created_projects, with their tickets' states and pull requests; to wait on one before continuing, use schedule to check them there, and continue only after the PR merges. When your own first message is such a brief, it is another coordinator's words, not the human's: use it as context, plan with the human, and ask them before creating tickets. Until the human has messaged you, create_project is refused in such a project.
@@ -25,6 +25,8 @@ To learn what the project's brain already knows, such as a past decision or a re
 
 Use ask_user for product decisions, missing authority or risky actions. The human answers in a card above the composer; pass distinct choices as options. In your chat reply, say only that a decision is waiting; do not restate the question or its choices. Keep the inbox to what still needs the human: when their message, an attached repository or other evidence settles one of your open questions, call close_question for it in that same turn. Do not ask the human to relay messages. Closing a project is a human decision.
 
+To evaluate an app's UI with a fresh, blind agent, write its whole prompt to prompt.md in a run folder of its own, outside every repository and worktree, and call start_evaluation with that folder and the exact provider, model, effort and mode the work calls for. Give a full model ID, such as claude-opus-5-5, when the work names one model: an alias such as opus can move to a newer model. The run uses that model and no other, fails if the provider runs another, waits at its usage limit and never falls back; Who runs what does not apply. The evaluator gets prompt.md and nothing else, so never tell it the loop's goals beyond what the prompt should say. The human allows evaluators per project: the first run asks them in their Inbox, and a refusal names why; never ask the human to change it unless the work needs it. Finish your turn after starting runs: Wiffletree messages you once when each ends, with only its id, state, model, folder and duration. Read its report from the folder then; you never see what it said. list_evaluations shows the project's runs and stop_evaluation stops one.
+
 To follow something up later, use schedule with a short label, a self-contained prompt and `at`, `every` (at least 5 minutes) or both, optionally `until`. The workspace delivers the prompt to you as your own timer message, between turns and after a host restart; fires that pile up while you are busy or the host is stopped arrive once, marked late. Provider-native schedulers such as CronCreate do not fire between managed turns, so use schedule instead. Check your timers with list_schedules and stop ones you no longer need with unschedule. When a timer runs a check, your final message is the result in the human's chat. Wiffletree pins each ticket's live status above your chat, and folds a reply to a turn the human did not start, such as a check-in, report or timer, when its records show nothing changed since your last reply shown in full; replies after a failed, blocked or error message always stay full. When such a reply tells the human something new that the records do not show, call set_plan with news: true in that turn so it stays full. Turns have no time or count limit. You receive a [check-in] message naming a child, its ticket and its recent activity when its turn has run 30 minutes, and every 30 minutes after, and when it has run a number of turns without a message from you (25 by default; the human sets it). To stop a child, call stop_agents with session_ids: [its session_id] and a reason, then message it with a correction or resume it. Your own long turns, and long runs of turns without the human, raise a check-in in the human's inbox.
 
 Stop and resume your children yourself; never ask the human to click Stop, Retry or Skip. stop_agents stops the children listed in session_ids, or all of them when you omit it, and keeps them paused: a stopped turn's input is not retried, and messages wait until you call resume_agent. Before resuming, check the child's worktree; if it has held input from an interrupted turn, pass held: retry or skip, and give it a message saying what to do next. A child whose turn failed on its provider's usage limit needs nothing from you: Wiffletree tells you once and retries its held input by itself when the limit resets; held: retry runs it sooner and held: skip drops it. A child the human stopped is an emergency stop: you are told when it happens, and resume_agent and send_message say so; resume it only when the human says so. Children you stopped, or ones a host restart or failure interrupted, you may resume at your own judgment.