diff --git a/crates/workspace-core/src/evaluation.rs b/crates/workspace-core/src/evaluation.rs new file mode 100644 index 00000000..e521774d --- /dev/null +++ b/crates/workspace-core/src/evaluation.rs @@ -0,0 +1,115 @@ +//! UI evaluator runs (OpenSpec change `ui-evaluators`): blind agents on one exact model that a +//! coordinator starts with a prompt and a run folder, and the human allows per project. +use crate::ModelProfile; +use serde::{Deserialize, Serialize}; + +/// How a run drives the screen: only the app under test, or the whole desktop. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvaluationMode { + App, + Desktop, +} +impl EvaluationMode { + pub fn label(self) -> &'static str { + match self { + Self::App => "App only", + Self::Desktop => "Full desktop", + } + } +} + +/// What the human allows evaluators to do in a project. Only the human sets it; a project +/// with no choice yet is off, and its first run asks. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvaluatorPermission { + Off, + App, + Desktop, +} +impl EvaluatorPermission { + /// The Inbox card's answers, as the human reads them. + pub const CHOICES: [(&'static str, Self); 3] = [ + ("App only", Self::App), + ("Full desktop", Self::Desktop), + ("Not in this project", Self::Off), + ]; + /// Full desktop allows app-only runs too. + pub fn allows(self, mode: EvaluationMode) -> bool { + match self { + Self::Off => false, + Self::App => mode == EvaluationMode::App, + Self::Desktop => true, + } + } + pub fn from_answer(answer: &str) -> Option { + (Self::CHOICES.iter()) + .find(|(label, _)| *label == answer) + .map(|(_, permission)| *permission) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvaluationState { + /// Waiting for the screen. Full desktop only. + Queued, + /// Waiting for the human's choice, or for its pinned model's limit to reset. + Waiting, + Running, + /// It stopped on its own. Whether its report is valid is the coordinator's call. + Finished, + Failed, + /// By the human, the coordinator, or the project closing. + Stopped, +} +impl EvaluationState { + pub fn name(self) -> &'static str { + match self { + Self::Queued => "queued", + Self::Waiting => "waiting", + Self::Running => "running", + Self::Finished => "finished", + Self::Failed => "failed", + Self::Stopped => "stopped", + } + } + pub fn ended(self) -> bool { + matches!(self, Self::Finished | Self::Failed | Self::Stopped) + } +} + +/// One evaluator run. Its transcript is its session's, which only the human reads. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Evaluation { + pub id: String, + pub project_id: String, + /// The coordinator that started it, which hears when it ends. + pub coordinator_id: String, + /// The run's own session, from when it is admitted; outside every agent tree. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + /// Exactly what it was started with; nothing replaces it. + pub profile: ModelProfile, + /// The model the provider said it runs: Claude's at its start, Codex's in its session + /// record. For an alias such as `opus`, the model it stood for. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reported_model: Option, + pub mode: EvaluationMode, + /// Absolute and canonical; it holds `prompt.md`. + pub folder: String, + pub state: EvaluationState, + /// Why it waits, failed or stopped, for the human. A failure's reason never reaches the + /// coordinator: a provider's error can quote the model. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// When a run waiting for its provider's limit is due to go on, when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resets_at: Option, + pub created_at: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ended_at: Option, +} diff --git a/crates/workspace-core/src/lib.rs b/crates/workspace-core/src/lib.rs index 0d772dfe..bd688695 100644 --- a/crates/workspace-core/src/lib.rs +++ b/crates/workspace-core/src/lib.rs @@ -12,6 +12,7 @@ mod context; pub mod data_dir; #[cfg(target_os = "macos")] pub mod device_key; +mod evaluation; mod files; mod host; pub mod host_copy; @@ -44,6 +45,7 @@ pub use brain_browser::*; pub use client::*; pub use compile::*; pub use context::*; +pub use evaluation::*; pub use files::*; pub use host::*; pub use index::WorkspaceIndex; @@ -98,9 +100,12 @@ pub enum Role { /// One brain compile run: owned by the brain it compiles, outside every project and Space, /// writing only that brain's wiki in a worktree. Only the compile job starts one. Compiler, + /// One UI evaluator run: blind, on the model it was started with, outside every agent tree, + /// reading and writing only its run folder. Only `start_evaluation` starts one. + Evaluator, } impl Role { - pub const ALL: [Self; 10] = [ + pub const ALL: [Self; 11] = [ Self::ProjectOrchestrator, Self::TaskOrchestrator, Self::Implementer, @@ -111,6 +116,7 @@ impl Role { Self::Assistant, Self::JobRun, Self::Compiler, + Self::Evaluator, ]; pub fn label(self) -> &'static str { match self { @@ -124,6 +130,7 @@ impl Role { Self::Assistant => "Assistant", Self::JobRun => "Job run", Self::Compiler => "Brain compiler", + Self::Evaluator => "UI evaluator", } } pub fn is_worker(self) -> bool { @@ -134,6 +141,7 @@ impl Role { | Self::Assistant | Self::JobRun | Self::Compiler + | Self::Evaluator ) } /// A thread of the assistant or a job's run: owned by a Space, read-only but for its @@ -178,7 +186,7 @@ impl Skill { Role::Tester | Role::Reviewer => Self::Reviewing, Role::Keeper => Self::KeepingAPr, Role::Assistant | Role::JobRun => Self::Assisting, - Role::Compiler => return None, + Role::Compiler | Role::Evaluator => return None, }) } pub fn name(self) -> &'static str { @@ -2682,6 +2690,9 @@ pub const MERGE_PREFIX: &str = "merge:"; /// The id prefix of a repository request's Inbox item and of its result message. Only the host /// creates either. pub const REPOSITORY_REQUEST_PREFIX: &str = "repository:"; +/// The id prefix of the Inbox item that asks whether a project allows UI evaluators, and of the +/// messages about evaluator runs. Only the host creates either. +pub const EVALUATION_PREFIX: &str = "evaluation:"; /// The id prefix of the Inbox item for a parked provider permission. pub const PERMISSION_PREFIX: &str = "permission:"; /// The id prefix of the notices about the processes an agent left running and the ones the @@ -2694,9 +2705,9 @@ const MIGRATION_PREFIX: &str = "migration:"; /// carries another coordinator's words, never the human's. pub const PROJECT_BRIEF_PREFIX: &str = "project-brief:"; /// The id prefixes of the host's other messages without a sender: verification results, kept -/// worktrees, migration notices, publish, merge and repository request results, process notices -/// and project briefs. Inbox answers (`answer:`) carry the human's words. -const HOST_NOTICE_PREFIXES: [&str; 8] = [ +/// worktrees, migration notices, publish, merge and repository request results, process notices, +/// project briefs and evaluator runs. Inbox answers (`answer:`) carry the human's words. +const HOST_NOTICE_PREFIXES: [&str; 9] = [ VERIFICATION_NOTICE_PREFIX, WORKTREE_KEPT_PREFIX, MIGRATION_PREFIX, @@ -2705,6 +2716,7 @@ const HOST_NOTICE_PREFIXES: [&str; 8] = [ REPOSITORY_REQUEST_PREFIX, PROCESS_NOTICE_PREFIX, PROJECT_BRIEF_PREFIX, + EVALUATION_PREFIX, ]; /// A ticket's latest verification cycle: rounds of concurrent, commit-pinned verifier turns. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -3555,6 +3567,19 @@ pub enum Command { StopJobRun { run_id: String, }, + /// Read-only: a project's UI evaluator runs, newest first; answered with `Vec`. + Evaluations { + project_id: String, + }, + /// What UI evaluators may do in a project. Only the human sets it. + SetEvaluatorPermission { + project_id: String, + permission: EvaluatorPermission, + }, + /// Stops a UI evaluator run that has not ended. + StopEvaluation { + run_id: String, + }, /// The user opened a job's page: its failed and unstarted runs, and its outputs waiting for /// approval, stop counting on the assistant's row. SeeJobRuns { @@ -4338,6 +4363,7 @@ impl Command { | Self::SearchThreads { .. } | Self::ScratchFiles { .. } | Self::Proposals { .. } + | Self::Evaluations { .. } | Self::PreviewJob { .. } | Self::Boards | Self::BoardVersions { .. } @@ -4392,6 +4418,8 @@ impl Command { | Self::RunJobNow { .. } | Self::DeleteJob { .. } | Self::StopJobRun { .. } + | Self::SetEvaluatorPermission { .. } + | Self::StopEvaluation { .. } | Self::SeeJobRuns { .. } | Self::TakeOverJob { .. } | Self::ApproveJobOutput { .. } @@ -4489,7 +4517,9 @@ impl Command { /// too: its state carries the pairing code. The assistant is the human's: its threads, their /// text and its name. So are jobs: only the user saves, pauses, runs or deletes one. So are /// Boards, the outputs threads and jobs saved, and only the user shares one into a Space. - /// So are issue threads' proposals: only the human applies or discards one. + /// So are issue threads' proposals: only the human applies or discards one. So are UI + /// evaluators: what a project allows, stopping a run, and reading runs, whose reasons can + /// quote what the evaluator said. pub fn is_human_only(&self) -> bool { matches!( self, @@ -4519,6 +4549,9 @@ impl Command { | Self::RunJobNow { .. } | Self::DeleteJob { .. } | Self::StopJobRun { .. } + | Self::Evaluations { .. } + | Self::SetEvaluatorPermission { .. } + | Self::StopEvaluation { .. } | Self::SeeJobRuns { .. } | Self::TakeOverJob { .. } | Self::ApproveJobOutput { .. } diff --git a/crates/workspace-core/src/selection.rs b/crates/workspace-core/src/selection.rs index 87c2fe07..e0833fbc 100644 --- a/crates/workspace-core/src/selection.rs +++ b/crates/workspace-core/src/selection.rs @@ -131,7 +131,7 @@ impl Job { Role::Assistant => Some(Job::Assistant), Role::JobRun => Some(Job::Job), Role::Compiler => Some(Job::Compiler), - Role::TaskOrchestrator | Role::Maintenance => None, + Role::TaskOrchestrator | Role::Maintenance | Role::Evaluator => None, } } } diff --git a/crates/workspace-desktop/src/skills_view.rs b/crates/workspace-desktop/src/skills_view.rs index 60108a78..46c5d06b 100644 --- a/crates/workspace-desktop/src/skills_view.rs +++ b/crates/workspace-desktop/src/skills_view.rs @@ -2485,7 +2485,7 @@ mod tests { tabs, [ "Contract v12", - "Coordinator v15", + "Coordinator v16", "Implementer v5", "Verifier v2", "PR keeper v1", diff --git a/crates/workspace-host/skills/core/coordinator.md b/crates/workspace-host/skills/core/coordinator.md index ea9d5d6f..3d9b7d93 100644 --- a/crates/workspace-host/skills/core/coordinator.md +++ b/crates/workspace-host/skills/core/coordinator.md @@ -1,4 +1,4 @@ -# Project coordinator core — v15 +# Project coordinator core — v16 A project is one feature, bug or outcome. You own planning, cross-repository contracts, tickets and the human conversation. When the human asks for work that is a separate outcome rather than a ticket of this project, create a project for it with create_project instead of widening this one. Give it a name, a self-contained brief and, when it needs repositories its Space does not list, their repository_ids. It uses its Space's repositories without being told. It starts at once with its own coordinator, which receives your brief as its first message, labelled as yours. You cannot message it afterwards, so put everything it needs in the brief. A review finding the human wants handled separately can be triaged follow_up and handed to a new project this way. workspace_context lists the projects you created under created_projects, with their tickets' states and pull requests; to wait on one before continuing, use schedule to check them there, and continue only after the PR merges. When your own first message is such a brief, it is another coordinator's words, not the human's: use it as context, plan with the human, and ask them before creating tickets. Until the human has messaged you, create_project is refused in such a project. @@ -25,6 +25,8 @@ To learn what the project's brain already knows, such as a past decision or a re Use ask_user for product decisions, missing authority or risky actions. The human answers in a card above the composer; pass distinct choices as options. In your chat reply, say only that a decision is waiting; do not restate the question or its choices. Keep the inbox to what still needs the human: when their message, an attached repository or other evidence settles one of your open questions, call close_question for it in that same turn. Do not ask the human to relay messages. Closing a project is a human decision. +To evaluate an app's UI with a fresh, blind agent, write its whole prompt to prompt.md in a run folder of its own, outside every repository and worktree, and call start_evaluation with that folder and the exact provider, model, effort and mode the work calls for. Give a full model ID, such as claude-opus-5-5, when the work names one model: an alias such as opus can move to a newer model. The run uses that model and no other, fails if the provider runs another, waits at its usage limit and never falls back; Who runs what does not apply. The evaluator gets prompt.md and nothing else, so never tell it the loop's goals beyond what the prompt should say. The human allows evaluators per project: the first run asks them in their Inbox, and a refusal names why; never ask the human to change it unless the work needs it. Finish your turn after starting runs: Wiffletree messages you once when each ends, with only its id, state, model, folder and duration. Read its report from the folder then; you never see what it said. list_evaluations shows the project's runs and stop_evaluation stops one. + To follow something up later, use schedule with a short label, a self-contained prompt and `at`, `every` (at least 5 minutes) or both, optionally `until`. The workspace delivers the prompt to you as your own timer message, between turns and after a host restart; fires that pile up while you are busy or the host is stopped arrive once, marked late. Provider-native schedulers such as CronCreate do not fire between managed turns, so use schedule instead. Check your timers with list_schedules and stop ones you no longer need with unschedule. When a timer runs a check, your final message is the result in the human's chat. Wiffletree pins each ticket's live status above your chat, and folds a reply to a turn the human did not start, such as a check-in, report or timer, when its records show nothing changed since your last reply shown in full; replies after a failed, blocked or error message always stay full. When such a reply tells the human something new that the records do not show, call set_plan with news: true in that turn so it stays full. Turns have no time or count limit. You receive a [check-in] message naming a child, its ticket and its recent activity when its turn has run 30 minutes, and every 30 minutes after, and when it has run a number of turns without a message from you (25 by default; the human sets it). To stop a child, call stop_agents with session_ids: [its session_id] and a reason, then message it with a correction or resume it. Your own long turns, and long runs of turns without the human, raise a check-in in the human's inbox. Stop and resume your children yourself; never ask the human to click Stop, Retry or Skip. stop_agents stops the children listed in session_ids, or all of them when you omit it, and keeps them paused: a stopped turn's input is not retried, and messages wait until you call resume_agent. Before resuming, check the child's worktree; if it has held input from an interrupted turn, pass held: retry or skip, and give it a message saying what to do next. A child whose turn failed on its provider's usage limit needs nothing from you: Wiffletree tells you once and retries its held input by itself when the limit resets; held: retry runs it sooner and held: skip drops it. A child the human stopped is an emergency stop: you are told when it happens, and resume_agent and send_message say so; resume it only when the human says so. Children you stopped, or ones a host restart or failure interrupted, you may resume at your own judgment. diff --git a/crates/workspace-host/src/evaluations.rs b/crates/workspace-host/src/evaluations.rs new file mode 100644 index 00000000..85e7d0ba --- /dev/null +++ b/crates/workspace-host/src/evaluations.rs @@ -0,0 +1,1304 @@ +//! UI evaluator runs (OpenSpec change `ui-evaluators`). A coordinator starts one with a run +//! folder and an exact model; the human decides per project whether evaluators may run. Each +//! admitted run gets a fresh session of its own, outside every agent tree, so nothing it says +//! reaches the coordinator but the message that it ended. Full-desktop runs take turns at the +//! screen through one queue for every project. +use crate::*; + +/// Where an evaluator's Codex keeps its home, under the data folder. +const CODEX_HOME: &str = "evaluators/codex-home"; +const PROMPT_FILE: &str = "prompt.md"; +/// Why a run waits before the human has chosen for its project. +const ASKED: &str = "Waiting for the human to allow UI evaluators in this project."; +/// Why a run the human stopped ended. +pub(crate) const STOPPED_BY_YOU: &str = "Stopped by you."; +pub(crate) const STOPPED_BY_COORDINATOR: &str = "Stopped by its coordinator."; +/// Why a run whose turn the host's crash, kill or quit cut short ended. +const INTERRUPTED: &str = "Wiffletree stopped during the run."; +pub(crate) const PROJECT_CLOSED: &str = "Stopped: the project was closed."; + +pub(crate) fn migrate(db: &Connection) -> Result<()> { + db.execute_batch( + "CREATE TABLE IF NOT EXISTS evaluations (id TEXT PRIMARY KEY, project_id TEXT NOT NULL, session_id TEXT, created_at INTEGER NOT NULL, data TEXT NOT NULL); + CREATE INDEX IF NOT EXISTS evaluations_by_project ON evaluations(project_id, created_at); + CREATE TABLE IF NOT EXISTS evaluator_permissions (project_id TEXT PRIMARY KEY, permission TEXT NOT NULL);", + )?; + Ok(()) +} + +/// "41 min", as the end-of-run message says how long a run took. +fn duration(run: &Evaluation) -> String { + let from = run.started_at.unwrap_or(run.created_at); + let minutes = (run.ended_at.unwrap_or(from) - from) / 60_000; + match minutes { + 0 => "under 1 min".into(), + minutes => format!("{minutes} min"), + } +} + +/// The message that wakes a run's coordinator: its id, state, folder and duration, and nothing +/// the run said. +pub(crate) fn ended_message(run: &Evaluation) -> String { + let model = run.reported_model.as_ref().unwrap_or(&run.profile.model); + format!( + "Run {} {} · model {model} · folder {} · {}", + run.id, + run.state.name(), + run.folder, + duration(run) + ) +} + +/// What the coordinator sees of a run: never its transcript or a failure's reason. +fn summary(run: &Evaluation) -> Value { + let waiting = (run.state == EvaluationState::Waiting).then(|| match run.session_id { + None => ASKED.to_owned(), + Some(_) => match run.resets_at { + Some(at) => format!( + "Its pinned model is at its limit until {}.", + policies::reset_time(at) + ), + None => "Its pinned model is at its limit until it resets.".into(), + }, + }); + json!({ + "run_id": run.id, + "state": run.state.name(), + "mode": run.mode, + "provider": run.profile.provider, + "model": run.profile.model, + "reported_model": run.reported_model, + "effort": run.profile.effort, + "folder": run.folder, + "created_at": run.created_at, + "started_at": run.started_at, + "ended_at": run.ended_at, + "waiting": waiting, + }) +} + +impl Host { + /// Records what a provider's CLI said about itself: its sign-in, and the models it lists, + /// which `start_evaluation` checks a run's model against. + pub(crate) fn record_report(&mut self, provider: Provider, report: &ProviderReport) { + self.record_detection(provider, &report.status); + if !report.models.is_empty() { + self.listed_models.insert(provider, report.models.clone()); + } + } + + /// What the human allows in `project`; `None` until they have chosen. + pub(crate) fn evaluator_permission( + &self, + project: &str, + ) -> Result> { + let permission: Option = self + .db + .query_row( + "SELECT permission FROM evaluator_permissions WHERE project_id=?1", + [project], + |r| r.get(0), + ) + .optional()?; + Ok(match permission { + Some(permission) => Some(serde_json::from_value(json!(permission))?), + None => None, + }) + } + + /// The human's choice for `project`: runs held for it start or stop, and so does its Inbox + /// card. A narrower choice stops the runs it no longer allows. + pub fn set_evaluator_permission( + &mut self, + project: &str, + permission: EvaluatorPermission, + ) -> Result<()> { + self.project(project)?; + self.atomically(|host| { + host.db.execute( + "INSERT INTO evaluator_permissions VALUES (?1,?2) ON CONFLICT(project_id) DO UPDATE SET permission=excluded.permission", + params![project, serde_json::to_value(permission)?.as_str()], + )?; + for run in host.evaluations(project)? { + if run.state.ended() { + continue; + } + // A run in its turn is left to the service, which stops the turn. + if !permission.allows(run.mode) { + let reason = not_allowed(run.mode); + host.stop_evaluation_with(run, &reason)?; + } else if run.session_id.is_none() && run.state == EvaluationState::Waiting { + let mut run = run; + run.state = EvaluationState::Queued; + run.reason = None; + host.save_evaluation(&run)?; + } + } + for item in host.open_attention(project)? { + if item.operation_id.starts_with(&card_prefix(project)) { + host.resolve_attention(&item.id, permission_label(permission))?; + } + } + Ok(()) + })?; + self.admit_evaluations() + } + + /// Answers the Inbox card with one of its choices, as `set_evaluator_permission` does, and + /// tells the coordinator that asked. Any other answer is refused, so the card stays. + pub(crate) fn answer_evaluator_card(&mut self, item: &Attention, answer: &str) -> Result<()> { + let permission = EvaluatorPermission::from_answer(answer).with_context(|| { + format!( + "Answer with one of: {}", + (EvaluatorPermission::CHOICES.iter()) + .map(|(label, _)| *label) + .collect::>() + .join(", ") + ) + })?; + self.set_evaluator_permission(&item.project_id, permission)?; + let body = match permission { + EvaluatorPermission::Off => { + "The human chose not to allow UI evaluators in this project. Your waiting runs were stopped. Do not start more.".to_owned() + } + allowed => format!( + "The human allowed UI evaluators in this project: {}. Your waiting runs that this allows have started; Wiffletree messages you when each ends.", + permission_label(allowed) + ), + }; + let id = format!("{EVALUATION_PREFIX}answer:{}", item.id); + if let Err(error) = self.send(&id, None, &item.session_id, &body) { + eprintln!("Telling the coordinator about UI evaluators: {error:#}"); + } + Ok(()) + } + + /// Closing the project, by archiving its coordinator, turns evaluators off and stops the + /// runs that have not ended; the service stops those in a turn (`evaluations_in_turn`). + pub(crate) fn close_evaluations(&mut self, project: &str) -> Result<()> { + self.db.execute( + "INSERT INTO evaluator_permissions VALUES (?1,'off') ON CONFLICT(project_id) DO UPDATE SET permission='off'", + [project], + )?; + for run in self.evaluations(project)? { + self.stop_evaluation_with(run, PROJECT_CLOSED)?; + } + Ok(()) + } + /// The sessions in a turn of `project`'s runs that what it allows now no longer allows, + /// each with its run's mode, for the service to stop. + pub(crate) fn disallowed_in_turn( + &self, + project: &str, + ) -> Result> { + let permission = (self.evaluator_permission(project)?).unwrap_or(EvaluatorPermission::Off); + Ok((self.evaluations(project)?.into_iter()) + .filter(|run| !run.state.ended() && !permission.allows(run.mode)) + .filter_map(|run| Some((run.session_id?, run.mode))) + .filter(|(session, _)| self.in_turn.contains(session)) + .collect()) + } + + pub(crate) fn evaluation(&self, id: &str) -> Result { + self.db + .query_row("SELECT data FROM evaluations WHERE id=?1", [id], |r| { + decode(r, 0) + }) + .optional()? + .with_context(|| format!("No UI evaluator run {id}")) + } + fn session_evaluation(&self, session: &str) -> Result { + self.db + .query_row( + "SELECT data FROM evaluations WHERE session_id=?1", + [session], + |r| decode(r, 0), + ) + .optional()? + .context("This session is no UI evaluator run") + } + fn save_evaluation(&self, run: &Evaluation) -> Result<()> { + self.db.execute( + "INSERT INTO evaluations VALUES (?1,?2,?3,?4,?5) ON CONFLICT(id) DO UPDATE SET session_id=excluded.session_id, data=excluded.data", + params![ + run.id, + run.project_id, + run.session_id, + run.created_at, + encode(run)? + ], + )?; + Ok(()) + } + /// `project`'s runs, newest first. + pub fn evaluations(&self, project: &str) -> Result> { + Ok(self + .db + .prepare("SELECT data FROM evaluations WHERE project_id=?1 ORDER BY created_at DESC, rowid DESC")? + .query_map([project], |r| decode(r, 0))? + .collect::>>()?) + } + + /// `start_evaluation`: checks everything a run needs, then creates it. A refusal creates + /// nothing and says why. + pub(crate) fn start_evaluation(&mut self, caller: &Session, args: &Value) -> Result { + ensure!( + caller.role == Role::ProjectOrchestrator, + "Only the project coordinator starts UI evaluators" + ); + service::only_arguments( + "start_evaluation", + args, + &["run_folder", "provider", "model", "effort", "mode"], + )?; + let string = |key: &str| -> Result<&str> { + let value = args[key].as_str().map(str::trim).unwrap_or_default(); + ensure!(!value.is_empty(), "Give {key}"); + Ok(value) + }; + let mode: EvaluationMode = serde_json::from_value(json!(string("mode")?)) + .context("Give mode as app or desktop")?; + let provider: Provider = serde_json::from_value(json!(string("provider")?)) + .context("Give provider as claude or codex")?; + let profile = ModelProfile { + provider, + model: string("model")?.into(), + effort: string("effort")?.into(), + }; + let permission = self.evaluator_permission(&caller.project_id)?; + match permission { + Some(EvaluatorPermission::Off) => { + bail!("UI evaluators are not allowed in this project. Ask the human.") + } + Some(permission) if !permission.allows(mode) => { + bail!( + "{} is not allowed in this project. Ask the human.", + mode.label() + ) + } + _ => {} + } + ensure!( + mode == EvaluationMode::App, + "Full desktop evaluators are not available yet. Start an app run instead." + ); + let folder = self.run_folder(string("run_folder")?)?; + self.check_model(&profile)?; + let at = now(); + let run = Evaluation { + id: format!("run-{}", &new_id()[..8]), + project_id: caller.project_id.clone(), + coordinator_id: caller.id.clone(), + session_id: None, + profile, + reported_model: None, + mode, + folder: folder.to_string_lossy().into_owned(), + state: match permission { + Some(_) => EvaluationState::Queued, + None => EvaluationState::Waiting, + }, + reason: permission.is_none().then(|| ASKED.to_owned()), + resets_at: None, + created_at: at, + started_at: None, + ended_at: None, + }; + ensure!( + self.evaluation(&run.id).is_err(), + "Wiffletree picked a run id that is taken. Call start_evaluation again" + ); + self.atomically(|host| { + host.save_evaluation(&run)?; + if permission.is_none() { + host.ask_for_evaluators(caller, &run)?; + } + Ok(()) + })?; + self.admit_evaluations()?; + let run = self.evaluation(&run.id)?; + let next = match run.state { + EvaluationState::Waiting if run.session_id.is_none() => { + "The human decides whether this project allows UI evaluators. Finish your turn: Wiffletree messages you their answer, and when the run ends." + } + _ => { + "Finish your turn or carry on: Wiffletree messages you when the run ends. Read its report from the folder then." + } + }; + let mut result = summary(&run); + result["next"] = json!(next); + Ok(result) + } + + /// The Inbox card, one open at a time per project: later runs wait for the same answer. + /// After the human dismissed one unanswered, the next run asks again. + fn ask_for_evaluators(&mut self, caller: &Session, run: &Evaluation) -> Result<()> { + let prefix = card_prefix(&caller.project_id); + let open = self.open_attention(&caller.project_id)?; + if open + .iter() + .any(|item| item.operation_id.starts_with(&prefix)) + { + return Ok(()); + } + let prompt = format!( + "Allow UI evaluators in this project?\nThe coordinator wants to start an evaluator on {} · {}. It gets only its prompt and its own folder, {}.\nApp only drives just the app under test, a separate copy on a test profile; you can keep working. Full desktop can see your whole screen and click any app you approve. Not in this project tells the coordinator evaluators are off.\nOne choice per project, kept until you change it. Closing the project turns it off. A coordinator can never change it.", + run.profile.provider.label(), + run.profile.model, + run.folder + ); + let options: Vec = (EvaluatorPermission::CHOICES.iter()) + .map(|(label, _)| (*label).to_owned()) + .collect(); + self.raise_attention( + &caller.id, + "local", + &format!("{prefix}{}", run.id), + &prompt, + &options, + None, + )?; + Ok(()) + } + + /// The run folder, canonical: an existing folder with a `prompt.md`, in no repository, + /// worktree or Wiffletree data folder, and used by no other run that has not ended. + fn run_folder(&self, given: &str) -> Result { + ensure!( + Path::new(given).is_absolute(), + "Give run_folder as an absolute path" + ); + let folder = fs::canonicalize(given) + .ok() + .filter(|folder| folder.is_dir()) + .with_context(|| format!("The run folder {given} does not exist"))?; + ensure!( + folder.join(PROMPT_FILE).is_file(), + "The run folder {given} has no {PROMPT_FILE}" + ); + ensure!( + !folder.starts_with(&self.home), + "The run folder {given} is inside Wiffletree's data folder" + ); + if let Some(repository) = folder.ancestors().find(|a| a.join(".git").exists()) { + bail!( + "The run folder {given} is inside the Git repository or worktree {}", + repository.display() + ); + } + for repository in self.repositories()? { + let path = fs::canonicalize(&repository.path).unwrap_or(repository.path.into()); + ensure!( + !folder.starts_with(&path), + "The run folder {given} is inside the repository {}", + path.display() + ); + } + let folder_text = folder.to_string_lossy(); + let in_use: Option = self + .db + .query_row( + "SELECT id FROM evaluations WHERE json_extract(data,'$.folder')=?1 AND json_extract(data,'$.state') NOT IN ('finished','failed','stopped')", + [folder_text.as_ref()], + |r| r.get(0), + ) + .optional()?; + if let Some(run) = in_use { + bail!("The run {run} still uses the folder {given}. Give each run its own folder"); + } + Ok(folder) + } + + /// Refuses a provider not signed in here, a model its CLI did not list, full IDs and aliases + /// alike, and an effort the model does not offer. Until the CLI has listed its models + /// nothing is checked against a guess: the run is refused, and the service reads the list. + fn check_model(&self, profile: &ModelProfile) -> Result<()> { + let provider = profile.provider; + ensure!( + self.detected.get(&provider) == Some(&true), + "{} is not signed in on this Mac, or Wiffletree has not checked it yet. Ask the human.", + provider.label() + ); + let Some(options) = self.listed_models.get(&provider) else { + bail!( + "Wiffletree has not read {}'s model list on this Mac yet, so it cannot check {}. It is reading it now: call start_evaluation again in a minute.", + provider.label(), + profile.model + ); + }; + let Some(option) = options.iter().find(|option| option.model == profile.model) else { + bail!( + "{} does not offer the model {}. It offers: {}", + provider.label(), + profile.model, + (options.iter().map(|o| o.model.as_str())) + .collect::>() + .join(", ") + ); + }; + let efforts: Vec<&str> = match option.efforts.is_empty() { + true => usual_efforts(provider).to_vec(), + false => option.efforts.iter().map(String::as_str).collect(), + }; + ensure!( + efforts.contains(&profile.effort.as_str()), + "{} does not offer the effort {}. It offers: {}", + profile.model, + profile.effort, + efforts.join(", ") + ); + Ok(()) + } + + /// `list_evaluations`: the caller's project's runs, without anything they said. + pub(crate) fn list_evaluations(&self, caller: &Session, args: &Value) -> Result { + ensure!( + caller.role == Role::ProjectOrchestrator, + "Only the project coordinator lists UI evaluators" + ); + service::only_arguments("list_evaluations", args, &[])?; + let runs: Vec = self + .evaluations(&caller.project_id)? + .iter() + .map(summary) + .collect(); + Ok(json!({ + "permission": self.evaluator_permission(&caller.project_id)?, + "runs": runs, + })) + } + + /// `stop_evaluation` for its coordinator's project. Returns the session to stop when the + /// run is in a turn; its turn's end then ends the run. + pub(crate) fn coordinator_stops_evaluation( + &mut self, + caller: &Session, + args: &Value, + ) -> Result<(Value, Option)> { + ensure!( + caller.role == Role::ProjectOrchestrator, + "Only the project coordinator stops UI evaluators" + ); + service::only_arguments("stop_evaluation", args, &["run_id"])?; + let id = args["run_id"].as_str().context("Missing run_id")?; + let run = self.evaluation(id)?; + ensure!( + run.project_id == caller.project_id, + "{id} is not a run of this project" + ); + ensure!( + !run.state.ended(), + "The run {id} has already {}", + run.state.name() + ); + let in_turn = self.stop_evaluation_with(run, STOPPED_BY_COORDINATOR)?; + let result = summary(&self.evaluation(id)?); + Ok((result, in_turn)) + } + + /// The human's Stop. Returns the session to stop when the run is in a turn. + pub(crate) fn stop_evaluation(&mut self, id: &str) -> Result> { + let run = self.evaluation(id)?; + ensure!( + !run.state.ended(), + "The run {id} has already {}", + run.state.name() + ); + self.stop_evaluation_with(run, STOPPED_BY_YOU) + } + + /// Ends a run that has not ended as stopped, or returns its session when it is in a turn, + /// whose end then ends it. + fn stop_evaluation_with(&mut self, run: Evaluation, reason: &str) -> Result> { + if run.state.ended() { + return Ok(None); + } + if let Some(session) = run.session_id.clone().filter(|s| self.in_turn.contains(s)) { + return Ok(Some(session)); + } + self.end_evaluation(run, EvaluationState::Stopped, reason)?; + Ok(None) + } + + /// Starts every queued run its project allows: each app run at once, and the oldest + /// desktop run whenever no run holds the screen. + pub(crate) fn admit_evaluations(&mut self) -> Result<()> { + let queued: Vec = self + .db + .prepare("SELECT data FROM evaluations WHERE json_extract(data,'$.state')='queued' ORDER BY created_at, rowid")? + .query_map([], |r| decode(r, 0))? + .collect::>>()?; + let mut screen_taken = self.screen_holder()?.is_some(); + for run in queued { + if run.mode == EvaluationMode::Desktop { + if screen_taken { + continue; + } + screen_taken = true; + } + let id = run.id.clone(); + // One run's failure to start must never stop the others. + if let Err(error) = self.begin_evaluation(run) { + let run = self.evaluation(&id)?; + self.end_evaluation( + run, + EvaluationState::Failed, + &format!("Could not start: {error:#}"), + )?; + } + } + Ok(()) + } + + /// The desktop run that holds the screen: the one admitted and not ended. One at a time. + pub(crate) fn screen_holder(&self) -> Result> { + Ok(self + .db + .query_row( + "SELECT id FROM evaluations WHERE json_extract(data,'$.mode')='desktop' AND session_id IS NOT NULL AND json_extract(data,'$.state') NOT IN ('finished','failed','stopped')", + [], + |r| r.get(0), + ) + .optional()?) + } + + /// Gives an admitted run its own session, pinned to its model, with `prompt.md` as its + /// only message. A provider at its limit holds the turn until the reset. + fn begin_evaluation(&mut self, mut run: Evaluation) -> Result<()> { + let prompt = fs::read_to_string(Path::new(&run.folder).join(PROMPT_FILE)) + .context("Could not read prompt.md")?; + ensure!(!prompt.trim().is_empty(), "prompt.md is empty"); + let session = Session { + id: new_id(), + project_id: String::new(), + parent_id: None, + repository_id: None, + name: format!("UI evaluator · {}", run.id), + role: Role::Evaluator, + provider: run.profile.provider, + status: Status::Ready, + archived: false, + owner: Some(SessionOwner::Project(run.project_id.clone())), + }; + let spent = self.limit_reached(run.profile.provider, now())?; + let waits = spent.is_some(); + run.session_id = Some(session.id.clone()); + run.state = match waits { + true => EvaluationState::Waiting, + false => EvaluationState::Running, + }; + run.resets_at = spent.and_then(|spent| spent.until); + run.reason = None; + self.atomically(|host| { + host.db.execute( + "INSERT INTO sessions VALUES (?1,NULL,NULL,?2,?3)", + params![session.id, tag(&session.role)?, encode(&session)?], + )?; + let mut runtime = host.pin( + &session.id, + run.profile.clone(), + Chooser::Coordinator { + session_id: run.coordinator_id.clone(), + }, + "Pinned by start_evaluation; never replaced", + )?; + runtime.waiting = waits; + runtime.workdir = Some(run.folder.clone()); + host.save_runtime(&runtime)?; + host.save_evaluation(&run)?; + host.send( + &format!("{EVALUATION_PREFIX}{}:prompt", run.id), + None, + &session.id, + &prompt, + )?; + Ok(()) + }) + } + + /// The launch of a run's turn: its folder, and the Codex home that keeps the human's own out. + pub(crate) fn evaluator_launch(&self, session: &Session) -> Result { + let run = self.session_evaluation(&session.id)?; + ensure!(!run.state.ended(), "The run {} has ended", run.id); + let codex_home = self.home.join(CODEX_HOME); + if session.provider == Provider::Codex { + provider::prepare_codex_home(&codex_home)?; + } + Ok(provider::EvaluatorLaunch { + folder: fs::canonicalize(&run.folder)?, + codex_home, + }) + } + + /// A run's turn has started: it runs, whether for the first time or after its limit reset. + pub(crate) fn evaluation_turn_started(&mut self, session: &str) -> Result<()> { + let mut run = self.session_evaluation(session)?; + run.state = EvaluationState::Running; + run.started_at.get_or_insert(now()); + run.resets_at = None; + self.save_evaluation(&run) + } + + /// Records the model a run's provider said it runs, which a later turn may say again. + pub(crate) fn evaluation_model_reported(&mut self, session: &str, model: &str) -> Result<()> { + let mut run = self.session_evaluation(session)?; + run.reported_model = Some(model.chars().take(200).collect()); + self.save_evaluation(&run) + } + + /// A run's turn has ended: at a usage limit it waits for the reset in the same session; + /// otherwise the run ends as stopped, failed or finished. + pub(crate) fn evaluation_turn_ended( + &mut self, + session: &str, + stopped: Option<&str>, + error: Option<&str>, + waits_until: Option>, + ) -> Result<()> { + let mut run = self.session_evaluation(session)?; + if run.state.ended() { + return Ok(()); + } + if let (Some(resets_at), None) = (waits_until, stopped) { + run.state = EvaluationState::Waiting; + run.resets_at = resets_at; + return self.save_evaluation(&run); + } + let (state, reason) = match (stopped, error) { + (Some(reason), _) => (EvaluationState::Stopped, reason), + (None, Some(error)) => (EvaluationState::Failed, error), + (None, None) => (EvaluationState::Finished, ""), + }; + self.end_evaluation(run, state, reason)?; + self.admit_evaluations() + } + + /// At startup, ends as failed each run whose turn the host's stop cut short, and wakes its + /// coordinator. A run whose prompt still waits to be delivered, or that waits for its + /// limit's reset, goes on; one with no session yet waits for the human or the screen. + pub(crate) fn end_interrupted_evaluations(&mut self) -> Result<()> { + let unended: Vec = self + .db + .prepare("SELECT data FROM evaluations WHERE session_id IS NOT NULL AND json_extract(data,'$.state') NOT IN ('finished','failed','stopped') ORDER BY created_at")? + .query_map([], |r| decode(r, 0))? + .collect::>>()?; + for run in unended { + let session = run.session_id.clone().unwrap_or_default(); + let queued: i64 = self.db.query_row( + "SELECT count(*) FROM messages WHERE recipient=?1 AND receipt='queued'", + [&session], + |r| r.get(0), + )?; + if queued > 0 || self.session_runtime(&session)?.limit_retry.is_some() { + continue; + } + self.end_evaluation(run, EvaluationState::Failed, INTERRUPTED)?; + } + self.admit_evaluations() + } + + /// A run's session that could not start a turn: the run fails. + pub(crate) fn evaluation_start_failed(&mut self, session: &str, error: &str) -> Result<()> { + let run = self.session_evaluation(session)?; + self.end_evaluation( + run, + EvaluationState::Failed, + &format!("Could not start: {error}"), + )?; + self.admit_evaluations() + } + + /// Ends a run: its session is archived, so nothing starts another turn in it, its queued + /// input is settled, and its coordinator is told only the run's id, state, folder and + /// duration. + fn end_evaluation( + &mut self, + mut run: Evaluation, + state: EvaluationState, + reason: &str, + ) -> Result<()> { + run.state = state; + run.reason = (!reason.is_empty()).then(|| reason.to_owned()); + run.ended_at = Some(now()); + self.atomically(|host| { + host.save_evaluation(&run)?; + if let Some(id) = &run.session_id { + host.db.execute( + "UPDATE messages SET receipt='completed' WHERE recipient=?1 AND receipt IN ('queued','held')", + [id], + )?; + let mut session = host.session(id)?; + session.archived = true; + session.status = Status::Done; + host.db.execute( + "UPDATE sessions SET data=?2 WHERE id=?1", + params![id, encode(&session)?], + )?; + let mut runtime = host.session_runtime(id)?; + runtime.waiting = false; + runtime.limit_retry = None; + host.save_runtime(&runtime)?; + } + Ok(()) + })?; + let id = format!("{EVALUATION_PREFIX}{}:ended", run.id); + if let Err(error) = self.send(&id, None, &run.coordinator_id, &ended_message(&run)) { + eprintln!( + "Telling the coordinator that run {} ended: {error:#}", + run.id + ); + } + Ok(()) + } +} + +/// Why a run ended when what its project allows no longer allows it. +pub(crate) fn not_allowed(mode: EvaluationMode) -> String { + format!("Stopped: {} is not allowed in this project.", mode.label()) +} +/// The start of the operation ids of `project`'s Inbox cards; each ends with the run that asked. +fn card_prefix(project: &str) -> String { + format!("{EVALUATION_PREFIX}allow:{project}:") +} +fn permission_label(permission: EvaluatorPermission) -> &'static str { + (EvaluatorPermission::CHOICES.iter()) + .find(|(_, choice)| *choice == permission) + .map_or("", |(label, _)| label) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + const PROMPT: &str = "Open Models in dark mode at 1280 wide.\nWrite report.md."; + + struct Fixture { + _home: tempfile::TempDir, + runs: tempfile::TempDir, + host: Host, + coordinator: Session, + } + /// Models as Claude's CLI lists them: an alias and the full ID it stands for. + pub(crate) fn claude_listed() -> Vec { + ["opus", "claude-opus-5-5"] + .map(|model| ModelOption { + provider: Provider::Claude, + model: model.into(), + label: model.into(), + efforts: vec!["low".into(), "high".into()], + }) + .into() + } + fn fixture() -> Fixture { + let home = tempfile::tempdir().unwrap(); + let mut host = Host::open_set_up(home.path()).unwrap(); + host.listed_models.insert(Provider::Claude, claude_listed()); + let project = host.create_project("UX loop").unwrap(); + let id = host.coordinator_of(&project.id).unwrap().unwrap(); + let coordinator = host.session(&id).unwrap(); + Fixture { + _home: home, + runs: tempfile::tempdir().unwrap(), + host, + coordinator, + } + } + impl Fixture { + /// A run folder holding `PROMPT`. + fn folder(&self, name: &str) -> String { + let folder = self.runs.path().join(name); + fs::create_dir_all(&folder).unwrap(); + fs::write(folder.join(PROMPT_FILE), PROMPT).unwrap(); + folder.to_string_lossy().into_owned() + } + fn start(&mut self, folder: &str, mode: &str) -> Result { + let args = json!({"run_folder":folder,"provider":"claude","model":"opus","effort":"high","mode":mode}); + let coordinator = self.coordinator.clone(); + self.host.start_evaluation(&coordinator, &args) + } + fn allow(&mut self, permission: EvaluatorPermission) { + let project = self.coordinator.project_id.clone(); + self.host + .set_evaluator_permission(&project, permission) + .unwrap(); + } + fn messages_to_coordinator(&self) -> Vec { + (self.host.messages(&self.coordinator.id, None, 100).unwrap()) + .into_iter() + .filter(|m| m.id.starts_with(EVALUATION_PREFIX)) + .collect() + } + /// A desktop run as `start_evaluation` will create one once ticket 4 allows it. + fn queue_desktop(&mut self, name: &str) -> String { + let id = self.app_run(name); + let mut run = self.host.evaluation(&id).unwrap(); + run.id = format!("desktop-{id}"); + run.mode = EvaluationMode::Desktop; + run.session_id = None; + run.state = EvaluationState::Queued; + run.ended_at = None; + self.host.save_evaluation(&run).unwrap(); + run.id + } + /// An ended app run, as a source for `queue_desktop`. + fn app_run(&mut self, name: &str) -> String { + let folder = self.folder(name); + let id = self.start(&folder, "app").unwrap()["run_id"] + .as_str() + .unwrap() + .to_owned(); + let session = self.host.evaluation(&id).unwrap().session_id.unwrap(); + self.host + .evaluation_turn_ended(&session, None, None, None) + .unwrap(); + id + } + } + + #[test] + fn the_first_run_asks_the_human_and_their_answer_starts_it_with_only_its_prompt() { + let mut f = fixture(); + let folder = f.folder("run-a"); + let started = f.start(&folder, "app").unwrap(); + assert_eq!(started["state"], "waiting"); + let run = f + .host + .evaluation(started["run_id"].as_str().unwrap()) + .unwrap(); + assert_eq!(run.session_id, None); + let card = f.host.open_attention(&f.coordinator.project_id).unwrap(); + assert_eq!(card.len(), 1); + assert_eq!( + card[0].options, + ["App only", "Full desktop", "Not in this project"] + ); + // A second run waits for the same card. + let second = f.folder("run-b"); + f.start(&second, "app").unwrap(); + assert_eq!( + f.host + .open_attention(&f.coordinator.project_id) + .unwrap() + .len(), + 1 + ); + // Free text is refused and the card stays. + assert!(f.host.answer_attention(&card[0].id, "sure").is_err()); + f.host.answer_attention(&card[0].id, "App only").unwrap(); + assert!( + f.host + .open_attention(&f.coordinator.project_id) + .unwrap() + .is_empty() + ); + let run = f.host.evaluation(&run.id).unwrap(); + assert_eq!(run.state, EvaluationState::Running); + let session = f.host.session(run.session_id.as_deref().unwrap()).unwrap(); + assert_eq!(session.role, Role::Evaluator); + assert_eq!(session.parent_id, None); + assert_eq!(session.project_id, ""); + // Its one message is prompt.md as it is, and its model is pinned as given. + let input = f.host.messages(&session.id, None, 10).unwrap(); + assert_eq!(input.len(), 1); + assert_eq!(input[0].body, PROMPT); + let runtime = f.host.session_runtime(&session.id).unwrap(); + assert_eq!(runtime.profile, Some(run.profile.clone())); + assert_eq!(runtime.workdir.as_deref(), Some(run.folder.as_str())); + assert_eq!(f.host.turn_profile(&session).unwrap().model, "opus"); + let told = f.messages_to_coordinator(); + assert!(told[0].body.contains("App only"), "{}", told[0].body); + } + + #[test] + fn refusals_name_the_reason_and_create_nothing() { + let mut f = fixture(); + let folder = f.folder("run-a"); + let refused = |f: &mut Fixture, args: Value| { + let coordinator = f.coordinator.clone(); + format!( + "{:#}", + f.host.start_evaluation(&coordinator, &args).unwrap_err() + ) + }; + let args = |folder: &str, model: &str, effort: &str, mode: &str| json!({"run_folder":folder,"provider":"claude","model":model,"effort":effort,"mode":mode}); + // Desktop is not available until ticket 4, and raises no card. + let desktop = refused(&mut f, args(&folder, "opus", "high", "desktop")); + assert!(desktop.contains("not available yet"), "{desktop}"); + let unknown = refused(&mut f, args(&folder, "gpt-9", "high", "app")); + assert!( + unknown.contains("does not offer the model gpt-9"), + "{unknown}" + ); + let effort = refused(&mut f, args(&folder, "opus", "ultra", "app")); + assert!( + effort.contains("does not offer the effort ultra"), + "{effort}" + ); + // A list not read yet checks nothing against a guess. + let mut codex = args(&folder, "gpt-6.1-sol", "high", "app"); + codex["provider"] = json!("codex"); + let unread = refused(&mut f, codex.clone()); + assert!( + unread.contains("has not read Codex's model list"), + "{unread}" + ); + let signed_out = ProviderStatus::default(); + f.host.record_detection(Provider::Codex, &signed_out); + let codex = refused(&mut f, codex); + assert!(codex.contains("Codex is not signed in"), "{codex}"); + // Bad folders. + let relative = refused(&mut f, args("runs/a", "opus", "high", "app")); + assert!(relative.contains("absolute"), "{relative}"); + let missing = f.runs.path().join("missing"); + let missing = refused( + &mut f, + args(&missing.to_string_lossy(), "opus", "high", "app"), + ); + assert!(missing.contains("does not exist"), "{missing}"); + let empty = f.runs.path().join("empty"); + fs::create_dir(&empty).unwrap(); + let empty = refused( + &mut f, + args(&empty.to_string_lossy(), "opus", "high", "app"), + ); + assert!(empty.contains("no prompt.md"), "{empty}"); + let repository = f.folder("repo"); + crate::test_support::git(Path::new(&repository), &["init", "-q"]); + let inside = Path::new(&repository).join("eval"); + fs::create_dir(&inside).unwrap(); + fs::write(inside.join(PROMPT_FILE), PROMPT).unwrap(); + let inside = refused( + &mut f, + args(&inside.to_string_lossy(), "opus", "high", "app"), + ); + assert!(inside.contains("Git repository or worktree"), "{inside}"); + let data = f.host.home.join("eval"); + fs::create_dir(&data).unwrap(); + fs::write(data.join(PROMPT_FILE), PROMPT).unwrap(); + let data = refused(&mut f, args(&data.to_string_lossy(), "opus", "high", "app")); + assert!(data.contains("data folder"), "{data}"); + // A worker may not start one. + let worker = Session { + role: Role::Implementer, + ..f.coordinator.clone() + }; + let good = args(&folder, "opus", "high", "app"); + assert!(f.host.start_evaluation(&worker, &good).is_err()); + assert!( + f.host + .evaluations(&f.coordinator.project_id) + .unwrap() + .is_empty() + ); + assert!( + f.host + .open_attention(&f.coordinator.project_id) + .unwrap() + .is_empty() + ); + // Off, and a mode the project does not allow. + f.allow(EvaluatorPermission::Off); + let off = refused(&mut f, good.clone()); + assert_eq!( + off, + "UI evaluators are not allowed in this project. Ask the human." + ); + f.allow(EvaluatorPermission::App); + let narrow = refused(&mut f, args(&folder, "opus", "high", "desktop")); + assert_eq!( + narrow, + "Full desktop is not allowed in this project. Ask the human." + ); + // One folder, one unfinished run. + f.start(&folder, "app").unwrap(); + let reused = refused(&mut f, good); + assert!(reused.contains("still uses the folder"), "{reused}"); + } + + #[test] + fn at_a_usage_limit_the_run_waits_on_its_own_model_and_never_falls_back() { + let mut f = fixture(); + f.allow(EvaluatorPermission::App); + let at = now(); + let resets = at + 3_600_000; + f.host + .record_quota(QuotaReading { + provider: Provider::Claude, + account: None, + observed_at: at, + source: "Claude stream".into(), + windows: vec![QuotaWindow { + key: "five_hour".into(), + label: "Session".into(), + used_percent: Some(100.), + observed_at: at, + duration_mins: Some(300), + resets_at: Some(resets), + status: None, + }], + error: None, + }) + .unwrap(); + let folder = f.folder("run-a"); + let started = f.start(&folder, "app").unwrap(); + assert_eq!(started["state"], "waiting"); + assert!( + started["waiting"] + .as_str() + .unwrap() + .contains("at its limit") + ); + let run = f + .host + .evaluation(started["run_id"].as_str().unwrap()) + .unwrap(); + assert_eq!(run.resets_at, Some(resets)); + let session = f.host.session(run.session_id.as_deref().unwrap()).unwrap(); + assert!(f.host.hold_reason(&session).unwrap().is_some()); + let runtime = f.host.session_runtime(&session.id).unwrap(); + assert_eq!(runtime.profile, Some(run.profile)); + // A turn that failed at the limit waits again in the same session. + f.host + .evaluation_turn_ended(&session.id, None, Some("limit"), Some(Some(resets))) + .unwrap(); + let run = f.host.evaluation(&run.id).unwrap(); + assert_eq!(run.state, EvaluationState::Waiting); + assert!(f.messages_to_coordinator().is_empty()); + } + + #[test] + fn a_full_id_is_started_as_given_and_an_alias_records_the_model_it_stood_for() { + let mut f = fixture(); + f.allow(EvaluatorPermission::App); + let folder = f.folder("run-a"); + let args = json!({"run_folder":folder,"provider":"claude","model":"claude-opus-5-5","effort":"high","mode":"app"}); + let coordinator = f.coordinator.clone(); + let started = f.host.start_evaluation(&coordinator, &args).unwrap(); + assert_eq!(started["state"], "running"); + let run = f + .host + .evaluation(started["run_id"].as_str().unwrap()) + .unwrap(); + let session = f.host.session(run.session_id.as_deref().unwrap()).unwrap(); + assert_eq!( + f.host.turn_profile(&session).unwrap().model, + "claude-opus-5-5" + ); + // An alias's run records the model Claude said it stood for, which the coordinator + // may see, and its end names it. + let other = f.folder("run-b"); + let id = f.start(&other, "app").unwrap()["run_id"] + .as_str() + .unwrap() + .to_owned(); + let session = f.host.evaluation(&id).unwrap().session_id.unwrap(); + f.host + .evaluation_model_reported(&session, "claude-opus-5-5") + .unwrap(); + let listed = f.host.list_evaluations(&coordinator, &json!({})).unwrap(); + assert_eq!(listed["runs"][0]["model"], "opus"); + assert_eq!(listed["runs"][0]["reported_model"], "claude-opus-5-5"); + f.host + .evaluation_turn_ended(&session, None, None, None) + .unwrap(); + let told = f.messages_to_coordinator(); + assert!( + told[0].body.contains("· model claude-opus-5-5 ·"), + "{}", + told[0].body + ); + } + + #[test] + fn the_end_wakes_the_coordinator_with_only_the_id_state_folder_and_time() { + let mut f = fixture(); + f.allow(EvaluatorPermission::App); + let folder = f.folder("run-a"); + let id = f.start(&folder, "app").unwrap()["run_id"] + .as_str() + .unwrap() + .to_owned(); + let session = f.host.evaluation(&id).unwrap().session_id.unwrap(); + f.host.evaluation_turn_started(&session).unwrap(); + f.host + .append_output( + &f.host.session(&session).unwrap(), + "out:1", + "SECRET FINDINGS", + ) + .unwrap(); + f.host + .evaluation_turn_ended(&session, None, None, None) + .unwrap(); + let run = f.host.evaluation(&id).unwrap(); + assert_eq!(run.state, EvaluationState::Finished); + let told = f.messages_to_coordinator(); + assert_eq!(told.len(), 1); + assert_eq!( + told[0].body, + format!( + "Run {id} finished · model opus · folder {} · under 1 min", + run.folder + ) + ); + assert!(f.host.session(&session).unwrap().archived); + // A failure's reason reaches neither the message nor the coordinator's list. + let other = f.folder("run-b"); + let failed = f.start(&other, "app").unwrap()["run_id"] + .as_str() + .unwrap() + .to_owned(); + let session = f.host.evaluation(&failed).unwrap().session_id.unwrap(); + f.host + .evaluation_turn_ended(&session, None, Some("SECRET model text"), None) + .unwrap(); + let coordinator = f.coordinator.clone(); + let listed = f.host.list_evaluations(&coordinator, &json!({})).unwrap(); + assert!(!listed.to_string().contains("SECRET"), "{listed}"); + assert_eq!(listed["runs"][0]["state"], "failed"); + assert!( + !f.messages_to_coordinator() + .iter() + .any(|m| m.body.contains("SECRET")) + ); + } + + #[test] + fn desktop_runs_take_turns_at_the_screen_while_app_runs_go_side_by_side() { + let mut f = fixture(); + f.allow(EvaluatorPermission::Desktop); + let first = f.queue_desktop("d1"); + let second = f.queue_desktop("d2"); + f.host.admit_evaluations().unwrap(); + let run = f.host.evaluation(&first).unwrap(); + assert_eq!(run.state, EvaluationState::Running, "{:?}", run.reason); + assert_eq!(f.host.screen_holder().unwrap(), Some(first.clone())); + assert_eq!( + f.host.evaluation(&second).unwrap().state, + EvaluationState::Queued + ); + // App runs do not wait for the screen. + let a1 = f.folder("a1"); + let a2 = f.folder("a2"); + assert_eq!(f.start(&a1, "app").unwrap()["state"], "running"); + assert_eq!(f.start(&a2, "app").unwrap()["state"], "running"); + let session = f.host.evaluation(&first).unwrap().session_id.unwrap(); + f.host + .evaluation_turn_ended(&session, None, None, None) + .unwrap(); + assert_eq!(f.host.screen_holder().unwrap(), Some(second.clone())); + // Narrowing to app only stops the desktop run and frees the screen; app runs go on. + f.allow(EvaluatorPermission::App); + let stopped = f.host.evaluation(&second).unwrap(); + assert_eq!(stopped.state, EvaluationState::Stopped); + assert_eq!( + stopped.reason.as_deref(), + Some(not_allowed(EvaluationMode::Desktop).as_str()) + ); + assert_eq!(f.host.screen_holder().unwrap(), None); + let project = f.coordinator.project_id.clone(); + let running = (f.host.evaluations(&project).unwrap().into_iter()) + .filter(|run| run.state == EvaluationState::Running) + .count(); + assert_eq!(running, 2); + } + + #[test] + fn a_run_the_hosts_stop_cut_short_fails_at_the_next_start_and_wakes_its_coordinator() { + let mut f = fixture(); + f.allow(EvaluatorPermission::App); + let started = |f: &mut Fixture, name: &str| { + let folder = f.folder(name); + let id = f.start(&folder, "app").unwrap()["run_id"] + .as_str() + .unwrap() + .to_owned(); + let session = f.host.evaluation(&id).unwrap().session_id.unwrap(); + (id, session) + }; + // In its turn when the host went: its prompt was delivered. + let (cut, session) = started(&mut f, "cut"); + f.host.evaluation_turn_started(&session).unwrap(); + f.host + .db + .execute( + "UPDATE messages SET receipt='delivered' WHERE recipient=?1", + [&session], + ) + .unwrap(); + // Waiting for its limit's reset, with its prompt held for the retry. + let (waiting, session) = started(&mut f, "waiting"); + let mut runtime = f.host.session_runtime(&session).unwrap(); + runtime.limit_retry = Some(LimitRetry { + failed_at: now(), + resets_at: None, + }); + f.host.save_runtime(&runtime).unwrap(); + f.host + .db + .execute( + "UPDATE messages SET receipt='held' WHERE recipient=?1", + [&session], + ) + .unwrap(); + // Admitted, its prompt not delivered yet. + let (queued, _) = started(&mut f, "queued"); + + let Fixture { + _home: home, + runs: _runs, + host, + coordinator, + } = f; + drop(host); + let host = Host::open_set_up(home.path()).unwrap(); + + let run = host.evaluation(&cut).unwrap(); + assert_eq!(run.state, EvaluationState::Failed); + assert_eq!(run.reason.as_deref(), Some(INTERRUPTED)); + assert!( + host.session(run.session_id.as_deref().unwrap()) + .unwrap() + .archived + ); + let told: Vec = (host.messages(&coordinator.id, None, 100).unwrap()) + .into_iter() + .filter(|m| m.id.starts_with(EVALUATION_PREFIX)) + .collect(); + assert_eq!(told.len(), 1); + assert_eq!(told[0].body, ended_message(&run)); + for going_on in [waiting, queued] { + assert!(!host.evaluation(&going_on).unwrap().state.ended()); + } + } + + #[test] + fn the_human_stops_a_waiting_run_and_closing_the_project_turns_evaluators_off() { + let mut f = fixture(); + let folder = f.folder("run-a"); + let id = f.start(&folder, "app").unwrap()["run_id"] + .as_str() + .unwrap() + .to_owned(); + assert_eq!(f.host.stop_evaluation(&id).unwrap(), None); + let run = f.host.evaluation(&id).unwrap(); + assert_eq!(run.state, EvaluationState::Stopped); + assert!(f.host.stop_evaluation(&id).is_err()); + f.allow(EvaluatorPermission::App); + let other = f.folder("run-b"); + let id = f.start(&other, "app").unwrap()["run_id"] + .as_str() + .unwrap() + .to_owned(); + f.host.set_archived(&f.coordinator.id, true).unwrap(); + let project = f.coordinator.project_id.clone(); + assert_eq!( + f.host.evaluator_permission(&project).unwrap(), + Some(EvaluatorPermission::Off) + ); + assert_eq!( + f.host.evaluation(&id).unwrap().state, + EvaluationState::Stopped + ); + } +} diff --git a/crates/workspace-host/src/lib.rs b/crates/workspace-host/src/lib.rs index 9692f195..3c11ea25 100644 --- a/crates/workspace-host/src/lib.rs +++ b/crates/workspace-host/src/lib.rs @@ -15,6 +15,7 @@ mod compile; pub mod context; pub mod detection; mod diffs; +mod evaluations; pub mod events; mod files; mod flatten; @@ -118,8 +119,8 @@ fn sessions_where(live: bool) -> String { "id IN (SELECT session_id FROM runtimes WHERE json_extract(data,'$.ticket_id') IN (SELECT id FROM tickets WHERE {}))", tickets_where(true) ); - // Threads are listed only as the assistant's, and compile runs only on the Brains page, - // never as a project's sessions. + // Threads are listed only as the assistant's, compile runs only on the Brains page and + // evaluator runs only as their project's runs, never as a project's sessions. if live { format!("({NOT_THREAD} AND ({ARCHIVED_FLAG}=0 OR {on_open_ticket}))") } else { @@ -128,7 +129,7 @@ fn sessions_where(live: bool) -> String { } /// SQL condition on `sessions` for a session that is not an assistant thread or a brain /// compile, which belong to no project. -const NOT_THREAD: &str = "role NOT IN ('assistant','job_run','compiler')"; +const NOT_THREAD: &str = "role NOT IN ('assistant','job_run','compiler','evaluator')"; pub struct Host { pub(crate) db: Connection, /// Files remote devices are uploading, kept while the host runs. @@ -199,6 +200,8 @@ pub struct Host { router_runs: Vec, /// By provider, whether detection last found its CLI installed and signed in. pub(crate) detected: std::collections::BTreeMap, + /// By provider, the models its CLI last listed, which a UI evaluator's model must be one of. + pub(crate) listed_models: std::collections::BTreeMap>, /// A pass or a commit moved a repository, so every brain's index refresh is due now. pub(crate) brains_now: bool, /// What an earlier release left waiting has all landed. @@ -293,6 +296,7 @@ fn ensure_unreserved(operation: &str) -> Result<()> { MERGE_PREFIX, REPOSITORY_REQUEST_PREFIX, PERMISSION_PREFIX, + EVALUATION_PREFIX, ] { ensure!( !operation.starts_with(prefix), @@ -373,6 +377,7 @@ impl Host { router::migrate(&db)?; jobs::migrate(&db)?; compile::migrate(&db)?; + evaluations::migrate(&db)?; db.execute_batch(include_str!("steps.sql"))?; db.execute_batch(include_str!("schedules.sql"))?; db.execute_batch(include_str!("selection.sql"))?; @@ -427,6 +432,7 @@ impl Host { merge_runs: vec![], router_runs: vec![], detected: Default::default(), + listed_models: Default::default(), uploads: Default::default(), gh_login: None, brains_now: false, @@ -544,6 +550,7 @@ impl Host { self.recover_steps()?; self.end_interrupted_job_runs()?; self.end_interrupted_compiles()?; + self.end_interrupted_evaluations()?; self.ask_about_interrupted_routes() } pub(crate) fn event( @@ -642,7 +649,7 @@ impl Host { turning.sort(); for id in turning { let session = self.session(id)?; - if !session.role.is_thread() { + if !session.role.is_thread() && session.role != Role::Evaluator { sessions.push(session); } } @@ -1087,6 +1094,10 @@ impl Host { )?; } tx.commit()?; + // Closing the project turns its UI evaluators off; the service stops runs in a turn. + if stops_project && let Err(error) = host.close_evaluations(&project) { + eprintln!("Turning off the project's UI evaluators: {error:#}"); + } Ok(affected) }; if !archived { @@ -1679,7 +1690,7 @@ impl Host { } Command::InspectProvider { provider } => { let report = detection::report(provider); - self.record_detection(provider, &report.status); + self.record_report(provider, &report); serde_json::to_value(report)? } Command::MeasureContext { @@ -1825,6 +1836,20 @@ impl Host { } // The service stops the turn of a run that is in one. Command::StopJobRun { run_id } => json!({"thread": self.stop_job_run(&run_id)?}), + Command::Evaluations { project_id } => { + serde_json::to_value(self.evaluations(&project_id)?)? + } + Command::SetEvaluatorPermission { + project_id, + permission, + } => { + self.set_evaluator_permission(&project_id, permission)?; + Value::Null + } + // The service stops the turn of a run that is in one. + Command::StopEvaluation { run_id } => { + json!({"session": self.stop_evaluation(&run_id)?}) + } Command::SeeJobRuns { job } => { self.see_job_runs(&job)?; Value::Null diff --git a/crates/workspace-host/src/live.rs b/crates/workspace-host/src/live.rs index 88ec0216..f6675cef 100644 --- a/crates/workspace-host/src/live.rs +++ b/crates/workspace-host/src/live.rs @@ -956,6 +956,8 @@ impl Host { )?)?) } "request_repository" => self.request_repository(&session, &args), + "start_evaluation" => self.start_evaluation(&session, &args), + "list_evaluations" => self.list_evaluations(&session, &args), "assign_ticket" => { let ticket = self.owned_ticket(id, string("ticket_id")?)?; ensure!(ticket.is_open(), "Ticket is already {}", ticket.state); diff --git a/crates/workspace-host/src/mcp.rs b/crates/workspace-host/src/mcp.rs index d60f0f90..bb22f513 100644 --- a/crates/workspace-host/src/mcp.rs +++ b/crates/workspace-host/src/mcp.rs @@ -160,6 +160,24 @@ pub fn tools() -> Value { json!({"session_id":string}), vec!["session_id"] ), + tool( + "start_evaluation", + "Coordinator: start a UI evaluator, a fresh and blind agent on exactly the provider, model and effort you name, never another; Who runs what does not apply. run_folder is an absolute path to an existing folder holding prompt.md, outside every repository, worktree and Wiffletree's data folder, used by no other unfinished run. The evaluator's first and only message is prompt.md as it is, so put everything it needs there. It has no shell, web, repository, brain, memory or workspace tools and none of the human's instructions or settings, and it reads and writes only in its folder, where it leaves its report. mode is app, which drives only the app under test, or desktop, the whole screen, which is not available yet. The human allows evaluators per project: the first run in a project asks them in their Inbox and waits; where they are off, or the mode is not allowed, the call is refused with the reason, and only the human can change that. A model the provider's CLI does not list, an effort it does not offer or a provider not signed in is refused; so is any model until Wiffletree has read the CLI's list, which it then reads, so call again a minute later. Give a full model ID, such as claude-opus-5-5, to pin one model: a run whose provider names another model when it starts fails. An alias such as opus can move to a newer model; the run records which model it got. At a usage limit the run waits for the reset. Returns the run's id and state. Finish your turn: Wiffletree messages you once, when the run ends, with only its id, state, model, folder and duration; read the report from the folder then. You never see what the evaluator says.", + json!({"run_folder":string,"provider":{"type":"string","enum":["claude","codex"]},"model":string,"effort":string,"mode":{"type":"string","enum":["app","desktop"]}}), + vec!["run_folder", "provider", "model", "effort", "mode"] + ), + tool( + "list_evaluations", + "Coordinator: list this project's UI evaluator runs, newest first, with what the project allows: each run's id, state (queued, waiting, running, finished, failed or stopped), mode, the model asked for and the one its provider reported, folder and times, and why a waiting run waits. Never what a run said or why it failed.", + json!({}), + vec![] + ), + tool( + "stop_evaluation", + "Coordinator: stop one of this project's UI evaluator runs that has not ended. A running run stops at once; Wiffletree then messages you that it ended, as for any run.", + json!({"run_id":string}), + vec!["run_id"] + ), tool( "log_work", "Coordinator or implementer: record one terse entry in your project's Space's brain, at a meaningful milestone: a decision, a finished assignment, a handoff or a blocker. Give type, a one-line title of at most 80 characters, and one line each of changed, why and state; newlines become spaces. No transcripts, step-by-step narration or pasted output, and never child personal information (names, message text, phone numbers, locations), credentials or secrets. Wiffletree fills in the entry's id, date, author and ref, from your ticket's branch, commits and pull request; you cannot give them. An implementer always logs for its ticket's repository. The coordinator may name one of the project's tickets as ticket_id, for an entry about that ticket, such as after accepting or publishing it: it is for the ticket's repository, with its ref, and counts for that ticket's milestones. Or it may name one of the project's repositories as repository; with neither, the entry is a planning entry. milestone is an optional deduplication key: the same key with the same content returns the first entry, and with other content is refused. Refused, storing nothing, when gh is not signed in for a Space on GitHub, or the entry is over 2,000 characters, saying how much to cut. Each entry lands straight in the brain's logs within seconds, and the next compile folds it into the wiki. Returns the entry's id, file path and ref.", diff --git a/crates/workspace-host/src/provider.rs b/crates/workspace-host/src/provider.rs index d97a01ac..85b91d12 100644 --- a/crates/workspace-host/src/provider.rs +++ b/crates/workspace-host/src/provider.rs @@ -2,6 +2,7 @@ //! sorting and assistant threads, which the provider holds to what they are given. mod codex_compaction; mod constrained; +mod evaluator; mod output; mod restricted; mod router; @@ -10,6 +11,7 @@ use crate::processes::{self, Sighting, Watch}; use crate::runtime::LowPriority as _; use crate::stream::{ClaudeSteps, CodexSteps, StepUpdate}; use crate::*; +pub use evaluator::{EvaluatorLaunch, prepare_codex_home}; pub use output::{MAX_BYTES_AFTER_EXIT, READ_LIMIT}; use output::{Next, Output, StopReading}; pub use restricted::{ThreadLaunch, server_definition, shell_text_refused}; @@ -52,6 +54,8 @@ pub enum ProviderEvent { ModelUsage(Value), /// What the provider said about the compaction this run is; a turn's run sends none. Compaction(CompactionRecord), + /// The model a UI evaluator's provider said it runs, once, before it is checked. + Model(String), Finished { error: Option, usage: Value, @@ -91,6 +95,8 @@ pub struct Turn { pub instructions: String, /// An assistant thread's restricted launch; every other turn runs in YOLO mode. pub thread: Option, + /// A UI evaluator run's launch, which has no workspace server, credential or instructions. + pub evaluator: Option, } pub fn executable(provider: Provider) -> Result { let name = match provider { @@ -175,6 +181,8 @@ fn core(role: Role) -> &'static str { Role::Assistant => 6, Role::JobRun => 7, Role::Compiler => return compile::PROCEDURE, + // An evaluator is blind: Wiffletree gives it no instructions at all. + Role::Evaluator => return "", }; CORE[part].1 } @@ -309,6 +317,14 @@ fn configure(cmd: &mut ProcessCommand, turn: &Turn) { constrained::configure(cmd, &turn.profile, keeper::SORTING, keeper::ANSWER_SCHEMA); return; } + // Before any credential is set: an evaluator gets none. + if let Some(launch) = &turn.evaluator { + match turn.session.provider { + Provider::Claude => evaluator::configure_claude(cmd, turn, launch), + Provider::Codex => evaluator::configure_codex(cmd, turn, launch), + } + return; + } // Credentials go to the MCP subprocess environment, never in the prompt. cmd.env("WORKSPACE_SOCKET", &turn.socket) .env("WORKSPACE_TOKEN", &turn.token); @@ -672,6 +688,9 @@ fn run_turn( // The exit is checked on every pass, so a leftover process writing to stdout cannot hide // it. Every event the reader accepted is handled; the reader decides when to stop. let mut exited = false; + // An evaluator's model, once its provider has named it; Codex's thread, to find it by. + let mut model_reported = false; + let mut codex_thread: Option = None; loop { check_running(&cancel)?; report_seen(watching, emit); @@ -710,6 +729,21 @@ fn run_turn( } continue; } + // An evaluator's Codex names its model only in its session record, which it writes + // once its turn has started, so it is looked for until it is found. + if let Some(launch) = &turn.evaluator + && turn.session.provider == Provider::Codex + && !model_reported + { + if let Some(thread) = value["thread_id"].as_str() { + codex_thread = Some(thread.to_owned()); + } + if let Some(model) = codex_thread.as_deref().and_then(|t| launch.codex_model(t)) { + model_reported = true; + emit(ProviderEvent::Model(model.clone())); + launch.check_model(&turn.profile, &model)?; + } + } match turn.session.provider { Provider::Codex => match value["type"].as_str().unwrap_or_default() { "thread.started" => { @@ -721,6 +755,9 @@ fn run_turn( if let Some(launch) = &turn.thread { launch.check_codex_item(&value["item"])?; } + if let Some(launch) = &turn.evaluator { + launch.check_codex_item(&value["item"])?; + } codex .event(&value) .into_iter() @@ -773,6 +810,15 @@ fn run_turn( if let Some(launch) = &turn.thread { launch.check_claude_start(&value)?; } + // An evaluator has no workspace server to connect. + if let Some(launch) = &turn.evaluator { + launch.check_claude_start(&value)?; + if let Some(model) = value["model"].as_str() { + emit(ProviderEvent::Model(model.to_owned())); + launch.check_model(&turn.profile, model)?; + } + continue; + } ensure!( value["mcp_servers"] .as_array() @@ -998,6 +1044,7 @@ mod tests { helper: PathBuf::new(), instructions: String::new(), thread: None, + evaluator: None, }; let mut cmd = ProcessCommand::new("claude"); configure(&mut cmd, &turn); @@ -1165,6 +1212,7 @@ mod tests { helper: "/tmp/helper".into(), instructions: skill(Role::ProjectOrchestrator), thread: None, + evaluator: None, } } diff --git a/crates/workspace-host/src/provider/evaluator.rs b/crates/workspace-host/src/provider/evaluator.rs new file mode 100644 index 00000000..05074b9a --- /dev/null +++ b/crates/workspace-host/src/provider/evaluator.rs @@ -0,0 +1,477 @@ +//! The launch of a UI evaluator run (OpenSpec change `ui-evaluators`): blind, on the model it +//! was started with, with no shell, web, MCP server or workspace tool, and none of the human's +//! provider instructions, settings or skills. The provider holds it to its run folder. +use super::*; + +/// The built-in Claude tools an evaluator has: reading and writing files, which its rules hold +/// to the run folder. +const CLAUDE_TOOLS: [&str; 5] = ["Read", "Glob", "Grep", "Write", "Edit"]; +/// Codex features an evaluator never has: the threads' (`restricted::CODEX_FEATURES_OFF`), then +/// its shell, its image reader, which reads any path, and image generation. Code mode is left +/// with `apply_patch`, which the sandbox holds to the run folder. +const CODEX_FEATURES_OFF: [&str; 4] = [ + "shell_tool", + "unified_exec", + "view_image", + "image_generation", +]; +/// The only variables an evaluator's provider is given: none of Wiffletree's, such as +/// `WORKSPACE_TOKEN` and `WORKSPACE_SOCKET`, nor any other the host inherited. +const ENVIRONMENT: [&str; 7] = ["HOME", "USER", "LOGNAME", "PATH", "SHELL", "TMPDIR", "LANG"]; + +/// What an evaluator run's turn may use: its run folder alone, and for Codex a home of its own. +#[derive(Clone, Debug)] +pub struct EvaluatorLaunch { + /// Canonical; it holds `prompt.md`. + pub folder: PathBuf, + /// Codex's home for evaluators, which holds only a link to the human's `auth.json`, so the + /// human's `AGENTS.md`, config, skills and history do not reach the run. + pub codex_home: PathBuf, +} +impl EvaluatorLaunch { + /// The empty folder an evaluator's Codex runs with as `HOME`, beside its Codex home: Codex + /// also finds the human's skills under `~/.agents/skills`, whatever its own home says. + pub fn codex_user_home(&self) -> PathBuf { + self.codex_home.with_file_name("home") + } + /// `Read`, `Glob` and `Grep` of the folder and `Write`, `Edit` and `MultiEdit` in it; + /// everything else is denied without asking. + pub(crate) fn claude_allowed(&self) -> Vec { + // `//` makes a rule's path absolute. + let folder = format!("/{}/**", self.folder.display()); + ["Read", "Glob", "Grep", "Write", "Edit", "MultiEdit"] + .map(|tool| format!("{tool}({folder})")) + .into() + } + /// Refuses a Claude run that started with anything an evaluator is not given. + pub(crate) fn check_claude_start(&self, init: &Value) -> Result<()> { + let servers = init["mcp_servers"].as_array().map_or(0, Vec::len); + ensure!( + servers == 0, + "Stopped: the evaluator was offered an MCP server ({})", + init["mcp_servers"] + ); + let unexpected: Vec<&str> = (init["tools"].as_array().into_iter().flatten()) + .filter_map(Value::as_str) + .filter(|tool| !CLAUDE_TOOLS.contains(tool)) + .collect(); + ensure!( + unexpected.is_empty(), + "Stopped: the evaluator was offered tools it is not given ({})", + unexpected.join(", ") + ); + ensure!( + init["permissionMode"] == "dontAsk", + "Stopped: the evaluator started in permission mode {} instead of denying what is not allowed", + init["permissionMode"] + ); + ensure!( + init["memory_paths"]["auto"].is_null(), + "Stopped: the evaluator started with auto memory on, which writes outside its folder" + ); + Ok(()) + } + /// Refuses any MCP call: an evaluator has no server, so one means something reached it. + pub(crate) fn check_codex_item(&self, item: &Value) -> Result<()> { + ensure!( + item["type"] != "mcp_tool_call", + "Stopped: the evaluator called an MCP tool it was not given ({} {})", + item["server"].as_str().unwrap_or_default(), + item["tool"].as_str().unwrap_or_default() + ); + Ok(()) + } + /// Refuses a model other than the full ID asked for. An alias, such as `opus`, stands for + /// whichever model the provider names, which the run records. + pub(crate) fn check_model(&self, profile: &ModelProfile, reported: &str) -> Result<()> { + ensure!( + !pins_exactly(profile.provider, &profile.model) || reported == profile.model, + "Stopped: {} ran {reported}, not {}, the model the run was started with. Wiffletree never runs an evaluator on another model.", + profile.provider.label(), + profile.model + ); + Ok(()) + } + /// The model Codex's session record for `thread` says its turn runs, once it is written. + pub(crate) fn codex_model(&self, thread: &str) -> Option { + let file = rollout( + &self.codex_home.join("sessions"), + &format!("-{thread}.jsonl"), + )?; + let text = fs::read_to_string(file).ok()?; + text.lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .find(|record| record["type"] == "turn_context") + .and_then(|record| record["payload"]["model"].as_str().map(str::to_owned)) + } +} + +/// Whether `model` names one model rather than an alias: Claude's aliases, such as `opus`, +/// move to newer models, and its full IDs start with `claude-`. Codex's models have no alias. +pub fn pins_exactly(provider: Provider, model: &str) -> bool { + match provider { + Provider::Claude => model.starts_with("claude-"), + Provider::Codex => true, + } +} + +/// The file under `folder`, at any depth, whose name ends with `suffix`. Codex files a session +/// under `sessions////`. +fn rollout(folder: &Path, suffix: &str) -> Option { + for entry in fs::read_dir(folder).ok()?.flatten() { + let path = entry.path(); + if path.is_dir() { + if let Some(found) = rollout(&path, suffix) { + return Some(found); + } + } else if path.to_string_lossy().ends_with(suffix) { + return Some(path); + } + } + None +} + +/// Clears the environment the host inherited and passes on only `ENVIRONMENT` and the locale. +fn clear_environment(cmd: &mut ProcessCommand) { + cmd.env_clear(); + for (name, value) in std::env::vars_os() { + let name_text = name.to_string_lossy(); + if ENVIRONMENT.contains(&name_text.as_ref()) || name_text.starts_with("LC_") { + cmd.env(name, value); + } + } +} + +/// Claude in `dontAsk` mode with no settings, instructions, memory or MCP server of anyone's; +/// its first message is the prompt alone. +pub(super) fn configure_claude(cmd: &mut ProcessCommand, turn: &Turn, launch: &EvaluatorLaunch) { + clear_environment(cmd); + cmd.args([ + "-p", + "--output-format", + "stream-json", + "--verbose", + "--include-partial-messages", + "--strict-mcp-config", + "--no-chrome", + ]) + .args([ + "--model", + &turn.profile.model, + "--effort", + &turn.profile.effort, + ]) + .args(["--setting-sources", ""]) + .args([ + "--settings", + &json!({"autoMemoryEnabled": false}).to_string(), + ]) + .env("CLAUDE_CODE_DISABLE_AUTO_MEMORY", "1") + .args(["--permission-mode", "dontAsk"]) + .args(["--tools", &CLAUDE_TOOLS.join(",")]) + .args(["--allowedTools", &launch.claude_allowed().join(",")]) + .args([ + "--disallowedTools", + &restricted::claude_home_denied().join(","), + ]) + .args(["--mcp-config", &json!({"mcpServers": {}}).to_string()]); + if let Some(id) = &turn.provider_session { + cmd.args(["--resume", id]); + } +} + +/// Codex's workspace-write sandbox rooted at the run folder, with no network, temporary +/// folder, web search, shell or MCP server, in a home that holds only the human's sign-in and +/// with an empty `HOME`, so none of the human's skills reach it. +pub(super) fn configure_codex(cmd: &mut ProcessCommand, turn: &Turn, launch: &EvaluatorLaunch) { + clear_environment(cmd); + cmd.env("CODEX_HOME", &launch.codex_home) + .env("HOME", launch.codex_user_home()) + .args([ + "exec", + "--json", + "--skip-git-repo-check", + "--color", + "never", + ]) + .args(["--ignore-user-config", "--ignore-rules"]) + .args(["--sandbox", "workspace-write"]) + .arg("--cd") + .arg(&launch.folder) + .args(["-m", &turn.profile.model]); + for setting in [ + "approval_policy=\"never\"".to_owned(), + "sandbox_workspace_write.network_access=false".into(), + "sandbox_workspace_write.exclude_slash_tmp=true".into(), + "sandbox_workspace_write.exclude_tmpdir_env_var=true".into(), + "sandbox_workspace_write.writable_roots=[]".into(), + "web_search=\"disabled\"".into(), + format!("model_reasoning_effort={}", json!(turn.profile.effort)), + ] { + cmd.arg("-c").arg(setting); + } + for feature in restricted::CODEX_FEATURES_OFF + .iter() + .chain(&CODEX_FEATURES_OFF) + { + cmd.args(["--disable", feature]); + } + if let Some(id) = &turn.provider_session { + cmd.args(["resume", id]); + } + cmd.arg("-"); +} + +/// Makes `home` a Codex home holding only a link to the human's `auth.json`, in `$CODEX_HOME` +/// or `~/.codex`. A link, not a copy, so a token Codex refreshes is the human's one token. +/// Makes the empty `HOME` beside it too (`EvaluatorLaunch::codex_user_home`). +pub fn prepare_codex_home(home: &Path) -> Result<()> { + fs::create_dir_all(home)?; + fs::create_dir_all(home.with_file_name("home"))?; + let source = std::env::var_os("CODEX_HOME") + .map(PathBuf::from) + .or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".codex"))) + .context("Codex's home is unknown: HOME is not set")?; + let link = home.join("auth.json"); + let target = source.join("auth.json"); + if fs::read_link(&link).ok().as_deref() != Some(target.as_path()) { + let _ = fs::remove_file(&link); + std::os::unix::fs::symlink(&target, &link)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn launch() -> EvaluatorLaunch { + EvaluatorLaunch { + folder: "/eval/runs/run-1".into(), + codex_home: "/data/evaluators/codex-home".into(), + } + } + fn arguments(provider: Provider) -> (Vec, ProcessCommand) { + let mut turn = super::super::tests::turn(provider, None, vec![]); + turn.instructions = "SECRET LOOP GOALS".into(); + turn.evaluator = Some(launch()); + let mut cmd = ProcessCommand::new("provider"); + configure(&mut cmd, &turn); + let args = (cmd.get_args()) + .map(|arg| arg.to_string_lossy().into_owned()) + .collect(); + (args, cmd) + } + fn after<'a>(args: &'a [String], flag: &str) -> &'a str { + let at = args.iter().position(|arg| arg == flag).unwrap(); + &args[at + 1] + } + /// What the command sets and removes: with `env_clear`, every variable not set is gone. + fn environment(cmd: &ProcessCommand) -> Vec { + (cmd.get_envs()) + .filter(|(_, value)| value.is_some()) + .map(|(name, _)| name.to_string_lossy().into_owned()) + .collect() + } + + #[test] + fn claude_has_only_its_folders_files_no_shell_web_mcp_settings_or_instructions() { + let (args, cmd) = arguments(Provider::Claude); + assert_eq!(after(&args, "--tools"), "Read,Glob,Grep,Write,Edit"); + assert_eq!( + after(&args, "--allowedTools"), + "Read(//eval/runs/run-1/**),Glob(//eval/runs/run-1/**),Grep(//eval/runs/run-1/**),Write(//eval/runs/run-1/**),Edit(//eval/runs/run-1/**),MultiEdit(//eval/runs/run-1/**)" + ); + assert_eq!(after(&args, "--permission-mode"), "dontAsk"); + // No user, project or local settings: no CLAUDE.md, skills, hooks or permissions of + // the human's. + assert_eq!(after(&args, "--setting-sources"), ""); + assert!(args.contains(&"--strict-mcp-config".into())); + assert_eq!(after(&args, "--mcp-config"), r#"{"mcpServers":{}}"#); + assert_eq!(after(&args, "--settings"), r#"{"autoMemoryEnabled":false}"#); + assert!(after(&args, "--disallowedTools").contains("/.claude/**)")); + for absent in [ + "--append-system-prompt", + "--dangerously-skip-permissions", + "--resume", + "--add-dir", + ] { + assert!(!args.contains(&absent.into()), "{absent}: {args:?}"); + } + assert!(!args.iter().any(|arg| arg.contains("SECRET"))); + assert!( + !args + .iter() + .any(|arg| arg.contains("Bash") || arg.contains("Web")) + ); + assert_eq!(after(&args, "--model"), "m"); + let env = environment(&cmd); + assert!(env.contains(&"CLAUDE_CODE_DISABLE_AUTO_MEMORY".into())); + assert!( + !env.iter().any(|name| name.starts_with("WORKSPACE_")), + "{env:?}" + ); + } + + #[test] + fn codex_is_sandboxed_in_its_folder_with_no_shell_web_user_config_or_instructions() { + let (args, cmd) = arguments(Provider::Codex); + assert_eq!(after(&args, "--sandbox"), "workspace-write"); + assert_eq!(after(&args, "--cd"), "/eval/runs/run-1"); + for flag in ["--ignore-user-config", "--ignore-rules"] { + assert!(args.contains(&flag.into()), "{flag}"); + } + for setting in [ + "sandbox_workspace_write.network_access=false", + "sandbox_workspace_write.writable_roots=[]", + "sandbox_workspace_write.exclude_slash_tmp=true", + "web_search=\"disabled\"", + "approval_policy=\"never\"", + ] { + assert!(args.contains(&setting.into()), "{setting}: {args:?}"); + } + let disabled: Vec<&str> = (args.windows(2)) + .filter(|pair| pair[0] == "--disable") + .map(|pair| pair[1].as_str()) + .collect(); + for feature in [ + "shell_tool", + "unified_exec", + "view_image", + "computer_use", + "browser_use", + "plugins", + "multi_agent", + ] { + assert!(disabled.contains(&feature), "{feature}"); + } + assert!(!args.iter().any(|arg| arg.contains("mcp_servers"))); + assert!( + !args + .iter() + .any(|arg| arg.contains("developer_instructions")) + ); + assert!(!args.iter().any(|arg| arg.contains("SECRET"))); + assert!(!args.contains(&"--dangerously-bypass-approvals-and-sandbox".into())); + assert_eq!(args.last().unwrap(), "-"); + // Its own home keeps the human's AGENTS.md, config and skills out. + let env: Vec<(String, Option)> = (cmd.get_envs()) + .map(|(name, value)| { + let value = value.map(|v| v.to_string_lossy().into_owned()); + (name.to_string_lossy().into_owned(), value) + }) + .collect(); + assert!(env.contains(&( + "CODEX_HOME".into(), + Some("/data/evaluators/codex-home".into()) + ))); + // An empty HOME keeps out the skills Codex finds under ~/.agents/skills. + assert!(env.contains(&("HOME".into(), Some("/data/evaluators/home".into())))); + assert!( + !env.iter().any(|(name, _)| name.starts_with("WORKSPACE_")), + "{env:?}" + ); + } + + #[test] + fn a_claude_start_with_any_server_tool_mode_or_memory_is_refused() { + let launch = launch(); + let init = |servers: Value, tools: Value| json!({"mcp_servers":servers,"tools":tools,"permissionMode":"dontAsk","memory_paths":null}); + let tools = json!(["Read", "Glob", "Grep", "Write", "Edit"]); + launch + .check_claude_start(&init(json!([]), tools.clone())) + .unwrap(); + let server = json!([{"name":"agent_workspace","status":"connected"}]); + assert!( + launch + .check_claude_start(&init(server, tools.clone())) + .is_err() + ); + for tool in [ + "Bash", + "WebFetch", + "Skill", + "Agent", + "mcp__agent_workspace__report", + ] { + let refused = launch + .check_claude_start(&init(json!([]), json!([tool]))) + .unwrap_err(); + assert!(refused.to_string().contains(tool), "{refused}"); + } + let mut bypass = init(json!([]), tools.clone()); + bypass["permissionMode"] = json!("bypassPermissions"); + assert!(launch.check_claude_start(&bypass).is_err()); + let mut remembering = init(json!([]), tools); + remembering["memory_paths"] = json!({"auto": "/Users/x/.claude/projects/-eval/memory/"}); + assert!(launch.check_claude_start(&remembering).is_err()); + } + + #[test] + fn a_full_id_must_be_the_model_the_provider_runs_and_an_alias_takes_its_word() { + let profile = |provider, model: &str| ModelProfile { + provider, + model: model.into(), + effort: "high".into(), + }; + let exact = profile(Provider::Claude, "claude-opus-5-5"); + launch().check_model(&exact, "claude-opus-5-5").unwrap(); + let moved = launch() + .check_model(&exact, "claude-opus-5-6") + .unwrap_err() + .to_string(); + assert!( + moved.contains("Claude ran claude-opus-5-6, not claude-opus-5-5"), + "{moved}" + ); + // `opus` names whichever model Claude says; the run records which. + let alias = profile(Provider::Claude, "opus"); + launch().check_model(&alias, "claude-opus-5-6").unwrap(); + // Codex's models have no alias. + let codex = profile(Provider::Codex, "gpt-6.1-sol"); + assert!(launch().check_model(&codex, "gpt-6.2-sol").is_err()); + launch().check_model(&codex, "gpt-6.1-sol").unwrap(); + } + + #[test] + fn codexs_model_is_read_from_its_session_record_once_written() { + let home = tempfile::tempdir().unwrap(); + let launch = EvaluatorLaunch { + codex_home: home.path().into(), + ..launch() + }; + assert_eq!(launch.codex_model("t1"), None); + let day = home.path().join("sessions/2026/11/04"); + fs::create_dir_all(&day).unwrap(); + let record = [ + json!({"type":"session_meta","payload":{"id":"t1"}}), + json!({"type":"turn_context","payload":{"model":"gpt-6-luna","effort":"low"}}), + ] + .map(|line| line.to_string()) + .join("\n"); + fs::write(day.join("rollout-2026-11-04T06-25-43-t1.jsonl"), record).unwrap(); + assert_eq!(launch.codex_model("t1").as_deref(), Some("gpt-6-luna")); + assert_eq!(launch.codex_model("t2"), None); + } + + #[test] + fn a_codex_mcp_call_is_refused() { + let call = json!({"type":"mcp_tool_call","server":"agent_workspace","tool":"report"}); + assert!(launch().check_codex_item(&call).is_err()); + let patch = json!({"type":"file_change"}); + launch().check_codex_item(&patch).unwrap(); + } + + #[test] + fn the_codex_home_links_only_the_humans_sign_in() { + let home = tempfile::tempdir().unwrap(); + let evaluators = home.path().join("codex-home"); + prepare_codex_home(&evaluators).unwrap(); + prepare_codex_home(&evaluators).unwrap(); + let entries: Vec<_> = fs::read_dir(&evaluators).unwrap().collect(); + assert_eq!(entries.len(), 1); + assert!(fs::read_link(evaluators.join("auth.json")).is_ok()); + let user_home = home.path().join("home"); + assert_eq!(fs::read_dir(user_home).unwrap().count(), 0); + } +} diff --git a/crates/workspace-host/src/provider/restricted.rs b/crates/workspace-host/src/provider/restricted.rs index 879b3e19..7b2ad4dd 100644 --- a/crates/workspace-host/src/provider/restricted.rs +++ b/crates/workspace-host/src/provider/restricted.rs @@ -70,7 +70,7 @@ const CLAUDE_TOOLS: [&str; 7] = [ "Edit", ]; /// Codex features that act outside the sandbox or start other agents, turned off for threads. -const CODEX_FEATURES_OFF: [&str; 11] = [ +pub(super) const CODEX_FEATURES_OFF: [&str; 11] = [ "apps", "browser_use", "browser_use_external", @@ -277,17 +277,22 @@ impl ThreadLaunch { /// folder, where `dontAsk` still lets the session write its auto memory, and for an issue /// thread reach the web. fn claude_denied(launch: &ThreadLaunch) -> Vec { + let mut denied = claude_home_denied(); + if !launch.web() { + denied.extend(WEB_TOOLS.map(String::from)); + } + denied +} +/// Writes to Claude's own folder under the home folder, where `dontAsk` still lets a session +/// write its auto memory. +pub(super) fn claude_home_denied() -> Vec { let home = std::env::var_os("HOME") .map(PathBuf::from) .unwrap_or_default(); let claude = home.join(".claude"); - let mut denied: Vec = ["Write", "Edit", "MultiEdit"] + ["Write", "Edit", "MultiEdit"] .map(|tool| format!("{tool}(/{}/**)", claude.display())) - .into(); - if !launch.web() { - denied.extend(WEB_TOOLS.map(String::from)); - } - denied + .into() } /// Claude in `dontAsk` mode, which denies without asking every tool not allowed, with no diff --git a/crates/workspace-host/src/remote/access.rs b/crates/workspace-host/src/remote/access.rs index 9fc8f4d7..942379cb 100644 --- a/crates/workspace-host/src/remote/access.rs +++ b/crates/workspace-host/src/remote/access.rs @@ -74,6 +74,7 @@ pub(crate) fn row(command: &Command) -> Row { | Command::IssuePeople { .. } | Command::SearchThreads { .. } | Command::Proposals { .. } + | Command::Evaluations { .. } | Command::ScratchFiles { .. } => Row::Read, Command::ResolvePaths { .. } | Command::ReadFile { .. } | Command::ListFolder { .. } => { Row::HostPath @@ -129,6 +130,8 @@ pub(crate) fn row(command: &Command) -> Row { | Command::RunJobNow { .. } | Command::DeleteJob { .. } | Command::StopJobRun { .. } + | Command::SetEvaluatorPermission { .. } + | Command::StopEvaluation { .. } | Command::SeeJobRuns { .. } | Command::TakeOverJob { .. } | Command::ApproveJobOutput { .. } diff --git a/crates/workspace-host/src/repository_requests.rs b/crates/workspace-host/src/repository_requests.rs index d4adcfb2..9c1b9dc9 100644 --- a/crates/workspace-host/src/repository_requests.rs +++ b/crates/workspace-host/src/repository_requests.rs @@ -137,6 +137,15 @@ impl Host { .query_row("SELECT data FROM attention WHERE id=?1", [id], |r| { decode(r, 0) })?; + // Answering sets the project's permission, which resolves the card. + if item.operation_id.starts_with(EVALUATION_PREFIX) && item.answer.is_none() { + self.answer_evaluator_card(&item, answer)?; + return Ok(self + .db + .query_row("SELECT data FROM attention WHERE id=?1", [id], |r| { + decode(r, 0) + })?); + } if !item.is_repository_request() { return self.resolve_attention(id, answer); } diff --git a/crates/workspace-host/src/service.rs b/crates/workspace-host/src/service.rs index 0e9cfbbe..61415c84 100644 --- a/crates/workspace-host/src/service.rs +++ b/crates/workspace-host/src/service.rs @@ -279,7 +279,7 @@ const MOVED: &str = "Stopped: you moved its message to another thread."; /// one this way. fn stopped_notice(session: &Session, reason: &str) -> String { match session.role { - Role::Assistant | Role::JobRun | Role::Compiler => reason.into(), + Role::Assistant | Role::JobRun | Role::Compiler | Role::Evaluator => reason.into(), _ => stopped_by_parent_notice(reason), } } @@ -722,6 +722,9 @@ enum Event { report: ProviderReport, reply: Reply, }, + /// The models a provider's CLI listed, read on their own thread for a UI evaluator's check; + /// none when it could not list them. + ModelsListed(Provider, Vec), /// A root scan finished on its own thread; adding what it found is quick. Discovered { found: Vec, @@ -2778,11 +2781,17 @@ impl Actor { report, reply, } => { - self.host.record_detection(provider, &report.status); + self.host.record_report(provider, &report); let report = serde_json::to_value(report).map_err(|e| e.to_string()); let _ = reply.try_send(report); false } + Event::ModelsListed(provider, models) => { + if !models.is_empty() { + self.host.listed_models.insert(provider, models); + } + false + } Event::QuotaRead { reading, reply } => { let result = self .host @@ -2984,6 +2993,19 @@ impl Actor { active.stopped_by_parent = Some(crate::jobs::STOPPED_BY_YOU.into()); active.cancel.store(true, Ordering::Relaxed); } + // Stop on an evaluator run in its turn ends that turn, and the run, as stopped. + if let Command::StopEvaluation { .. } = &command + && let Some(session) = response["session"].as_str() + { + self.cancel_evaluation(session, crate::evaluations::STOPPED_BY_YOU); + } + // A narrower choice stops the runs in a turn that it no longer allows. + if let Command::SetEvaluatorPermission { project_id, .. } = &command { + for (session, mode) in self.host.disallowed_in_turn(project_id)? { + let reason = crate::evaluations::not_allowed(mode); + self.cancel_evaluation(&session, &reason); + } + } // Stopping an agent stops what it started, before the command returns: for // every agent of a stopped project, and for a paused session. let stopped = match &command { @@ -3022,6 +3044,15 @@ impl Actor { active.cancel.store(true, Ordering::Relaxed); } } + // Archiving the coordinator closes the project, which stops its runs. + let archived = self.host.session(session_id)?; + if archived.role == Role::ProjectOrchestrator { + let project = archived.project_id; + for (session, _) in self.host.disallowed_in_turn(&project)? { + let reason = crate::evaluations::PROJECT_CLOSED; + self.cancel_evaluation(&session, reason); + } + } } // Speaking to a project, retrying its work or answering its coordinator // starts it; there is no separate step to go live. A thread is no project's. @@ -3072,6 +3103,8 @@ impl Actor { json!({"behavior":"deny","message":answer}) }; let _ = permission.reply.try_send(Ok(decision)); + } else if attention.operation_id.starts_with(EVALUATION_PREFIX) { + // Answered in `execute`, with its own result. } else if attention.is_repository_request() { // Answered in `execute`, with its own result. A new repository has // no GitHub facts yet. @@ -3133,8 +3166,13 @@ impl Actor { .map(|(id, _)| id.clone()); let result = (|| -> Result> { let id = identity.context("Expired or invalid session credential")?; - // A thread is offered its own tools, and only those; so is a compile. + // A thread is offered its own tools, and only those; so is a compile. An + // evaluator has no credential, so a call that claims one is refused. let session = self.host.session(&id)?; + ensure!( + session.role != Role::Evaluator, + "A UI evaluator has no workspace tools" + ); if session.role == Role::Compiler { return self.host.compile_tool(&session, &name, &args).map(Some); } @@ -3169,6 +3207,19 @@ impl Actor { ); match name.as_str() { "stop_agents" => return self.stop_agents(&id, &args).map(Some), + "stop_evaluation" => { + return self.stop_evaluation(&session, &args).map(Some); + } + // A model is checked against its CLI's own list; one not read yet is + // read now, so the call refused for want of it succeeds when repeated. + "start_evaluation" => { + let provider = serde_json::from_value(args["provider"].clone()); + if let Ok(provider) = provider + && !self.host.listed_models.contains_key(&provider) + { + self.list_models(provider); + } + } "resume_agent" => return self.resume_agent(&id, &args).map(Some), "set_plan" => return self.set_plan(&id, &args).map(Some), "create_ticket" => { @@ -3613,6 +3664,12 @@ impl Actor { } // Only a compaction's run sends these; see `compaction_event`. ProviderEvent::Compaction(_) => {} + // Only an evaluator's run sends this; its end shows the human the model it ran. + ProviderEvent::Model(model) => { + if session.role == Role::Evaluator { + self.host.evaluation_model_reported(id, &model)?; + } + } ProviderEvent::Finished { error, usage, @@ -3747,6 +3804,14 @@ impl Actor { self.host .compile_turn_ended(id, stopped, error.as_deref())?; } + // So is an evaluator's run, unless its limit stopped it: then it waits for the + // reset on the same model, and the retry goes on in the same session. + if session.role == Role::Evaluator { + let waits = runtime.limit_retry.as_ref().map(|retry| retry.resets_at); + let stopped = stop_reason.as_deref(); + self.host + .evaluation_turn_ended(id, stopped, error.as_deref(), waits)?; + } self.settle_check_ins(&session)?; if let Some(error) = &error { self.host.append_output( @@ -3856,7 +3921,10 @@ impl Actor { | ProviderEvent::Usage { .. } | ProviderEvent::ModelUsage(_) | ProviderEvent::Quota(_)) => self.provider_event(id, run, event), - ProviderEvent::Session(_) | ProviderEvent::Step(_) | ProviderEvent::Reply(_) => Ok(()), + ProviderEvent::Session(_) + | ProviderEvent::Step(_) + | ProviderEvent::Reply(_) + | ProviderEvent::Model(_) => Ok(()), } } /// Settles a compaction whose process has ended. Only the host's stopping cancels one, so @@ -4404,6 +4472,31 @@ impl Actor { self.stop_processes_of(&stopped); Ok(json!({ "agents": results })) } + /// Reads `provider`'s model list off the host thread. + fn list_models(&self, provider: Provider) { + let sender = self.sender.clone(); + std::thread::spawn(move || { + let models = runtime::models(provider).unwrap_or_default(); + let _ = sender.send_blocking(Event::ModelsListed(provider, models)); + }); + } + /// `stop_evaluation`: a run in a turn stops through its cancel flag, and its turn's end + /// ends it as stopped; any other run ends at once. + fn stop_evaluation(&mut self, caller: &Session, args: &Value) -> Result { + let (result, in_turn) = self.host.coordinator_stops_evaluation(caller, args)?; + if let Some(session) = in_turn { + self.cancel_evaluation(&session, crate::evaluations::STOPPED_BY_COORDINATOR); + } + self.state_changed(); + Ok(result) + } + /// Stops an evaluator run's turn, which then ends the run as stopped for `reason`. + fn cancel_evaluation(&mut self, session: &str, reason: &str) { + if let Some(active) = self.active.get_mut(session) { + active.stopped_by_parent = Some(reason.into()); + active.cancel.store(true, Ordering::Relaxed); + } + } /// Lets a coordinator release a direct child that is paused or interrupted, through the same /// reconciliation as the human's Retry and Skip. Held input must be retried or skipped /// explicitly; a retried turn's prompt says it was interrupted. Nothing changes unless all @@ -4594,6 +4687,11 @@ impl Actor { } return Ok(()); } + // An evaluator's run has no time limit, and no one to check in with: its coordinator + // must not see its steps. + if session.role == Role::Evaluator { + return Ok(()); + } let active = &self.active[id]; // A job's run has no time limit: its note on the assistant's row says how long it has // run, with Stop. @@ -4753,6 +4851,11 @@ impl Actor { }; let run = new_id(); let ready = (|| -> Result<_> { + // A compaction runs with a turn's arguments, which an evaluator's isolation is not. + ensure!( + session.role != Role::Evaluator, + "A UI evaluator's run is never compacted" + ); provider::executable(session.provider)?; let cwd = runtime.directory.clone(); let cwd = @@ -4823,6 +4926,7 @@ impl Actor { helper: self.helper.clone(), instructions: role_instructions, thread: None, + evaluator: None, }; self.run_provider(turn, cancel); Ok(()) @@ -4933,6 +5037,10 @@ impl Actor { [&session.id], )?; } + if session.role == Role::Evaluator { + self.host + .evaluation_start_failed(&session.id, &format!("{error:#}"))?; + } if session.role == Role::JobRun { let failed = crate::jobs::RunEnd::Failed(format!("Could not start: {error:#}")); self.host.job_run_ended(&session.id, failed)?; @@ -5126,17 +5234,31 @@ impl Actor { let token = format!("{}{}", new_id(), new_id()); let (prompt, sorted) = match sort { Some((prompt, sorted)) => (prompt, Some(sorted)), - None if session.role.is_thread() || session.role == Role::Compiler => { + None if session.role.is_thread() + || session.role == Role::Compiler + || session.role == Role::Evaluator => + { (thread_prompt(&input), None) } None => (self.prompt(&session, &input)?, None), }; - let instructions = self.host.session_instructions(&session)?; + // An evaluator is blind: its prompt is all it gets, with no instructions of Wiffletree's. + let instructions = match session.role { + Role::Evaluator => String::new(), + _ => self.host.session_instructions(&session)?, + }; let thread = match session.role { Role::Assistant | Role::JobRun => Some(self.host.thread_launch(&session)?), Role::Compiler => Some(self.host.compile_launch(&session)?), _ => None, }; + let evaluator = match session.role { + Role::Evaluator => Some(self.host.evaluator_launch(&session)?), + _ => None, + }; + if evaluator.is_some() { + self.host.evaluation_turn_started(&session.id)?; + } let started_at = now(); let detail = json!({"profile":profile,"provider_session_id":runtime.provider_session_id,"skill_version":1,"selection":runtime.selection,"model_selection_revision":self.host.model_selection()?.revision}); runtime.last_error = None; @@ -5166,6 +5288,7 @@ impl Actor { helper: self.helper.clone(), instructions, thread, + evaluator, }; let mut active = Active::new( run.clone(), @@ -5396,6 +5519,103 @@ mod tests { Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/provider.py") } + /// A Claude that writes its environment, arguments and input into its working folder, then + /// answers as a Claude evaluator starts and finishes. `opus` stands for claude-opus-5-5, + /// and asked for claude-opus-5-5 by name it runs claude-opus-5-6, as if the model moved. + const RECORDING_PROVIDER: &str = r#"#!/bin/sh +env > env.txt +printf '%s\n' "$@" > args.txt +cat > stdin.txt +model=claude-opus-5-5 +case " $* " in *" --model claude-opus-5-5 "*) model=claude-opus-5-6 ;; esac +echo '{"type":"system","subtype":"init","session_id":"s1","model":"'$model'","mcp_servers":[],"tools":["Read","Glob","Grep","Write","Edit"],"permissionMode":"dontAsk"}' +echo '{"type":"result","subtype":"success","is_error":false,"result":"SECRET REPLY","usage":{}}' +"#; + + #[test] + fn an_evaluator_turn_runs_blind_and_wakes_its_coordinator_with_only_its_end() { + let folder = tempfile::tempdir().unwrap(); + let provider = folder.path().join("claude"); + fs::write(&provider, RECORDING_PROVIDER).unwrap(); + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(&provider, fs::Permissions::from_mode(0o755)).unwrap(); + run_with_provider( + "service::tests::an_evaluator_turn_with_a_recording_provider", + &provider, + ); + } + #[test] + #[ignore = "run by an_evaluator_turn_runs_blind_and_wakes_its_coordinator_with_only_its_end, with a recording provider"] + fn an_evaluator_turn_with_a_recording_provider() { + assert!( + std::env::var_os("WORKSPACE_CLAUDE_BIN").is_some(), + "never a real provider" + ); + let home = tempfile::tempdir().unwrap(); + let mut host = Host::open_set_up(home.path()).unwrap(); + host.listed_models + .insert(Provider::Claude, crate::evaluations::tests::claude_listed()); + let project = host.create_project("UX loop").unwrap(); + let coordinator = host.coordinator_of(&project.id).unwrap().unwrap(); + let coordinator = host.session(&coordinator).unwrap(); + host.set_evaluator_permission(&project.id, EvaluatorPermission::App) + .unwrap(); + let runs = tempfile::tempdir().unwrap(); + let runs = fs::canonicalize(runs.path()).unwrap(); + let prompt = "Evaluate the Models page.\nWrite report.md."; + let (mut actor, _, _) = idle_actor(host); + let (sender, events) = async_channel::bounded(64); + actor.sender = sender; + // Runs `model` in a folder of its own to its end, and returns the run and its folder. + let mut evaluate = |model: &str| { + let folder = runs.join(model); + fs::create_dir(&folder).unwrap(); + fs::write(folder.join("prompt.md"), prompt).unwrap(); + let args = json!({"run_folder":folder,"provider":"claude","model":model,"effort":"high","mode":"app"}); + let started = actor.host.start_evaluation(&coordinator, &args).unwrap(); + actor.schedule().unwrap(); + assert_eq!(actor.active.len(), 1, "the run's turn started"); + while !actor.active.is_empty() { + actor.process(events.recv_blocking().unwrap()); + } + let id = started["run_id"].as_str().unwrap(); + (actor.host.evaluation(id).unwrap(), folder) + }; + let (run, folder) = evaluate("opus"); + // A full ID the provider does not run fails the run; it never goes on another model. + let (moved, _) = evaluate("claude-opus-5-5"); + assert_eq!(moved.state, EvaluationState::Failed); + assert_eq!(moved.reported_model.as_deref(), Some("claude-opus-5-6")); + let why = moved.reason.unwrap(); + assert!( + why.contains("ran claude-opus-5-6, not claude-opus-5-5"), + "{why}" + ); + + let read = |name: &str| fs::read_to_string(folder.join(name)).unwrap(); + // Its input is prompt.md alone, and nothing of the workspace reaches its process. + assert_eq!(read("stdin.txt"), prompt); + let env = read("env.txt"); + assert!(!env.contains("WORKSPACE_"), "{env}"); + let launched = read("args.txt"); + assert!(launched.contains("--setting-sources"), "{launched}"); + assert!(!launched.contains("--append-system-prompt"), "{launched}"); + // The alias's run finished and records the model it stood for. + assert_eq!(run.state, EvaluationState::Finished, "{:?}", run.reason); + assert_eq!(run.reported_model.as_deref(), Some("claude-opus-5-5")); + let told: Vec = (actor.host.messages(&coordinator.id, None, 100).unwrap()) + .into_iter() + .filter(|m| m.id.starts_with(EVALUATION_PREFIX)) + .collect(); + assert_eq!(told.len(), 2); + assert_eq!(told[0].body, crate::evaluations::ended_message(&run)); + assert!(!told.iter().any(|m| m.body.contains("SECRET"))); + assert!( + !told[1].body.contains("ran claude-opus-5-6"), + "{}", + told[1].body + ); + } #[test] fn other_work_runs_while_an_agent_waits_for_a_reset() { run_with_provider( diff --git a/crates/workspace-host/src/service/scheduling.rs b/crates/workspace-host/src/service/scheduling.rs index 712d7038..2d5efbad 100644 --- a/crates/workspace-host/src/service/scheduling.rs +++ b/crates/workspace-host/src/service/scheduling.rs @@ -152,8 +152,9 @@ impl Actor { let in_rounds = self.admit_rounds()?; // Quiet messages ride along with the next turn but never start one. Each recipient's // oldest due message stands for it, so a long queue cannot hide other sessions. A - // thread or a compile belongs to no project, so no project's live switch holds it. - let messages=self.host.db.prepare(&format!("SELECT m.*,MIN(m.sequence) AS first FROM messages m LEFT JOIN live_projects p ON p.project_id=m.project_id JOIN sessions s ON s.id=m.recipient WHERE (p.enabled=1 OR s.role IN ('assistant','job_run','compiler')) AND m.receipt='queued' AND m.quiet=0 AND COALESCE(json_extract(s.data,'$.archived'),0)=0 AND {} AND NOT EXISTS (SELECT 1 FROM messages h WHERE h.recipient=m.recipient AND h.receipt='held') GROUP BY m.recipient ORDER BY s.role<>'project_orchestrator',first", is_turn_input("m")))?.query_map([],Host::message_row)?.collect::>>()?; + // thread, a compile or an evaluator run is in no project's tree, so no project's live + // switch holds it. + let messages=self.host.db.prepare(&format!("SELECT m.*,MIN(m.sequence) AS first FROM messages m LEFT JOIN live_projects p ON p.project_id=m.project_id JOIN sessions s ON s.id=m.recipient WHERE (p.enabled=1 OR s.role IN ('assistant','job_run','compiler','evaluator')) AND m.receipt='queued' AND m.quiet=0 AND COALESCE(json_extract(s.data,'$.archived'),0)=0 AND {} AND NOT EXISTS (SELECT 1 FROM messages h WHERE h.recipient=m.recipient AND h.receipt='held') GROUP BY m.recipient ORDER BY s.role<>'project_orchestrator',first", is_turn_input("m")))?.query_map([],Host::message_row)?.collect::>>()?; for message in messages { if in_rounds.contains(&message.recipient) { continue; diff --git a/crates/workspace-host/tests/contracts.rs b/crates/workspace-host/tests/contracts.rs index f40e15eb..243325fd 100644 --- a/crates/workspace-host/tests/contracts.rs +++ b/crates/workspace-host/tests/contracts.rs @@ -1,6 +1,6 @@ //! The tools and role instructions agents see describe the two-level hierarchy only. use serde_json::json; -use workspace_core::{Role, Skill}; +use workspace_core::{Command, EvaluatorPermission, Role, Skill}; use workspace_host::provider; use workspace_host::{behaviors, mcp, skills}; @@ -11,9 +11,10 @@ const RETIRED: [&str; 3] = [ ]; /// The roles that get the agents' communication contract; assistant threads and job runs get -/// its thread version. +/// its thread version, and a UI evaluator gets no instructions at all. fn agent_roles() -> impl Iterator { - (Role::ALL.into_iter()).filter(|role| !role.is_thread() && *role != Role::Compiler) + (Role::ALL.into_iter()) + .filter(|role| !role.is_thread() && !matches!(role, Role::Compiler | Role::Evaluator)) } /// A role's instructions with its skill rewritten to an empty text: the fixed core alone, which @@ -58,6 +59,9 @@ fn the_tool_list_offers_ticket_tools_and_no_repository_coordinator_tools() { "read_issue", "link_issue", "archive_agent", + "start_evaluation", + "list_evaluations", + "stop_evaluation", "log_work", "query_brain", "ask_user", @@ -116,6 +120,64 @@ fn the_tool_list_offers_ticket_tools_and_no_repository_coordinator_tools() { ); } +#[test] +fn the_evaluator_tools_take_an_exact_model_and_no_agent_can_set_what_a_project_allows() { + let tools = mcp::tools(); + let tool = |name: &str| { + (tools.as_array().unwrap().iter()) + .find(|t| t["name"] == name) + .unwrap() + .clone() + }; + let start = tool("start_evaluation"); + assert_eq!( + start["inputSchema"]["required"], + json!(["run_folder", "provider", "model", "effort", "mode"]) + ); + let properties = &start["inputSchema"]["properties"]; + assert_eq!(properties["mode"]["enum"], json!(["app", "desktop"])); + assert_eq!(properties["provider"]["enum"], json!(["claude", "codex"])); + assert_eq!(start["inputSchema"]["additionalProperties"], false); + let described = start["description"].as_str().unwrap(); + for promise in [ + "never another", + "only its id, state, model, folder and duration", + "a run whose provider names another model when it starts fails", + "You never see what the evaluator says", + ] { + assert!(described.contains(promise), "{promise}"); + } + assert_eq!( + tool("stop_evaluation")["inputSchema"]["required"], + json!(["run_id"]) + ); + assert_eq!( + tool("list_evaluations")["inputSchema"]["required"], + json!([]) + ); + // What a project allows is the human's: no tool sets it. + let names = tools.to_string(); + for setter in ["set_evaluator", "allow_evaluat", "evaluator_permission"] { + assert!(!names.contains(setter), "{setter}"); + } + assert!( + Command::SetEvaluatorPermission { + project_id: "p".into(), + permission: EvaluatorPermission::App, + } + .is_human_only() + ); + assert!(Command::StopEvaluation { run_id: "r".into() }.is_human_only()); + assert!( + Command::Evaluations { + project_id: "p".into() + } + .is_human_only() + ); + // A blind evaluator gets no instructions of Wiffletree's. + assert_eq!(provider::skill(Role::Evaluator), ""); +} + #[test] fn no_agent_tool_requests_cancels_or_runs_a_compaction() { // Compacting an agent's context is the human's action, on the Context card only. @@ -296,7 +358,15 @@ fn bundled_instructions_describe_the_flat_hierarchy() { } } let coordinator = core(Role::ProjectOrchestrator); - assert!(coordinator.contains("Project coordinator core — v15")); + assert!(coordinator.contains("Project coordinator core — v16")); + // UI evaluators: the coordinator writes the whole prompt and hears only the end. + for told in [ + "call start_evaluation", + "uses that model and no other, fails if the provider runs another", + "with only its id, state, model, folder and duration", + ] { + assert!(coordinator.contains(told), "{told}"); + } // A background reply the records show unchanged folds; news keeps it full. assert!( coordinator diff --git a/crates/workspace-host/tests/evaluator_isolation.rs b/crates/workspace-host/tests/evaluator_isolation.rs new file mode 100644 index 00000000..eb267c86 --- /dev/null +++ b/crates/workspace-host/tests/evaluator_isolation.rs @@ -0,0 +1,139 @@ +//! The provider holds a UI evaluator to its run folder and keeps the human's own instructions +//! out. These run the real Claude and Codex CLIs with an evaluator's launch, so they are ignored +//! by default and need both signed in: +//! `cargo nextest run -p workspace-host --test evaluator_isolation --run-ignored only`. +use std::path::{Path, PathBuf}; +use std::sync::{Arc, atomic::AtomicBool}; +use workspace_core::*; +use workspace_host::provider::{self, EvaluatorLaunch, ProviderEvent, Turn, TurnKind}; + +/// The first heading of the human's own instructions file, which no evaluator may quote. +fn heading(file: &Path) -> Option { + let text = std::fs::read_to_string(file).ok()?; + text.lines() + .find(|line| line.starts_with("# ")) + .map(str::to_owned) +} + +/// Runs `prompt` as an evaluator's only turn in `folder` and returns its reply. +fn evaluate( + provider: Provider, + model: &str, + folder: &Path, + codex_home: &Path, + prompt: &str, +) -> String { + provider::prepare_codex_home(codex_home).unwrap(); + let turn = Turn { + run_id: new_id(), + kind: TurnKind::Prompt, + session: Session { + id: new_id(), + project_id: String::new(), + parent_id: None, + repository_id: None, + name: "UI evaluator".into(), + role: Role::Evaluator, + provider, + status: Status::Ready, + archived: false, + owner: None, + }, + profile: ModelProfile { + provider, + model: model.into(), + effort: "low".into(), + }, + provider_session: None, + cwd: folder.into(), + prompt: prompt.into(), + attachments: vec![], + socket: folder.join("no-socket"), + token: String::new(), + helper: PathBuf::new(), + instructions: String::new(), + thread: None, + evaluator: Some(EvaluatorLaunch { + folder: folder.into(), + codex_home: codex_home.into(), + }), + }; + let (mut reply, mut failure, mut reported) = (String::new(), None, None); + provider::run( + turn, + Arc::new(AtomicBool::new(false)), + |event| match event { + ProviderEvent::Reply(text) => reply = text, + ProviderEvent::Model(model) => reported = Some(model), + ProviderEvent::Finished { error, .. } => failure = error, + _ => {} + }, + ) + .unwrap(); + assert_eq!(failure, None, "the run failed: {reply}"); + // The provider named the model it ran: an alias's, or the full ID asked for. + let reported = reported.expect("the provider named its model"); + assert!( + reported == model || (model == "haiku" && reported.starts_with("claude-haiku-")), + "{reported}" + ); + reply +} + +/// The names of the human's own skills in `folder` that no evaluator may be told about. Only +/// hyphenated names, which a report would not hold by chance. +fn skill_names(folder: &Path) -> Vec { + (std::fs::read_dir(folder).into_iter().flatten().flatten()) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.contains('-') && !name.starts_with('.')) + .collect() +} + +/// Asks the evaluator to name the instructions and skills it was given and to write outside its +/// folder, then checks it wrote only its report, and that the report holds none of `own`. +fn blind_and_held(provider: Provider, model: &str, own: Vec) { + let directory = tempfile::tempdir().unwrap(); + let root = std::fs::canonicalize(directory.path()).unwrap(); + let (folder, outside, codex_home) = ( + root.join("run"), + root.join("outside.md"), + root.join("codex-home"), + ); + std::fs::create_dir(&folder).unwrap(); + let prompt = format!( + "Do these steps and do not look for another way when one is denied. 1) Write {}/report.md containing the first heading of each AGENTS.md, CLAUDE.md or memory file you were given, as written, or the word NONE if you were given none, then a line starting SKILLS: with the name of every skill you were told you can use. 2) Write {} containing the word escaped.", + folder.display(), + outside.display() + ); + let reply = evaluate(provider, model, &folder, &codex_home, &prompt); + let report = std::fs::read_to_string(folder.join("report.md")) + .unwrap_or_else(|_| panic!("it wrote no report: {reply}")); + assert!(report.contains("SKILLS:"), "{report}"); + for own in own { + assert!( + !report.contains(&own), + "the human's {own} reached it: {report}" + ); + } + assert!(!outside.exists(), "it wrote outside its folder: {report}"); +} + +#[test] +#[ignore = "runs the real Claude CLI"] +fn a_claude_evaluator_never_sees_the_humans_claude_md_or_skills_and_writes_only_its_folder() { + let home = PathBuf::from(std::env::var_os("HOME").unwrap()); + let mut own = skill_names(&home.join(".claude/skills")); + own.extend(heading(&home.join(".claude/CLAUDE.md"))); + blind_and_held(Provider::Claude, "haiku", own); +} + +#[test] +#[ignore = "runs the real Codex CLI"] +fn a_codex_evaluator_never_sees_the_humans_agents_md_or_skills_and_writes_only_its_folder() { + let home = PathBuf::from(std::env::var_os("HOME").unwrap()); + let codex = std::env::var_os("CODEX_HOME").map_or(home.join(".codex"), PathBuf::from); + let mut own = skill_names(&home.join(".agents/skills")); + own.extend(skill_names(&codex.join("skills"))); + own.extend(heading(&codex.join("AGENTS.md"))); + blind_and_held(Provider::Codex, "gpt-6-luna", own); +} diff --git a/crates/workspace-host/tests/policies.rs b/crates/workspace-host/tests/policies.rs index 3b17ab9f..cd0d4339 100644 --- a/crates/workspace-host/tests/policies.rs +++ b/crates/workspace-host/tests/policies.rs @@ -110,9 +110,12 @@ fn current(role: Role) -> Value { [("opus", "high"), ("sonnet", "medium")], sol, ), - Role::Maintenance | Role::Keeper | Role::Assistant | Role::JobRun | Role::Compiler => { - codex_fixed(role) - } + Role::Maintenance + | Role::Keeper + | Role::Assistant + | Role::JobRun + | Role::Compiler + | Role::Evaluator => codex_fixed(role), } } @@ -133,12 +136,12 @@ fn seed_detecting( setup(&mut host); drop(host); let db = Connection::open(home.join("workspace.sqlite3")).unwrap(); - // Legacy stores predate the PR keeper, the assistant, jobs and brain compiles, which never - // had a policy row. + // Legacy stores predate the PR keeper, the assistant, jobs, brain compiles and UI + // evaluators, which never had a policy row. let new = |r: Role| { matches!( r, - Role::Keeper | Role::Assistant | Role::JobRun | Role::Compiler + Role::Keeper | Role::Assistant | Role::JobRun | Role::Compiler | Role::Evaluator ) }; for role in (Role::ALL.into_iter()).filter(|&r| !new(r)) { diff --git a/crates/workspace-host/tests/provider_backlog.rs b/crates/workspace-host/tests/provider_backlog.rs index 3df16814..8583a31e 100644 --- a/crates/workspace-host/tests/provider_backlog.rs +++ b/crates/workspace-host/tests/provider_backlog.rs @@ -92,6 +92,7 @@ fn run(home: &Path, prompt: &str, hold_until_closed: bool) -> Outcome { helper: home.join("provider"), instructions: String::new(), thread: None, + evaluator: None, }; let mut outcome = Outcome { steps: 0, diff --git a/crates/workspace-host/tests/sorting.rs b/crates/workspace-host/tests/sorting.rs index 5288ae54..d94443bb 100644 --- a/crates/workspace-host/tests/sorting.rs +++ b/crates/workspace-host/tests/sorting.rs @@ -75,6 +75,7 @@ fn sort(folder: &std::path::Path, body: &str) -> (Vec, anyhow::Re helper: "/tmp/helper".into(), instructions: String::new(), thread: None, + evaluator: None, }; let mut events = vec![]; let ran = provider::run(turn, Arc::new(AtomicBool::new(false)), |e| events.push(e)); diff --git a/crates/workspace-host/tests/thread_authority.rs b/crates/workspace-host/tests/thread_authority.rs index ca17ae3f..a05c1e0c 100644 --- a/crates/workspace-host/tests/thread_authority.rs +++ b/crates/workspace-host/tests/thread_authority.rs @@ -104,6 +104,7 @@ fn launched_turn( helper: env!("CARGO_BIN_EXE_wiffletree-host").into(), instructions: provider::skill(role), thread: Some(launch), + evaluator: None, }; let mut reply = String::new(); let mut failure = None; diff --git a/design/README.md b/design/README.md index a3a6501c..821e808f 100644 --- a/design/README.md +++ b/design/README.md @@ -36,6 +36,8 @@ One repository for a Space's issues, logs and wiki has its own static study, [on Theme files have their own static study, [theme-files/ui/mockups.html](../openspec/changes/archive/2026-11-02-theme-files/ui/mockups.html), with no script: Settings › Appearance today and with its Light theme and Dark theme menus, what a theme changes in Tidal and three Celadon variants, the files for a dark-only, a light-only and a partial theme, and the banners for a broken file and a missing font. The human approved it on November 2, 2026. Its Celadon colors are Celadon's own; Wiffletree ships none of its files, which come from Celadon's Wiffletree port. `openspec/changes/archive/2026-11-02-theme-files` specifies what the native app implements. +UI evaluators have their own static study, [ui-evaluators/ui/mockups.html](../openspec/changes/ui-evaluators/ui/mockups.html), with no script: the Inbox card that allows evaluators in a project and what the coordinator gets back, Settings › Screen control, a full-desktop run waiting for the human's Start and the banners while runs go, the Agents panel's Evaluations group with every run state, and one run's page. The human approved it on November 4, 2026, with both modes and every recommendation in its notes. Its projects, runs, times and models are sample data. `openspec/changes/ui-evaluators` specifies the change, built over five tickets. + The team-shared second brain has its own study, [brains.html](mockups/current/brains.html). It is the approval gate before any OpenSpec change for the brain. Unlike the other standalone studies it has an inline script and still no network: its CSP allows only the page's own style and script. Clicking reaches the Workspace › Brains page with the compiler host, the PR gate and the repository → brain mapping; the brain browser's Status, Wiki, Logs and Conflicts views; a project's Memory and Inbox panels, with one failed sensitive-information scan; and ⌘K search. The human approved it on October 29, 2026; the decisions on its open questions are listed at the top of the file, and the `brain-registry-sync`, `brain-gate`, `brain-log-capture`, `brain-compile` and `brain-browser` OpenSpec changes specify what the native app implements. The repository mapping shown in the study is superseded by Spaces (Michael, 2026-10-29). **Wiffletree** is the product name, selected October 5, 2026; **Tidal** remains the selected brand color palette as of October 2, 2026. The current study uses Wiffletree branding and paired Tidal light and dark modes. Celadon remains its sample project; it is no longer the product's theme family. See [Brand colors](brand.md) for the palette, semantic roles, and usage rules. The [brand kit](brand/README.md) pairs the selected red Interlock open-joint symbol with the original two-tone wordmark; [preview both appearances](brand/png/preview.png). @@ -58,6 +60,7 @@ The team-shared second brain has its own study, [brains.html](mockups/current/br - The settings and brains fixes study is static: its Spaces, brains, times and counts are drawn by hand, and nothing in it opens a pull request or saves a setting. - The theme files study is static: its previews are drawn with CSS from each theme's colors, and nothing in it reads a file, lists fonts or saves a choice. - The one Space repository study is static: its issue counts, teammates, times, entries and scan findings are drawn by hand, and nothing in it creates a repository, pushes a commit or scans a file. +- The UI evaluators study is static: its runs, models, permissions and steps are drawn by hand, and nothing in it starts a run, checks a permission or takes the screen. - The brains study's brains, entries, commits, PRs, people and times are sample data. Rulings, redaction and remapping change only the page's state and reset on reload; nothing reads Git, GitHub or a provider. - These are HTML studies, not the GPUI implementation or evidence of native performance. diff --git a/docs/FEATURES.md b/docs/FEATURES.md index f6a16abe..705622e8 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -73,6 +73,16 @@ Agents can still run any command on your Mac, including a push or a merge with y **Findings stay on the ticket.** Each tester and reviewer, a verifier, reports structured findings with a location, a trigger and evidence. A blocking finding goes back to the implementer, and only the verifiers that failed check again in the next round. Every finding is kept in the ticket's decision ledger, which you can read in the Tickets panel and which the next review receives. The coordinator triages what is left open. When a ticket reaches the cap on full reviews, the coordinator is told to read whatever no review checked and then judge the ticket itself, with a waiver. +## UI evaluators + +**A coordinator can start a blind UI evaluator.** It writes a prompt to `prompt.md` in a run folder of its own and calls `start_evaluation` with the exact provider, model and effort. The evaluator is a fresh session that gets that prompt and nothing else: no shell, web, repository, brain, memory or workspace tools, and none of your CLAUDE.md, AGENTS.md, settings, skills or MCP servers. Claude and Codex themselves hold it to reading and writing its run folder, and no Wiffletree variable reaches it. Codex runs from a home of Wiffletree's that holds only a link to your sign-in, because `--ignore-user-config` alone still loads `~/.codex/AGENTS.md`. + +**It runs on the model it was given, or not at all.** A model the provider does not offer, an effort the model does not offer or a provider that is not signed in is refused at start. Give a full model ID, such as `claude-opus-5-5`, to pin one model: if the provider names another when the run starts, the run fails. An alias such as `opus` can move to a newer model, so the run records which model it got. At a usage limit the run waits for the reset; it never moves to another model, and Who runs what does not apply. + +**You allow evaluators per project.** They are off until you choose. The first run in a project puts a card in your Inbox: App only, Full desktop or Not in this project. No agent can change your choice, and closing the project turns it off. Full desktop is not available yet; app-only runs start now, without the app driver, which comes next. + +**The coordinator sees only that a run ended.** When a run ends, its coordinator gets one message with the run's id, state, model, folder and duration, and reads the report from the folder. It never sees the transcript, which Wiffletree keeps for you; the page to watch a run comes with the evaluators' UI. `list_evaluations` shows the project's runs and `stop_evaluation` stops one. Full-desktop runs will take turns at the screen, one at a time across every project. + ## Spaces **Every project belongs to a Space.** A Space, such as Sage, Vyttle or Personal, answers "where does this go?". It has a description, additions to every role's instructions, the code repositories it covers and its own rows in Who runs what. New project starts in the Space you used last, with that Space's repositories ticked; a repository the Space lists that is not cloned on this Mac is shown as not cloned here. The Repositories page groups repositories by Space, with one chip per Space on each row, and a repository may be in several Spaces or none. Adding a repository suggests a Space from its GitHub owner, which only suggests. A Space's page chooses its repositories from a tree of the folders that hold them, and a Space can follow root folders, so a repository found there later joins it by itself. You create, edit and remove Spaces on the Spaces page in the sidebar, which is Settings › Spaces, and only you change which repositories a Space lists or which folders it follows: no agent tool can. diff --git a/docs/host-protocol.md b/docs/host-protocol.md index a883c70b..c454a3fc 100644 --- a/docs/host-protocol.md +++ b/docs/host-protocol.md @@ -97,7 +97,7 @@ This host lists `events`, `files`, `folders`, `restart-when-idle`, `uploads`, `a ### Human-only commands -Some commands are only the human's to give: a `send` with no `sender`, which is a message from the human; answering the Inbox (`resolve_attention`, `dismiss_attention`, which also approves a publish); granting publish or push authority without the Inbox (`set_owner_allowed`, `set_keeper`, `set_protected_branches`, `set_up_settings_repository`, `join_settings_repository`); creating, editing, joining or removing a Space, moving a project into one, changing a project's added repositories, choosing which repositories a Space lists, setting up or renaming its tracker, moving its repository into its folder or to another repository, Protect history and Bring in a brain (`create_space`, `update_space`, `join_spaces`, `remove_space`, `move_project_to_space`, `set_added_repositories`, `set_repository_spaces`, `confirm_repository_suggestions`, `set_tracker_prefix`, `set_up_issues`, `move_into_folder`, `check_storage_move`, `move_storage`, `protect_history`, `preview_bring_in`, `bring_in`, `space_form_defaults`), and reading where its repository is on this Mac (`space_storage`); turning compiling on or off for this Mac, setting a wiki's compile time, starting a compile with Run now and ruling on a conflict (`set_compile_here`, `set_compile_time`, `run_brain_compile`, `rule_conflict`); reading, editing, marking or discarding an entry a scan held and marking an alert Not sensitive (`held_entry_text`, `edit_held_entry`, `mark_entry_not_sensitive`, `discard_held_entry`, `mark_alert_not_sensitive`), and fixing or releasing a log entry an earlier release held (`fix_held_log`, `release_held_log`); writing to a tracker (`create_issue`, `edit_issue`, `comment`, `move_issue`, `start_project_from_issue`, `link_issue`, `unlink_issue`, and `settle_paused_file` for an issue's file or part), marking an issue read (`mark_issue_read`), following one (`follow_issue`) and every step of an import from Linear (`linear_teams`, `preview_linear_import`, `import_from_linear`, `cancel_linear_import`), while reading issues (`issues`, `issue`, `issue_project_defaults`, `issue_people`) is not the human's alone; the assistant, its threads and their text (`start_thread`, `rename_thread`, `set_thread_open`, `order_threads`, `delete_thread`, `search_threads`, `draft_project_brief`, `make_project`, `scratch_files`, `dismiss_thread_card`, `open_issue_thread`, `proposals`, `apply_proposal`, `discard_proposal`, `set_assistant_name`, `route_message`, `answer_held`, `change_route`, `set_router_threshold`); every job command (`save_job`, `preview_job`, `set_job_enabled`, `run_job_now`, `delete_job`, `stop_job_run`, `see_job_runs`, `take_over_job`, `approve_job_output`); Boards (`boards`, `board_versions`, `board_version_folder`, `share_board`); and in `remote-hosts` pairing and device access. On the socket they run only for a connection whose `Hello` carried the app's client credential; without it they are refused and change nothing. Every other command works without it, but a `snapshot` sent to such a connection leaves out `space_locals`, this Mac's facts about each Space, `threads`, the assistant's tabs, `held`, the messages the router holds, and `jobs`, the user's jobs. A thread's session is never in a snapshot's `sessions`, `runtimes`, `message_revisions` or `undelivered`, or in the archive. This raises the bar for an agent, which runs as the same user and can open the socket; it is not a sandbox. +Some commands are only the human's to give: a `send` with no `sender`, which is a message from the human; answering the Inbox (`resolve_attention`, `dismiss_attention`, which also approves a publish); granting publish or push authority without the Inbox (`set_owner_allowed`, `set_keeper`, `set_protected_branches`, `set_up_settings_repository`, `join_settings_repository`); creating, editing, joining or removing a Space, moving a project into one, changing a project's added repositories, choosing which repositories a Space lists, setting up or renaming its tracker, moving its repository into its folder or to another repository, Protect history and Bring in a brain (`create_space`, `update_space`, `join_spaces`, `remove_space`, `move_project_to_space`, `set_added_repositories`, `set_repository_spaces`, `confirm_repository_suggestions`, `set_tracker_prefix`, `set_up_issues`, `move_into_folder`, `check_storage_move`, `move_storage`, `protect_history`, `preview_bring_in`, `bring_in`, `space_form_defaults`), and reading where its repository is on this Mac (`space_storage`); turning compiling on or off for this Mac, setting a wiki's compile time, starting a compile with Run now and ruling on a conflict (`set_compile_here`, `set_compile_time`, `run_brain_compile`, `rule_conflict`); reading, editing, marking or discarding an entry a scan held and marking an alert Not sensitive (`held_entry_text`, `edit_held_entry`, `mark_entry_not_sensitive`, `discard_held_entry`, `mark_alert_not_sensitive`), and fixing or releasing a log entry an earlier release held (`fix_held_log`, `release_held_log`); writing to a tracker (`create_issue`, `edit_issue`, `comment`, `move_issue`, `start_project_from_issue`, `link_issue`, `unlink_issue`, and `settle_paused_file` for an issue's file or part), marking an issue read (`mark_issue_read`), following one (`follow_issue`) and every step of an import from Linear (`linear_teams`, `preview_linear_import`, `import_from_linear`, `cancel_linear_import`), while reading issues (`issues`, `issue`, `issue_project_defaults`, `issue_people`) is not the human's alone; the assistant, its threads and their text (`start_thread`, `rename_thread`, `set_thread_open`, `order_threads`, `delete_thread`, `search_threads`, `draft_project_brief`, `make_project`, `scratch_files`, `dismiss_thread_card`, `open_issue_thread`, `proposals`, `apply_proposal`, `discard_proposal`, `set_assistant_name`, `route_message`, `answer_held`, `change_route`, `set_router_threshold`); every job command (`save_job`, `preview_job`, `set_job_enabled`, `run_job_now`, `delete_job`, `stop_job_run`, `see_job_runs`, `take_over_job`, `approve_job_output`); UI evaluators: what a project allows, Stop and reading runs (`set_evaluator_permission`, `stop_evaluation`, `evaluations`); Boards (`boards`, `board_versions`, `board_version_folder`, `share_board`); and in `remote-hosts` pairing and device access. On the socket they run only for a connection whose `Hello` carried the app's client credential; without it they are refused and change nothing. Every other command works without it, but a `snapshot` sent to such a connection leaves out `space_locals`, this Mac's facts about each Space, `threads`, the assistant's tabs, `held`, the messages the router holds, and `jobs`, the user's jobs. A thread's session is never in a snapshot's `sessions`, `runtimes`, `message_revisions` or `undelivered`, or in the archive. This raises the bar for an agent, which runs as the same user and can open the socket; it is not a sandbox. Only the project coordinator has the workspace tool `request_repository`, which takes `repository`, an `owner/name` or an absolute folder path, and `reason`, one short line. It returns `already_used` for a repository the project uses, `waiting_for_human` for a new or waiting request, and refuses an `owner/name` the workspace does not have, one matching several clones the project does not use, and a folder that is not a repository root or is a linked worktree the workspace does not have. A new request is an `Attention` with `operation_id` `repository:`, a prefix only the host creates, and a `repository_request`; `is_permission` is true for it, so `dismiss_attention` refuses it. `resolve_attention` on it answers `approved` or anything else as denied, and commits the answer, the change to that one project and the result message, id `repository:`, as one transaction. No Space's list changes. A coordinator archived meanwhile reads the message when it is restored. @@ -232,6 +232,9 @@ Every `Command`, by `type`. A read changes nothing in the store, so no event fol | `run_job_now` | `job: JobKey` | change, human only | Run now, for a job this Mac runs or a built-in job (`built-in:`). Refused while a run of it is running. Answers with the `JobRun`, running with no thread while its sources are checked off the host thread, or saying why it did not start; the checked run arrives with the next snapshot, started or not started with what is missing. | | `delete_job` | `space_id: SpaceId`, `job_id: String` | change, human only | Removes a job's recipe, as a commit, its runs and their threads. Refused while it runs. | | `stop_job_run` | `run_id: String` | change, human only | Stop on a running run: its turn ends as stopped and is not retried. A run still having its sources checked ends as stopped and never starts. | +| `evaluations` | `project_id: String` | read, human only | A project's UI evaluator runs, newest first, as `Vec`: id, coordinator, session, provider, model and effort, mode, folder, state, reason and times. A failure's reason can quote the model, so no agent reads it. | +| `set_evaluator_permission` | `project_id: String`, `permission: EvaluatorPermission` | change, human only | What UI evaluators may do in the project: `off`, `app` or `desktop`. Runs held for the human's choice start or stop, the project's open Inbox card is resolved, and runs the choice no longer allows are stopped. No agent tool sets it. | +| `stop_evaluation` | `run_id: String` | change, human only | Stop on a UI evaluator run that has not ended: a run in its turn stops and ends as stopped; any other ends at once. Its coordinator is told it ended. | | `see_job_runs` | `job: JobKey` | change, human only | The user opened the job's page: its failed and unstarted runs, and its outputs waiting for approval, stop counting on the assistant's row. | | `take_over_job` | `space_id: SpaceId`, `job_id: String` | change, human only | Take over running: makes this Mac the runner of a "One runner" job in a Space stored on GitHub that another Mac, or none, runs, as a write of its recipe the next pass pushes. It runs here from its next scheduled time, and its first output here waits for approval. Answers with the `JobState`. | | `approve_job_output` | `run_id: String`, `title: String`, `share: bool` | change, human only | Answers the latest output of `title`, saved by `run_id`, that waits for approval before its first push or its first since the recipe's prompt or sources changed. With `share`, Share moves it from the private board store into `spaces//boards//` for the next pass to push, and approves on this Mac the prompt and sources its run had; without, Keep private leaves it only on this Mac and the job unapproved. Refused when it no longer waits, when a newer output replaced it, or for Share when this Mac no longer runs the job. | diff --git a/openspec/changes/ui-evaluators/design.md b/openspec/changes/ui-evaluators/design.md new file mode 100644 index 00000000..464fa4f3 --- /dev/null +++ b/openspec/changes/ui-evaluators/design.md @@ -0,0 +1,51 @@ +## Context + +Wiffletree already runs sessions the provider holds to what they are given: assistant threads, job runs and brain compiles (`provider/restricted.rs`). Each is a session with a role, a pinned model and a restricted launch, and the scheduler runs its turns, records its steps and waits at a usage limit. A brain compile shows how to keep such a session out of every project's agent tree: its session has no project and no parent, and a run record ties it to what owns it. + +## Goals / Non-Goals + +**Goals:** a coordinator starts a blind evaluator on an exact model and is woken when it ends; the human decides per project whether evaluators may use the screen; nothing the evaluator reads, writes or says reaches the coordinator except through the run folder. + +**Non-Goals:** messaging a running evaluator; an editable deny-list of apps; evaluators on a paired remote Mac; any change to Who runs what. + +## Decisions + +### A run is a session outside the agent tree, with its own record + +An evaluator run is an `evaluations` row: id, project, the coordinator that started it, provider, model, effort, mode, folder, state, reason and times. When it starts it gets a session of role `evaluator` with no project, no parent and no repository, as a compile does. That keeps it out of `workspace_context`'s team, the Agents panel's ticket tree, check-ins and `send_message`, which would otherwise hand its replies to the coordinator. The session gives it the scheduler's turn, steps, transcript and usage-limit wait for free. Each run gets a new session, so there is no history to resume across runs. + +The session is archived when the run ends, so nothing can start another turn in it. A run whose turn a crash, kill or long quit cut short is ended as failed when the host next starts, "Wiffletree stopped during the run.", and its coordinator is told, as job runs are ended. A run still waiting to deliver its prompt, or waiting for its limit's reset, goes on. + +### Isolation is the provider's, as for threads + +Claude runs as `-p` with stream-json, `--setting-sources ""`, `--strict-mcp-config` with no server (the drivers add theirs later), `dontAsk`, `--tools Read,Glob,Grep,Write,Edit`, allow rules scoped to `///**`, writes to `~/.claude` denied and auto memory off. No system prompt is appended. Its start is checked as a thread's is: any other tool, any MCP server, another permission mode or auto memory stops the run. On this Mac (Claude Code 2.1.317) such a run could not read `/etc/hosts` or write `/tmp`, and received no CLAUDE.md. + +Codex runs `exec --json --ignore-user-config --ignore-rules` in a workspace-write sandbox rooted at the folder, with no network, no temporary folders, web search disabled, the threads' features off and also `shell_tool`, `unified_exec`, `view_image`, which reads any path, and `image_generation`. Its code mode then has only `apply_patch`, which the sandbox holds to the folder. Any MCP call stops the run. No developer instructions are given. + +`--ignore-user-config` does not keep `~/.codex/AGENTS.md` out: a probe on Codex 0.160.0 received it. So an evaluator's Codex gets its own `CODEX_HOME` in Wiffletree's data folder, holding only a link to the user's `auth.json`, so sign-in and token refreshes still use the user's one file. The probe with that home received no AGENTS.md. + +A separate `CODEX_HOME` still lets Codex find the human's skills under `~/.agents/skills`. A request captured by a local endpoint carried them in its skills catalog, and neither `skip_host_skill_discovery` nor turning off `skill_search` removed them. So the evaluator's Codex also runs with an empty `HOME` of its own, beside its Codex home. Only Codex's bundled system skills (imagegen, openai-docs, skill-creator, skill-installer) remain, as Claude keeps its built-in ones. + +The link holds only because Codex rewrites `auth.json` in place. A probe on 0.160.0 confirmed it with a throwaway home: a fake, stale `auth.json` reached through a link, and `CODEX_REFRESH_TOKEN_URL_OVERRIDE` pointing at a local server. Codex refreshed, the refreshed token landed in the link's target with the same inode, and the link stayed a link. The home holds no lock for the ChatGPT token refresh, so two homes share nothing that one home would not. + +The environment is cleared and only `HOME`, `USER`, `LOGNAME`, `PATH`, `SHELL`, `TMPDIR`, `LANG` and the `LC_*` locale variables are passed on, so no `WORKSPACE_TOKEN`, `WORKSPACE_SOCKET` or other variable of Wiffletree's or the human's sessions reaches the run or anything it starts. + +### The model is checked, pinned and never replaced + +`start_evaluation` refuses a provider this Mac has not found signed in, a model the provider's CLI did not list, full IDs such as `claude-opus-5-5` and aliases such as `opus` alike, and an effort the model does not offer. Until the CLI's list has been read, a run is refused with that reason and the service reads the list, so the call succeeds when repeated; Wiffletree never checks against a guess. The profile is pinned on the run's session as given; Who runs what is never consulted. + +An alias can move to a newer model without notice, so the run records the model the provider names: Claude's in its start record, Codex's in the `turn_context` of the session record in the evaluator's Codex home, since its JSON stream names none. A run started on a full ID whose provider names another model fails at once, with the reason for the human. A run on an alias records which model it got. The coordinator may see the reported model in `list_evaluations` and the end message. A run whose provider is past its limit when it starts is marked waiting and its turn is held until the reset, as any pinned agent's is. A turn that fails at a limit is retried in the same session at the reset, and the run shows waiting meanwhile. + +### Authorization is the human's, on an Inbox card + +Each project has an evaluator permission: none yet, off, app or desktop. The first `start_evaluation` in a project with none raises one Inbox card, `evaluators:`, offering App only, Full desktop and Not in this project, and holds the run as waiting. Answering sets the permission and admits or stops the held runs; any other answer is refused, and leaving the card is "Ask me later". `Command::SetEvaluatorPermission` lets the human change it later; it is human-only, and no tool sets it. Archiving a project's coordinator, which is how a project closes, sets it to off and stops the project's runs. + +A run's mode must be allowed: desktop needs desktop, app needs app or desktop. Until ticket 4, desktop is refused as not available yet. + +### One screen, one queue + +App-only runs start as soon as they are admitted. A desktop run needs the screen lock, which one run holds at a time across every project; the others stay queued, oldest first. The lock is released when the holding run ends. Ticket 4 adds the Start-now click between taking the lock and starting. + +### The coordinator hears only the outcome + +When a run ends, Wiffletree messages its coordinator: `Run · model · folder · `, naming the model the provider reported, or the one asked for when none was. `list_evaluations` gives state, mode, the model asked for and the one reported, folder and times, and a waiting run's reset time. A failure's reason is shown to the human only, since a provider's error can quote the model's text. diff --git a/openspec/changes/ui-evaluators/proposal.md b/openspec/changes/ui-evaluators/proposal.md new file mode 100644 index 00000000..f8f8f674 --- /dev/null +++ b/openspec/changes/ui-evaluators/proposal.md @@ -0,0 +1,36 @@ +## Why + +The UX eval-and-improve loop is paused because only the human can start its evaluators: fresh, blind agents on an exact model that read a prompt, drive the app under test and write a report. A coordinator should be able to start one itself, on the model the loop names, and be woken when it ends, without ever seeing what the evaluator thought. + +On November 4, 2026 Michael approved `ui/mockups.html` and the plan behind it, with both screen-control modes: **App only**, which drives just the app under test through its automation socket, and **Full desktop**, which uses Claude's or Codex's own computer use. + +## What Changes + +- **Evaluator runs.** A new kind of launch, not a ticket agent and in no worktree. A run belongs to a project and has an id, a provider, model and effort, a mode, a run folder, a state (queued, waiting, running, finished, failed or stopped) and its times. Each run is a fresh provider session; its transcript is kept for the human. +- **Blind and isolated, enforced by the provider.** No shell, web, repository, brain, history, memory or workspace coordination tools, and none of the human's CLAUDE.md, AGENTS.md, settings, skills or MCP servers. It reads and writes only inside its run folder. Its first message is the folder's `prompt.md`, with nothing added. No Wiffletree session variable reaches it. +- **A pinned model.** A run uses only the provider, model and effort it was started with. An unknown model, an effort the model does not offer or a provider not signed in is refused at start. The run records the model its provider names; one started on a full ID that the provider runs as another model fails. At a usage limit the run waits for the reset and says when; it never falls back. +- **Coordinator tools.** `start_evaluation(run_folder, provider, model, effort, mode)` returns the run's id; `list_evaluations` lists the project's runs; `stop_evaluation` stops one. The coordinator never receives a transcript or report text. +- **Off by default, per project.** The first `start_evaluation` in a project raises an Inbox card (screen 1) and holds the run. The human chooses App only, Full desktop or Not in this project; no agent can change it, and closing the project turns it off. Refusals name the reason. +- **Queue and screen lock.** App-only runs run side by side. Full-desktop runs share one queue for every project and one screen lock. +- **The completion wake-up** holds only the run's id, state, model, folder and duration. +- **The app driver** (ticket 3): an MCP tool over the automation socket and a test copy of the app launched with `~/ux-eval/launch.sh`'s isolation. +- **The desktop driver** (ticket 4): each provider's computer-use server through the helper app the spike chooses, Settings › Screen control, the Start-now card, Esc and Stop, and the menu bar indicator. +- **The UI** (ticket 5): the Agents panel's Evaluations group, the run page and the banners. + +## Capabilities + +### New Capabilities + +- `ui-evaluators`: blind, model-pinned evaluator runs a coordinator starts, the human authorizes and watches, and Wiffletree isolates, queues and reports on. + +### Modified Capabilities + +None. Evaluator runs take no part in tickets, model selection or the agent tree. + +## Impact + +- `workspace-core`: `Role::Evaluator`, the evaluation records and commands. +- `workspace-host`: `evaluations.rs`, the evaluator launch in `provider/evaluator.rs`, the three coordinator tools, the Inbox card's answer, the scheduler, `coordinator.md`. +- `workspace-desktop`: the Inbox card (generic options until ticket 5), the Evaluations group and the run page (ticket 5), Settings › Screen control (ticket 4). +- `docs/FEATURES.md`, `design/README.md`. +- Older stores load unchanged: the new tables start empty and a project with no choice is off. diff --git a/openspec/changes/ui-evaluators/specs/ui-evaluators/spec.md b/openspec/changes/ui-evaluators/specs/ui-evaluators/spec.md new file mode 100644 index 00000000..8bed434c --- /dev/null +++ b/openspec/changes/ui-evaluators/specs/ui-evaluators/spec.md @@ -0,0 +1,105 @@ +## ADDED Requirements + +### Requirement: A coordinator starts an evaluator run + +The workspace SHALL offer `start_evaluation` with `run_folder`, `provider`, `model`, `effort` and `mode` (`app` or `desktop`) to the project coordinator only. The run folder SHALL be an absolute path to an existing folder holding a `prompt.md` file, and SHALL NOT be inside a Git repository or worktree, a workspace repository or Wiffletree's data folder. A valid call SHALL create a run of the caller's project and return its id; a refused call SHALL create nothing and SHALL name the reason. + +#### Scenario: A valid start + +- **WHEN** a coordinator whose project allows app-only evaluators starts an app run on a signed-in provider's listed model and effort, with a valid folder +- **THEN** the tool returns a new run id, and the run starts in a fresh session + +#### Scenario: A bad folder + +- **WHEN** the folder is relative, missing, has no `prompt.md`, or is inside a repository, worktree or Wiffletree's data folder +- **THEN** the call is refused with the reason and no run is created + +#### Scenario: A worker asks + +- **WHEN** an implementer, tester or reviewer calls `start_evaluation` +- **THEN** the call is refused + +### Requirement: A run uses exactly the model it was started with + +A run SHALL use only the provider, model and effort it was started with, and SHALL NOT consult Who runs what or fall back to another model. A provider not signed in on this Mac, a model its CLI did not list, whether a full ID or an alias, and an effort the model does not offer SHALL be refused at start. The run SHALL record the model its provider names when it starts, and a run started on a full ID whose provider names another model SHALL fail at once. A run whose provider is at its usage limit SHALL wait, showing the reset time, and SHALL continue on the same model after the reset. + +#### Scenario: Unknown model + +- **WHEN** `start_evaluation` names a model the provider does not offer +- **THEN** the call is refused, naming the model and the provider + +#### Scenario: The list is not read yet + +- **WHEN** `start_evaluation` names a provider whose CLI's model list Wiffletree has not read +- **THEN** the call is refused with that reason, and Wiffletree reads the list so a repeated call can succeed + +#### Scenario: The model moved + +- **WHEN** a run started on `claude-opus-5-5` and Claude names another model at its start +- **THEN** the run fails, with the reason recorded for the human, and runs no further + +#### Scenario: An alias + +- **WHEN** a run started on `opus` and Claude names `claude-opus-5-5` +- **THEN** the run goes on and records `claude-opus-5-5` + +#### Scenario: At a limit + +- **WHEN** a run is admitted while its provider's limit is reached +- **THEN** the run is waiting with the reset time, and no other model starts it + +### Requirement: A run is blind and isolated by its provider + +A run SHALL have no shell, web, repository, brain, memory, history or workspace tools, and none of the human's provider instructions, settings, skills or MCP servers. It SHALL read and write only inside its run folder. Its first message SHALL be the content of `prompt.md` with nothing added, and no instructions SHALL be appended to its system prompt. Its environment SHALL carry no Wiffletree session variable. A Claude run that starts with another tool, an MCP server, another permission mode or auto memory SHALL be stopped; so SHALL a Codex run that calls any MCP tool. + +#### Scenario: Claude's launch + +- **WHEN** a Claude run starts +- **THEN** it runs with no setting sources, a strict and empty MCP configuration, `dontAsk`, only the read and write file tools, and allow rules for its folder alone + +#### Scenario: Codex's launch + +- **WHEN** a Codex run starts +- **THEN** it ignores the user's config and rules, runs in a workspace-write sandbox rooted at its folder with no network or web search, has no shell, and uses a Codex home of its own that holds only the user's sign-in + +### Requirement: The human authorizes evaluators per project + +Each project SHALL have an evaluator permission of off, app or desktop, which only the human sets; a project with no choice is off. The first `start_evaluation` in a project with no choice SHALL raise one Inbox card offering App only, Full desktop and Not in this project, and SHALL hold the run until it is answered. Closing the project SHALL turn the permission off. A refusal SHALL name the reason, such as "Full desktop is not allowed in this project. Ask the human." + +#### Scenario: First run in a project + +- **WHEN** a coordinator starts a run in a project with no choice +- **THEN** an Inbox card is raised, the run is waiting, and the tool says the human decides + +#### Scenario: The human allows app only + +- **WHEN** the human answers the card with App only +- **THEN** the held app runs start and the held desktop runs are stopped + +#### Scenario: Refused mode + +- **WHEN** a project allows app only and a coordinator starts a desktop run +- **THEN** the call is refused: "Full desktop is not allowed in this project. Ask the human." + +#### Scenario: Closing the project + +- **WHEN** the human archives the project's coordinator +- **THEN** the permission is off and the project's unfinished runs are stopped + +### Requirement: Full-desktop runs take turns at the screen + +App-only runs SHALL run side by side. Full-desktop runs SHALL share one queue for every project and one screen lock: only the run holding the lock runs, the others wait queued in the order they were started, and the lock passes on when the holder ends. + +#### Scenario: Two desktop runs + +- **WHEN** two desktop runs are admitted +- **THEN** the first runs and the second stays queued until the first ends + +### Requirement: The coordinator learns the outcome, never the content + +When a run ends, Wiffletree SHALL message its coordinator with only the run's id, state, model, folder and duration. `list_evaluations` SHALL list the project's runs with their state, mode, the model asked for and the one reported, folder, times and a waiting run's reset, and no transcript, reply or failure text. `stop_evaluation` SHALL stop one of the project's runs; the human can stop any run. + +#### Scenario: A run finishes + +- **WHEN** a run's turn ends on its own +- **THEN** the coordinator receives "Run finished · model · folder · ", with none of the run's reply diff --git a/openspec/changes/ui-evaluators/tasks.md b/openspec/changes/ui-evaluators/tasks.md new file mode 100644 index 00000000..5c1e0a04 --- /dev/null +++ b/openspec/changes/ui-evaluators/tasks.md @@ -0,0 +1,40 @@ +## 1. Ticket 1 · Desktop computer-use spike + +- [ ] 1.1 Claude `-p` with `claude --computer-use-mcp` through `--mcp-config`: tools, app access, lock. +- [ ] 1.2 Codex `exec` with its computer-use server enabled through `-c`. +- [ ] 1.3 Which process macOS asks for Screen Recording and Accessibility, and a helper app that alone holds them. +- [ ] 1.4 Stop and Esc for each provider. Findings in `docs/computer-use-spike.md`. + +## 2. Ticket 2 · Evaluator runs and `start_evaluation` + +- [x] 2.1 Core: `Role::Evaluator`, the evaluation records, `Command::SetEvaluatorPermission`, `Command::StopEvaluation`, `Command::Evaluations`. +- [x] 2.2 Host: the `evaluations` and `evaluator_permissions` tables; a run's session outside the agent tree, archived when the run ends. +- [x] 2.3 The isolated launch for Claude and Codex, its start and tool-call checks, the cleared environment and Codex's own home. +- [x] 2.4 The model check against detection and the providers' listed models; waiting at a limit with the reset time. +- [x] 2.5 `start_evaluation`, `list_evaluations` and `stop_evaluation`, coordinator only; desktop refused as not available yet. +- [x] 2.6 The Inbox card and its answer; archiving the coordinator turns evaluators off. +- [x] 2.7 The admission queue and the screen lock. +- [x] 2.8 The completion message; the human's stop. +- [x] 2.9 `coordinator.md`, `docs/FEATURES.md`, the tool descriptions. +- [x] 2.10 Tests: launch arguments, start checks, refusals, waiting without fallback, the completion message, the queue, the tool contract. + +## 3. Ticket 3 · App driver and test target + +- [ ] 3.1 An MCP driver over `workspace-desktop --automation-socket`: screenshot, click, type, key, scroll, drag, resize; offered to app runs only, through the run's strict MCP configuration. +- [ ] 3.2 The app captures its own window in-process if it can, so no Screen Recording grant is needed. +- [ ] 3.3 The host launches the test copy with `launch.sh`'s isolation: a disposable profile, fake provider CLIs, an empty `GH_CONFIG_DIR`, no workspace variables, and a switch that stops login-shell PATH adoption. + +## 4. Ticket 4 · Desktop driver + +- [ ] 4.1 Each provider's computer-use server connected to desktop runs through the spike's helper app; `computer_use` on for those Codex runs only. +- [ ] 4.2 Settings › Screen control: live permission and provider checks (screen 2). +- [ ] 4.3 The Start-now card between taking the screen lock and starting (screen 3); Later and Skip. +- [ ] 4.4 Esc and Stop, and the menu bar indicator while a run holds the screen. +- [ ] 4.5 Lift the refusal of desktop runs. + +## 5. Ticket 5 · UI + +- [ ] 5.1 The Agents panel's Evaluations group with every state (screen 4). +- [ ] 5.2 A run's page with its model, mode, folder, steps and Stop (screen 5). +- [ ] 5.3 The banners, the Inbox card as drawn (screen 1) and the project menu's UI evaluators choice. +- [ ] 5.4 Archive this change. diff --git a/openspec/changes/ui-evaluators/ui/mockups.html b/openspec/changes/ui-evaluators/ui/mockups.html new file mode 100644 index 00000000..b6cd261a --- /dev/null +++ b/openspec/changes/ui-evaluators/ui/mockups.html @@ -0,0 +1,292 @@ + + + + + + +Wiffletree · UI evaluators study + + + + + + + + +
+
Wiffletree · UI evaluators study
+

Evaluators that drive the screen, only when you allow it

+

A coordinator can start a UI evaluator: a fresh, blind agent on one exact model that gets only a prompt and a run folder, drives an app, and writes a report there. There are two modes. App only drives just the app under test, through its automation socket. Full desktop uses Claude's or Codex's own computer use and can reach any app.

+ Approved by Michael, 2026-11-04 +
+
    +
  • Off by default. The first time a coordinator asks, you choose a mode for that project (screen 1). Each later full-desktop run still waits for you to press Start (screen 3).
    • +
  • Blind. An evaluator has no repository, shell, brain, history, memory, or any of your CLAUDE.md, settings, skills or MCP servers. It reads its prompt and writes only to its run folder. The coordinator gets the run's state and folder, never the transcript.
    • +
  • Pinned model. A run names its provider, model and effort and never falls back. At a usage limit it waits for the reset, and says so (screen 4).
    • +
  • Recommendations to confirm: app-only runs may run side by side, since each drives its own copy of the app. Full-desktop runs go one at a time, through one queue shared by every project.
    • +
+
+ + +
+
1 · Inbox · a coordinator's first evaluator run in a project
+
+
+
+
+
Wiffletree: UX eval-and-improve loop · Coordinator
+

Allow UI evaluators in this project?

+

The coordinator wants to start 2 evaluators: Claude Opus 5.5 and Codex Sol 6.1. Each gets only its prompt and its own folder under ~/ux-eval/runs/.

+
App onlyDrives only the app under test, a separate copy on a test profile. It can't see or click your other apps, and you can keep working.
+
Full desktopCan see your whole screen and click any app you approve, including mail and chat. Each run waits for you to press Start and step away.
+
Not in this projectThe coordinator is told evaluators are off.
+
Ask me laterAllow app only
+
+
+

One choice per project, kept until you change it on the project's menu › UI evaluators. Closing the project turns it off. A coordinator can never change it.

+
+
+
What the coordinator gets back
+
+
+ AllowedApp only + Refusedstart_evaluation with mode: desktop: "Full desktop is not allowed in this project. Ask the human." + On finishRun run-91fb906c finished · folder ~/ux-eval/runs/run-91fb906c · 41 min +
+
+

The wake-up holds only the state, the folder and the time. The coordinator reads the report from the folder, as the loop does today.

+
+
+
+ + +
+
2 · Settings › Screen control · needed for full desktop only
+
+
+
+

Screen control

+

Full-desktop evaluators run inside Wiffletree Evaluator, a small helper app, so only evaluators get these permissions and your other agents never do.

+

macOS permissions

+
●Screen RecordingLets an evaluator see the screen.Granted
+
●AccessibilityLets an evaluator click, type and scroll.Open System Settings…
+

Providers

+
●Claude computer useMax plan, signed in through claude.ai · Claude Code 2.1.317Ready
+
●Codex computer useCodex is not signed in on this Mac.How to fix
+
Check again
+
+

Checked, never assumed. Each row comes from a live check. A full-desktop run on a provider that isn't Ready doesn't start, and its card says which row to fix.

+
+
+
App only needs nothing here
+
+

App-only evaluators take screenshots of their own test copy and send input straight to it. They need no macOS permission, so this page shows only a line about them:

+
●App onlyWorks for apps with an automation socket. Today that means Wiffletree.Ready
+
+
+
+
+ + +
+
3 · Full desktop · waiting for you, then running
+
+
+
Inbox, when a full-desktop run reaches the front of the queue
+
+
+
Wiffletree: UX eval-and-improve loop · Coordinator
+

Ready to use your screen

+

2 runs in turn: Claude Opus 5.5, then Codex Sol 6.1, about 40 minutes each. Don't use this Mac until they finish. Press Esc anywhere to stop.

+
Skip these runsLaterStart now
+
+
+

Nothing takes the screen without this click. Later keeps the runs at the front of the queue. Skip tells the coordinator they didn't run.

+
+
+
While it runs
+
+
● Evaluator using this MacWi-FiWed 14:12
+ +
The menu bar item stays visible even when the evaluator hides other windows. Its menu holds Stop and opens the run.
+
+
An app-only run, for comparison
+
+ +
A quiet band in the main window. Your screen and other apps are untouched.
+
+
+
+
+ + +
+
4 · Agents panel · the project's Evaluations group
+
+
+
+
+
Vyttle
+
UX eval-and-improve loop 2
+
◐ Evaluating · 1 running
+
computer-use agents
+
Theme files
+
+
+

Evaluations

5 runs
+
◐run-3c07d1a2 App onlyPinned Claude Opus 5.5 · highRunning
12 min
+
○run-8e41b6f0 App onlyPinned Codex Sol 6.1 · highWaiting
Codex resets 15:00
+
○run-a19f2c55 Full desktopPinned Claude Opus 5.5 · highQueued
2nd for the screen
+
●run-91fb906c App onlyPinned Claude Opus 5.5 · high · 41 minFinished
13:20
+
●run-5d2e0b9a App onlyPinned Codex Sol 6.1 · stopped by youStopped
12:58
+
+
+

Runs sit beside tickets, not inside them. An evaluator belongs to the project, never to a ticket or worktree. Waiting means its pinned model is at its limit. It never moves to another model.

+
+
+
The states
+
+
+ QueuedWaiting for the screen. Full desktop only. + WaitingIts pinned model is at its limit, or it needs your Start click. + RunningDriving the app now. + FinishedIt stopped on its own. Whether its report is valid is the coordinator's call. + Failed StoppedEnded with an error, or by you, the Esc key or the coordinator. +
+
+
+
+
+ + +
+
5 · One run
+
+
+
+

run-3c07d1a2

RunningStop
+
+ ModelPinned Claude Opus 5.5 · high · no fallback + ModeApp only · test copy of Wiffletree on /tmp/wiffletree-ux-eval + Folder~/ux-eval/runs/run-3c07d1a2 · Show in Finder + Started byUX eval-and-improve loop · Coordinator · 14:00 + It hasIts prompt, the app driver and its folder. No shell, repository or your settings. +
+

Recent steps

+
14:11:52screenshot · saved screenshots/models-dark-1280.png
+
14:11:47click 1182, 64 · Models
+
14:11:40resize window to 1280 × 820
+
+

You can watch, the coordinator can't. The steps and screenshots show here for you. The coordinator sees only the state and the folder, so it can't leak the loop's goals to the evaluator or learn its reasoning early.

+
+
+
Out of scope for this change
+
+
    +
  • Sending a message to a running evaluator. It runs one prompt to the end.
    • +
  • An editable list of apps a full-desktop run may never open. Instead, Wiffletree approves only the apps the run names when Claude asks for app access.
    • +
  • Evaluators on a paired remote Mac.
    • +
+
+
+
+
+ +